Unverified Commit 2a5f00c9 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Consolidate TaskCache deletion state and reuse lookup hash

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent e4f5918b
......@@ -462,7 +462,10 @@ impl TurboTasksBackend {
Vec::new(),
Some(roots),
Vec::<Vec<SnapshotItem>>::new(),
|task_type_hash| self.storage.task_cache_ids(task_type_hash),
|hash, added_ids, deleted_ids| {
self.storage
.reconcile_task_cache_bucket(hash, added_ids, deleted_ids)
},
)
{
panic!("gc_for_testing: failed to persist GC roots: {err:?}");
......
......@@ -1406,8 +1406,6 @@ impl TurboTasksBackend {
.get_persistent_task_type()
.expect("a GC-deleted task must have a task type"),
);
self.storage
.note_task_cache_deletion(task_type_hash, task_id);
return SnapshotItem::Delete {
task_id,
task_type_hash,
......@@ -1502,7 +1500,10 @@ impl TurboTasksBackend {
suspended_operations,
gc_roots_to_persist,
task_snapshots,
|task_type_hash| self.storage.task_cache_ids(task_type_hash),
|task_type_hash, added_ids, deleted_ids| {
self.storage
.reconcile_task_cache_bucket(task_type_hash, added_ids, deleted_ids)
},
)?;
span.record("snapshot_meta", display(snapshot_meta));
......@@ -1780,14 +1781,10 @@ impl TurboTasksBackend {
// Step 2: Check backing storage using borrowed components (no box needed yet). A restore
// records every candidate in the in-memory hash bucket before returning.
let restored_task = if transient {
None
} else {
ctx.task_by_type(native_fn, this, arg_ref)
};
// Task exists in backing storage. task_by_type has inserted it into the in-memory cache.
let task_id = if let Some((task_id, _stored_type)) = restored_task {
let task_id = if !transient
&& let Some((task_id, _stored_type)) =
ctx.task_by_type(native_fn, this, arg_ref, task_type_hash)
{
self.track_cache_hit_by_fn(native_fn);
task_id
} else {
......
......@@ -36,6 +36,7 @@ use crate::{
storage::{SpecificTaskDataCategory, StorageWriteGuard, TaskEntryGuard, TrackOutcome},
storage_schema::{TaskStorage, TaskStorageAccessors},
},
backing_storage::TaskTypeHash,
data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState, TransientTask},
};
......@@ -209,6 +210,7 @@ pub trait ExecuteContext<'e>: Sized {
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn DynTaskInputs,
task_type_hash: Option<TaskTypeHash>,
) -> Option<(TaskId, CachedTaskTypeArc)>;
fn debug_get_task_description(&self, task_id: TaskId) -> String;
......@@ -967,7 +969,7 @@ impl<'e> ExecuteContextImpl<'e> {
this,
arg,
} = &**task_type;
let _ = self.task_by_type(native_fn, *this, arg.as_ref());
let _ = self.task_by_type(native_fn, *this, arg.as_ref(), None);
}
// Only call the callback if no category is still being restored by another thread.
// If so, Phase 3 calls the callback after all categories are fully restored.
......@@ -1400,13 +1402,15 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn DynTaskInputs,
task_type_hash: Option<TaskTypeHash>,
) -> Option<(TaskId, CachedTaskTypeArc)> {
if !self.backend.should_restore() {
return None;
}
let task_type_hash =
crate::backing_storage::compute_task_type_hash_from_components(native_fn, this, arg);
let task_type_hash = task_type_hash.unwrap_or_else(|| {
crate::backing_storage::compute_task_type_hash_from_components(native_fn, this, arg)
});
// With persistence enabled, a published bucket includes every on-disk candidate.
// Direct-by-ID restores during eviction must not repeatedly read the DB or recurse into
// task storage when that bucket is already in memory.
......@@ -1415,10 +1419,10 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
}
// Get candidates from backing storage (hash-based lookup may return multiple).
let (task_type_hash, candidates) = self
let candidates = self
.backend
.backing_storage
.lookup_task_candidates(native_fn, this, arg)
.lookup_task_candidates(native_fn, this, task_type_hash)
.expect("Failed to lookup task ids");
// Restore every candidate's full type before locking the TaskCache bucket, even when
......@@ -1443,11 +1447,12 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
.task_cache
.entry(task_type_hash)
.or_default();
// This deduplication appears O(N²), but hash collisions are so rare that buckets almost
// always contain one item; more than four entries would be an effectively impossible event.
for (stored_type, candidate_id) in restored_candidates {
bucket.insert(stored_type, candidate_id);
if !bucket.is_empty() {
// A concurrent restore populated this complete bucket while we read from disk.
// Prefer the current bucket: reinserting our older candidates could revive a deletion.
return bucket.find(native_fn, this, arg);
}
bucket.insert_all(restored_candidates);
matching_candidate
}
......
......@@ -221,9 +221,6 @@ pub struct Storage {
pub task_cache: TaskCache,
/// Transient task types do not participate in persisted hash-collision buckets.
pub transient_task_cache: FxDashMap<CachedTaskTypeArc, TaskId>,
/// GC tombstones retained until their IDs are evicted (or shutdown), so a later
/// collision-bucket write cannot re-persist a deleted ID when eviction was skipped.
task_cache_deleted: FxDashMap<TaskTypeHash, TaskIdBucket>,
}
impl Storage {
......@@ -258,7 +255,6 @@ impl Storage {
restored: Event::new(|| || "Storage::restored".to_string()),
task_cache: FxDashMap::default(),
transient_task_cache: FxDashMap::default(),
task_cache_deleted: FxDashMap::default(),
}
}
......@@ -292,36 +288,37 @@ impl Storage {
}
}
#[cfg(test)]
pub fn task_cache_ids(&self, task_type_hash: TaskTypeHash) -> TaskIdBucket {
let mut ids = self
.task_cache
self.task_cache
.get(&task_type_hash)
.map(|bucket| bucket.task_ids())
.unwrap_or_default();
self.filter_deleted_task_cache_ids(task_type_hash, &mut ids);
ids
}
fn filter_deleted_task_cache_ids(&self, hash: TaskTypeHash, ids: &mut TaskIdBucket) {
if let Some(deleted) = self.task_cache_deleted.get(&hash) {
ids.retain(|id| !deleted.contains(id));
}
.unwrap_or_default()
}
pub fn note_task_cache_deletion(&self, hash: TaskTypeHash, task_id: TaskId) {
let mut deleted = self.task_cache_deleted.entry(hash).or_default();
if !deleted.contains(&task_id) {
deleted.push(task_id);
/// Reconcile a snapshot's changes after the task-storage shard locks are released. The
/// bucket stays complete and keeps soft-deleted IDs for resurrection, while its disk ID list
/// omits them until a put marks them live or eviction removes them. An absent bucket must stay
/// absent: creating an empty one could hide colliding candidates still on disk.
pub fn reconcile_task_cache_bucket(
&self,
hash: TaskTypeHash,
added_ids: &[TaskId],
deleted_ids: &[TaskId],
) -> TaskIdBucket {
if let Some(mut bucket) = self.task_cache.get_mut(&hash) {
for task_id in added_ids {
bucket.mark_live(*task_id);
}
for task_id in deleted_ids {
bucket.mark_deleted(*task_id);
}
bucket.task_ids()
} else {
TaskIdBucket::new()
}
}
fn clear_task_cache_deletion(&self, hash: TaskTypeHash, task_id: TaskId) {
self.task_cache_deleted.remove_if_mut(&hash, |_, ids| {
ids.retain(|id| *id != task_id);
ids.is_empty()
});
}
/// Eviction holds a task-storage shard, while creation locks the transient cache first.
/// Try the matching cache shard without blocking and defer the removal on contention.
fn try_evict_transient_task_cache_id(
......@@ -381,10 +378,6 @@ impl Storage {
} else {
false
};
drop(shard);
if deleted {
self.clear_task_cache_deletion(hash, task_id);
}
Some(removed)
}
......@@ -399,9 +392,6 @@ impl Storage {
removed
}
});
if deleted {
self.clear_task_cache_deletion(hash, task_id);
}
removed
}
......@@ -732,7 +722,6 @@ impl Storage {
pub(crate) fn drop_task_cache(&self) {
drop_contents(&self.task_cache);
drop_contents(&self.transient_task_cache);
drop_contents(&self.task_cache_deleted);
}
/// Evict tasks from in-memory storage after a successful snapshot.
......@@ -1304,10 +1293,14 @@ impl<P> Drop for SnapshotShardIter<'_, P> {
#[cfg(test)]
mod tests {
use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::TaskId;
use turbo_tasks::{
RawVc, TaskId,
backend::{CachedTaskType, CachedTaskTypeArc},
macro_helpers::VTABLE_DEFAULT,
};
use super::{SpecificTaskDataCategory, Storage, TrackOutcome};
use crate::backing_storage::SnapshotItem;
use crate::backing_storage::{SnapshotItem, TaskCacheBucket};
fn non_transient_task(id: u32) -> TaskId {
// TRANSIENT_TASK_BIT is 0x2000_0000; any id without that bit is non-transient.
......@@ -1333,21 +1326,80 @@ mod tests {
let deleted = non_transient_task(1);
let survivor = non_transient_task(2);
let created_later = non_transient_task(3);
storage.note_task_cache_deletion(hash, deleted);
// The in-memory bucket still contains the deleted ID until eviction (as on canary),
// but both the initial delete and a later colliding put must filter it on disk.
let mut first = smallvec::smallvec![deleted, survivor];
storage.filter_deleted_task_cache_ids(hash, &mut first);
assert_eq!(first.as_slice(), &[survivor]);
let mut later = smallvec::smallvec![deleted, survivor, created_later];
storage.filter_deleted_task_cache_ids(hash, &mut later);
assert_eq!(later.as_slice(), &[survivor, created_later]);
// Once eviction removes the deleted task from the in-memory cache, its tombstone
// can be forgotten. A missing bucket is also safe: disk has already been filtered.
assert!(!storage.evict_task_cache_id(hash, deleted, true));
assert!(storage.task_cache_deleted.get(&hash).is_none());
let task_type = |this| {
CachedTaskTypeArc::new(CachedTaskType {
native_fn: &VTABLE_DEFAULT,
this,
arg: Box::new(()),
})
};
let first_type = task_type(None);
let second_type = task_type(Some(RawVc::task_output(non_transient_task(91))));
let third_type = task_type(Some(RawVc::task_output(non_transient_task(92))));
let mut bucket = TaskCacheBucket::default();
bucket.insert(first_type.clone(), deleted);
bucket.insert(second_type, survivor);
storage.task_cache.insert(hash, bucket);
assert_eq!(
storage
.reconcile_task_cache_bucket(hash, &[], &[deleted])
.as_slice(),
&[survivor]
);
// The deleted ID remains discoverable for resurrection, but is absent from disk writes.
assert_eq!(storage.task_cache_ids(hash).as_slice(), &[survivor]);
assert_eq!(
storage
.task_cache
.get(&hash)
.unwrap()
.find(&VTABLE_DEFAULT, None, &()),
Some((deleted, first_type))
);
storage
.task_cache
.get_mut(&hash)
.unwrap()
.insert(third_type, created_later);
assert_eq!(
storage.task_cache_ids(hash).as_slice(),
&[survivor, created_later],
"later colliding writes must not re-persist a deleted ID"
);
// A snapshot of the resurrected task marks the same ID live again.
assert_eq!(
storage
.reconcile_task_cache_bucket(hash, &[deleted], &[])
.as_slice(),
&[deleted, survivor, created_later]
);
assert_eq!(
storage.task_cache_ids(hash).as_slice(),
&[deleted, survivor, created_later]
);
storage.reconcile_task_cache_bucket(hash, &[], &[deleted]);
assert!(storage.evict_task_cache_id(hash, deleted, true));
assert_eq!(
storage.task_cache_ids(hash).as_slice(),
&[survivor, created_later]
);
// An absent bucket is not replaced by an empty one, which would mask unseen disk IDs.
let absent_hash = 42u64.to_le_bytes();
storage.reconcile_task_cache_bucket(absent_hash, &[], &[deleted]);
assert!(storage.task_cache.get(&absent_hash).is_none());
let singleton_hash = 43u64.to_le_bytes();
let mut singleton = TaskCacheBucket::default();
singleton.insert(task_type(None), deleted);
storage.task_cache.insert(singleton_hash, singleton);
storage.reconcile_task_cache_bucket(singleton_hash, &[], &[deleted]);
assert!(storage.task_cache.get(&singleton_hash).is_some());
assert!(storage.task_cache_ids(singleton_hash).is_empty());
assert!(storage.evict_task_cache_id(singleton_hash, deleted, true));
assert!(storage.task_cache.get(&singleton_hash).is_none());
}
/// A process fn that returns a non-empty SnapshotItem so the iterator doesn't
......
......@@ -16,12 +16,19 @@ pub type TaskCache = FxDashMap<TaskTypeHash, TaskCacheBucket>;
/// All task types and IDs sharing one persistent task-type hash.
///
/// Hash collisions are exceptionally rare, so almost every bucket contains exactly one entry.
/// One `(CachedTaskTypeArc, TaskId)` pair fits inline; a second, exceptionally rare collision
/// spills to the heap. When persistence is enabled, buckets are populated after every
/// One `(CachedTaskTypeArc, CachedTaskId)` pair fits inline; a second, exceptionally rare
/// collision spills to the heap. When persistence is enabled, buckets are populated after every
/// on-disk candidate for the hash has been read, so lookups need no further persistence read.
#[derive(Clone, Copy)]
struct CachedTaskId {
id: TaskId,
/// Soft-deleted tasks remain discoverable for resurrection, but not in snapshot writes.
deleted: bool,
}
#[derive(Clone, Default)]
pub struct TaskCacheBucket {
entries: SmallVec<[(CachedTaskTypeArc, TaskId); 1]>,
entries: SmallVec<[(CachedTaskTypeArc, CachedTaskId); 1]>,
}
impl TaskCacheBucket {
......@@ -34,7 +41,7 @@ impl TaskCacheBucket {
self.entries
.iter()
.find(|(task_type, _)| task_type.eq_components(native_fn, this, arg))
.map(|(task_type, task_id)| (*task_id, task_type.clone()))
.map(|(task_type, cached)| (cached.id, task_type.clone()))
}
pub fn insert(&mut self, task_type: CachedTaskTypeArc, task_id: TaskId) {
......@@ -43,17 +50,58 @@ impl TaskCacheBucket {
.iter()
.any(|(candidate, _)| candidate == &task_type)
{
self.entries.push((task_type, task_id));
self.entries.push((
task_type,
CachedTaskId {
id: task_id,
deleted: false,
},
));
}
}
/// Insert all candidates after verifying that no concurrent restore filled this bucket.
pub fn insert_all(
&mut self,
candidates: impl IntoIterator<Item = (CachedTaskTypeArc, TaskId)>,
) {
// Deduplication is cheap: a hash collision is exceptional and most buckets contain one ID.
for (task_type, task_id) in candidates {
self.insert(task_type, task_id);
}
}
pub fn mark_deleted(&mut self, task_id: TaskId) {
if let Some((_, cached)) = self
.entries
.iter_mut()
.find(|(_, cached)| cached.id == task_id)
{
cached.deleted = true;
}
}
pub fn mark_live(&mut self, task_id: TaskId) {
if let Some((_, cached)) = self
.entries
.iter_mut()
.find(|(_, cached)| cached.id == task_id)
{
cached.deleted = false;
}
}
pub fn task_ids(&self) -> TaskIdBucket {
self.entries.iter().map(|(_, task_id)| *task_id).collect()
self.entries
.iter()
.filter(|(_, cached)| !cached.deleted)
.map(|(_, cached)| cached.id)
.collect()
}
pub fn remove_id(&mut self, task_id: TaskId) -> bool {
let len = self.entries.len();
self.entries.retain(|(_, id)| *id != task_id);
self.entries.retain(|(_, cached)| cached.id != task_id);
self.entries.len() != len
}
......@@ -62,7 +110,24 @@ impl TaskCacheBucket {
}
pub fn is_singleton_id(&self, task_id: TaskId) -> bool {
self.entries.len() == 1 && self.entries[0].1 == task_id
self.entries.len() == 1 && self.entries[0].1.id == task_id
}
}
#[cfg(test)]
mod tests {
use std::mem::size_of;
use turbo_tasks::{TaskId, backend::CachedTaskTypeArc};
use super::CachedTaskId;
#[test]
fn deletion_marker_fits_in_existing_bucket_entry_padding() {
assert_eq!(
size_of::<(CachedTaskTypeArc, CachedTaskId)>(),
size_of::<(CachedTaskTypeArc, TaskId)>(),
);
}
}
......
......@@ -14,7 +14,7 @@ use turbo_bincode::{
};
use turbo_persistence::CommitStats;
use turbo_tasks::{
DynTaskInputs, RawVc, TaskId,
RawVc, TaskId,
macro_helpers::NativeFunction,
panic_hooks::{PanicHookGuard, register_panic_hook},
parallel,
......@@ -23,10 +23,7 @@ use turbo_tasks::{
use crate::{
GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, TtlCounter, storage_schema::TaskStorage},
backing_storage::{
SnapshotItem, SnapshotMeta, TaskIdBucket, TaskTypeHash,
compute_task_type_hash_from_components,
},
backing_storage::{SnapshotItem, SnapshotMeta, TaskIdBucket, TaskTypeHash},
database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
db_versioning::handle_db_versioning,
......@@ -298,7 +295,7 @@ impl TurboBackingStorage {
operations: Vec<Arc<AnyOperation>>,
roots: Option<Vec<(TaskId, TtlCounter)>>,
snapshots: Vec<I>,
task_cache_bucket: impl Fn(TaskTypeHash) -> TaskIdBucket + Sync,
task_cache_bucket: impl Fn(TaskTypeHash, &[TaskId], &[TaskId]) -> TaskIdBucket + Sync,
) -> Result<SnapshotMeta>
where
I: IntoIterator<Item = SnapshotItem> + Send + Sync,
......@@ -421,7 +418,7 @@ impl TurboBackingStorage {
)
.entered();
for (hash, (added_ids, deleted_ids)) in &task_cache_changes {
let mut task_ids = task_cache_bucket(*hash);
let mut task_ids = task_cache_bucket(*hash, added_ids, deleted_ids);
for task_id in added_ids {
if !task_ids.contains(task_id) {
task_ids.push(*task_id);
......@@ -460,12 +457,11 @@ impl TurboBackingStorage {
&self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn DynTaskInputs,
) -> Result<(TaskTypeHash, TaskIdBucket)> {
hash: TaskTypeHash,
) -> Result<TaskIdBucket> {
let inner = &*self.inner;
let hash = compute_task_type_hash_from_components(native_fn, this, arg);
if inner.database.is_empty() {
return Ok((hash, SmallVec::new()));
return Ok(SmallVec::new());
}
let Some(buffer) = inner
.database
......@@ -474,10 +470,10 @@ impl TurboBackingStorage {
format!("Looking up task id for {native_fn:?}(this={this:?}) from database failed")
})?
else {
return Ok((hash, SmallVec::new()));
return Ok(SmallVec::new());
};
Ok((hash, decode_task_ids(Borrow::<[u8]>::borrow(&buffer))?))
decode_task_ids(Borrow::<[u8]>::borrow(&buffer))
}
/// Reads the stored `category` for `task_id`.
......@@ -729,7 +725,7 @@ mod tests {
task_type_hash: Some(collision_hash),
},
]],
|_| bucket.clone(),
|_, _, _| bucket.clone(),
)?;
assert_eq!(
......@@ -772,7 +768,7 @@ mod tests {
}],
],
// Match canary's in-memory lifecycle: deletion has not evicted the stale ID yet.
|_| smallvec::smallvec![deleted, survivor, created],
|_, _, _| smallvec::smallvec![deleted, survivor, created],
)?;
assert_eq!(
task_cache_ids(&storage.inner.database, hash)?,
......@@ -904,7 +900,7 @@ mod tests {
task_id: deleted_id,
task_type_hash: collision_hash.to_le_bytes(),
}]],
|_| smallvec::smallvec![survivor_id],
|_, _, _| smallvec::smallvec![survivor_id],
)?;
let db = &storage.inner.database;
......@@ -929,7 +925,7 @@ mod tests {
task_id: survivor_id,
task_type_hash: collision_hash.to_le_bytes(),
}]],
|_| SmallVec::new(),
|_, _, _| SmallVec::new(),
)?;
assert!(
db.get(KeySpace::TaskCache, &collision_hash.to_le_bytes())?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment