Unverified Commit 42771179 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Preserve cold TaskCache collision siblings and compact deletion flag

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 2a5f00c9
...@@ -299,14 +299,15 @@ impl Storage { ...@@ -299,14 +299,15 @@ impl Storage {
/// Reconcile a snapshot's changes after the task-storage shard locks are released. The /// Reconcile a snapshot's changes after the task-storage shard locks are released. The
/// bucket stays complete and keeps soft-deleted IDs for resurrection, while its disk ID list /// bucket stays complete and keeps soft-deleted IDs for resurrection, while its disk ID list
/// omits them until a put marks them live or eviction removes them. An absent bucket must stay /// omits them until a put marks them live or eviction removes them. An absent bucket must stay
/// absent: creating an empty one could hide colliding candidates still on disk. /// absent: creating an empty one could hide colliding candidates still on disk. The writer
/// reads the existing disk bucket on this cold path before applying the snapshot changes.
pub fn reconcile_task_cache_bucket( pub fn reconcile_task_cache_bucket(
&self, &self,
hash: TaskTypeHash, hash: TaskTypeHash,
added_ids: &[TaskId], added_ids: &[TaskId],
deleted_ids: &[TaskId], deleted_ids: &[TaskId],
) -> TaskIdBucket { ) -> Option<TaskIdBucket> {
if let Some(mut bucket) = self.task_cache.get_mut(&hash) { self.task_cache.get_mut(&hash).map(|mut bucket| {
for task_id in added_ids { for task_id in added_ids {
bucket.mark_live(*task_id); bucket.mark_live(*task_id);
} }
...@@ -314,9 +315,7 @@ impl Storage { ...@@ -314,9 +315,7 @@ impl Storage {
bucket.mark_deleted(*task_id); bucket.mark_deleted(*task_id);
} }
bucket.task_ids() bucket.task_ids()
} else { })
TaskIdBucket::new()
}
} }
/// Eviction holds a task-storage shard, while creation locks the transient cache first. /// Eviction holds a task-storage shard, while creation locks the transient cache first.
...@@ -1343,6 +1342,7 @@ mod tests { ...@@ -1343,6 +1342,7 @@ mod tests {
assert_eq!( assert_eq!(
storage storage
.reconcile_task_cache_bucket(hash, &[], &[deleted]) .reconcile_task_cache_bucket(hash, &[], &[deleted])
.unwrap()
.as_slice(), .as_slice(),
&[survivor] &[survivor]
); );
...@@ -1372,6 +1372,7 @@ mod tests { ...@@ -1372,6 +1372,7 @@ mod tests {
assert_eq!( assert_eq!(
storage storage
.reconcile_task_cache_bucket(hash, &[deleted], &[]) .reconcile_task_cache_bucket(hash, &[deleted], &[])
.unwrap()
.as_slice(), .as_slice(),
&[deleted, survivor, created_later] &[deleted, survivor, created_later]
); );
...@@ -1388,7 +1389,11 @@ mod tests { ...@@ -1388,7 +1389,11 @@ mod tests {
// An absent bucket is not replaced by an empty one, which would mask unseen disk IDs. // An absent bucket is not replaced by an empty one, which would mask unseen disk IDs.
let absent_hash = 42u64.to_le_bytes(); let absent_hash = 42u64.to_le_bytes();
storage.reconcile_task_cache_bucket(absent_hash, &[], &[deleted]); assert!(
storage
.reconcile_task_cache_bucket(absent_hash, &[], &[deleted])
.is_none()
);
assert!(storage.task_cache.get(&absent_hash).is_none()); assert!(storage.task_cache.get(&absent_hash).is_none());
let singleton_hash = 43u64.to_le_bytes(); let singleton_hash = 43u64.to_le_bytes();
......
...@@ -17,13 +17,43 @@ pub type TaskCache = FxDashMap<TaskTypeHash, TaskCacheBucket>; ...@@ -17,13 +17,43 @@ pub type TaskCache = FxDashMap<TaskTypeHash, TaskCacheBucket>;
/// ///
/// Hash collisions are exceptionally rare, so almost every bucket contains exactly one entry. /// Hash collisions are exceptionally rare, so almost every bucket contains exactly one entry.
/// One `(CachedTaskTypeArc, CachedTaskId)` pair fits inline; a second, exceptionally rare /// One `(CachedTaskTypeArc, CachedTaskId)` pair fits inline; a second, exceptionally rare
/// collision spills to the heap. When persistence is enabled, buckets are populated after every /// collision spills to the heap. `TaskId` values use at most 31 bits, so the unused high bit
/// on-disk candidate for the hash has been read, so lookups need no further persistence read. /// stores the deletion marker without growing bucket entries on either 32- or 64-bit targets.
/// When persistence is enabled, buckets are populated after every on-disk candidate for the
/// hash has been read, so lookups need no further persistence read.
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
struct CachedTaskId { struct CachedTaskId(u32);
id: TaskId,
/// Soft-deleted tasks remain discoverable for resurrection, but not in snapshot writes. // Assert at compile time on every target: the marker must not grow a cache entry on WASI.
deleted: bool, const _: () = assert!(
std::mem::size_of::<(CachedTaskTypeArc, CachedTaskId)>()
== std::mem::size_of::<(CachedTaskTypeArc, TaskId)>()
);
impl CachedTaskId {
const DELETED_BIT: u32 = 1 << 31;
fn new(id: TaskId) -> Self {
let raw = *id;
debug_assert_eq!(raw & Self::DELETED_BIT, 0);
Self(raw)
}
fn id(self) -> TaskId {
TaskId::new(self.0 & !Self::DELETED_BIT).expect("a cached task ID must be non-zero")
}
fn is_deleted(self) -> bool {
self.0 & Self::DELETED_BIT != 0
}
fn set_deleted(&mut self, deleted: bool) {
if deleted {
self.0 |= Self::DELETED_BIT;
} else {
self.0 &= !Self::DELETED_BIT;
}
}
} }
#[derive(Clone, Default)] #[derive(Clone, Default)]
...@@ -41,7 +71,7 @@ impl TaskCacheBucket { ...@@ -41,7 +71,7 @@ impl TaskCacheBucket {
self.entries self.entries
.iter() .iter()
.find(|(task_type, _)| task_type.eq_components(native_fn, this, arg)) .find(|(task_type, _)| task_type.eq_components(native_fn, this, arg))
.map(|(task_type, cached)| (cached.id, task_type.clone())) .map(|(task_type, cached)| (cached.id(), task_type.clone()))
} }
pub fn insert(&mut self, task_type: CachedTaskTypeArc, task_id: TaskId) { pub fn insert(&mut self, task_type: CachedTaskTypeArc, task_id: TaskId) {
...@@ -50,13 +80,7 @@ impl TaskCacheBucket { ...@@ -50,13 +80,7 @@ impl TaskCacheBucket {
.iter() .iter()
.any(|(candidate, _)| candidate == &task_type) .any(|(candidate, _)| candidate == &task_type)
{ {
self.entries.push(( self.entries.push((task_type, CachedTaskId::new(task_id)));
task_type,
CachedTaskId {
id: task_id,
deleted: false,
},
));
} }
} }
...@@ -75,9 +99,9 @@ impl TaskCacheBucket { ...@@ -75,9 +99,9 @@ impl TaskCacheBucket {
if let Some((_, cached)) = self if let Some((_, cached)) = self
.entries .entries
.iter_mut() .iter_mut()
.find(|(_, cached)| cached.id == task_id) .find(|(_, cached)| cached.id() == task_id)
{ {
cached.deleted = true; cached.set_deleted(true);
} }
} }
...@@ -85,23 +109,23 @@ impl TaskCacheBucket { ...@@ -85,23 +109,23 @@ impl TaskCacheBucket {
if let Some((_, cached)) = self if let Some((_, cached)) = self
.entries .entries
.iter_mut() .iter_mut()
.find(|(_, cached)| cached.id == task_id) .find(|(_, cached)| cached.id() == task_id)
{ {
cached.deleted = false; cached.set_deleted(false);
} }
} }
pub fn task_ids(&self) -> TaskIdBucket { pub fn task_ids(&self) -> TaskIdBucket {
self.entries self.entries
.iter() .iter()
.filter(|(_, cached)| !cached.deleted) .filter(|(_, cached)| !cached.is_deleted())
.map(|(_, cached)| cached.id) .map(|(_, cached)| cached.id())
.collect() .collect()
} }
pub fn remove_id(&mut self, task_id: TaskId) -> bool { pub fn remove_id(&mut self, task_id: TaskId) -> bool {
let len = self.entries.len(); let len = self.entries.len();
self.entries.retain(|(_, cached)| cached.id != task_id); self.entries.retain(|(_, cached)| cached.id() != task_id);
self.entries.len() != len self.entries.len() != len
} }
...@@ -110,24 +134,7 @@ impl TaskCacheBucket { ...@@ -110,24 +134,7 @@ impl TaskCacheBucket {
} }
pub fn is_singleton_id(&self, task_id: TaskId) -> bool { pub fn is_singleton_id(&self, task_id: TaskId) -> bool {
self.entries.len() == 1 && self.entries[0].1.id == task_id self.entries.len() == 1 && self.entries[0].1.id() == task_id
}
}
#[cfg(test)]
mod tests {
use std::mem::size_of;
use turbo_tasks::{TaskId, backend::CachedTaskTypeArc};
use super::CachedTaskId;
#[test]
fn deletion_marker_fits_in_existing_bucket_entry_padding() {
assert_eq!(
size_of::<(CachedTaskTypeArc, CachedTaskId)>(),
size_of::<(CachedTaskTypeArc, TaskId)>(),
);
} }
} }
......
...@@ -295,7 +295,7 @@ impl TurboBackingStorage { ...@@ -295,7 +295,7 @@ impl TurboBackingStorage {
operations: Vec<Arc<AnyOperation>>, operations: Vec<Arc<AnyOperation>>,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
snapshots: Vec<I>, snapshots: Vec<I>,
task_cache_bucket: impl Fn(TaskTypeHash, &[TaskId], &[TaskId]) -> TaskIdBucket + Sync, task_cache_bucket: impl Fn(TaskTypeHash, &[TaskId], &[TaskId]) -> Option<TaskIdBucket> + Sync,
) -> Result<SnapshotMeta> ) -> Result<SnapshotMeta>
where where
I: IntoIterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
...@@ -418,7 +418,17 @@ impl TurboBackingStorage { ...@@ -418,7 +418,17 @@ impl TurboBackingStorage {
) )
.entered(); .entered();
for (hash, (added_ids, deleted_ids)) in &task_cache_changes { for (hash, (added_ids, deleted_ids)) in &task_cache_changes {
let mut task_ids = task_cache_bucket(*hash, added_ids, deleted_ids); // Direct-by-ID restore can leave this hash absent from the in-memory cache.
// Only on that cold path, read the old disk bucket before applying changes;
// otherwise a GC deletion or colliding put would erase unseen siblings.
let mut task_ids = match task_cache_bucket(*hash, added_ids, deleted_ids) {
Some(ids) => ids,
None => batch
.get(KeySpace::TaskCache, hash)?
.map(|bytes| decode_task_ids(Borrow::<[u8]>::borrow(&bytes)))
.transpose()?
.unwrap_or_default(),
};
for task_id in added_ids { for task_id in added_ids {
if !task_ids.contains(task_id) { if !task_ids.contains(task_id) {
task_ids.push(*task_id); task_ids.push(*task_id);
...@@ -725,7 +735,7 @@ mod tests { ...@@ -725,7 +735,7 @@ mod tests {
task_type_hash: Some(collision_hash), task_type_hash: Some(collision_hash),
}, },
]], ]],
|_, _, _| bucket.clone(), |_, _, _| Some(bucket.clone()),
)?; )?;
assert_eq!( assert_eq!(
...@@ -736,6 +746,73 @@ mod tests { ...@@ -736,6 +746,73 @@ mod tests {
Ok(()) Ok(())
} }
/// A GC deletion reached through a direct-by-ID restore may have no in-memory type bucket.
/// The snapshot must still retain siblings that live only in the on-disk collision bucket.
#[cfg_attr(
target_os = "wasi",
ignore = "WASI test host cannot run disk-backed persistence"
)]
#[tokio::test(flavor = "multi_thread")]
async fn snapshot_cold_bucket_keeps_disk_only_collision_siblings() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let db = TurboKeyValueDatabase::new(tempdir.path().to_path_buf(), TEST_STORAGE_OPTIONS)?;
let hash = 0xC0111DEu64;
let deleted = TaskId::try_from(100u32).unwrap();
let sibling = TaskId::try_from(200u32).unwrap();
let another_sibling = TaskId::try_from(300u32).unwrap();
write_task_cache_entry(&db, hash, &[deleted, sibling, another_sibling])?;
let storage = TurboBackingStorage::new_in_memory(db);
storage.save_snapshot(
Vec::new(),
None,
vec![vec![SnapshotItem::Delete {
task_id: deleted,
task_type_hash: hash.to_le_bytes(),
}]],
|_, _, _| None,
)?;
assert_eq!(
task_cache_ids(&storage.inner.database, hash)?,
vec![sibling, another_sibling]
);
storage.inner.database.shutdown()?;
Ok(())
}
#[cfg_attr(
target_os = "wasi",
ignore = "WASI test host cannot run disk-backed persistence"
)]
#[tokio::test(flavor = "multi_thread")]
async fn snapshot_cold_bucket_put_keeps_disk_only_collision_siblings() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let db = TurboKeyValueDatabase::new(tempdir.path().to_path_buf(), TEST_STORAGE_OPTIONS)?;
let hash = 0xC0111DEu64;
let sibling = TaskId::try_from(200u32).unwrap();
let created = TaskId::try_from(300u32).unwrap();
write_task_cache_entry(&db, hash, &[sibling])?;
let storage = TurboBackingStorage::new_in_memory(db);
storage.save_snapshot(
Vec::new(),
None,
vec![vec![SnapshotItem::Put {
task_id: created,
meta: None,
data: None,
task_type_hash: Some(hash.to_le_bytes()),
}]],
|_, _, _| None,
)?;
assert_eq!(
task_cache_ids(&storage.inner.database, hash)?,
vec![sibling, created]
);
storage.inner.database.shutdown()?;
Ok(())
}
#[cfg_attr( #[cfg_attr(
target_os = "wasi", target_os = "wasi",
ignore = "WASI test host cannot run disk-backed persistence" ignore = "WASI test host cannot run disk-backed persistence"
...@@ -768,7 +845,7 @@ mod tests { ...@@ -768,7 +845,7 @@ mod tests {
}], }],
], ],
// Match canary's in-memory lifecycle: deletion has not evicted the stale ID yet. // Match canary's in-memory lifecycle: deletion has not evicted the stale ID yet.
|_, _, _| smallvec::smallvec![deleted, survivor, created], |_, _, _| Some(smallvec::smallvec![deleted, survivor, created]),
)?; )?;
assert_eq!( assert_eq!(
task_cache_ids(&storage.inner.database, hash)?, task_cache_ids(&storage.inner.database, hash)?,
...@@ -900,7 +977,7 @@ mod tests { ...@@ -900,7 +977,7 @@ mod tests {
task_id: deleted_id, task_id: deleted_id,
task_type_hash: collision_hash.to_le_bytes(), task_type_hash: collision_hash.to_le_bytes(),
}]], }]],
|_, _, _| smallvec::smallvec![survivor_id], |_, _, _| Some(smallvec::smallvec![survivor_id]),
)?; )?;
let db = &storage.inner.database; let db = &storage.inner.database;
...@@ -925,7 +1002,7 @@ mod tests { ...@@ -925,7 +1002,7 @@ mod tests {
task_id: survivor_id, task_id: survivor_id,
task_type_hash: collision_hash.to_le_bytes(), task_type_hash: collision_hash.to_le_bytes(),
}]], }]],
|_, _, _| SmallVec::new(), |_, _, _| Some(SmallVec::new()),
)?; )?;
assert!( assert!(
db.get(KeySpace::TaskCache, &collision_hash.to_le_bytes())? db.get(KeySpace::TaskCache, &collision_hash.to_le_bytes())?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment