Unverified Commit 804ceca2 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Resolve TaskCache hash collisions with single-value snapshot buckets

<!-- NEXT_JS_LLM -->
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 50199e6b
...@@ -147,10 +147,13 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> { ...@@ -147,10 +147,13 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
/// tombstones last (see [`CollectorEntryValue::sort_rank`]). /// tombstones last (see [`CollectorEntryValue::sort_rank`]).
/// This method does not deduplicate entries. /// This method does not deduplicate entries.
/// ///
/// In debug builds, asserts that SingleValue families have no duplicate keys. /// A SingleValue family may contain duplicate keys here: which of them a reader sees is
pub fn sorted(&mut self, family_kind: FamilyKind) -> (&[CollectorEntry<K>], usize) { /// undefined, so a caller that writes a key twice must supersede both entries with a later
/// write after [`WriteBatch::flush`](crate::WriteBatch::flush) (see
/// [`WriteBatch::put`](crate::WriteBatch::put)).
pub fn sorted(&mut self, _family_kind: FamilyKind) -> (&[CollectorEntry<K>], usize) {
// We can use unstable sort because the relative order of equal elements // We can use unstable sort because the relative order of equal elements
// doesn't matter — duplicates are either disallowed (SingleValue) or // doesn't matter — duplicates are either superseded (SingleValue) or
// allowed without deduplication (MultiValue). // allowed without deduplication (MultiValue).
self.entries.sort_unstable_by(|a, b| { self.entries.sort_unstable_by(|a, b| {
a.key a.key
...@@ -158,29 +161,6 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> { ...@@ -158,29 +161,6 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
.then_with(|| a.value.sort_rank().cmp(&b.value.sort_rank())) .then_with(|| a.value.sort_rank().cmp(&b.value.sort_rank()))
}); });
#[cfg(debug_assertions)]
if family_kind == FamilyKind::SingleValue {
// WriteBatch callers must not insert duplicate keys for SingleValue families.
for w in self.entries.windows(2) {
if w[0].key == w[1].key {
let mut key_buf = Vec::new();
w[0].key.data.write_to(&mut key_buf);
panic!(
"WriteBatch invariant violation: SingleValue family has duplicate key \
(hash={:#018x}, key={})",
w[0].key.hash,
key_buf
.iter()
.map(|b| format!("{b:02x}"))
.collect::<String>(),
);
}
}
}
// Suppress unused variable warning in release builds
let _ = family_kind;
(&self.entries, self.total_key_size) (&self.entries, self.total_key_size)
} }
......
...@@ -1894,29 +1894,52 @@ fn multi_value_delete_with_compaction_interleaved(#[case] mmap: bool) -> Result< ...@@ -1894,29 +1894,52 @@ fn multi_value_delete_with_compaction_interleaved(#[case] mmap: bool) -> Result<
} }
// Tests for the new WriteBatch semantics: // Tests for the new WriteBatch semantics:
// - SingleValue: duplicate keys in the same batch is a user error (panics in debug builds) // - SingleValue: a write after `flush` supersedes earlier writes of the same key in the batch
// - MultiValue: tombstone only shadows entries from older SSTs, not entries in the same batch // - MultiValue: tombstone only shadows entries from older SSTs, not entries in the same batch
#[test] #[rstest]
#[cfg(debug_assertions)] #[case(true)]
#[should_panic(expected = "WriteBatch invariant violation: SingleValue family has duplicate key")] #[case(false)]
fn single_value_duplicate_key_panics() { fn single_value_write_after_flush_supersedes_duplicates(#[case] mmap: bool) -> Result<()> {
let tempdir = tempfile::tempdir().unwrap(); let tempdir = tempfile::tempdir()?;
let path = tempdir.path(); let path = tempdir.path();
let duplicated = vec![1u8];
let deleted = vec![2u8];
let open = || {
let mut config = multi_value_config_with_mmap(mmap);
config.family_configs[0].kind = FamilyKind::SingleValue;
open_db_with_config(path, config)
};
// For SingleValue, writing the same key twice in one batch should panic in debug builds. {
let key = vec![1u8]; let db = open()?;
let batch = db.write_batch()?;
// Two writes of one key between flushes: either may win until superseded.
batch.put(0, duplicated.clone(), vec![10u8].into())?;
batch.put(0, duplicated.clone(), vec![20u8].into())?;
batch.put(0, deleted.clone(), vec![30u8].into())?;
unsafe { batch.flush(0)? };
batch.put(0, duplicated.clone(), vec![40u8].into())?;
batch.delete(0, deleted.clone())?;
db.commit_write_batch(batch)?;
let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler( assert_eq!(db.get(0, &duplicated)?.as_deref(), Some(&[40u8][..]));
path.to_path_buf(), assert!(db.get(0, &deleted)?.is_none());
RayonParallelScheduler, let results = db.batch_get(0, &[duplicated.clone(), deleted.clone()])?;
) assert_eq!(results[0].as_deref(), Some(&[40u8][..]));
.unwrap(); assert!(results[1].is_none());
let batch = db.write_batch().unwrap(); db.full_compact()?;
batch.put(0, key.clone(), vec![10u8].into()).unwrap(); assert_eq!(db.get(0, &duplicated)?.as_deref(), Some(&[40u8][..]));
batch.put(0, key.clone(), vec![20u8].into()).unwrap(); // should panic assert!(db.get(0, &deleted)?.is_none());
db.commit_write_batch(batch).unwrap(); // panics during commit db.shutdown()?;
}
let db = open()?;
assert_eq!(db.get(0, &duplicated)?.as_deref(), Some(&[40u8][..]));
assert!(db.get(0, &deleted)?.is_none());
db.shutdown()?;
Ok(())
} }
#[rstest] #[rstest]
......
...@@ -232,6 +232,12 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize ...@@ -232,6 +232,12 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
} }
/// Puts a key-value pair into the write batch. /// Puts a key-value pair into the write batch.
///
/// In a [`FamilyKind::SingleValue`] family each key should be written once per batch. A key may
/// be written again only after [`WriteBatch::flush`] of the family: entries flushed earlier are
/// stored in older SST files, so the later write supersedes them for reads and compaction.
/// Which of several writes of one key between two flushes wins is undefined; a caller that
/// produces such duplicates must supersede all of them after a flush.
pub fn put(&self, family: u32, key: K, value: ValueBuffer<'_>) -> Result<()> { pub fn put(&self, family: u32, key: K, value: ValueBuffer<'_>) -> Result<()> {
let state = self.thread_local_state(); let state = self.thread_local_state();
let collector = self.thread_local_collector_mut(state, family)?; let collector = self.thread_local_collector_mut(state, family)?;
......
...@@ -46,6 +46,9 @@ pub(super) fn resurrect_deleted<'e, C: ExecuteContext<'e>>( ...@@ -46,6 +46,9 @@ pub(super) fn resurrect_deleted<'e, C: ExecuteContext<'e>>(
// restores All before marking it deleted, and eviction removes deleted tasks as // restores All before marking it deleted, and eviction removes deleted tasks as
// whole entries. // whole entries.
task.set_new_task(true); task.set_new_task(true);
// The old type's disk entry may have been tombstoned while collision siblings remain.
// A resurrected task is not a freshly allocated ID with a canonical empty-bucket read.
task.set_task_cache_needs_read(true);
let _ = task.track_modification(SpecificTaskDataCategory::Data, "gc_resurrected"); let _ = task.track_modification(SpecificTaskDataCategory::Data, "gc_resurrected");
// Mark dirty so it is rescheduled. GC has already dropped its edges and data, so it needs // Mark dirty so it is rescheduled. GC has already dropped its edges and data, so it needs
......
...@@ -10,7 +10,7 @@ mod update_collectible; ...@@ -10,7 +10,7 @@ mod update_collectible;
use std::{ use std::{
fmt::{Debug, Display, Formatter}, fmt::{Debug, Display, Formatter},
ops::{Deref, DerefMut}, ops::{Deref, DerefMut},
sync::Arc, sync::{Arc, atomic::Ordering},
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
...@@ -198,17 +198,16 @@ pub trait ExecuteContext<'e>: Sized { ...@@ -198,17 +198,16 @@ pub trait ExecuteContext<'e>: Sized {
/// ///
/// Uses hash-based lookup which may return multiple candidates due to hash collisions, /// Uses hash-based lookup which may return multiple candidates due to hash collisions,
/// then verifies each candidate by comparing the stored `persistent_task_type`. /// then verifies each candidate by comparing the stored `persistent_task_type`.
/// Returns `Some((task_id, task_type))` if a matching task is found, where `task_type` is /// Returns `(matching_task, disk_bucket_was_empty)`. The latter is true only if a guarded
/// the existing `CachedTaskTypeArc` from storage (avoiding a duplicate /// lookup proved that the entire committed hash bucket was empty; a freshly allocated task
/// allocation). /// may carry it into its first snapshot. It is false for collisions and noncanonical reads.
///
/// Accepts exploded components so the caller does not need to box the argument before calling. /// Accepts exploded components so the caller does not need to box the argument before calling.
fn task_by_type( fn task_by_type(
&mut self, &mut self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: &dyn DynTaskInputs, arg: &dyn DynTaskInputs,
) -> Option<(TaskId, CachedTaskTypeArc)>; ) -> (Option<(TaskId, CachedTaskTypeArc)>, bool);
fn debug_get_task_description(&self, task_id: TaskId) -> String; fn debug_get_task_description(&self, task_id: TaskId) -> String;
} }
...@@ -1395,12 +1394,17 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> { ...@@ -1395,12 +1394,17 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: &dyn DynTaskInputs, arg: &dyn DynTaskInputs,
) -> Option<(TaskId, CachedTaskTypeArc)> { ) -> (Option<(TaskId, CachedTaskTypeArc)>, bool) {
if !self.backend.should_restore() { if !self.backend.should_restore() {
return None; return (None, false);
} }
// Get candidates from backing storage (hash-based lookup may return multiple) // The exclusion phase waits for this guarded operation to finish or suspend. No suspend
// point occurs before the caller publishes the newly allocated task/cache entry. If a
// snapshot has already released exclusion, its disk writes are still in flight, so a
// read during that interval cannot prove the next snapshot's disk baseline.
let canonical = matches!(self.phase, ExecutePhase::Normal { guard: Some(_) })
&& !self.backend.pending_db_commit.load(Ordering::Acquire);
let candidates = self let candidates = self
.backend .backend
.backing_storage .backing_storage
...@@ -1409,15 +1413,18 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> { ...@@ -1409,15 +1413,18 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
// Verify each candidate by comparing the stored persistent_task_type. // Verify each candidate by comparing the stored persistent_task_type.
// Only rarely is there more than one candidate, so no need for parallelization. // Only rarely is there more than one candidate, so no need for parallelization.
if candidates.is_empty() {
return (None, canonical);
}
for candidate_id in candidates { for candidate_id in candidates {
let task = self.task(candidate_id, TaskDataCategory::Data); let task = self.task(candidate_id, TaskDataCategory::Data);
if let Some(stored_type) = task.get_persistent_task_type() if let Some(stored_type) = task.get_persistent_task_type()
&& stored_type.eq_components(native_fn, this, arg) && stored_type.eq_components(native_fn, this, arg)
{ {
return Some((candidate_id, stored_type.clone())); return (Some((candidate_id, stored_type.clone())), false);
} }
} }
None (None, false)
} }
fn debug_get_task_description(&self, task_id: TaskId) -> String { fn debug_get_task_description(&self, task_id: TaskId) -> String {
......
...@@ -230,6 +230,11 @@ struct TaskStorageSchema { ...@@ -230,6 +230,11 @@ struct TaskStorageSchema {
#[field(storage = "flag", category = "transient")] #[field(storage = "flag", category = "transient")]
pub new_task: bool, pub new_task: bool,
/// Whether a fresh task's lookup did not prove its entire TaskCache hash bucket empty.
/// Stored lazily because all 16 TaskFlags bits are in use; absent means false.
#[field(storage = "direct", category = "transient")]
pub task_cache_needs_read: bool,
/// GC soft-deletion marker. Set by the garbage collector when a task is marked for deletion. /// GC soft-deletion marker. Set by the garbage collector when a task is marked for deletion.
#[field(storage = "flag", category = "transient")] #[field(storage = "flag", category = "transient")]
deleted: bool, deleted: bool,
......
...@@ -6,6 +6,8 @@ use turbo_tasks::{ ...@@ -6,6 +6,8 @@ use turbo_tasks::{
}; };
use turbo_tasks_hash::Xxh3Hash64Hasher; use turbo_tasks_hash::Xxh3Hash64Hasher;
// The TaskCache key is a stable hash, so different task types may share one encoded
// list-valued disk entry. Snapshot reconciliation merges their IDs by hash.
pub type TaskTypeHash = [u8; 8]; pub type TaskTypeHash = [u8; 8];
/// A single item yielded by the snapshot iterator during persistence: either a put (persist a /// A single item yielded by the snapshot iterator during persistence: either a put (persist a
...@@ -21,6 +23,9 @@ pub enum SnapshotItem { ...@@ -21,6 +23,9 @@ pub enum SnapshotItem {
data: Option<TurboBincodeBuffer>, data: Option<TurboBincodeBuffer>,
/// Task type for new tasks that need to be added to the task cache /// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>, task_type_hash: Option<TaskTypeHash>,
/// Only a freshly allocated task whose guarded lookup proved the entire disk hash bucket
/// empty may omit the TaskCache read on its first snapshot. Resurrection sets this false.
task_cache_empty_on_creation: bool,
}, },
Delete { Delete {
task_id: TaskId, task_id: TaskId,
......
...@@ -47,8 +47,8 @@ impl KeySpace { ...@@ -47,8 +47,8 @@ impl KeySpace {
}, },
KeySpace::TaskCache => FamilyConfig { KeySpace::TaskCache => FamilyConfig {
name: self.name(), name: self.name(),
// TaskCache uses hash-based lookups with potential collisions. // TaskCache maps each hash to one list of TaskIds to handle rare collisions.
kind: FamilyKind::MultiValue, kind: FamilyKind::SingleValue,
compression: Compression::Lz4, compression: Compression::Lz4,
}, },
} }
......
...@@ -6,7 +6,6 @@ use std::{ ...@@ -6,7 +6,6 @@ use std::{
}; };
use anyhow::{Ok, Result}; use anyhow::{Ok, Result};
use smallvec::SmallVec;
use turbo_persistence::{ use turbo_persistence::{
ArcBytes, CommitStats, CompactConfig, DbConfig, KeyBase, StoreKey, TurboPersistence, ArcBytes, CommitStats, CompactConfig, DbConfig, KeyBase, StoreKey, TurboPersistence,
ValueBuffer, ValueBuffer,
...@@ -99,13 +98,6 @@ impl TurboKeyValueDatabase { ...@@ -99,13 +98,6 @@ impl TurboKeyValueDatabase {
self.db.batch_get(key_space as usize, keys) self.db.batch_get(key_space as usize, keys)
} }
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
pub fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
self.db.get_multiple(key_space as usize, &key)
}
pub fn write_batch(&self) -> Result<TurboWriteBatch<'_>> { pub fn write_batch(&self) -> Result<TurboWriteBatch<'_>> {
Ok(TurboWriteBatch { Ok(TurboWriteBatch {
batch: self.db.write_batch()?, batch: self.db.write_batch()?,
...@@ -226,24 +218,10 @@ impl<'a> TurboWriteBatch<'a> { ...@@ -226,24 +218,10 @@ impl<'a> TurboWriteBatch<'a> {
} }
/// Writes a delete (tombstone) for `key` into the write batch. /// Writes a delete (tombstone) for `key` into the write batch.
///
/// Use [`Self::delete_value`] to remove a single mapping from a MultiValue KeySpace
pub fn delete(&self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> { pub fn delete(&self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
self.batch.delete(key_space as u32, key.into_static()) self.batch.delete(key_space as u32, key.into_static())
} }
/// Writes a tombstone for a single `key` -> `value` mapping, leaving other values under `key`
/// intact. Only valid for `MultiValue` families (`TaskCache`).
pub fn delete_value(
&self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
self.batch
.delete_value(key_space as u32, key.into_static(), value.into())
}
/// Flushes a key space of the write batch, reducing the amount of buffered memory used. /// Flushes a key space of the write batch, reducing the amount of buffered memory used.
/// Does not commit any data persistently. /// Does not commit any data persistently.
/// ///
......
...@@ -60,26 +60,26 @@ pub fn with_entry_in_shard<K: Eq + Hash, V, S: BuildHasher + Clone, Q: ?Sized, R ...@@ -60,26 +60,26 @@ pub fn with_entry_in_shard<K: Eq + Hash, V, S: BuildHasher + Clone, Q: ?Sized, R
pub enum TryLockAndRemove { pub enum TryLockAndRemove {
/// The shard lock was acquired and a matching entry was removed. /// The shard lock was acquired and a matching entry was removed.
Removed, Removed,
/// The shard lock was acquired but no matching entry was present. /// The shard lock was acquired but the key was absent or held another value.
NotFound, NotFound,
/// The shard lock was contended; the caller should retry later after releasing /// The shard lock was contended; the caller should retry later after releasing
/// any other locks they are holding. /// any other locks they are holding.
WouldBlock, WouldBlock,
} }
/// Remove `key` from `map` without blocking on shard contention. /// Remove `key` only if it still maps to `expected`, without blocking on shard contention.
/// ///
/// Intended for call sites that already hold another lock and want to avoid a /// The value check prevents an eviction of an old TaskId from removing a newer mapping for
/// cyclic wait. On contention (`WouldBlock`), the caller is expected to defer the /// the same task type. On contention the caller defers removal until releasing other locks.
/// removal and retry after dropping the other lock.
pub fn try_lock_and_remove< pub fn try_lock_and_remove<
K: Eq + Hash + AsRef<Q>, K: Eq + Hash + AsRef<Q>,
V, V: Copy + PartialEq,
Q: Eq + Hash + ?Sized, Q: Eq + Hash + ?Sized,
S: BuildHasher + Clone, S: BuildHasher + Clone,
>( >(
map: &DashMap<K, V, S>, map: &DashMap<K, V, S>,
key: &Q, key: &Q,
expected: V,
) -> TryLockAndRemove { ) -> TryLockAndRemove {
let hasher = map.hasher(); let hasher = map.hasher();
let hash = hasher.hash_one(key); let hash = hasher.hash_one(key);
...@@ -88,10 +88,54 @@ pub fn try_lock_and_remove< ...@@ -88,10 +88,54 @@ pub fn try_lock_and_remove<
return TryLockAndRemove::WouldBlock; return TryLockAndRemove::WouldBlock;
}; };
match shard.find_entry(hash, |(k, _v)| k.as_ref() == key) { match shard.find_entry(hash, |(k, _v)| k.as_ref() == key) {
Ok(entry) => { Ok(entry) if entry.get().1 == expected => {
entry.remove(); entry.remove();
TryLockAndRemove::Removed TryLockAndRemove::Removed
} }
Err(_) => TryLockAndRemove::NotFound, _ => TryLockAndRemove::NotFound,
}
}
#[cfg(test)]
mod tests {
use std::hash::BuildHasher;
use turbo_tasks::FxDashMap;
use super::{TryLockAndRemove, get_shard, try_lock_and_remove};
#[test]
fn try_lock_and_remove_requires_matching_values() {
let map = FxDashMap::<String, u32>::default();
map.insert("task".to_string(), 2);
assert!(matches!(
try_lock_and_remove(&map, "task", 1),
TryLockAndRemove::NotFound
));
assert_eq!(map.get("task").map(|id| *id), Some(2));
assert!(matches!(
try_lock_and_remove(&map, "task", 2),
TryLockAndRemove::Removed
));
assert!(map.get("task").is_none());
}
#[test]
fn contended_removal_rechecks_the_expected_value_after_unlocking() {
let map = FxDashMap::<String, u32>::default();
map.insert("task".to_string(), 1);
let hash = map.hasher().hash_one("task");
let shard_guard = get_shard(&map, hash).write();
assert!(matches!(
try_lock_and_remove(&map, "task", 1),
TryLockAndRemove::WouldBlock
));
drop(shard_guard);
// A new mapping can be published after the contended eviction releases its other lock.
map.insert("task".to_string(), 2);
assert!(map.remove_if("task", |_, id| *id == 1).is_none());
assert_eq!(map.get("task").map(|id| *id), Some(2));
assert!(map.remove_if("task", |_, id| *id == 2).is_some());
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment