Unverified Commit 804ceca2 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Resolve TaskCache hash collisions with single-value snapshot buckets

<!-- NEXT_JS_LLM -->
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 50199e6b
......@@ -147,10 +147,13 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
/// tombstones last (see [`CollectorEntryValue::sort_rank`]).
/// This method does not deduplicate entries.
///
/// In debug builds, asserts that SingleValue families have no duplicate keys.
pub fn sorted(&mut self, family_kind: FamilyKind) -> (&[CollectorEntry<K>], usize) {
/// A SingleValue family may contain duplicate keys here: which of them a reader sees is
/// undefined, so a caller that writes a key twice must supersede both entries with a later
/// write after [`WriteBatch::flush`](crate::WriteBatch::flush) (see
/// [`WriteBatch::put`](crate::WriteBatch::put)).
pub fn sorted(&mut self, _family_kind: FamilyKind) -> (&[CollectorEntry<K>], usize) {
// We can use unstable sort because the relative order of equal elements
// doesn't matter — duplicates are either disallowed (SingleValue) or
// doesn't matter — duplicates are either superseded (SingleValue) or
// allowed without deduplication (MultiValue).
self.entries.sort_unstable_by(|a, b| {
a.key
......@@ -158,29 +161,6 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
.then_with(|| a.value.sort_rank().cmp(&b.value.sort_rank()))
});
#[cfg(debug_assertions)]
if family_kind == FamilyKind::SingleValue {
// WriteBatch callers must not insert duplicate keys for SingleValue families.
for w in self.entries.windows(2) {
if w[0].key == w[1].key {
let mut key_buf = Vec::new();
w[0].key.data.write_to(&mut key_buf);
panic!(
"WriteBatch invariant violation: SingleValue family has duplicate key \
(hash={:#018x}, key={})",
w[0].key.hash,
key_buf
.iter()
.map(|b| format!("{b:02x}"))
.collect::<String>(),
);
}
}
}
// Suppress unused variable warning in release builds
let _ = family_kind;
(&self.entries, self.total_key_size)
}
......
......@@ -1894,29 +1894,52 @@ fn multi_value_delete_with_compaction_interleaved(#[case] mmap: bool) -> Result<
}
// Tests for the new WriteBatch semantics:
// - SingleValue: duplicate keys in the same batch is a user error (panics in debug builds)
// - SingleValue: a write after `flush` supersedes earlier writes of the same key in the batch
// - MultiValue: tombstone only shadows entries from older SSTs, not entries in the same batch
#[test]
#[cfg(debug_assertions)]
#[should_panic(expected = "WriteBatch invariant violation: SingleValue family has duplicate key")]
fn single_value_duplicate_key_panics() {
let tempdir = tempfile::tempdir().unwrap();
#[rstest]
#[case(true)]
#[case(false)]
fn single_value_write_after_flush_supersedes_duplicates(#[case] mmap: bool) -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
let duplicated = vec![1u8];
let deleted = vec![2u8];
let open = || {
let mut config = multi_value_config_with_mmap(mmap);
config.family_configs[0].kind = FamilyKind::SingleValue;
open_db_with_config(path, config)
};
// For SingleValue, writing the same key twice in one batch should panic in debug builds.
let key = vec![1u8];
{
let db = open()?;
let batch = db.write_batch()?;
// Two writes of one key between flushes: either may win until superseded.
batch.put(0, duplicated.clone(), vec![10u8].into())?;
batch.put(0, duplicated.clone(), vec![20u8].into())?;
batch.put(0, deleted.clone(), vec![30u8].into())?;
unsafe { batch.flush(0)? };
batch.put(0, duplicated.clone(), vec![40u8].into())?;
batch.delete(0, deleted.clone())?;
db.commit_write_batch(batch)?;
let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)
.unwrap();
assert_eq!(db.get(0, &duplicated)?.as_deref(), Some(&[40u8][..]));
assert!(db.get(0, &deleted)?.is_none());
let results = db.batch_get(0, &[duplicated.clone(), deleted.clone()])?;
assert_eq!(results[0].as_deref(), Some(&[40u8][..]));
assert!(results[1].is_none());
let batch = db.write_batch().unwrap();
batch.put(0, key.clone(), vec![10u8].into()).unwrap();
batch.put(0, key.clone(), vec![20u8].into()).unwrap(); // should panic
db.commit_write_batch(batch).unwrap(); // panics during commit
db.full_compact()?;
assert_eq!(db.get(0, &duplicated)?.as_deref(), Some(&[40u8][..]));
assert!(db.get(0, &deleted)?.is_none());
db.shutdown()?;
}
let db = open()?;
assert_eq!(db.get(0, &duplicated)?.as_deref(), Some(&[40u8][..]));
assert!(db.get(0, &deleted)?.is_none());
db.shutdown()?;
Ok(())
}
#[rstest]
......
......@@ -232,6 +232,12 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
}
/// Puts a key-value pair into the write batch.
///
/// In a [`FamilyKind::SingleValue`] family each key should be written once per batch. A key may
/// be written again only after [`WriteBatch::flush`] of the family: entries flushed earlier are
/// stored in older SST files, so the later write supersedes them for reads and compaction.
/// Which of several writes of one key between two flushes wins is undefined; a caller that
/// produces such duplicates must supersede all of them after a flush.
pub fn put(&self, family: u32, key: K, value: ValueBuffer<'_>) -> Result<()> {
let state = self.thread_local_state();
let collector = self.thread_local_collector_mut(state, family)?;
......
......@@ -46,6 +46,9 @@ pub(super) fn resurrect_deleted<'e, C: ExecuteContext<'e>>(
// restores All before marking it deleted, and eviction removes deleted tasks as
// whole entries.
task.set_new_task(true);
// The old type's disk entry may have been tombstoned while collision siblings remain.
// A resurrected task is not a freshly allocated ID with a canonical empty-bucket read.
task.set_task_cache_needs_read(true);
let _ = task.track_modification(SpecificTaskDataCategory::Data, "gc_resurrected");
// Mark dirty so it is rescheduled. GC has already dropped its edges and data, so it needs
......
......@@ -10,7 +10,7 @@ mod update_collectible;
use std::{
fmt::{Debug, Display, Formatter},
ops::{Deref, DerefMut},
sync::Arc,
sync::{Arc, atomic::Ordering},
};
use anyhow::{Context, Result};
......@@ -198,17 +198,16 @@ pub trait ExecuteContext<'e>: Sized {
///
/// Uses hash-based lookup which may return multiple candidates due to hash collisions,
/// then verifies each candidate by comparing the stored `persistent_task_type`.
/// Returns `Some((task_id, task_type))` if a matching task is found, where `task_type` is
/// the existing `CachedTaskTypeArc` from storage (avoiding a duplicate
/// allocation).
///
/// Returns `(matching_task, disk_bucket_was_empty)`. The latter is true only if a guarded
/// lookup proved that the entire committed hash bucket was empty; a freshly allocated task
/// may carry it into its first snapshot. It is false for collisions and noncanonical reads.
/// Accepts exploded components so the caller does not need to box the argument before calling.
fn task_by_type(
&mut self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn DynTaskInputs,
) -> Option<(TaskId, CachedTaskTypeArc)>;
) -> (Option<(TaskId, CachedTaskTypeArc)>, bool);
fn debug_get_task_description(&self, task_id: TaskId) -> String;
}
......@@ -1395,12 +1394,17 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn DynTaskInputs,
) -> Option<(TaskId, CachedTaskTypeArc)> {
) -> (Option<(TaskId, CachedTaskTypeArc)>, bool) {
if !self.backend.should_restore() {
return None;
return (None, false);
}
// Get candidates from backing storage (hash-based lookup may return multiple)
// The exclusion phase waits for this guarded operation to finish or suspend. No suspend
// point occurs before the caller publishes the newly allocated task/cache entry. If a
// snapshot has already released exclusion, its disk writes are still in flight, so a
// read during that interval cannot prove the next snapshot's disk baseline.
let canonical = matches!(self.phase, ExecutePhase::Normal { guard: Some(_) })
&& !self.backend.pending_db_commit.load(Ordering::Acquire);
let candidates = self
.backend
.backing_storage
......@@ -1409,15 +1413,18 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
// Verify each candidate by comparing the stored persistent_task_type.
// Only rarely is there more than one candidate, so no need for parallelization.
if candidates.is_empty() {
return (None, canonical);
}
for candidate_id in candidates {
let task = self.task(candidate_id, TaskDataCategory::Data);
if let Some(stored_type) = task.get_persistent_task_type()
&& stored_type.eq_components(native_fn, this, arg)
{
return Some((candidate_id, stored_type.clone()));
return (Some((candidate_id, stored_type.clone())), false);
}
}
None
(None, false)
}
fn debug_get_task_description(&self, task_id: TaskId) -> String {
......
......@@ -230,6 +230,11 @@ struct TaskStorageSchema {
#[field(storage = "flag", category = "transient")]
pub new_task: bool,
/// Whether a fresh task's lookup did not prove its entire TaskCache hash bucket empty.
/// Stored lazily because all 16 TaskFlags bits are in use; absent means false.
#[field(storage = "direct", category = "transient")]
pub task_cache_needs_read: bool,
/// GC soft-deletion marker. Set by the garbage collector when a task is marked for deletion.
#[field(storage = "flag", category = "transient")]
deleted: bool,
......
......@@ -6,6 +6,8 @@ use turbo_tasks::{
};
use turbo_tasks_hash::Xxh3Hash64Hasher;
// The TaskCache key is a stable hash, so different task types may share one encoded
// list-valued disk entry. Snapshot reconciliation merges their IDs by hash.
pub type TaskTypeHash = [u8; 8];
/// A single item yielded by the snapshot iterator during persistence: either a put (persist a
......@@ -21,6 +23,9 @@ pub enum SnapshotItem {
data: Option<TurboBincodeBuffer>,
/// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>,
/// Only a freshly allocated task whose guarded lookup proved the entire disk hash bucket
/// empty may omit the TaskCache read on its first snapshot. Resurrection sets this false.
task_cache_empty_on_creation: bool,
},
Delete {
task_id: TaskId,
......
......@@ -47,8 +47,8 @@ impl KeySpace {
},
KeySpace::TaskCache => FamilyConfig {
name: self.name(),
// TaskCache uses hash-based lookups with potential collisions.
kind: FamilyKind::MultiValue,
// TaskCache maps each hash to one list of TaskIds to handle rare collisions.
kind: FamilyKind::SingleValue,
compression: Compression::Lz4,
},
}
......
......@@ -6,7 +6,6 @@ use std::{
};
use anyhow::{Ok, Result};
use smallvec::SmallVec;
use turbo_persistence::{
ArcBytes, CommitStats, CompactConfig, DbConfig, KeyBase, StoreKey, TurboPersistence,
ValueBuffer,
......@@ -99,13 +98,6 @@ impl TurboKeyValueDatabase {
self.db.batch_get(key_space as usize, keys)
}
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
pub fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
self.db.get_multiple(key_space as usize, &key)
}
pub fn write_batch(&self) -> Result<TurboWriteBatch<'_>> {
Ok(TurboWriteBatch {
batch: self.db.write_batch()?,
......@@ -226,24 +218,10 @@ impl<'a> TurboWriteBatch<'a> {
}
/// Writes a delete (tombstone) for `key` into the write batch.
///
/// Use [`Self::delete_value`] to remove a single mapping from a MultiValue KeySpace
pub fn delete(&self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
self.batch.delete(key_space as u32, key.into_static())
}
/// Writes a tombstone for a single `key` -> `value` mapping, leaving other values under `key`
/// intact. Only valid for `MultiValue` families (`TaskCache`).
pub fn delete_value(
&self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
self.batch
.delete_value(key_space as u32, key.into_static(), value.into())
}
/// Flushes a key space of the write batch, reducing the amount of buffered memory used.
/// Does not commit any data persistently.
///
......
......@@ -60,26 +60,26 @@ pub fn with_entry_in_shard<K: Eq + Hash, V, S: BuildHasher + Clone, Q: ?Sized, R
pub enum TryLockAndRemove {
/// The shard lock was acquired and a matching entry was removed.
Removed,
/// The shard lock was acquired but no matching entry was present.
/// The shard lock was acquired but the key was absent or held another value.
NotFound,
/// The shard lock was contended; the caller should retry later after releasing
/// any other locks they are holding.
WouldBlock,
}
/// Remove `key` from `map` without blocking on shard contention.
/// Remove `key` only if it still maps to `expected`, without blocking on shard contention.
///
/// Intended for call sites that already hold another lock and want to avoid a
/// cyclic wait. On contention (`WouldBlock`), the caller is expected to defer the
/// removal and retry after dropping the other lock.
/// The value check prevents an eviction of an old TaskId from removing a newer mapping for
/// the same task type. On contention the caller defers removal until releasing other locks.
pub fn try_lock_and_remove<
K: Eq + Hash + AsRef<Q>,
V,
V: Copy + PartialEq,
Q: Eq + Hash + ?Sized,
S: BuildHasher + Clone,
>(
map: &DashMap<K, V, S>,
key: &Q,
expected: V,
) -> TryLockAndRemove {
let hasher = map.hasher();
let hash = hasher.hash_one(key);
......@@ -88,10 +88,54 @@ pub fn try_lock_and_remove<
return TryLockAndRemove::WouldBlock;
};
match shard.find_entry(hash, |(k, _v)| k.as_ref() == key) {
Ok(entry) => {
Ok(entry) if entry.get().1 == expected => {
entry.remove();
TryLockAndRemove::Removed
}
Err(_) => TryLockAndRemove::NotFound,
_ => TryLockAndRemove::NotFound,
}
}
#[cfg(test)]
mod tests {
use std::hash::BuildHasher;
use turbo_tasks::FxDashMap;
use super::{TryLockAndRemove, get_shard, try_lock_and_remove};
#[test]
fn try_lock_and_remove_requires_matching_values() {
let map = FxDashMap::<String, u32>::default();
map.insert("task".to_string(), 2);
assert!(matches!(
try_lock_and_remove(&map, "task", 1),
TryLockAndRemove::NotFound
));
assert_eq!(map.get("task").map(|id| *id), Some(2));
assert!(matches!(
try_lock_and_remove(&map, "task", 2),
TryLockAndRemove::Removed
));
assert!(map.get("task").is_none());
}
#[test]
fn contended_removal_rechecks_the_expected_value_after_unlocking() {
let map = FxDashMap::<String, u32>::default();
map.insert("task".to_string(), 1);
let hash = map.hasher().hash_one("task");
let shard_guard = get_shard(&map, hash).write();
assert!(matches!(
try_lock_and_remove(&map, "task", 1),
TryLockAndRemove::WouldBlock
));
drop(shard_guard);
// A new mapping can be published after the contended eviction releases its other lock.
map.insert("task".to_string(), 2);
assert!(map.remove_if("task", |_, id| *id == 1).is_none());
assert_eq!(map.get("task").map(|id| *id), Some(2));
assert!(map.remove_if("task", |_, id| *id == 2).is_some());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment