Commit 42ccc8d6 authored by Luke Sandberg's avatar Luke Sandberg

Stop using TaskTypes as keys!

instead store hashes, this makes lookups slightly more complex but greatly reduces storage requirements.

clean up the old lookup methods

Use smallvecs, add tracing and share more implementation details in the db lookup methods

Add some safety to the new APIs and resolve a small race condition

fix some clippy issues

Remove obsolete info size metrics

revert a feature flag flip

fix stats collection and improve logic shaing

restore the scratch buffer

typo

fix silly bug

port back some simplifications

port back another batch of simplifications that landed on the wrong branch
parent b9e9f111
...@@ -1354,8 +1354,69 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1354,8 +1354,69 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
result_size = tracing::field::Empty result_size = tracing::field::Empty
) )
.entered(); .entered();
let results = self.get_impl(family, key, false)?;
debug_assert!(results.len() <= 1, "get() should return at most one result");
match results.into_iter().next() {
Some(value) => {
span.record("result_size", value.len());
Ok(Some(value))
}
None => {
span.record("result_size", "not found");
Ok(None)
}
}
}
/// Looks up a key and returns all matching values.
///
/// This is useful for keyspaces where keys are not unique and collisions are possible.
/// Unlike `get`, which returns only the first match, this method returns all
/// entries with the same key from all SST files. By default however we assume these collisions
/// are extremely rare and thus optimize for there being exactly 0 or 1 results.
///
/// Note: This method does NOT deduplicate values. If the same key-value pair exists
/// in multiple SST files (e.g., before compaction), it will be returned multiple times.
pub fn get_multiple<K: QueryKey>(
&self,
family: usize,
key: &K,
) -> Result<SmallVec<[ArcSlice<u8>; 1]>> {
debug_assert!(family < FAMILIES, "Family index out of bounds");
let span = tracing::trace_span!(
"database read multiple",
name = family,
result_count = tracing::field::Empty,
result_size = tracing::field::Empty
)
.entered();
let results = self.get_impl(family, key, true)?;
span.record("result_count", results.len());
span.record(
"result_size",
results.iter().map(|r| r.len()).sum::<usize>(),
);
Ok(results)
}
/// Shared implementation for `get` and `get_multiple`.
///
/// If `find_all` is false, stops after finding the first match.
/// If `find_all` is true, continues to find all matches across all meta files.
fn get_impl<K: QueryKey>(
&self,
family: usize,
key: &K,
find_all: bool,
) -> Result<SmallVec<[ArcSlice<u8>; 1]>> {
let hash = hash_key(key); let hash = hash_key(key);
let inner = self.inner.read(); let inner = self.inner.read();
let mut output: SmallVec<[ArcSlice<u8>; 1]> = SmallVec::new();
// Track whether we found the key in any SST (even if deleted).
// Used for miss_global stat: only fires if key was never found anywhere.
#[cfg(feature = "stats")]
let mut found_in_sst = false;
for meta in inner.meta_files.iter().rev() { for meta in inner.meta_files.iter().rev() {
match meta.lookup( match meta.lookup(
family as u32, family as u32,
...@@ -1363,6 +1424,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1363,6 +1424,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
key, key,
&self.key_block_cache, &self.key_block_cache,
&self.value_block_cache, &self.value_block_cache,
find_all,
)? { )? {
MetaLookupResult::FamilyMiss => { MetaLookupResult::FamilyMiss => {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -1377,29 +1439,40 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1377,29 +1439,40 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
self.stats.miss_amqf.fetch_add(1, Ordering::Relaxed); self.stats.miss_amqf.fetch_add(1, Ordering::Relaxed);
} }
MetaLookupResult::SstLookup(result) => match result { MetaLookupResult::SstLookup(result) => match result {
SstLookupResult::Found(result) => { SstLookupResult::Found(values) => {
#[cfg(feature = "stats")]
{
found_in_sst = true;
}
inner.accessed_key_hashes[family].insert(hash); inner.accessed_key_hashes[family].insert(hash);
match result { for value in values {
match value {
LookupValue::Deleted => { LookupValue::Deleted => {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed); self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed);
span.record("result_size", "deleted"); if !find_all {
return Ok(None); // For get, deleted means "key was deleted", stop looking
return Ok(SmallVec::new());
}
// For get_multiple, skip deleted entries but keep looking
} }
LookupValue::Slice { value } => { LookupValue::Slice { value } => {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
self.stats.hits_small.fetch_add(1, Ordering::Relaxed); self.stats.hits_small.fetch_add(1, Ordering::Relaxed);
span.record("result_size", value.len()); output.push(value);
return Ok(Some(value));
} }
LookupValue::Blob { sequence_number } => { LookupValue::Blob { sequence_number } => {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
self.stats.hits_blob.fetch_add(1, Ordering::Relaxed); self.stats.hits_blob.fetch_add(1, Ordering::Relaxed);
let blob = self.read_blob(sequence_number)?; let blob = self.read_blob(sequence_number)?;
span.record("result_size", blob.len()); output.push(blob);
return Ok(Some(blob)); }
} }
} }
if !find_all {
// For get, we found a non-deleted value, return it
return Ok(output);
}
} }
SstLookupResult::NotFound => { SstLookupResult::NotFound => {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -1408,10 +1481,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1408,10 +1481,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
}, },
} }
} }
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
if !found_in_sst {
self.stats.miss_global.fetch_add(1, Ordering::Relaxed); self.stats.miss_global.fetch_add(1, Ordering::Relaxed);
span.record("result_size", "not found"); }
Ok(None)
Ok(output)
} }
pub fn batch_get<K: QueryKey>( pub fn batch_get<K: QueryKey>(
......
...@@ -13,6 +13,7 @@ use bincode::{Decode, Encode}; ...@@ -13,6 +13,7 @@ use bincode::{Decode, Encode};
use bitfield::bitfield; use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use smallvec::SmallVec;
use turbo_bincode::turbo_bincode_decode; use turbo_bincode::turbo_bincode_decode;
use crate::{ use crate::{
...@@ -373,6 +374,11 @@ impl MetaFile { ...@@ -373,6 +374,11 @@ impl MetaFile {
&self.obsolete_sst_files &self.obsolete_sst_files
} }
/// Looks up a key in this meta file.
///
/// If `find_all` is false, returns after finding the first match.
/// If `find_all` is true, returns all entries with the same key from all SST files
/// (useful for keyspaces where keys are hashes and collisions are possible).
pub fn lookup<K: QueryKey>( pub fn lookup<K: QueryKey>(
&self, &self,
key_family: u32, key_family: u32,
...@@ -380,30 +386,54 @@ impl MetaFile { ...@@ -380,30 +386,54 @@ impl MetaFile {
key: &K, key: &K,
key_block_cache: &BlockCache, key_block_cache: &BlockCache,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
find_all: bool,
) -> Result<MetaLookupResult> { ) -> Result<MetaLookupResult> {
if key_family != self.family { if key_family != self.family {
return Ok(MetaLookupResult::FamilyMiss); return Ok(MetaLookupResult::FamilyMiss);
} }
let mut miss_result = MetaLookupResult::RangeMiss; let mut miss_result = MetaLookupResult::RangeMiss;
let mut all_results: SmallVec<[LookupValue; 1]> = SmallVec::new();
for entry in self.entries.iter().rev() { for entry in self.entries.iter().rev() {
if key_hash < entry.min_hash || key_hash > entry.max_hash { if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue; continue;
} }
{
let amqf = entry.amqf(self)?; let amqf = entry.amqf(self)?;
if !amqf.contains_fingerprint(key_hash) { if !amqf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss; miss_result = MetaLookupResult::QuickFilterMiss;
continue; continue;
} }
let result = entry.sst(self)?.lookup(
key_hash,
key,
key_block_cache,
value_block_cache,
find_all,
)?;
match result {
SstLookupResult::NotFound => {
// continue searching other sst files
}
SstLookupResult::Found(values) => {
if !find_all {
// Return immediately with the first result
return Ok(MetaLookupResult::SstLookup(SstLookupResult::Found(values)));
} }
let result = // Accumulate results
entry all_results.extend(values);
.sst(self)?
.lookup(key_hash, key, key_block_cache, value_block_cache)?;
if !matches!(result, SstLookupResult::NotFound) {
return Ok(MetaLookupResult::SstLookup(result));
} }
} }
}
if !all_results.is_empty() {
return Ok(MetaLookupResult::SstLookup(SstLookupResult::Found(
all_results,
)));
}
Ok(miss_result) Ok(miss_result)
} }
...@@ -483,8 +513,14 @@ impl MetaFile { ...@@ -483,8 +513,14 @@ impl MetaFile {
&keys[*index], &keys[*index],
key_block_cache, key_block_cache,
value_block_cache, value_block_cache,
false, // find_all: batch_lookup returns first match per key
)?; )?;
if let SstLookupResult::Found(value) = sst_result { if let SstLookupResult::Found(mut values) = sst_result {
// find_all=false guarantees exactly one result
debug_assert!(values.len() == 1);
let Some(value) = values.pop() else {
unreachable!()
};
*result = Some(value); *result = Some(value);
*empty_cells -= 1; *empty_cells -= 1;
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
......
...@@ -11,6 +11,7 @@ use byteorder::{BE, ReadBytesExt}; ...@@ -11,6 +11,7 @@ use byteorder::{BE, ReadBytesExt};
use memmap2::Mmap; use memmap2::Mmap;
use quick_cache::sync::GuardResult; use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use smallvec::SmallVec;
use crate::{ use crate::{
QueryKey, QueryKey,
...@@ -47,15 +48,15 @@ const _: () = assert!( ...@@ -47,15 +48,15 @@ const _: () = assert!(
/// The result of a lookup operation. /// The result of a lookup operation.
pub enum SstLookupResult { pub enum SstLookupResult {
/// The key was found. /// One or more values were found.
Found(LookupValue), Found(SmallVec<[LookupValue; 1]>),
/// The key was not found. /// The key was not found.
NotFound, NotFound,
} }
impl From<LookupValue> for SstLookupResult { impl From<LookupValue> for SstLookupResult {
fn from(value: LookupValue) -> Self { fn from(value: LookupValue) -> Self {
SstLookupResult::Found(value) SstLookupResult::Found(smallvec::smallvec![value])
} }
} }
...@@ -148,14 +149,20 @@ impl StaticSortedFile { ...@@ -148,14 +149,20 @@ impl StaticSortedFile {
} }
/// Looks up a key in this file. /// Looks up a key in this file.
///
/// If `find_all` is false, returns after finding the first match.
/// If `find_all` is true, returns all entries with the same key (useful for
/// keyspaces where keys are hashes and collisions are possible).
pub fn lookup<K: QueryKey>( pub fn lookup<K: QueryKey>(
&self, &self,
key_hash: u64, key_hash: u64,
key: &K, key: &K,
key_block_cache: &BlockCache, key_block_cache: &BlockCache,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
find_all: bool,
) -> Result<SstLookupResult> { ) -> Result<SstLookupResult> {
let mut current_block = self.meta.block_count - 1; let this = &self;
let mut current_block = this.meta.block_count - 1;
loop { loop {
let mut key_block_arc = self.get_key_block(current_block, key_block_cache)?; let mut key_block_arc = self.get_key_block(current_block, key_block_cache)?;
let block_type = key_block_arc.read_u8()?; let block_type = key_block_arc.read_u8()?;
...@@ -171,6 +178,7 @@ impl StaticSortedFile { ...@@ -171,6 +178,7 @@ impl StaticSortedFile {
key, key,
has_hash, has_hash,
value_block_cache, value_block_cache,
find_all,
); );
} }
_ => { _ => {
...@@ -227,6 +235,9 @@ impl StaticSortedFile { ...@@ -227,6 +235,9 @@ impl StaticSortedFile {
} }
/// Looks up a key in a key block and the value in a value block. /// Looks up a key in a key block and the value in a value block.
///
/// If `find_all` is false, returns after finding the first match.
/// If `find_all` is true, collects all entries with the same key.
fn lookup_key_block<K: QueryKey>( fn lookup_key_block<K: QueryKey>(
&self, &self,
mut block: ArcSlice<u8>, mut block: ArcSlice<u8>,
...@@ -234,6 +245,7 @@ impl StaticSortedFile { ...@@ -234,6 +245,7 @@ impl StaticSortedFile {
key: &K, key: &K,
has_hash: bool, has_hash: bool,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
find_all: bool,
) -> Result<SstLookupResult> { ) -> Result<SstLookupResult> {
let hash_len: u8 = if has_hash { 8 } else { 0 }; let hash_len: u8 = if has_hash { 8 } else { 0 };
let entry_count = block.read_u24::<BE>()? as usize; let entry_count = block.read_u24::<BE>()? as usize;
...@@ -242,33 +254,58 @@ impl StaticSortedFile { ...@@ -242,33 +254,58 @@ impl StaticSortedFile {
let mut l = 0; let mut l = 0;
let mut r = entry_count; let mut r = entry_count;
// binary search for the key let mut found = false;
// binary search for the first matching key
// TODO: we actually need to find the last matching key
while l < r { while l < r {
let m = (l + r) / 2; let m = (l + r) / 2;
let GetKeyEntryResult { let GetKeyEntryResult {
hash: mid_hash, hash: mid_hash,
key: mid_key, key: mid_key,
ty, ..
val: mid_val,
} = get_key_entry(offsets, entries, entry_count, m, hash_len)?; } = get_key_entry(offsets, entries, entry_count, m, hash_len)?;
let comparison = compare_hash_key(mid_hash, mid_key, key_hash, key); let comparison = compare_hash_key(mid_hash, mid_key, key_hash, key);
match comparison { match comparison {
Ordering::Less => { Ordering::Less => r = m,
Ordering::Equal => {
found = true;
r = m; r = m;
} }
Ordering::Equal => { Ordering::Greater => l = m + 1,
return Ok(self
.handle_key_match(ty, mid_val, &block, value_block_cache)?
.into());
} }
Ordering::Greater => {
l = m + 1;
} }
if !found {
return Ok(SstLookupResult::NotFound);
} }
// l is now the index of the first matching element
let GetKeyEntryResult { ty, val, .. } =
get_key_entry(offsets, entries, entry_count, l, hash_len)?;
// Collect all entries with matching key starting from l
let mut results = SmallVec::new();
results.push(self.handle_key_match(ty, val, &block, value_block_cache)?);
if find_all {
// duplicates, if there are anyu will be immediately following this one.
for i in (l + 1)..entry_count {
let GetKeyEntryResult {
hash,
key: entry_key,
ty,
val,
} = get_key_entry(offsets, entries, entry_count, i, hash_len)?;
let comparison = compare_hash_key(hash, entry_key, key_hash, key);
if comparison != Ordering::Equal {
break;
} }
Ok(SstLookupResult::NotFound) results.push(self.handle_key_match(ty, val, &block, value_block_cache)?);
}
}
Ok(SstLookupResult::Found(results))
} }
/// Handles a key match by looking up the value. /// Handles a key match by looking up the value.
......
...@@ -1303,3 +1303,211 @@ fn batch_get_after_restore() -> Result<()> { ...@@ -1303,3 +1303,211 @@ fn batch_get_after_restore() -> Result<()> {
Ok(()) Ok(())
} }
#[test]
fn get_multiple_same_key_different_batches() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Create a database and write the same key with different values in separate batches.
// Before compaction, all values should be retrievable via get_multiple.
let key = vec![42u8, 42u8, 42u8]; // The key we'll write multiple times
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
// Write first value for the key
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![1u8].into())?;
db.commit_write_batch(batch)?;
// Write second value for the same key in a new batch (new SST file)
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![2u8].into())?;
db.commit_write_batch(batch)?;
// Write third value for the same key
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![3u8].into())?;
db.commit_write_batch(batch)?;
// Now get_multiple should return all three values (newest first, as we iterate in reverse)
let results = db.get_multiple(0, &key.as_slice())?;
// All three values should be present
assert_eq!(results.len(), 3, "Should have 3 values before compaction");
// Convert to sorted Vec for comparison (order may vary based on SST iteration order)
let mut values: Vec<u8> = results.iter().map(|r| r[0]).collect();
values.sort();
assert_eq!(values, vec![1, 2, 3], "Should have all three values");
// Regular get should return only the newest value
let single = db.get(0, &key.as_slice())?;
assert_eq!(
single.as_deref(),
Some(&[3u8][..]),
"Regular get should return newest value"
);
db.shutdown()?;
}
// Reopen and verify persistence
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(results.len(), 3, "Should still have 3 values after reopen");
db.shutdown()?;
}
// After compaction, only the newest value should remain
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
db.full_compact()?;
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(
results.len(),
1,
"Should have only 1 value after compaction"
);
assert_eq!(
results[0].as_ref(),
&[3u8],
"Should be the newest value after compaction"
);
db.shutdown()?;
}
Ok(())
}
#[test]
fn get_multiple_same_key_same_batch() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Test writing the same key multiple times within a single batch.
// The SST file should contain all entries, and get_multiple should return them all.
let key = vec![99u8];
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let batch = db.write_batch()?;
// Write multiple values for the same key in the same batch
batch.put(0, key.clone(), vec![10u8].into())?;
batch.put(0, key.clone(), vec![20u8].into())?;
batch.put(0, key.clone(), vec![30u8].into())?;
db.commit_write_batch(batch)?;
// Within a single SST file, entries with same key are sorted and all present
let results = db.get_multiple(0, &key.as_slice())?;
// All three values should be present in the SST (no compaction yet)
assert_eq!(
results.len(),
3,
"Should have 3 values in same batch before compaction"
);
let mut values: Vec<u8> = results.iter().map(|r| r[0]).collect();
values.sort();
assert_eq!(values, vec![10, 20, 30]);
db.shutdown()?;
}
Ok(())
}
#[test]
fn get_multiple_different_keys() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Test that get_multiple only returns values for the specific key,
// not values for other keys.
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![100u8].into())?;
batch.put(0, vec![2u8], vec![200u8].into())?;
batch.put(0, vec![3u8], vec![200u8].into())?; // Same value, different key
db.commit_write_batch(batch)?;
// get_multiple for key [1] should only return value [100]
let results = db.get_multiple(0, &[1u8].as_slice())?;
assert_eq!(results.len(), 1);
assert_eq!(results[0].as_ref(), &[100u8]);
// get_multiple for key [2] should only return value [200]
let results = db.get_multiple(0, &[2u8].as_slice())?;
assert_eq!(results.len(), 1);
assert_eq!(results[0].as_ref(), &[200u8]);
// get_multiple for non-existent key should return empty
let results = db.get_multiple(0, &[99u8].as_slice())?;
assert_eq!(results.len(), 0);
db.shutdown()?;
}
Ok(())
}
#[test]
fn get_multiple_empty_result() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![100u8].into())?;
db.commit_write_batch(batch)?;
// Query for a key that doesn't exist
let results = db.get_multiple(0, &[99u8].as_slice())?;
assert!(
results.is_empty(),
"Should return empty vec for non-existent key"
);
// Query empty database for different family
let results = db.get_multiple(1, &[1u8].as_slice())?;
assert!(
results.is_empty(),
"Should return empty vec for different family"
);
db.shutdown()?;
}
Ok(())
}
...@@ -282,21 +282,6 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -282,21 +282,6 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
ExecuteContextImpl::new(self, turbo_tasks) ExecuteContextImpl::new(self, turbo_tasks)
} }
/// # Safety
///
/// `tx` must be a transaction from this TurboTasksBackendInner instance.
unsafe fn execute_context_with_tx<'e, 'tx>(
&'e self,
tx: Option<&'e B::ReadTransaction<'tx>>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) -> impl ExecuteContext<'e> + use<'e, 'tx, B>
where
'tx: 'e,
{
// Safety: `tx` is from `self`.
unsafe { ExecuteContextImpl::new_with_tx(self, tx, turbo_tasks) }
}
fn suspending_requested(&self) -> bool { fn suspending_requested(&self) -> bool {
self.should_persist() self.should_persist()
&& (self.in_progress_operations.load(Ordering::Relaxed) & SNAPSHOT_REQUESTED_BIT) != 0 && (self.in_progress_operations.load(Ordering::Relaxed) & SNAPSHOT_REQUESTED_BIT) != 0
...@@ -418,22 +403,6 @@ struct TaskExecutionCompletePrepareResult { ...@@ -418,22 +403,6 @@ struct TaskExecutionCompletePrepareResult {
// Operations // Operations
impl<B: BackingStorage> TurboTasksBackendInner<B> { impl<B: BackingStorage> TurboTasksBackendInner<B> {
/// # Safety
///
/// `tx` must be a transaction from this TurboTasksBackendInner instance.
unsafe fn connect_child_with_tx<'l, 'tx: 'l>(
&'l self,
tx: Option<&'l B::ReadTransaction<'tx>>,
parent_task: Option<TaskId>,
child_task: TaskId,
task_type: Option<ArcOrOwned<CachedTaskType>>,
turbo_tasks: &'l dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) {
operation::ConnectChildOperation::run(parent_task, child_task, task_type, unsafe {
self.execute_context_with_tx(tx, turbo_tasks)
});
}
fn connect_child( fn connect_child(
&self, &self,
parent_task: Option<TaskId>, parent_task: Option<TaskId>,
...@@ -1428,22 +1397,13 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1428,22 +1397,13 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
return task_id; return task_id;
} }
let check_backing_storage = // Create a single ExecuteContext for both lookup and connect_child
self.should_restore() && self.local_is_partial.load(Ordering::Acquire); let mut ctx = self.execute_context(turbo_tasks);
let tx = check_backing_storage
.then(|| self.backing_storage.start_read_transaction()) // Look up task from backing storage (handles hash-based lookup and verification)
.flatten(); let found_task_id = ctx.task_by_type(&task_type);
let (task_id, task_type) = {
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`. let (task_id, task_type) = if let Some(task_id) = found_task_id {
if let Some(task_id) = unsafe {
check_backing_storage
.then(|| {
self.backing_storage
.forward_lookup_task_cache(tx.as_ref(), &task_type)
.expect("Failed to lookup task id")
})
.flatten()
} {
// Task exists in backing storage // Task exists in backing storage
// So we only need to insert it into the in-memory cache // So we only need to insert it into the in-memory cache
self.track_cache_hit(&task_type); self.track_cache_hit(&task_type);
...@@ -1459,40 +1419,33 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1459,40 +1419,33 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
} else { } else {
// Task doesn't exist in memory cache or backing storage // Task doesn't exist in memory cache or backing storage
// So we might need to create a new task // So we might need to create a new task
let (task_id, mut task_type) = match raw_entry(&self.task_cache, &task_type) { let (task_id, task_type) = match raw_entry(&self.task_cache, &task_type) {
RawEntry::Occupied(e) => { RawEntry::Occupied(e) => {
// Another thread beat us to creating this task - use their task_id.
// They will handle logging to persisted_task_cache_log.
let task_id = *e.get(); let task_id = *e.get();
drop(e); drop(e);
self.track_cache_hit(&task_type); self.track_cache_hit(&task_type);
(task_id, ArcOrOwned::Owned(task_type)) (task_id, ArcOrOwned::Owned(task_type))
} }
RawEntry::Vacant(e) => { RawEntry::Vacant(e) => {
// We're creating a new task.
let task_type = Arc::new(task_type); let task_type = Arc::new(task_type);
let task_id = self.persisted_task_id_factory.get(); let task_id = self.persisted_task_id_factory.get();
e.insert(task_type.clone(), task_id); e.insert(task_type.clone(), task_id);
// insert() consumes e, releasing the lock
self.track_cache_miss(&task_type); self.track_cache_miss(&task_type);
if let Some(log) = &self.persisted_task_cache_log {
log.lock(task_id).push((task_type.clone(), task_id));
}
(task_id, ArcOrOwned::Arc(task_type)) (task_id, ArcOrOwned::Arc(task_type))
} }
}; };
if let Some(log) = &self.persisted_task_cache_log {
let task_type_arc: Arc<CachedTaskType> = Arc::from(task_type);
log.lock(task_id).push((task_type_arc.clone(), task_id));
task_type = ArcOrOwned::Arc(task_type_arc);
}
(task_id, task_type) (task_id, task_type)
}
}; };
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`. // Reuse the same ExecuteContext for connect_child
unsafe { operation::ConnectChildOperation::run(parent_task, task_id, Some(task_type), ctx);
self.connect_child_with_tx(
tx.as_ref(),
parent_task,
task_id,
Some(task_type),
turbo_tasks,
)
};
task_id task_id
} }
......
...@@ -34,10 +34,8 @@ pub trait Operation: Encode + Decode<()> + Default + TryFrom<AnyOperation, Error ...@@ -34,10 +34,8 @@ pub trait Operation: Encode + Decode<()> + Default + TryFrom<AnyOperation, Error
fn execute(self, ctx: &mut impl ExecuteContext<'_>); fn execute(self, ctx: &mut impl ExecuteContext<'_>);
} }
#[derive(Copy, Clone)] enum TransactionState<'tx, B: BackingStorage> {
enum TransactionState<'a, 'tx, B: BackingStorage> {
None, None,
Borrowed(Option<&'a B::ReadTransaction<'tx>>),
Owned(Option<B::ReadTransaction<'tx>>), Owned(Option<B::ReadTransaction<'tx>>),
} }
...@@ -93,6 +91,12 @@ pub trait ExecuteContext<'e>: Sized { ...@@ -93,6 +91,12 @@ pub trait ExecuteContext<'e>: Sized {
fn suspending_requested(&self) -> bool; fn suspending_requested(&self) -> bool;
fn should_track_dependencies(&self) -> bool; fn should_track_dependencies(&self) -> bool;
fn should_track_activeness(&self) -> bool; fn should_track_activeness(&self) -> bool;
/// Look up a TaskId from the backing storage for a given task type.
///
/// Uses hash-based lookup which may return multiple candidates due to hash collisions,
/// then verifies each candidate by comparing the stored `persistent_task_type`.
/// Returns `Some(task_id)` if a matching task is found, `None` otherwise.
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId>;
} }
pub trait ChildExecuteContext<'e>: Send + Sized { pub trait ChildExecuteContext<'e>: Send + Sized {
...@@ -107,7 +111,7 @@ where ...@@ -107,7 +111,7 @@ where
backend: &'e TurboTasksBackendInner<B>, backend: &'e TurboTasksBackendInner<B>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>, turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
_operation_guard: Option<OperationGuard<'e, B>>, _operation_guard: Option<OperationGuard<'e, B>>,
transaction: TransactionState<'e, 'tx, B>, transaction: TransactionState<'tx, B>,
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
active_task_locks: std::sync::Arc<std::sync::atomic::AtomicU8>, active_task_locks: std::sync::Arc<std::sync::atomic::AtomicU8>,
} }
...@@ -130,21 +134,6 @@ where ...@@ -130,21 +134,6 @@ where
} }
} }
pub(super) unsafe fn new_with_tx(
backend: &'e TurboTasksBackendInner<B>,
transaction: Option<&'e B::ReadTransaction<'tx>>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) -> Self {
Self {
backend,
turbo_tasks,
_operation_guard: Some(backend.start_operation()),
transaction: TransactionState::Borrowed(transaction),
#[cfg(debug_assertions)]
active_task_locks: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(0)),
}
}
fn ensure_transaction(&mut self) -> bool { fn ensure_transaction(&mut self) -> bool {
if matches!(self.transaction, TransactionState::None) { if matches!(self.transaction, TransactionState::None) {
let check_backing_storage = self.backend.should_restore() let check_backing_storage = self.backend.should_restore()
...@@ -228,7 +217,6 @@ where ...@@ -228,7 +217,6 @@ where
fn get_tx(&self) -> Option<&<B as BackingStorageSealed>::ReadTransaction<'tx>> { fn get_tx(&self) -> Option<&<B as BackingStorageSealed>::ReadTransaction<'tx>> {
match &self.transaction { match &self.transaction {
TransactionState::None => unreachable!(), TransactionState::None => unreachable!(),
TransactionState::Borrowed(tx) => *tx,
TransactionState::Owned(tx) => tx.as_ref(), TransactionState::Owned(tx) => tx.as_ref(),
} }
} }
...@@ -642,6 +630,35 @@ where ...@@ -642,6 +630,35 @@ where
fn should_track_activeness(&self) -> bool { fn should_track_activeness(&self) -> bool {
self.backend.should_track_activeness() self.backend.should_track_activeness()
} }
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId> {
// Ensure we have a transaction (this will be reused by subsequent task() calls)
if !self.ensure_transaction() {
return None;
}
let tx = self.get_tx();
// Get candidates from backing storage (hash-based lookup may return multiple)
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
let candidates = unsafe {
self.backend
.backing_storage
.lookup_task_candidate(tx, task_type)
.expect("Failed to lookup task ids")
};
// Verify each candidate by comparing the stored persistent_task_type.
// Only rarely is there more than one candidate, so no need for parallelization.
for candidate_id in candidates {
let task = self.task(candidate_id, TaskDataCategory::Data);
if let Some(stored_type) = task.get_persistent_task_type()
&& stored_type.as_ref() == task_type
{
return Some(candidate_id);
}
}
None
}
} }
struct ChildExecuteContextImpl<'e, B: BackingStorage> { struct ChildExecuteContextImpl<'e, B: BackingStorage> {
......
...@@ -60,14 +60,20 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -60,14 +60,20 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
> + Send > + Send
+ Sync; + Sync;
fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>; fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>;
/// Returns all task IDs that match the given task type (hash collision candidates).
///
/// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key.
/// The caller must verify each returned TaskId by comparing the stored task type which will
/// require a second database read
///
/// # Safety /// # Safety
/// ///
/// `tx` must be a transaction from this BackingStorage instance. /// `tx` must be a transaction from this BackingStorage instance.
unsafe fn forward_lookup_task_cache( unsafe fn lookup_task_candidate(
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType, key: &CachedTaskType,
) -> Result<Option<TaskId>>; ) -> Result<SmallVec<[TaskId; 1]>>;
/// # Safety /// # Safety
/// ///
/// `tx` must be a transaction from this BackingStorage instance. /// `tx` must be a transaction from this BackingStorage instance.
...@@ -151,19 +157,19 @@ where ...@@ -151,19 +157,19 @@ where
}) })
} }
unsafe fn forward_lookup_task_cache( unsafe fn lookup_task_candidate(
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType, key: &CachedTaskType,
) -> Result<Option<TaskId>> { ) -> Result<SmallVec<[TaskId; 1]>> {
match self { match self {
Either::Left(this) => { Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref())); let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
unsafe { this.forward_lookup_task_cache(tx, key) } unsafe { this.lookup_task_candidate(tx, key) }
} }
Either::Right(this) => { Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref())); let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
unsafe { this.forward_lookup_task_cache(tx, key) } unsafe { this.lookup_task_candidate(tx, key) }
} }
} }
} }
......
use anyhow::Result; use anyhow::Result;
use smallvec::SmallVec;
use crate::database::write_batch::{ use crate::database::write_batch::{
ConcurrentWriteBatch, SerialWriteBatch, UnimplementedWriteBatch, WriteBatch, ConcurrentWriteBatch, SerialWriteBatch, UnimplementedWriteBatch, WriteBatch,
}; };
/// Describes the lookup semantics for a keyspace.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LookupSemantics {
/// Single value per key. `get` returns the newest value.
/// Using `get_multiple` is allowed but unnecessary.
SingleValue,
/// Multiple values may exist per key (e.g., hash collisions).
/// Callers MUST use `get_multiple` and verify candidates.
MultipleValues,
}
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
pub enum KeySpace { pub enum KeySpace {
/// Infrastructure data. Single value per key, use `get`.
Infra = 0, Infra = 0,
/// Stores a map from TaskId->TaskStorage meta. Single value per key, use `get`.
TaskMeta = 1, TaskMeta = 1,
/// Stores a map from TaskId->TaskStorage data. Single value per key, use `get`.
TaskData = 2, TaskData = 2,
/// Stores a map from TaskType _hash_ -> TaskId.
/// **Multiple values possible due to hash collisions.**
/// MUST use `get_multiple` and verify candidates match the actual TaskType.
TaskCache = 3, TaskCache = 3,
} }
impl KeySpace {
/// Returns the lookup semantics for this keyspace.
///
/// This indicates whether the keyspace expects single or multiple values per key,
/// which determines whether callers should use `get` or `get_multiple`.
pub fn lookup_semantics(&self) -> LookupSemantics {
match self {
KeySpace::Infra | KeySpace::TaskMeta | KeySpace::TaskData => {
LookupSemantics::SingleValue
}
KeySpace::TaskCache => LookupSemantics::MultipleValues,
}
}
}
pub trait KeyValueDatabase { pub trait KeyValueDatabase {
type ReadTransaction<'l> type ReadTransaction<'l>
where where
...@@ -33,6 +66,19 @@ pub trait KeyValueDatabase { ...@@ -33,6 +66,19 @@ pub trait KeyValueDatabase {
key_space: KeySpace, key_space: KeySpace,
key: &[u8], key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>>; ) -> Result<Option<Self::ValueBuffer<'l>>>;
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
/// The default implementation returns at most one value (from `get`), but implementations
/// that support multiple values per key should override this.
fn get_multiple<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'l>; 1]>> {
Ok(self.get(transaction, key_space, key)?.into_iter().collect())
}
fn batch_get<'l, 'db: 'l>( fn batch_get<'l, 'db: 'l>(
&'l self, &'l self,
...@@ -48,6 +94,7 @@ pub trait KeyValueDatabase { ...@@ -48,6 +94,7 @@ pub trait KeyValueDatabase {
Ok(results) Ok(results)
} }
/// Looks up a key by its hash, confirming the match by comparing the value.
type SerialWriteBatch<'l>: SerialWriteBatch<'l> type SerialWriteBatch<'l>: SerialWriteBatch<'l>
= UnimplementedWriteBatch = UnimplementedWriteBatch
where where
......
...@@ -7,7 +7,7 @@ use lmdb::{ ...@@ -7,7 +7,7 @@ use lmdb::{
}; };
use crate::database::{ use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase}, key_value_database::{KeySpace, KeyValueDatabase, LookupSemantics},
write_batch::{BaseWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer}, write_batch::{BaseWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer},
}; };
...@@ -81,6 +81,11 @@ impl KeyValueDatabase for LmbdKeyValueDatabase { ...@@ -81,6 +81,11 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
key_space: super::key_value_database::KeySpace, key_space: super::key_value_database::KeySpace,
key: &[u8], key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> { ) -> Result<Option<Self::ValueBuffer<'l>>> {
debug_assert!(
key_space.lookup_semantics() != LookupSemantics::MultipleValues,
"KeySpace {:?} may have multiple values - use get_multiple instead",
key_space
);
let db = match key_space { let db = match key_space {
KeySpace::Infra => self.infra_db, KeySpace::Infra => self.infra_db,
KeySpace::TaskMeta => self.meta_db, KeySpace::TaskMeta => self.meta_db,
......
...@@ -8,13 +8,14 @@ use std::{ ...@@ -8,13 +8,14 @@ use std::{
use anyhow::{Ok, Result}; use anyhow::{Ok, Result};
use parking_lot::Mutex; use parking_lot::Mutex;
use smallvec::SmallVec;
use turbo_persistence::{ use turbo_persistence::{
ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer, ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer,
}; };
use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks}; use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks};
use crate::database::{ use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase}, key_value_database::{KeySpace, KeyValueDatabase, LookupSemantics},
turbo::parallel_scheduler::TurboTasksParallelScheduler, turbo::parallel_scheduler::TurboTasksParallelScheduler,
write_batch::{BaseWriteBatch, ConcurrentWriteBatch, WriteBatch, WriteBuffer}, write_batch::{BaseWriteBatch, ConcurrentWriteBatch, WriteBatch, WriteBuffer},
}; };
...@@ -81,6 +82,11 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -81,6 +82,11 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
key_space: KeySpace, key_space: KeySpace,
key: &[u8], key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> { ) -> Result<Option<Self::ValueBuffer<'l>>> {
debug_assert!(
key_space.lookup_semantics() != LookupSemantics::MultipleValues,
"KeySpace {:?} may have multiple values - use get_multiple instead",
key_space
);
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
...@@ -93,6 +99,20 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -93,6 +99,20 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.batch_get(key_space as usize, keys) self.db.batch_get(key_space as usize, keys)
} }
fn get_multiple<'l, 'db: 'l>(
&'l self,
_transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'l>; 1]>> {
debug_assert!(
key_space.lookup_semantics() != LookupSemantics::SingleValue,
"KeySpace {:?} has single values - use get instead of get_multiple",
key_space
);
self.db.get_multiple(key_space as usize, &key)
}
type ConcurrentWriteBatch<'l> type ConcurrentWriteBatch<'l>
= TurboWriteBatch<'l> = TurboWriteBatch<'l>
where where
......
...@@ -30,8 +30,8 @@ macro_rules! impl_write_number { ...@@ -30,8 +30,8 @@ macro_rules! impl_write_number {
/// Note that the default [`std::hash::Hash`] trait used by Rust allows for hashing that differs /// Note that the default [`std::hash::Hash`] trait used by Rust allows for hashing that differs
/// across process runs, so it is not suitable for filesystem cache with turbo-tasks. /// across process runs, so it is not suitable for filesystem cache with turbo-tasks.
/// ///
/// It's very important that `Vc`s never implement this, since they cannot be deterministic. The /// It's very important that `Vc`s never implement this, since they are not consistent from run to
/// value that they wrap, however, can implement the trait. /// run. The value that they wrap, however, can implement the trait.
pub trait DeterministicHash { pub trait DeterministicHash {
/// Adds `self`'s bytes to the [`DeterministicHasher`]'s state, in a way that is replicatable on /// Adds `self`'s bytes to the [`DeterministicHasher`]'s state, in a way that is replicatable on
/// any platform or process run. /// any platform or process run.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment