Commit 42ccc8d6 authored by Luke Sandberg's avatar Luke Sandberg

Stop using TaskTypes as keys!

instead store hashes, this makes lookups slightly more complex but greatly reduces storage requirements.

clean up the old lookup methods

Use smallvecs, add tracing and share more implementation details in the db lookup methods

Add some safety to the new APIs and resolve a small race condition

fix some clippy issues

Remove obsolete info size metrics

revert a feature flag flip

fix stats collection and improve logic shaing

restore the scratch buffer

typo

fix silly bug

port back some simplifications

port back another batch of simplifications that landed on the wrong branch
parent b9e9f111
......@@ -1354,8 +1354,69 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
result_size = tracing::field::Empty
)
.entered();
let results = self.get_impl(family, key, false)?;
debug_assert!(results.len() <= 1, "get() should return at most one result");
match results.into_iter().next() {
Some(value) => {
span.record("result_size", value.len());
Ok(Some(value))
}
None => {
span.record("result_size", "not found");
Ok(None)
}
}
}
/// Looks up a key and returns all matching values.
///
/// This is useful for keyspaces where keys are not unique and collisions are possible.
/// Unlike `get`, which returns only the first match, this method returns all
/// entries with the same key from all SST files. By default however we assume these collisions
/// are extremely rare and thus optimize for there being exactly 0 or 1 results.
///
/// Note: This method does NOT deduplicate values. If the same key-value pair exists
/// in multiple SST files (e.g., before compaction), it will be returned multiple times.
pub fn get_multiple<K: QueryKey>(
&self,
family: usize,
key: &K,
) -> Result<SmallVec<[ArcSlice<u8>; 1]>> {
debug_assert!(family < FAMILIES, "Family index out of bounds");
let span = tracing::trace_span!(
"database read multiple",
name = family,
result_count = tracing::field::Empty,
result_size = tracing::field::Empty
)
.entered();
let results = self.get_impl(family, key, true)?;
span.record("result_count", results.len());
span.record(
"result_size",
results.iter().map(|r| r.len()).sum::<usize>(),
);
Ok(results)
}
/// Shared implementation for `get` and `get_multiple`.
///
/// If `find_all` is false, stops after finding the first match.
/// If `find_all` is true, continues to find all matches across all meta files.
fn get_impl<K: QueryKey>(
&self,
family: usize,
key: &K,
find_all: bool,
) -> Result<SmallVec<[ArcSlice<u8>; 1]>> {
let hash = hash_key(key);
let inner = self.inner.read();
let mut output: SmallVec<[ArcSlice<u8>; 1]> = SmallVec::new();
// Track whether we found the key in any SST (even if deleted).
// Used for miss_global stat: only fires if key was never found anywhere.
#[cfg(feature = "stats")]
let mut found_in_sst = false;
for meta in inner.meta_files.iter().rev() {
match meta.lookup(
family as u32,
......@@ -1363,6 +1424,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
key,
&self.key_block_cache,
&self.value_block_cache,
find_all,
)? {
MetaLookupResult::FamilyMiss => {
#[cfg(feature = "stats")]
......@@ -1377,29 +1439,40 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
self.stats.miss_amqf.fetch_add(1, Ordering::Relaxed);
}
MetaLookupResult::SstLookup(result) => match result {
SstLookupResult::Found(result) => {
SstLookupResult::Found(values) => {
#[cfg(feature = "stats")]
{
found_in_sst = true;
}
inner.accessed_key_hashes[family].insert(hash);
match result {
LookupValue::Deleted => {
#[cfg(feature = "stats")]
self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed);
span.record("result_size", "deleted");
return Ok(None);
}
LookupValue::Slice { value } => {
#[cfg(feature = "stats")]
self.stats.hits_small.fetch_add(1, Ordering::Relaxed);
span.record("result_size", value.len());
return Ok(Some(value));
}
LookupValue::Blob { sequence_number } => {
#[cfg(feature = "stats")]
self.stats.hits_blob.fetch_add(1, Ordering::Relaxed);
let blob = self.read_blob(sequence_number)?;
span.record("result_size", blob.len());
return Ok(Some(blob));
for value in values {
match value {
LookupValue::Deleted => {
#[cfg(feature = "stats")]
self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed);
if !find_all {
// For get, deleted means "key was deleted", stop looking
return Ok(SmallVec::new());
}
// For get_multiple, skip deleted entries but keep looking
}
LookupValue::Slice { value } => {
#[cfg(feature = "stats")]
self.stats.hits_small.fetch_add(1, Ordering::Relaxed);
output.push(value);
}
LookupValue::Blob { sequence_number } => {
#[cfg(feature = "stats")]
self.stats.hits_blob.fetch_add(1, Ordering::Relaxed);
let blob = self.read_blob(sequence_number)?;
output.push(blob);
}
}
}
if !find_all {
// For get, we found a non-deleted value, return it
return Ok(output);
}
}
SstLookupResult::NotFound => {
#[cfg(feature = "stats")]
......@@ -1408,10 +1481,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
},
}
}
#[cfg(feature = "stats")]
self.stats.miss_global.fetch_add(1, Ordering::Relaxed);
span.record("result_size", "not found");
Ok(None)
if !found_in_sst {
self.stats.miss_global.fetch_add(1, Ordering::Relaxed);
}
Ok(output)
}
pub fn batch_get<K: QueryKey>(
......
......@@ -13,6 +13,7 @@ use bincode::{Decode, Encode};
use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt};
use memmap2::{Mmap, MmapOptions};
use smallvec::SmallVec;
use turbo_bincode::turbo_bincode_decode;
use crate::{
......@@ -373,6 +374,11 @@ impl MetaFile {
&self.obsolete_sst_files
}
/// Looks up a key in this meta file.
///
/// If `find_all` is false, returns after finding the first match.
/// If `find_all` is true, returns all entries with the same key from all SST files
/// (useful for keyspaces where keys are hashes and collisions are possible).
pub fn lookup<K: QueryKey>(
&self,
key_family: u32,
......@@ -380,30 +386,54 @@ impl MetaFile {
key: &K,
key_block_cache: &BlockCache,
value_block_cache: &BlockCache,
find_all: bool,
) -> Result<MetaLookupResult> {
if key_family != self.family {
return Ok(MetaLookupResult::FamilyMiss);
}
let mut miss_result = MetaLookupResult::RangeMiss;
let mut all_results: SmallVec<[LookupValue; 1]> = SmallVec::new();
for entry in self.entries.iter().rev() {
if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue;
}
{
let amqf = entry.amqf(self)?;
if !amqf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
let amqf = entry.amqf(self)?;
if !amqf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
let result =
entry
.sst(self)?
.lookup(key_hash, key, key_block_cache, value_block_cache)?;
if !matches!(result, SstLookupResult::NotFound) {
return Ok(MetaLookupResult::SstLookup(result));
let result = entry.sst(self)?.lookup(
key_hash,
key,
key_block_cache,
value_block_cache,
find_all,
)?;
match result {
SstLookupResult::NotFound => {
// continue searching other sst files
}
SstLookupResult::Found(values) => {
if !find_all {
// Return immediately with the first result
return Ok(MetaLookupResult::SstLookup(SstLookupResult::Found(values)));
}
// Accumulate results
all_results.extend(values);
}
}
}
if !all_results.is_empty() {
return Ok(MetaLookupResult::SstLookup(SstLookupResult::Found(
all_results,
)));
}
Ok(miss_result)
}
......@@ -483,8 +513,14 @@ impl MetaFile {
&keys[*index],
key_block_cache,
value_block_cache,
false, // find_all: batch_lookup returns first match per key
)?;
if let SstLookupResult::Found(value) = sst_result {
if let SstLookupResult::Found(mut values) = sst_result {
// find_all=false guarantees exactly one result
debug_assert!(values.len() == 1);
let Some(value) = values.pop() else {
unreachable!()
};
*result = Some(value);
*empty_cells -= 1;
#[cfg(feature = "stats")]
......
......@@ -11,6 +11,7 @@ use byteorder::{BE, ReadBytesExt};
use memmap2::Mmap;
use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher;
use smallvec::SmallVec;
use crate::{
QueryKey,
......@@ -47,15 +48,15 @@ const _: () = assert!(
/// The result of a lookup operation.
pub enum SstLookupResult {
/// The key was found.
Found(LookupValue),
/// One or more values were found.
Found(SmallVec<[LookupValue; 1]>),
/// The key was not found.
NotFound,
}
impl From<LookupValue> for SstLookupResult {
fn from(value: LookupValue) -> Self {
SstLookupResult::Found(value)
SstLookupResult::Found(smallvec::smallvec![value])
}
}
......@@ -148,14 +149,20 @@ impl StaticSortedFile {
}
/// Looks up a key in this file.
///
/// If `find_all` is false, returns after finding the first match.
/// If `find_all` is true, returns all entries with the same key (useful for
/// keyspaces where keys are hashes and collisions are possible).
pub fn lookup<K: QueryKey>(
&self,
key_hash: u64,
key: &K,
key_block_cache: &BlockCache,
value_block_cache: &BlockCache,
find_all: bool,
) -> Result<SstLookupResult> {
let mut current_block = self.meta.block_count - 1;
let this = &self;
let mut current_block = this.meta.block_count - 1;
loop {
let mut key_block_arc = self.get_key_block(current_block, key_block_cache)?;
let block_type = key_block_arc.read_u8()?;
......@@ -171,6 +178,7 @@ impl StaticSortedFile {
key,
has_hash,
value_block_cache,
find_all,
);
}
_ => {
......@@ -227,6 +235,9 @@ impl StaticSortedFile {
}
/// Looks up a key in a key block and the value in a value block.
///
/// If `find_all` is false, returns after finding the first match.
/// If `find_all` is true, collects all entries with the same key.
fn lookup_key_block<K: QueryKey>(
&self,
mut block: ArcSlice<u8>,
......@@ -234,6 +245,7 @@ impl StaticSortedFile {
key: &K,
has_hash: bool,
value_block_cache: &BlockCache,
find_all: bool,
) -> Result<SstLookupResult> {
let hash_len: u8 = if has_hash { 8 } else { 0 };
let entry_count = block.read_u24::<BE>()? as usize;
......@@ -242,33 +254,58 @@ impl StaticSortedFile {
let mut l = 0;
let mut r = entry_count;
// binary search for the key
let mut found = false;
// binary search for the first matching key
// TODO: we actually need to find the last matching key
while l < r {
let m = (l + r) / 2;
let GetKeyEntryResult {
hash: mid_hash,
key: mid_key,
ty,
val: mid_val,
..
} = get_key_entry(offsets, entries, entry_count, m, hash_len)?;
let comparison = compare_hash_key(mid_hash, mid_key, key_hash, key);
match comparison {
Ordering::Less => {
r = m;
}
Ordering::Less => r = m,
Ordering::Equal => {
return Ok(self
.handle_key_match(ty, mid_val, &block, value_block_cache)?
.into());
found = true;
r = m;
}
Ordering::Greater => {
l = m + 1;
Ordering::Greater => l = m + 1,
}
}
if !found {
return Ok(SstLookupResult::NotFound);
}
// l is now the index of the first matching element
let GetKeyEntryResult { ty, val, .. } =
get_key_entry(offsets, entries, entry_count, l, hash_len)?;
// Collect all entries with matching key starting from l
let mut results = SmallVec::new();
results.push(self.handle_key_match(ty, val, &block, value_block_cache)?);
if find_all {
// duplicates, if there are anyu will be immediately following this one.
for i in (l + 1)..entry_count {
let GetKeyEntryResult {
hash,
key: entry_key,
ty,
val,
} = get_key_entry(offsets, entries, entry_count, i, hash_len)?;
let comparison = compare_hash_key(hash, entry_key, key_hash, key);
if comparison != Ordering::Equal {
break;
}
results.push(self.handle_key_match(ty, val, &block, value_block_cache)?);
}
}
Ok(SstLookupResult::NotFound)
Ok(SstLookupResult::Found(results))
}
/// Handles a key match by looking up the value.
......
......@@ -1303,3 +1303,211 @@ fn batch_get_after_restore() -> Result<()> {
Ok(())
}
#[test]
fn get_multiple_same_key_different_batches() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Create a database and write the same key with different values in separate batches.
// Before compaction, all values should be retrievable via get_multiple.
let key = vec![42u8, 42u8, 42u8]; // The key we'll write multiple times
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
// Write first value for the key
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![1u8].into())?;
db.commit_write_batch(batch)?;
// Write second value for the same key in a new batch (new SST file)
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![2u8].into())?;
db.commit_write_batch(batch)?;
// Write third value for the same key
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![3u8].into())?;
db.commit_write_batch(batch)?;
// Now get_multiple should return all three values (newest first, as we iterate in reverse)
let results = db.get_multiple(0, &key.as_slice())?;
// All three values should be present
assert_eq!(results.len(), 3, "Should have 3 values before compaction");
// Convert to sorted Vec for comparison (order may vary based on SST iteration order)
let mut values: Vec<u8> = results.iter().map(|r| r[0]).collect();
values.sort();
assert_eq!(values, vec![1, 2, 3], "Should have all three values");
// Regular get should return only the newest value
let single = db.get(0, &key.as_slice())?;
assert_eq!(
single.as_deref(),
Some(&[3u8][..]),
"Regular get should return newest value"
);
db.shutdown()?;
}
// Reopen and verify persistence
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(results.len(), 3, "Should still have 3 values after reopen");
db.shutdown()?;
}
// After compaction, only the newest value should remain
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
db.full_compact()?;
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(
results.len(),
1,
"Should have only 1 value after compaction"
);
assert_eq!(
results[0].as_ref(),
&[3u8],
"Should be the newest value after compaction"
);
db.shutdown()?;
}
Ok(())
}
#[test]
fn get_multiple_same_key_same_batch() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Test writing the same key multiple times within a single batch.
// The SST file should contain all entries, and get_multiple should return them all.
let key = vec![99u8];
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let batch = db.write_batch()?;
// Write multiple values for the same key in the same batch
batch.put(0, key.clone(), vec![10u8].into())?;
batch.put(0, key.clone(), vec![20u8].into())?;
batch.put(0, key.clone(), vec![30u8].into())?;
db.commit_write_batch(batch)?;
// Within a single SST file, entries with same key are sorted and all present
let results = db.get_multiple(0, &key.as_slice())?;
// All three values should be present in the SST (no compaction yet)
assert_eq!(
results.len(),
3,
"Should have 3 values in same batch before compaction"
);
let mut values: Vec<u8> = results.iter().map(|r| r[0]).collect();
values.sort();
assert_eq!(values, vec![10, 20, 30]);
db.shutdown()?;
}
Ok(())
}
#[test]
fn get_multiple_different_keys() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Test that get_multiple only returns values for the specific key,
// not values for other keys.
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![100u8].into())?;
batch.put(0, vec![2u8], vec![200u8].into())?;
batch.put(0, vec![3u8], vec![200u8].into())?; // Same value, different key
db.commit_write_batch(batch)?;
// get_multiple for key [1] should only return value [100]
let results = db.get_multiple(0, &[1u8].as_slice())?;
assert_eq!(results.len(), 1);
assert_eq!(results[0].as_ref(), &[100u8]);
// get_multiple for key [2] should only return value [200]
let results = db.get_multiple(0, &[2u8].as_slice())?;
assert_eq!(results.len(), 1);
assert_eq!(results[0].as_ref(), &[200u8]);
// get_multiple for non-existent key should return empty
let results = db.get_multiple(0, &[99u8].as_slice())?;
assert_eq!(results.len(), 0);
db.shutdown()?;
}
Ok(())
}
#[test]
fn get_multiple_empty_result() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
{
let db = TurboPersistence::<_, 16>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![100u8].into())?;
db.commit_write_batch(batch)?;
// Query for a key that doesn't exist
let results = db.get_multiple(0, &[99u8].as_slice())?;
assert!(
results.is_empty(),
"Should return empty vec for non-existent key"
);
// Query empty database for different family
let results = db.get_multiple(1, &[1u8].as_slice())?;
assert!(
results.is_empty(),
"Should return empty vec for different family"
);
db.shutdown()?;
}
Ok(())
}
......@@ -282,21 +282,6 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
ExecuteContextImpl::new(self, turbo_tasks)
}
/// # Safety
///
/// `tx` must be a transaction from this TurboTasksBackendInner instance.
unsafe fn execute_context_with_tx<'e, 'tx>(
&'e self,
tx: Option<&'e B::ReadTransaction<'tx>>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) -> impl ExecuteContext<'e> + use<'e, 'tx, B>
where
'tx: 'e,
{
// Safety: `tx` is from `self`.
unsafe { ExecuteContextImpl::new_with_tx(self, tx, turbo_tasks) }
}
fn suspending_requested(&self) -> bool {
self.should_persist()
&& (self.in_progress_operations.load(Ordering::Relaxed) & SNAPSHOT_REQUESTED_BIT) != 0
......@@ -418,22 +403,6 @@ struct TaskExecutionCompletePrepareResult {
// Operations
impl<B: BackingStorage> TurboTasksBackendInner<B> {
/// # Safety
///
/// `tx` must be a transaction from this TurboTasksBackendInner instance.
unsafe fn connect_child_with_tx<'l, 'tx: 'l>(
&'l self,
tx: Option<&'l B::ReadTransaction<'tx>>,
parent_task: Option<TaskId>,
child_task: TaskId,
task_type: Option<ArcOrOwned<CachedTaskType>>,
turbo_tasks: &'l dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) {
operation::ConnectChildOperation::run(parent_task, child_task, task_type, unsafe {
self.execute_context_with_tx(tx, turbo_tasks)
});
}
fn connect_child(
&self,
parent_task: Option<TaskId>,
......@@ -1428,71 +1397,55 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
return task_id;
}
let check_backing_storage =
self.should_restore() && self.local_is_partial.load(Ordering::Acquire);
let tx = check_backing_storage
.then(|| self.backing_storage.start_read_transaction())
.flatten();
let (task_id, task_type) = {
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
if let Some(task_id) = unsafe {
check_backing_storage
.then(|| {
self.backing_storage
.forward_lookup_task_cache(tx.as_ref(), &task_type)
.expect("Failed to lookup task id")
})
.flatten()
} {
// Task exists in backing storage
// So we only need to insert it into the in-memory cache
self.track_cache_hit(&task_type);
let task_type = match raw_entry(&self.task_cache, &task_type) {
RawEntry::Occupied(_) => ArcOrOwned::Owned(task_type),
RawEntry::Vacant(e) => {
let task_type = Arc::new(task_type);
e.insert(task_type.clone(), task_id);
ArcOrOwned::Arc(task_type)
}
};
(task_id, task_type)
} else {
// Task doesn't exist in memory cache or backing storage
// So we might need to create a new task
let (task_id, mut task_type) = match raw_entry(&self.task_cache, &task_type) {
RawEntry::Occupied(e) => {
let task_id = *e.get();
drop(e);
self.track_cache_hit(&task_type);
(task_id, ArcOrOwned::Owned(task_type))
}
RawEntry::Vacant(e) => {
let task_type = Arc::new(task_type);
let task_id = self.persisted_task_id_factory.get();
e.insert(task_type.clone(), task_id);
self.track_cache_miss(&task_type);
(task_id, ArcOrOwned::Arc(task_type))
// Create a single ExecuteContext for both lookup and connect_child
let mut ctx = self.execute_context(turbo_tasks);
// Look up task from backing storage (handles hash-based lookup and verification)
let found_task_id = ctx.task_by_type(&task_type);
let (task_id, task_type) = if let Some(task_id) = found_task_id {
// Task exists in backing storage
// So we only need to insert it into the in-memory cache
self.track_cache_hit(&task_type);
let task_type = match raw_entry(&self.task_cache, &task_type) {
RawEntry::Occupied(_) => ArcOrOwned::Owned(task_type),
RawEntry::Vacant(e) => {
let task_type = Arc::new(task_type);
e.insert(task_type.clone(), task_id);
ArcOrOwned::Arc(task_type)
}
};
(task_id, task_type)
} else {
// Task doesn't exist in memory cache or backing storage
// So we might need to create a new task
let (task_id, task_type) = match raw_entry(&self.task_cache, &task_type) {
RawEntry::Occupied(e) => {
// Another thread beat us to creating this task - use their task_id.
// They will handle logging to persisted_task_cache_log.
let task_id = *e.get();
drop(e);
self.track_cache_hit(&task_type);
(task_id, ArcOrOwned::Owned(task_type))
}
RawEntry::Vacant(e) => {
// We're creating a new task.
let task_type = Arc::new(task_type);
let task_id = self.persisted_task_id_factory.get();
e.insert(task_type.clone(), task_id);
// insert() consumes e, releasing the lock
self.track_cache_miss(&task_type);
if let Some(log) = &self.persisted_task_cache_log {
log.lock(task_id).push((task_type.clone(), task_id));
}
};
if let Some(log) = &self.persisted_task_cache_log {
let task_type_arc: Arc<CachedTaskType> = Arc::from(task_type);
log.lock(task_id).push((task_type_arc.clone(), task_id));
task_type = ArcOrOwned::Arc(task_type_arc);
(task_id, ArcOrOwned::Arc(task_type))
}
(task_id, task_type)
}
};
(task_id, task_type)
};
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
unsafe {
self.connect_child_with_tx(
tx.as_ref(),
parent_task,
task_id,
Some(task_type),
turbo_tasks,
)
};
// Reuse the same ExecuteContext for connect_child
operation::ConnectChildOperation::run(parent_task, task_id, Some(task_type), ctx);
task_id
}
......
......@@ -34,10 +34,8 @@ pub trait Operation: Encode + Decode<()> + Default + TryFrom<AnyOperation, Error
fn execute(self, ctx: &mut impl ExecuteContext<'_>);
}
#[derive(Copy, Clone)]
enum TransactionState<'a, 'tx, B: BackingStorage> {
enum TransactionState<'tx, B: BackingStorage> {
None,
Borrowed(Option<&'a B::ReadTransaction<'tx>>),
Owned(Option<B::ReadTransaction<'tx>>),
}
......@@ -93,6 +91,12 @@ pub trait ExecuteContext<'e>: Sized {
fn suspending_requested(&self) -> bool;
fn should_track_dependencies(&self) -> bool;
fn should_track_activeness(&self) -> bool;
/// Look up a TaskId from the backing storage for a given task type.
///
/// Uses hash-based lookup which may return multiple candidates due to hash collisions,
/// then verifies each candidate by comparing the stored `persistent_task_type`.
/// Returns `Some(task_id)` if a matching task is found, `None` otherwise.
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId>;
}
pub trait ChildExecuteContext<'e>: Send + Sized {
......@@ -107,7 +111,7 @@ where
backend: &'e TurboTasksBackendInner<B>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
_operation_guard: Option<OperationGuard<'e, B>>,
transaction: TransactionState<'e, 'tx, B>,
transaction: TransactionState<'tx, B>,
#[cfg(debug_assertions)]
active_task_locks: std::sync::Arc<std::sync::atomic::AtomicU8>,
}
......@@ -130,21 +134,6 @@ where
}
}
pub(super) unsafe fn new_with_tx(
backend: &'e TurboTasksBackendInner<B>,
transaction: Option<&'e B::ReadTransaction<'tx>>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) -> Self {
Self {
backend,
turbo_tasks,
_operation_guard: Some(backend.start_operation()),
transaction: TransactionState::Borrowed(transaction),
#[cfg(debug_assertions)]
active_task_locks: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(0)),
}
}
fn ensure_transaction(&mut self) -> bool {
if matches!(self.transaction, TransactionState::None) {
let check_backing_storage = self.backend.should_restore()
......@@ -228,7 +217,6 @@ where
fn get_tx(&self) -> Option<&<B as BackingStorageSealed>::ReadTransaction<'tx>> {
match &self.transaction {
TransactionState::None => unreachable!(),
TransactionState::Borrowed(tx) => *tx,
TransactionState::Owned(tx) => tx.as_ref(),
}
}
......@@ -642,6 +630,35 @@ where
fn should_track_activeness(&self) -> bool {
self.backend.should_track_activeness()
}
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId> {
// Ensure we have a transaction (this will be reused by subsequent task() calls)
if !self.ensure_transaction() {
return None;
}
let tx = self.get_tx();
// Get candidates from backing storage (hash-based lookup may return multiple)
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
let candidates = unsafe {
self.backend
.backing_storage
.lookup_task_candidate(tx, task_type)
.expect("Failed to lookup task ids")
};
// Verify each candidate by comparing the stored persistent_task_type.
// Only rarely is there more than one candidate, so no need for parallelization.
for candidate_id in candidates {
let task = self.task(candidate_id, TaskDataCategory::Data);
if let Some(stored_type) = task.get_persistent_task_type()
&& stored_type.as_ref() == task_type
{
return Some(candidate_id);
}
}
None
}
}
struct ChildExecuteContextImpl<'e, B: BackingStorage> {
......
......@@ -60,14 +60,20 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
> + Send
+ Sync;
fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>;
/// Returns all task IDs that match the given task type (hash collision candidates).
///
/// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key.
/// The caller must verify each returned TaskId by comparing the stored task type which will
/// require a second database read
///
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
unsafe fn forward_lookup_task_cache(
unsafe fn lookup_task_candidate(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType,
) -> Result<Option<TaskId>>;
) -> Result<SmallVec<[TaskId; 1]>>;
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
......@@ -151,19 +157,19 @@ where
})
}
unsafe fn forward_lookup_task_cache(
unsafe fn lookup_task_candidate(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType,
) -> Result<Option<TaskId>> {
) -> Result<SmallVec<[TaskId; 1]>> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
unsafe { this.forward_lookup_task_cache(tx, key) }
unsafe { this.lookup_task_candidate(tx, key) }
}
Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
unsafe { this.forward_lookup_task_cache(tx, key) }
unsafe { this.lookup_task_candidate(tx, key) }
}
}
}
......
use anyhow::Result;
use smallvec::SmallVec;
use crate::database::write_batch::{
ConcurrentWriteBatch, SerialWriteBatch, UnimplementedWriteBatch, WriteBatch,
};
/// Describes the lookup semantics for a keyspace.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LookupSemantics {
/// Single value per key. `get` returns the newest value.
/// Using `get_multiple` is allowed but unnecessary.
SingleValue,
/// Multiple values may exist per key (e.g., hash collisions).
/// Callers MUST use `get_multiple` and verify candidates.
MultipleValues,
}
#[derive(Debug, Clone, Copy)]
pub enum KeySpace {
/// Infrastructure data. Single value per key, use `get`.
Infra = 0,
/// Stores a map from TaskId->TaskStorage meta. Single value per key, use `get`.
TaskMeta = 1,
/// Stores a map from TaskId->TaskStorage data. Single value per key, use `get`.
TaskData = 2,
/// Stores a map from TaskType _hash_ -> TaskId.
/// **Multiple values possible due to hash collisions.**
/// MUST use `get_multiple` and verify candidates match the actual TaskType.
TaskCache = 3,
}
impl KeySpace {
/// Returns the lookup semantics for this keyspace.
///
/// This indicates whether the keyspace expects single or multiple values per key,
/// which determines whether callers should use `get` or `get_multiple`.
pub fn lookup_semantics(&self) -> LookupSemantics {
match self {
KeySpace::Infra | KeySpace::TaskMeta | KeySpace::TaskData => {
LookupSemantics::SingleValue
}
KeySpace::TaskCache => LookupSemantics::MultipleValues,
}
}
}
pub trait KeyValueDatabase {
type ReadTransaction<'l>
where
......@@ -33,6 +66,19 @@ pub trait KeyValueDatabase {
key_space: KeySpace,
key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>>;
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
/// The default implementation returns at most one value (from `get`), but implementations
/// that support multiple values per key should override this.
fn get_multiple<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'l>; 1]>> {
Ok(self.get(transaction, key_space, key)?.into_iter().collect())
}
fn batch_get<'l, 'db: 'l>(
&'l self,
......@@ -48,6 +94,7 @@ pub trait KeyValueDatabase {
Ok(results)
}
/// Looks up a key by its hash, confirming the match by comparing the value.
type SerialWriteBatch<'l>: SerialWriteBatch<'l>
= UnimplementedWriteBatch
where
......
......@@ -7,7 +7,7 @@ use lmdb::{
};
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
key_value_database::{KeySpace, KeyValueDatabase, LookupSemantics},
write_batch::{BaseWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer},
};
......@@ -81,6 +81,11 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
key_space: super::key_value_database::KeySpace,
key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> {
debug_assert!(
key_space.lookup_semantics() != LookupSemantics::MultipleValues,
"KeySpace {:?} may have multiple values - use get_multiple instead",
key_space
);
let db = match key_space {
KeySpace::Infra => self.infra_db,
KeySpace::TaskMeta => self.meta_db,
......
......@@ -8,13 +8,14 @@ use std::{
use anyhow::{Ok, Result};
use parking_lot::Mutex;
use smallvec::SmallVec;
use turbo_persistence::{
ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer,
};
use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks};
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
key_value_database::{KeySpace, KeyValueDatabase, LookupSemantics},
turbo::parallel_scheduler::TurboTasksParallelScheduler,
write_batch::{BaseWriteBatch, ConcurrentWriteBatch, WriteBatch, WriteBuffer},
};
......@@ -81,6 +82,11 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
key_space: KeySpace,
key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> {
debug_assert!(
key_space.lookup_semantics() != LookupSemantics::MultipleValues,
"KeySpace {:?} may have multiple values - use get_multiple instead",
key_space
);
self.db.get(key_space as usize, &key)
}
......@@ -93,6 +99,20 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.batch_get(key_space as usize, keys)
}
fn get_multiple<'l, 'db: 'l>(
&'l self,
_transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'l>; 1]>> {
debug_assert!(
key_space.lookup_semantics() != LookupSemantics::SingleValue,
"KeySpace {:?} has single values - use get instead of get_multiple",
key_space
);
self.db.get_multiple(key_space as usize, &key)
}
type ConcurrentWriteBatch<'l>
= TurboWriteBatch<'l>
where
......
......@@ -30,8 +30,8 @@ macro_rules! impl_write_number {
/// Note that the default [`std::hash::Hash`] trait used by Rust allows for hashing that differs
/// across process runs, so it is not suitable for filesystem cache with turbo-tasks.
///
/// It's very important that `Vc`s never implement this, since they cannot be deterministic. The
/// value that they wrap, however, can implement the trait.
/// It's very important that `Vc`s never implement this, since they are not consistent from run to
/// run. The value that they wrap, however, can implement the trait.
pub trait DeterministicHash {
/// Adds `self`'s bytes to the [`DeterministicHasher`]'s state, in a way that is replicatable on
/// any platform or process run.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment