Commit 8769f4c3 authored by Luke Sandberg's avatar Luke Sandberg Committed by Luke Sandberg

reduce buffer lifetime

deserialize directly into the TaskStorage struct in the in memory cache

this means we need to hold the shard lock while doing it, but we save a bunch of temporary memory allocations from the snapshots
parent ed0b4ac5
...@@ -106,6 +106,45 @@ struct TrackedStats { ...@@ -106,6 +106,45 @@ struct TrackedStats {
miss_global: std::sync::atomic::AtomicU64, miss_global: std::sync::atomic::AtomicU64,
} }
/// A compact bitset for tracking which batch-lookup cells have been resolved.
///
/// Uses `u64` words (1 bit per cell) rather than `Vec<bool>` (1 byte per cell) for an 8x
/// reduction in memory and better cache behaviour on large batches.
pub struct FoundBitset {
words: Box<[u64]>,
len: usize,
}
impl FoundBitset {
pub(crate) fn new(len: usize) -> Self {
let words = vec![0u64; len.div_ceil(64)].into_boxed_slice();
Self { words, len }
}
#[inline]
pub(crate) fn get(&self, index: usize) -> bool {
debug_assert!(index < self.len);
(self.words[index / 64] >> (index % 64)) & 1 == 1
}
#[inline]
pub(crate) fn set(&mut self, index: usize) {
debug_assert!(index < self.len);
let wi = index / 64;
let bi = index % 64;
self.words[wi] |= 1 << bi;
}
/// Returns the number of bits that are set (i.e. not yet found).
pub(crate) fn count_ones(&self) -> usize {
self.words.iter().map(|w| w.count_ones()).sum::<u32>() as usize
}
pub(crate) fn len(&self) -> usize {
self.len
}
}
/// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time /// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time
/// using a single write batch. It allows for concurrent reads. /// using a single write batch. It allows for concurrent reads.
pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> { pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
...@@ -1603,11 +1642,36 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1603,11 +1642,36 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok(output) Ok(output)
} }
/// Looks up multiple keys in batch and collects results into a `Vec`.
///
/// For large batches where memory pressure matters, prefer
/// [`batch_get_with`][Self::batch_get_with] which calls a callback per entry without
/// accumulating all decoded bytes simultaneously.
pub fn batch_get<K: QueryKey>( pub fn batch_get<K: QueryKey>(
&self, &self,
family: usize, family: usize,
keys: &[K], keys: &[K],
) -> Result<Vec<Option<ArcBytes>>> { ) -> Result<Vec<Option<ArcBytes>>> {
let mut results = vec![None; keys.len()];
self.batch_get_with(family, keys, |index, opt_bytes| {
results[index] = opt_bytes.map(|b| ArcBytes::from(b.to_vec().into_boxed_slice()));
Ok(())
})?;
Ok(results)
}
/// Looks up multiple keys in batch, calling `callback(index, Option<&[u8]>)` for each entry
/// immediately after it is resolved rather than accumulating all results into a `Vec`.
///
/// This keeps at most one decompressed value block live at a time, significantly reducing
/// peak memory when the batch is large. The callback receives the 0-based key index and the
/// value bytes (`None` for not-found or deleted). Callback errors propagate immediately.
pub fn batch_get_with<K: QueryKey>(
&self,
family: usize,
keys: &[K],
mut callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<()> {
debug_assert!(family < FAMILIES, "Family index out of bounds"); debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue { if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic // This is an error in our caller so just panic
...@@ -1622,22 +1686,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1622,22 +1686,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
result_size = tracing::field::Empty result_size = tracing::field::Empty
) )
.entered(); .entered();
let mut cells: Vec<(u64, usize, Option<LookupValue>)> = Vec::with_capacity(keys.len()); let mut cells: Vec<(u64, usize)> = Vec::with_capacity(keys.len());
let mut empty_cells = keys.len();
for (index, key) in keys.iter().enumerate() { for (index, key) in keys.iter().enumerate() {
let hash = hash_key(key); let hash = hash_key(key);
cells.push((hash, index, None)); cells.push((hash, index));
} }
cells.sort_by_key(|(hash, _, _)| *hash); cells.sort_unstable_by_key(|(hash, _)| *hash);
let mut found = FoundBitset::new(cells.len());
let inner = self.inner.read(); let inner = self.inner.read();
let mut not_found = 0;
let mut deleted = 0;
let mut result_size = 0;
let mut read_blob = |seq| self.read_blob(seq);
// Wrap the callback to track stats (deleted/result_size) for found keys.
// not_found is tracked separately in the post-loop over cells.
let mut stats_callback = |index: usize, opt_bytes: Option<&[u8]>| -> Result<()> {
if let Some(bytes) = opt_bytes {
result_size += bytes.len();
} else {
deleted += 1;
}
callback(index, opt_bytes)
};
for meta in inner.meta_files.iter().rev() { for meta in inner.meta_files.iter().rev() {
let _result = meta.batch_lookup( let (all_found, _result) = meta.batch_lookup_with(
family as u32, family as u32,
keys, keys,
&mut cells, &cells,
&mut empty_cells, &mut found,
&self.key_block_cache, &self.key_block_cache,
&self.value_block_cache, &self.value_block_cache,
&mut read_blob,
&mut stats_callback,
)?; )?;
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -1669,49 +1749,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1669,49 +1749,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
} }
if empty_cells == 0 { if all_found {
break; break;
} }
} }
let mut deleted = 0; // Record accessed hashes for found keys, and fire callback for not-found keys.
let mut not_found = 0; for (hash, index) in cells.into_iter() {
let mut result_size = 0; if found.get(index) {
let mut results = vec![None; keys.len()];
for (hash, index, result) in cells {
if let Some(result) = result {
inner.accessed_key_hashes[family].insert(hash); inner.accessed_key_hashes[family].insert(hash);
let result = match result {
LookupValue::Deleted => {
#[cfg(feature = "stats")]
self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed);
deleted += 1;
None
}
LookupValue::Slice { value } => {
#[cfg(feature = "stats")]
self.stats.hits_small.fetch_add(1, Ordering::Relaxed);
result_size += value.len();
Some(value)
}
LookupValue::Blob { sequence_number } => {
#[cfg(feature = "stats")]
self.stats.hits_blob.fetch_add(1, Ordering::Relaxed);
let blob = self.read_blob(sequence_number)?;
result_size += blob.len();
Some(blob)
}
};
results[index] = result;
} else { } else {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
self.stats.miss_global.fetch_add(1, Ordering::Relaxed); self.stats.miss_global.fetch_add(1, Ordering::Relaxed);
not_found += 1; not_found += 1;
callback(index, None)?;
} }
} }
span.record("not_found", not_found); span.record("not_found", not_found);
span.record("deleted", deleted); span.record("deleted", deleted);
span.record("result_size", result_size); span.record("result_size", result_size);
Ok(results) Ok(())
} }
/// Returns database statistics. /// Returns database statistics.
......
...@@ -14,7 +14,8 @@ use smallvec::SmallVec; ...@@ -14,7 +14,8 @@ use smallvec::SmallVec;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be};
use crate::{ use crate::{
QueryKey, ArcBytes, QueryKey,
db::FoundBitset,
lookup_entry::LookupValue, lookup_entry::LookupValue,
mmap_helper::advise_mmap_for_persistence, mmap_helper::advise_mmap_for_persistence,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
...@@ -498,33 +499,44 @@ impl MetaFile { ...@@ -498,33 +499,44 @@ impl MetaFile {
Ok(miss_result) Ok(miss_result)
} }
pub fn batch_lookup<K: QueryKey>( pub fn batch_lookup_with<K: QueryKey>(
&self, &self,
key_family: u32, key_family: u32,
keys: &[K], keys: &[K],
cells: &mut [(u64, usize, Option<LookupValue>)], cells: &[(u64, usize)],
empty_cells: &mut usize, found: &mut FoundBitset,
key_block_cache: &BlockCache, key_block_cache: &BlockCache,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
) -> Result<MetaBatchLookupResult> { read_blob: &mut impl FnMut(u32) -> Result<ArcBytes>,
callback: &mut impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<(bool, MetaBatchLookupResult)> {
debug_assert_eq!(
cells.len(),
found.len(),
"cells and found must have the same length"
);
if key_family != self.family { if key_family != self.family {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
return Ok(MetaBatchLookupResult { return Ok((
family_miss: true, false,
..Default::default() MetaBatchLookupResult {
}); family_miss: true,
..Default::default()
},
));
#[cfg(not(feature = "stats"))] #[cfg(not(feature = "stats"))]
return Ok(MetaBatchLookupResult {}); return Ok((false, MetaBatchLookupResult {}));
} }
debug_assert!( debug_assert!(
cells.is_sorted_by_key(|(hash, _, _)| *hash), cells.is_sorted_by_key(|(hash, _)| *hash),
"Cells must be sorted by key hash" "Cells must be sorted by key hash"
); );
#[allow(unused_mut, reason = "It's used when stats are enabled")] #[allow(unused_mut, reason = "It's used when stats are enabled")]
let mut lookup_result = MetaBatchLookupResult::default(); let mut lookup_result = MetaBatchLookupResult::default();
let mut empty_cells = found.len() - found.count_ones();
for entry in self.entries.iter().rev() { for entry in self.entries.iter().rev() {
let start_index = cells let mut start_index = cells
.binary_search_by(|(hash, _, _)| hash.cmp(&entry.min_hash).then(Ordering::Greater)) .binary_search_by(|(hash, _)| hash.cmp(&entry.min_hash).then(Ordering::Greater))
.err() .err()
.unwrap(); .unwrap();
if start_index >= cells.len() { if start_index >= cells.len() {
...@@ -534,11 +546,23 @@ impl MetaFile { ...@@ -534,11 +546,23 @@ impl MetaFile {
} }
continue; continue;
} }
let end_index = cells // Advance to the first not found cell.
.binary_search_by(|(hash, _, _)| hash.cmp(&entry.max_hash).then(Ordering::Less)) while start_index < cells.len() && found.get(cells[start_index].1) {
start_index += 1;
}
if start_index >= cells.len() {
#[cfg(feature = "stats")]
{
lookup_result.range_misses += 1;
}
continue;
}
let end_index = cells[start_index..]
.binary_search_by(|(hash, _)| hash.cmp(&entry.max_hash).then(Ordering::Less))
.err() .err()
.unwrap() .unwrap()
.checked_sub(1); .checked_sub(1)
.map(|i| i + start_index);
let Some(end_index) = end_index else { let Some(end_index) = end_index else {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
{ {
...@@ -553,12 +577,13 @@ impl MetaFile { ...@@ -553,12 +577,13 @@ impl MetaFile {
} }
continue; continue;
} }
for (hash, index, result) in &mut cells[start_index..=end_index] { for (hash, index) in cells[start_index..=end_index].iter() {
debug_assert!( debug_assert!(
*hash >= entry.min_hash && *hash <= entry.max_hash, *hash >= entry.min_hash && *hash <= entry.max_hash,
"Key hash out of range" "Key hash out of range"
); );
if result.is_some() { if found.get(*index) {
// we already found this key in a different meta-file
continue; continue;
} }
if !entry.amqf.contains_fingerprint(*hash) { if !entry.amqf.contains_fingerprint(*hash) {
...@@ -580,14 +605,27 @@ impl MetaFile { ...@@ -580,14 +605,27 @@ impl MetaFile {
let Some(value) = values.pop() else { let Some(value) = values.pop() else {
unreachable!() unreachable!()
}; };
*result = Some(value); // Resolve and callback immediately — the ArcBytes/blob is dropped before
*empty_cells -= 1; // the next iteration so we never hold more than one decompressed block at a
// time.
let blob;
let opt_bytes: Option<&[u8]> = match &value {
LookupValue::Deleted => None,
LookupValue::Slice { value } => Some(value),
LookupValue::Blob { sequence_number } => {
blob = read_blob(*sequence_number)?;
Some(&*blob)
}
};
found.set(*index);
empty_cells -= 1;
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
{ {
lookup_result.hits += 1; lookup_result.hits += 1;
} }
if *empty_cells == 0 { callback(*index, opt_bytes)?;
return Ok(lookup_result); if empty_cells == 0 {
return Ok((true, lookup_result));
} }
} else { } else {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -597,6 +635,6 @@ impl MetaFile { ...@@ -597,6 +635,6 @@ impl MetaFile {
} }
} }
} }
Ok(lookup_result) Ok((false, lookup_result))
} }
} }
...@@ -4,10 +4,11 @@ use anyhow::Result; ...@@ -4,10 +4,11 @@ use anyhow::Result;
use either::Either; use either::Either;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
use turbo_persistence::ArcBytes;
use turbo_tasks::{TaskId, backend::CachedTaskType}; use turbo_tasks::{TaskId, backend::CachedTaskType};
use crate::{ use crate::{
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage}, backend::{AnyOperation, SpecificTaskDataCategory},
utils::chunked_vec::ChunkedVec, utils::chunked_vec::ChunkedVec,
}; };
...@@ -69,22 +70,33 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -69,22 +70,33 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
/// The caller must verify each returned TaskId by comparing the stored task type which will /// The caller must verify each returned TaskId by comparing the stored task type which will
/// require a second database read /// require a second database read
fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>>; fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>>;
/// Looks up and decodes persisted data for a single task, updating the provided storage with /// Looks up raw persisted bytes for a single task.
/// data from the database in the given category. ///
/// Returns `None` if the task has no persisted data for the given category.
/// The caller is responsible for decoding the returned bytes. The returned
/// [`ArcBytes`] is ref-counted and zero-copy when backed by a memory-mapped file.
fn lookup_data( fn lookup_data(
&self, &self,
task_id: TaskId, task_id: TaskId,
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
storage: &mut TaskStorage, ) -> Result<Option<ArcBytes>>;
) -> Result<()>;
/// Batch lookup and decode data for multiple tasks directly into TypedStorage instances. /// Batch lookup raw bytes for multiple tasks, calling
/// Returns a vector of TypedStorage, one for each task_id in the input slice. /// `callback(index, Option<&[u8]>)` for every task in `task_ids` immediately after its
/// entry is resolved. Index corresponds to the position in `task_ids`. `None` means the key
/// was not found in the database (no allocation is made for missing keys).
///
/// The caller is responsible for decoding the bytes. The byte slice is only valid for the
/// duration of the callback invocation and must not be stored.
///
/// This avoids allocating an intermediate `TaskStorage` per entry — callers can decode
/// directly into the live in-memory storage, reducing peak memory usage for large batches.
fn batch_lookup_data( fn batch_lookup_data(
&self, &self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>>; callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()>;
fn compact(&self) -> Result<bool> { fn compact(&self) -> Result<bool> {
Ok(false) Ok(false)
...@@ -142,17 +154,17 @@ where ...@@ -142,17 +154,17 @@ where
&self, &self,
task_id: TaskId, task_id: TaskId,
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
storage: &mut TaskStorage, ) -> Result<Option<ArcBytes>> {
) -> Result<()> { either::for_both!(self, this => this.lookup_data(task_id, category))
either::for_both!(self, this => this.lookup_data(task_id, category, storage))
} }
fn batch_lookup_data( fn batch_lookup_data(
&self, &self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> { callback: &mut dyn FnMut(usize, Option<&[u8]>),
either::for_both!(self, this => this.batch_lookup_data(task_ids, category)) ) -> Result<()> {
either::for_both!(self, this => this.batch_lookup_data(task_ids, category, callback))
} }
fn compact(&self) -> Result<bool> { fn compact(&self) -> Result<bool> {
......
use anyhow::Result; use anyhow::Result;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_persistence::{FamilyConfig, FamilyKind}; use turbo_persistence::{ArcBytes, FamilyConfig, FamilyKind};
use crate::database::write_batch::ConcurrentWriteBatch; use crate::database::write_batch::ConcurrentWriteBatch;
...@@ -57,36 +57,26 @@ pub trait KeyValueDatabase { ...@@ -57,36 +57,26 @@ pub trait KeyValueDatabase {
false false
} }
type ValueBuffer<'l>: std::borrow::Borrow<[u8]> fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>>;
where
Self: 'l;
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>>;
/// Looks up a key and returns all matching values. /// Looks up a key and returns all matching values.
/// ///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache). /// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
/// The default implementation returns at most one value (from `get`), but implementations /// The default implementation returns at most one value (from `get`), but implementations
/// that support multiple values per key should override this. /// that support multiple values per key should override this.
fn get_multiple( fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>>;
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'_>; 1]>> {
Ok(self.get(key_space, key)?.into_iter().collect())
}
fn batch_get( /// Calls `callback(index, Option<&[u8]>)` for each key immediately after it is resolved,
/// rather than accumulating all results into a `Vec`. This avoids holding every decompressed
/// value simultaneously in memory.
///
/// The default implementation falls back to sequential `get` calls. Implementations backed by
/// [`TurboPersistence`] override this to use the native streaming path.
fn batch_get_with(
&self, &self,
key_space: KeySpace, key_space: KeySpace,
keys: &[&[u8]], keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> { callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
let mut results = Vec::with_capacity(keys.len()); ) -> Result<()>;
for key in keys {
let value = self.get(key_space, key)?;
results.push(value);
}
Ok(results)
}
type ConcurrentWriteBatch<'l>: ConcurrentWriteBatch<'l> type ConcurrentWriteBatch<'l>: ConcurrentWriteBatch<'l>
where where
......
use anyhow::Result; use anyhow::Result;
use smallvec::SmallVec;
use turbo_persistence::ArcBytes;
use crate::database::{ use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase}, key_value_database::{KeySpace, KeyValueDatabase},
...@@ -8,14 +10,23 @@ use crate::database::{ ...@@ -8,14 +10,23 @@ use crate::database::{
pub struct NoopKvDb; pub struct NoopKvDb;
impl KeyValueDatabase for NoopKvDb { impl KeyValueDatabase for NoopKvDb {
type ValueBuffer<'l> fn get(&self, _key_space: KeySpace, _key: &[u8]) -> Result<Option<ArcBytes>> {
= &'l [u8]
where
Self: 'l;
fn get(&self, _key_space: KeySpace, _key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>> {
Ok(None) Ok(None)
} }
fn get_multiple(&self, _key_space: KeySpace, _key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
Ok(SmallVec::new())
}
fn batch_get_with(
&self,
_key_space: KeySpace,
keys: &[&[u8]],
mut callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<()> {
for (index, _key) in keys.iter().enumerate() {
callback(index, None)?;
}
Ok(())
}
type ConcurrentWriteBatch<'l> type ConcurrentWriteBatch<'l>
= NoopWriteBatch = NoopWriteBatch
......
...@@ -87,28 +87,20 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -87,28 +87,20 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.is_empty() self.db.is_empty()
} }
type ValueBuffer<'l> fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> {
= ArcBytes
where
Self: 'l;
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>> {
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
fn batch_get( fn batch_get_with(
&self, &self,
key_space: KeySpace, key_space: KeySpace,
keys: &[&[u8]], keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> { callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
self.db.batch_get(key_space as usize, keys) ) -> Result<()> {
self.db.batch_get_with(key_space as usize, keys, callback)
} }
fn get_multiple( fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'_>; 1]>> {
self.db.get_multiple(key_space as usize, &key) self.db.get_multiple(key_space as usize, &key)
} }
......
...@@ -7,7 +7,8 @@ use std::{ ...@@ -7,7 +7,8 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode}; use turbo_bincode::{turbo_bincode_decode, turbo_bincode_encode};
use turbo_persistence::ArcBytes;
use turbo_tasks::{ use turbo_tasks::{
TaskId, TaskId,
backend::CachedTaskType, backend::CachedTaskType,
...@@ -18,7 +19,7 @@ use turbo_tasks_hash::Xxh3Hash64Hasher; ...@@ -18,7 +19,7 @@ use turbo_tasks_hash::Xxh3Hash64Hasher;
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage}, backend::{AnyOperation, SpecificTaskDataCategory},
backing_storage::{BackingStorage, BackingStorageSealed, SnapshotItem}, backing_storage::{BackingStorage, BackingStorageSealed, SnapshotItem},
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
...@@ -47,11 +48,6 @@ impl AsRef<[u8]> for IntKey { ...@@ -47,11 +48,6 @@ impl AsRef<[u8]> for IntKey {
} }
} }
fn as_u32(bytes: impl Borrow<[u8]>) -> Result<u32> {
let n = u32::from_le_bytes(bytes.borrow().try_into()?);
Ok(n)
}
// We want to invalidate the cache on panic for most users, but this is a band-aid to underlying // We want to invalidate the cache on panic for most users, but this is a band-aid to underlying
// problems in turbo-tasks. // problems in turbo-tasks.
// //
...@@ -187,7 +183,7 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorageInner<T> { ...@@ -187,7 +183,7 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorageInner<T> {
fn get_infra_u32(&self, key: u32) -> Result<Option<u32>> { fn get_infra_u32(&self, key: u32) -> Result<Option<u32>> {
self.database self.database
.get(KeySpace::Infra, IntKey::new(key).as_ref())? .get(KeySpace::Infra, IntKey::new(key).as_ref())?
.map(as_u32) .map(|b| Ok(u32::from_le_bytes((*b).try_into()?)))
.transpose() .transpose()
} }
} }
...@@ -212,16 +208,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -212,16 +208,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
} }
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> { fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> {
fn get(database: &impl KeyValueDatabase) -> Result<Vec<AnyOperation>> { self.inner
let Some(operations) = .database
database.get(KeySpace::Infra, IntKey::new(META_KEY_OPERATIONS).as_ref())? .get(KeySpace::Infra, IntKey::new(META_KEY_OPERATIONS).as_ref())?
else { .map(|b| turbo_bincode_decode(&b))
return Ok(Vec::new()); .transpose()
}; .map(Option::unwrap_or_default)
let operations = turbo_bincode_decode(operations.borrow())?; .context("Unable to read uncompleted operations from database")
Ok(operations)
}
get(&self.inner.database).context("Unable to read uncompleted operations from database")
} }
fn save_snapshot<I>( fn save_snapshot<I>(
...@@ -298,14 +291,14 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -298,14 +291,14 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
} }
fn lookup_task_candidates(&self, task_type: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>> { fn lookup_task_candidates(&self, task_type: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>> {
let inner = &*self.inner; if self.inner.database.is_empty() {
if inner.database.is_empty() {
// Checking if the database is empty is a performance optimization // Checking if the database is empty is a performance optimization
// to avoid computing the hash. // to avoid computing the hash.
return Ok(SmallVec::new()); return Ok(SmallVec::new());
} }
let hash = compute_task_type_hash(task_type); let hash = compute_task_type_hash(task_type);
let buffers = inner let buffers = self
.inner
.database .database
.get_multiple(KeySpace::TaskCache, &hash.to_le_bytes()) .get_multiple(KeySpace::TaskCache, &hash.to_le_bytes())
.with_context(|| { .with_context(|| {
...@@ -314,7 +307,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -314,7 +307,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
let mut task_ids = SmallVec::with_capacity(buffers.len()); let mut task_ids = SmallVec::with_capacity(buffers.len());
for bytes in buffers { for bytes in buffers {
let bytes = bytes.borrow().try_into()?; let bytes = (*bytes).try_into()?;
let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap(); let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap();
task_ids.push(id); task_ids.push(id);
} }
...@@ -325,54 +318,35 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -325,54 +318,35 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&self, &self,
task_id: TaskId, task_id: TaskId,
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
storage: &mut TaskStorage, ) -> Result<Option<ArcBytes>> {
) -> Result<()> { self.inner
let inner = &*self.inner;
let Some(bytes) = inner
.database .database
.get(category.key_space(), IntKey::new(*task_id).as_ref()) .get(category.key_space(), IntKey::new(*task_id).as_ref())
.with_context(|| { .with_context(|| format!("Looking up task storage for {task_id} from database failed"))
format!("Looking up task storage for {task_id} from database failed")
})?
else {
return Ok(());
};
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))
} }
fn batch_lookup_data( fn batch_lookup_data(
&self, &self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> { callback: &mut dyn FnMut(usize, Option<&[u8]>),
let inner = &*self.inner; ) -> Result<()> {
let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect(); let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect();
let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>(); let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>();
let bytes = inner self.inner
.database .database
.batch_get(category.key_space(), &keys) .batch_get_with(category.key_space(), &keys, |index, opt_bytes| {
callback(index, opt_bytes);
// `opt_bytes` (and any underlying ArcBytes) is dropped here before the next
// iteration, so only one decompressed block is live at a time.
Ok(())
})
.with_context(|| { .with_context(|| {
format!( format!(
"Looking up typed data for {} tasks from database failed", "Looking up typed data for {} tasks from database failed",
task_ids.len() task_ids.len()
) )
})?;
bytes
.into_iter()
.map(|opt_bytes| {
let mut storage = TaskStorage::new();
if let Some(bytes) = opt_bytes {
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))?;
}
Ok(storage)
}) })
.collect::<Result<Vec<_>>>()
} }
fn compact(&self) -> Result<bool> { fn compact(&self) -> Result<bool> {
......
...@@ -662,18 +662,6 @@ fn gen_clone_inline_fields<'a>(fields: impl Iterator<Item = &'a FieldInfo>) -> V ...@@ -662,18 +662,6 @@ fn gen_clone_inline_fields<'a>(fields: impl Iterator<Item = &'a FieldInfo>) -> V
.collect() .collect()
} }
/// Generate inline field restore assignments: `self.field = source.field;`
fn gen_restore_inline_fields<'a>(fields: impl Iterator<Item = &'a FieldInfo>) -> Vec<TokenStream> {
fields
.map(|field| {
let field_name = &field.field_name;
quote! {
self.#field_name = source.#field_name;
}
})
.collect()
}
/// Generate lazy field match arms with a custom body that also receives the index. /// Generate lazy field match arms with a custom body that also receives the index.
/// `LazyField::Variant(data) => { <body> }` /// `LazyField::Variant(data) => { <body> }`
/// ///
...@@ -3080,14 +3068,6 @@ fn gen_clone_lazy_arms_for_category( ...@@ -3080,14 +3068,6 @@ fn gen_clone_lazy_arms_for_category(
}) })
} }
/// Generate restore inline statements for a category.
fn gen_restore_inline_for_category(
grouped_fields: &GroupedFields,
category: Category,
) -> Vec<TokenStream> {
gen_restore_inline_fields(grouped_fields.persistent_inline(category))
}
/// Generate snapshot clone and restore methods for TaskStorage. /// Generate snapshot clone and restore methods for TaskStorage.
/// ///
/// Generates: /// Generates:
...@@ -3098,56 +3078,12 @@ fn gen_restore_inline_for_category( ...@@ -3098,56 +3078,12 @@ fn gen_restore_inline_for_category(
/// - `restore_data_from(&mut self, source)` - Restore data fields from source /// - `restore_data_from(&mut self, source)` - Restore data fields from source
/// - `restore_all_from(&mut self, source)` - Restore all fields from source /// - `restore_all_from(&mut self, source)` - Restore all fields from source
fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStream { fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStream {
let has_meta_flags = grouped_fields.persisted_meta_flags().next().is_some();
let has_data_flags = grouped_fields.persisted_data_flags().next().is_some();
let has_any_flags = has_meta_flags || has_data_flags;
// Generate field operations by category // Generate field operations by category
let clone_meta_inline = gen_clone_inline_for_category(grouped_fields, Category::Meta); let clone_meta_inline = gen_clone_inline_for_category(grouped_fields, Category::Meta);
let clone_data_inline = gen_clone_inline_for_category(grouped_fields, Category::Data); let clone_data_inline = gen_clone_inline_for_category(grouped_fields, Category::Data);
let clone_meta_lazy_arms = gen_clone_lazy_arms_for_category(grouped_fields, Category::Meta); let clone_meta_lazy_arms = gen_clone_lazy_arms_for_category(grouped_fields, Category::Meta);
let clone_data_lazy_arms = gen_clone_lazy_arms_for_category(grouped_fields, Category::Data); let clone_data_lazy_arms = gen_clone_lazy_arms_for_category(grouped_fields, Category::Data);
let restore_meta_inline = gen_restore_inline_for_category(grouped_fields, Category::Meta);
let restore_data_inline = gen_restore_inline_for_category(grouped_fields, Category::Data);
let clone_all_flags = if has_any_flags {
quote! {
// Clone all persisted flags
snapshot.flags.set_persisted_bits(self.flags.persisted_bits());
}
} else {
quote! {}
};
// Generate flags handling for restore - per category
let restore_meta_flags = if has_meta_flags {
quote! {
// Restore persisted meta flags (preserve other flags)
self.flags.set_persisted_meta_bits(source.flags.persisted_meta_bits());
}
} else {
quote! {}
};
let restore_data_flags = if has_data_flags {
quote! {
// Restore persisted data flags (preserve other flags)
self.flags.set_persisted_data_bits(source.flags.persisted_data_bits());
}
} else {
quote! {}
};
let restore_all_flags = if has_any_flags {
quote! {
// Restore all persisted flags (preserve transient flags)
self.flags.set_persisted_bits(source.flags.persisted_bits());
}
} else {
quote! {}
};
quote! { quote! {
#[automatically_derived] #[automatically_derived]
impl TaskStorage { impl TaskStorage {
...@@ -3161,7 +3097,8 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3161,7 +3097,8 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
// Clone inline data fields // Clone inline data fields
#(#clone_data_inline)* #(#clone_data_inline)*
#clone_all_flags // Clone all persisted flags
snapshot.flags.set_persisted_bits(self.flags.persisted_bits());
// Pre-allocate lazy vec (upper bound - some may be transient and skipped) // Pre-allocate lazy vec (upper bound - some may be transient and skipped)
snapshot.lazy.reserve(self.lazy.len()); snapshot.lazy.reserve(self.lazy.len());
...@@ -3179,103 +3116,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3179,103 +3116,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
snapshot snapshot
} }
/// Restore persisted data from a decoded TaskStorage.
///
/// This is used during restore operations to copy decoded persisted data
/// into the task's existing storage. It preserves transient state (flags,
/// transient fields) while restoring the persisted data.
///
/// # Invariant
///
/// This method assumes the target does NOT already have the persistent fields
/// being restored. This is guaranteed by the restore protocol which only calls
/// this once per category when the task is first accessed. Debug assertions
/// verify this invariant.
///
/// The `category` parameter specifies which category of data to restore:
/// - `Meta`: Restore meta fields (aggregation_number, output, upper, dirty, etc.)
/// - `Data`: Restore data fields (output_dependent, dependencies, cell_data, etc.)
/// - `All`: Restore both meta and data fields
pub fn restore_from(
&mut self,
source: TaskStorage,
category: crate::backend::TaskDataCategory,
) {
match category {
crate::backend::TaskDataCategory::Meta => self.restore_meta_from(source),
crate::backend::TaskDataCategory::Data => self.restore_data_from(source),
crate::backend::TaskDataCategory::All => self.restore_all_from(source),
}
}
/// Restore meta category fields from source.
///
/// Debug assertions verify that the target doesn't already have the lazy fields
/// being restored.
fn restore_meta_from(&mut self, source: TaskStorage) {
// Debug assertion: verify target doesn't already have persistent meta lazy fields
debug_assert!(
!self.lazy.iter().any(|f| f.is_persistent() && f.is_meta()),
"restore_meta_from called on storage that already has persistent meta lazy fields"
);
// Inline meta fields - direct assignment
#(#restore_meta_inline)*
#restore_meta_flags
// Extend lazy vec with persistent meta fields from source
self.lazy.extend(
source.lazy.into_iter().filter(|f| f.is_persistent() && f.is_meta())
);
}
/// Restore data category fields from source.
///
/// Debug assertions verify that the target doesn't already have the lazy fields
/// being restored.
fn restore_data_from(&mut self, source: TaskStorage) {
// Debug assertion: verify target doesn't already have persistent data lazy fields
debug_assert!(
!self.lazy.iter().any(|f| f.is_persistent() && f.is_data()),
"restore_data_from called on storage that already has persistent data lazy fields"
);
// Inline data fields - direct assignment
#(#restore_data_inline)*
#restore_data_flags
// Extend lazy vec with persistent data fields from source
self.lazy.extend(
source.lazy.into_iter().filter(|f| f.is_persistent() && f.is_data())
);
}
/// Restore all fields from source (both meta and data).
///
/// Debug assertions verify that the target doesn't already have the lazy fields
/// being restored.
fn restore_all_from(&mut self, source: TaskStorage) {
// Debug assertion: verify target doesn't already have any persistent lazy fields
debug_assert!(
!self.lazy.iter().any(|f| f.is_persistent()),
"restore_all_from called on storage that already has persistent lazy fields"
);
// Inline meta fields - direct assignment
#(#restore_meta_inline)*
// Inline data fields - direct assignment
#(#restore_data_inline)*
#restore_all_flags
// Extend lazy vec with all persistent fields from source
self.lazy.extend(
source.lazy.into_iter().filter(|f| f.is_persistent())
);
}
} }
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment