Commit d8ef88ee authored by Luke Sandberg's avatar Luke Sandberg

streaming reads

parent ed631bbc
...@@ -35,6 +35,43 @@ export declare class ExternalObject<T> { ...@@ -35,6 +35,43 @@ export declare class ExternalObject<T> {
[K: symbol]: T [K: symbol]: T
} }
} }
/**
* Called by worker after WASM instantiation to register a dispatch callback.
*
* The `ops` object must have these methods:
* - transform(programPtr, programLen, unresolvedMark, commentsProxy) → number
* - getDiag() → number
* - readBuf(ptr, len) → Buffer
* - writeBuf(ptr, data: Buffer) → void
* - alloc(size) → number
* - free(ptr, size) → number
*
* A TSFN is created targeting this worker's event loop. When Rust needs to
* call a WASM export, it posts to this TSFN, which runs the appropriate
* method on `ops` and sends the result back via a sync channel.
*/
export declare function wasmWorkerRegisterCallback(
runtimeId: number,
instanceId: number,
ops: object
): void
/**
* Called by worker when WASM hits a host function import.
* Runs the Func closure synchronously on the worker thread and returns the result.
*
* The worker provides a `memoryAccessor` object with methods to read/write WASM memory:
* - readBuf(ptr, len) → Buffer
* - writeBuf(ptr, data: Buffer) → void
* - alloc(size) → number
* - free(ptr, size) → number
*/
export declare function wasmWorkerDispatchHostFn(
runtimeId: number,
instanceId: number,
fnIndex: number,
args: Array<number>,
memoryAccessor: object
): unknown
export declare function registerWorkerScheduler( export declare function registerWorkerScheduler(
creator: (arg: NapiWorkerCreation) => any, creator: (arg: NapiWorkerCreation) => any,
terminator: (arg: NapiWorkerTermination) => any terminator: (arg: NapiWorkerTermination) => any
......
...@@ -1206,7 +1206,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { ...@@ -1206,7 +1206,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
}, },
|(key, hash)| { |(key, hash)| {
let result = sst let result = sst
.lookup::<_, false>(hash, &key, key_block_cache, value_block_cache) .lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
.unwrap(); .unwrap();
black_box(result) black_box(result)
}, },
...@@ -1225,7 +1225,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { ...@@ -1225,7 +1225,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
|i| keys[i as usize % keys.len()], |i| keys[i as usize % keys.len()],
|(key, hash)| { |(key, hash)| {
let result = sst let result = sst
.lookup::<_, false>(hash, &key, key_block_cache, value_block_cache) .lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
.unwrap(); .unwrap();
black_box(result) black_box(result)
}, },
...@@ -1251,7 +1251,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { ...@@ -1251,7 +1251,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
}, },
|(key, hash)| { |(key, hash)| {
let result = sst let result = sst
.lookup::<_, false>(hash, &key, key_block_cache, value_block_cache) .lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
.unwrap(); .unwrap();
black_box(result) black_box(result)
}, },
...@@ -1284,7 +1284,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { ...@@ -1284,7 +1284,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
}, },
|(key, hash)| { |(key, hash)| {
let result = sst let result = sst
.lookup::<_, false>(*hash, &key, key_block_cache, value_block_cache) .lookup::<_, _, _, false>(*hash, &key, key_block_cache, value_block_cache)
.unwrap(); .unwrap();
black_box(result) black_box(result)
}, },
......
...@@ -106,6 +106,49 @@ struct TrackedStats { ...@@ -106,6 +106,49 @@ struct TrackedStats {
miss_global: std::sync::atomic::AtomicU64, miss_global: std::sync::atomic::AtomicU64,
} }
/// A compact bitset for tracking which batch-lookup cells have been resolved.
///
/// Uses `u64` words (1 bit per cell) rather than `Vec<bool>` (1 byte per cell) for an 8x
/// reduction in memory and better cache behaviour on large batches.
pub struct FoundBitset {
words: Box<[u64]>,
len: usize,
}
impl FoundBitset {
pub(crate) fn new(len: usize) -> Self {
let words = vec![0u64; len.div_ceil(64)].into_boxed_slice();
Self { words, len }
}
#[inline]
pub(crate) fn get(&self, index: usize) -> bool {
debug_assert!(index < self.len);
(self.words[index / 64] >> (index % 64)) & 1 == 1
}
#[inline]
pub(crate) fn set(&mut self, index: usize) {
debug_assert!(index < self.len);
self.words[index / 64] |= 1u64 << (index % 64);
}
/// Returns the number of bits that are NOT set (i.e. not yet found).
pub(crate) fn count_unset(&self) -> usize {
let set: usize = self.words.iter().map(|w| w.count_ones() as usize).sum();
self.len - set
}
pub(crate) fn len(&self) -> usize {
self.len
}
/// Iterates over all indices, yielding `(index, is_set)`.
pub(crate) fn iter(&self) -> impl Iterator<Item = bool> + '_ {
(0..self.len).map(|i| self.get(i))
}
}
/// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time /// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time
/// using a single write batch. It allows for concurrent reads. /// using a single write batch. It allows for concurrent reads.
pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> { pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
...@@ -1561,15 +1604,39 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1561,15 +1604,39 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok(output) Ok(output)
} }
/// Looks up multiple keys in batch and collects results into a `Vec`.
///
/// For large batches where memory pressure matters, prefer
/// [`batch_get_with`][Self::batch_get_with] which calls a callback per entry without
/// accumulating all decoded bytes simultaneously.
pub fn batch_get<K: QueryKey>( pub fn batch_get<K: QueryKey>(
&self, &self,
family: usize, family: usize,
keys: &[K], keys: &[K],
) -> Result<Vec<Option<ArcBytes>>> { ) -> Result<Vec<Option<ArcBytes>>> {
let mut results = vec![None; keys.len()];
self.batch_get_with(family, keys, |index, opt_bytes| {
results[index] = opt_bytes.map(|b| ArcBytes::from(b.to_vec().into_boxed_slice()));
Ok(())
})?;
Ok(results)
}
/// Looks up multiple keys in batch, calling `callback(index, Option<&[u8]>)` for each entry
/// immediately after it is resolved rather than accumulating all results into a `Vec`.
///
/// This keeps at most one decompressed value block live at a time, significantly reducing
/// peak memory when the batch is large. The callback receives the 0-based key index and the
/// value bytes (`None` for not-found or deleted). Callback errors propagate immediately.
pub fn batch_get_with<K: QueryKey>(
&self,
family: usize,
keys: &[K],
mut callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<()> {
debug_assert!(family < FAMILIES, "Family index out of bounds"); debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue { if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic panic!("only single valued tables can be queried with `batch_get_with'")
panic!("only single valued tables can be queried with `batch_get'")
} }
let span = tracing::trace_span!( let span = tracing::trace_span!(
"database batch read", "database batch read",
...@@ -1580,22 +1647,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1580,22 +1647,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
result_size = tracing::field::Empty result_size = tracing::field::Empty
) )
.entered(); .entered();
let mut cells: Vec<(u64, usize, Option<LookupValue>)> = Vec::with_capacity(keys.len()); let mut cells: Vec<(u64, usize)> = Vec::with_capacity(keys.len());
let mut empty_cells = keys.len();
for (index, key) in keys.iter().enumerate() { for (index, key) in keys.iter().enumerate() {
let hash = hash_key(key); let hash = hash_key(key);
cells.push((hash, index, None)); cells.push((hash, index));
} }
cells.sort_by_key(|(hash, _, _)| *hash); cells.sort_by_key(|(hash, _)| *hash);
let mut found = FoundBitset::new(cells.len());
let inner = self.inner.read(); let inner = self.inner.read();
let mut not_found = 0;
let mut deleted = 0;
let mut result_size = 0;
let mut read_blob = |seq| self.read_blob(seq);
// Wrap the callback to track stats (deleted/result_size) for found keys.
// not_found is tracked separately in the post-loop over cells.
let mut stats_callback = |index: usize, opt_bytes: Option<&[u8]>| -> Result<()> {
if let Some(bytes) = opt_bytes {
result_size += bytes.len();
} else {
deleted += 1;
}
callback(index, opt_bytes)
};
for meta in inner.meta_files.iter().rev() { for meta in inner.meta_files.iter().rev() {
let _result = meta.batch_lookup( let (all_found, _result) = meta.batch_lookup_with(
family as u32, family as u32,
keys, keys,
&mut cells, &cells,
&mut empty_cells, &mut found,
&self.key_block_cache, &self.key_block_cache,
&self.value_block_cache, &self.value_block_cache,
&mut read_blob,
&mut stats_callback,
)?; )?;
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -1627,49 +1710,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1627,49 +1710,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
} }
if empty_cells == 0 { if all_found {
break; break;
} }
} }
let mut deleted = 0; // Record accessed hashes for found keys, and fire callback for not-found keys.
let mut not_found = 0; for ((hash, index), is_found) in cells.iter().zip(found.iter()) {
let mut result_size = 0; if is_found {
let mut results = vec![None; keys.len()]; inner.accessed_key_hashes[family].insert(*hash);
for (hash, index, result) in cells {
if let Some(result) = result {
inner.accessed_key_hashes[family].insert(hash);
let result = match result {
LookupValue::Deleted => {
#[cfg(feature = "stats")]
self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed);
deleted += 1;
None
}
LookupValue::Slice { value } => {
#[cfg(feature = "stats")]
self.stats.hits_small.fetch_add(1, Ordering::Relaxed);
result_size += value.len();
Some(value)
}
LookupValue::Blob { sequence_number } => {
#[cfg(feature = "stats")]
self.stats.hits_blob.fetch_add(1, Ordering::Relaxed);
let blob = self.read_blob(sequence_number)?;
result_size += blob.len();
Some(blob)
}
};
results[index] = result;
} else { } else {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
self.stats.miss_global.fetch_add(1, Ordering::Relaxed); self.stats.miss_global.fetch_add(1, Ordering::Relaxed);
not_found += 1; not_found += 1;
callback(*index, None)?;
} }
} }
span.record("not_found", not_found); span.record("not_found", not_found);
span.record("deleted", deleted); span.record("deleted", deleted);
span.record("result_size", result_size); span.record("result_size", result_size);
Ok(results) Ok(())
} }
/// Returns database statistics. /// Returns database statistics.
......
...@@ -14,10 +14,13 @@ use smallvec::SmallVec; ...@@ -14,10 +14,13 @@ use smallvec::SmallVec;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be};
use crate::{ use crate::{
QueryKey, ArcBytes, QueryKey,
db::FoundBitset,
lookup_entry::LookupValue, lookup_entry::LookupValue,
mmap_helper::advise_mmap_for_persistence, mmap_helper::advise_mmap_for_persistence,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, static_sorted_file::{
BlockCache, LayeredBlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData,
},
}; };
bitfield! { bitfield! {
...@@ -459,7 +462,7 @@ impl MetaFile { ...@@ -459,7 +462,7 @@ impl MetaFile {
continue; continue;
} }
let result = entry.sst(self)?.lookup::<K, FIND_ALL>( let result = entry.sst(self)?.lookup::<K, _, _, FIND_ALL>(
key_hash, key_hash,
key, key,
key_block_cache, key_block_cache,
...@@ -498,33 +501,60 @@ impl MetaFile { ...@@ -498,33 +501,60 @@ impl MetaFile {
Ok(miss_result) Ok(miss_result)
} }
pub fn batch_lookup<K: QueryKey>( /// Looks up multiple keys in batch, calling `callback(key_index, Option<&[u8]>)` for each
/// found key immediately after resolution. This avoids keeping every decompressed value block
/// alive across all SST file iterations.
///
/// `cells` is `(hash, key_index)` sorted by hash. `found` is a parallel bitset — bit `i`
/// is set once `cells[i]` has been resolved (either by this call or a prior one on the same
/// batch). On return, bits are set for every cell resolved by this call.
///
/// Returns `true` if all cells are now resolved (caller can stop iterating meta files).
/// `read_blob` is called to obtain the bytes for large blob values.
pub fn batch_lookup_with<K: QueryKey>(
&self, &self,
key_family: u32, key_family: u32,
keys: &[K], keys: &[K],
cells: &mut [(u64, usize, Option<LookupValue>)], cells: &[(u64, usize)],
empty_cells: &mut usize, found: &mut FoundBitset,
key_block_cache: &BlockCache, key_block_cache: &BlockCache,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
) -> Result<MetaBatchLookupResult> { read_blob: &mut impl FnMut(u32) -> Result<ArcBytes>,
callback: &mut impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<(bool, MetaBatchLookupResult)> {
debug_assert_eq!(
cells.len(),
found.len(),
"cells and found must have the same length"
);
if key_family != self.family { if key_family != self.family {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
return Ok(MetaBatchLookupResult { return Ok((
family_miss: true, false,
..Default::default() MetaBatchLookupResult {
}); family_miss: true,
..Default::default()
},
));
#[cfg(not(feature = "stats"))] #[cfg(not(feature = "stats"))]
return Ok(MetaBatchLookupResult {}); return Ok((false, MetaBatchLookupResult {}));
} }
debug_assert!( debug_assert!(
cells.is_sorted_by_key(|(hash, _, _)| *hash), cells.is_sorted_by_key(|(hash, _)| *hash),
"Cells must be sorted by key hash" "Cells must be sorted by key hash"
); );
#[allow(unused_mut, reason = "It's used when stats are enabled")] #[allow(unused_mut, reason = "It's used when stats are enabled")]
let mut lookup_result = MetaBatchLookupResult::default(); let mut lookup_result = MetaBatchLookupResult::default();
let mut empty_cells = found.count_unset();
// Local single-entry caches layered on top of the global block caches.
// Sequential keys in a batch often hit the same key or value block; keeping
// the most recently used block locally avoids re-reading it if the global
// cache evicts it under concurrent pressure.
let mut local_key_cache = LayeredBlockCache::new(key_block_cache);
let mut local_value_cache = LayeredBlockCache::new(value_block_cache);
for entry in self.entries.iter().rev() { for entry in self.entries.iter().rev() {
let start_index = cells let start_index = cells
.binary_search_by(|(hash, _, _)| hash.cmp(&entry.min_hash).then(Ordering::Greater)) .binary_search_by(|(hash, _)| hash.cmp(&entry.min_hash).then(Ordering::Greater))
.err() .err()
.unwrap(); .unwrap();
if start_index >= cells.len() { if start_index >= cells.len() {
...@@ -535,7 +565,7 @@ impl MetaFile { ...@@ -535,7 +565,7 @@ impl MetaFile {
continue; continue;
} }
let end_index = cells let end_index = cells
.binary_search_by(|(hash, _, _)| hash.cmp(&entry.max_hash).then(Ordering::Less)) .binary_search_by(|(hash, _)| hash.cmp(&entry.max_hash).then(Ordering::Less))
.err() .err()
.unwrap() .unwrap()
.checked_sub(1); .checked_sub(1);
...@@ -553,12 +583,13 @@ impl MetaFile { ...@@ -553,12 +583,13 @@ impl MetaFile {
} }
continue; continue;
} }
for (hash, index, result) in &mut cells[start_index..=end_index] { for (cell_pos, (hash, index)) in cells[start_index..=end_index].iter().enumerate() {
let cell_pos = start_index + cell_pos;
debug_assert!( debug_assert!(
*hash >= entry.min_hash && *hash <= entry.max_hash, *hash >= entry.min_hash && *hash <= entry.max_hash,
"Key hash out of range" "Key hash out of range"
); );
if result.is_some() { if found.get(cell_pos) {
continue; continue;
} }
if !entry.amqf.contains_fingerprint(*hash) { if !entry.amqf.contains_fingerprint(*hash) {
...@@ -568,11 +599,11 @@ impl MetaFile { ...@@ -568,11 +599,11 @@ impl MetaFile {
} }
continue; continue;
} }
let sst_result = entry.sst(self)?.lookup::<_, false>( let sst_result = entry.sst(self)?.lookup::<_, _, _, false>(
*hash, *hash,
&keys[*index], &keys[*index],
key_block_cache, &mut local_key_cache,
value_block_cache, &mut local_value_cache,
)?; )?;
if let SstLookupResult::Found(mut values) = sst_result { if let SstLookupResult::Found(mut values) = sst_result {
// find_all=false guarantees exactly one result // find_all=false guarantees exactly one result
...@@ -580,14 +611,39 @@ impl MetaFile { ...@@ -580,14 +611,39 @@ impl MetaFile {
let Some(value) = values.pop() else { let Some(value) = values.pop() else {
unreachable!() unreachable!()
}; };
*result = Some(value); // Resolve and callback immediately — the ArcBytes is dropped before the
*empty_cells -= 1; // next iteration so we never hold more than one decompressed block at a time.
let opt_bytes: Option<&[u8]> = match &value {
LookupValue::Deleted => None,
LookupValue::Slice { value } => Some(value),
LookupValue::Blob { sequence_number } => {
// Blob resolution: read_blob is called inline. The blob ArcBytes
// must outlive the callback call, so we bind it here.
let blob = read_blob(*sequence_number)?;
found.set(cell_pos);
empty_cells -= 1;
#[cfg(feature = "stats")]
{
lookup_result.hits += 1;
}
callback(*index, Some(&*blob))?;
// blob dropped here
if empty_cells == 0 {
return Ok((true, lookup_result));
}
continue;
}
};
found.set(cell_pos);
empty_cells -= 1;
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
{ {
lookup_result.hits += 1; lookup_result.hits += 1;
} }
if *empty_cells == 0 { callback(*index, opt_bytes)?;
return Ok(lookup_result); // value (ArcBytes) dropped here
if empty_cells == 0 {
return Ok((true, lookup_result));
} }
} else { } else {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -597,6 +653,6 @@ impl MetaFile { ...@@ -597,6 +653,6 @@ impl MetaFile {
} }
} }
} }
Ok(lookup_result) Ok((false, lookup_result))
} }
} }
...@@ -1381,7 +1381,7 @@ mod tests { ...@@ -1381,7 +1381,7 @@ mod tests {
kc: &TestBlockCache, kc: &TestBlockCache,
vc: &TestBlockCache, vc: &TestBlockCache,
) -> Result<()> { ) -> Result<()> {
let result = sst.lookup::<_, false>(entry.hash, &entry.key, kc, vc)?; let result = sst.lookup::<_, _, _, false>(entry.hash, &entry.key, kc, vc)?;
match (&entry.value_kind, result) { match (&entry.value_kind, result) {
(_, SstLookupResult::Found(values)) (_, SstLookupResult::Found(values))
if values.len() == 1 && matches!(values[0], LookupValue::Slice { .. }) => if values.len() == 1 && matches!(values[0], LookupValue::Slice { .. }) =>
...@@ -1690,8 +1690,8 @@ mod tests { ...@@ -1690,8 +1690,8 @@ mod tests {
let vc = make_cache(); let vc = make_cache();
for entry in &entries { for entry in &entries {
let r1 = sst1.lookup::<_, false>(entry.hash, &entry.key, &kc, &vc)?; let r1 = sst1.lookup::<_, _, _, false>(entry.hash, &entry.key, &kc, &vc)?;
let r2 = sst2.lookup::<_, false>(entry.hash, &entry.key, &kc, &vc)?; let r2 = sst2.lookup::<_, _, _, false>(entry.hash, &entry.key, &kc, &vc)?;
match (&r1, &r2) { match (&r1, &r2) {
(SstLookupResult::Found(v1), SstLookupResult::Found(v2)) (SstLookupResult::Found(v1), SstLookupResult::Found(v2))
if v1.len() == 1 && v2.len() == 1 => if v1.len() == 1 && v2.len() == 1 =>
...@@ -1816,7 +1816,7 @@ mod tests { ...@@ -1816,7 +1816,7 @@ mod tests {
let sst = open_sst(dir, seq, meta).unwrap(); let sst = open_sst(dir, seq, meta).unwrap();
let kc = make_cache(); let kc = make_cache();
let vc = make_cache(); let vc = make_cache();
match sst.lookup::<_, false>(entries[0].hash, &entries[0].key, &kc, &vc) { match sst.lookup::<_, _, _, false>(entries[0].hash, &entries[0].key, &kc, &vc) {
Err(err) => { Err(err) => {
let msg = format!("{err}"); let msg = format!("{err}");
assert!( assert!(
......
...@@ -78,13 +78,22 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -78,13 +78,22 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
storage: &mut TaskStorage, storage: &mut TaskStorage,
) -> Result<()>; ) -> Result<()>;
/// Batch lookup and decode data for multiple tasks directly into TypedStorage instances. /// Batch lookup raw bytes for multiple tasks, calling
/// Returns a vector of TypedStorage, one for each task_id in the input slice. /// `callback(index, Option<&[u8]>)` for every task in `task_ids` immediately after its
/// entry is resolved. Index corresponds to the position in `task_ids`. `None` means the key
/// was not found in the database (no allocation is made for missing keys).
///
/// The caller is responsible for decoding the bytes. The byte slice is only valid for the
/// duration of the callback invocation and must not be stored.
///
/// This avoids allocating an intermediate `TaskStorage` per entry — callers can decode
/// directly into the live in-memory storage, reducing peak memory usage for large batches.
fn batch_lookup_data( fn batch_lookup_data(
&self, &self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>>; callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()>;
fn compact(&self) -> Result<bool> { fn compact(&self) -> Result<bool> {
Ok(false) Ok(false)
...@@ -151,8 +160,9 @@ where ...@@ -151,8 +160,9 @@ where
&self, &self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> { callback: &mut dyn FnMut(usize, Option<&[u8]>),
either::for_both!(self, this => this.batch_lookup_data(task_ids, category)) ) -> Result<()> {
either::for_both!(self, this => this.batch_lookup_data(task_ids, category, callback))
} }
fn compact(&self) -> Result<bool> { fn compact(&self) -> Result<bool> {
......
...@@ -75,17 +75,24 @@ pub trait KeyValueDatabase { ...@@ -75,17 +75,24 @@ pub trait KeyValueDatabase {
Ok(self.get(key_space, key)?.into_iter().collect()) Ok(self.get(key_space, key)?.into_iter().collect())
} }
fn batch_get( /// Calls `callback(index, Option<&[u8]>)` for each key immediately after it is resolved,
/// rather than accumulating all results into a `Vec`. This avoids holding every decompressed
/// value simultaneously in memory.
///
/// The default implementation falls back to sequential `get` calls. Implementations backed by
/// [`TurboPersistence`] override this to use the native streaming path.
fn batch_get_with(
&self, &self,
key_space: KeySpace, key_space: KeySpace,
keys: &[&[u8]], keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> { mut callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
let mut results = Vec::with_capacity(keys.len()); ) -> Result<()> {
for key in keys { for (index, key) in keys.iter().enumerate() {
let value = self.get(key_space, key)?; let value = self.get(key_space, key)?;
results.push(value); let opt_bytes = value.as_ref().map(std::borrow::Borrow::<[u8]>::borrow);
callback(index, opt_bytes)?;
} }
Ok(results) Ok(())
} }
type ConcurrentWriteBatch<'l>: ConcurrentWriteBatch<'l> type ConcurrentWriteBatch<'l>: ConcurrentWriteBatch<'l>
......
...@@ -96,12 +96,13 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -96,12 +96,13 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
fn batch_get( fn batch_get_with(
&self, &self,
key_space: KeySpace, key_space: KeySpace,
keys: &[&[u8]], keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> { callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
self.db.batch_get(key_space as usize, keys) ) -> Result<()> {
self.db.batch_get_with(key_space as usize, keys, callback)
} }
fn get_multiple( fn get_multiple(
......
...@@ -347,32 +347,25 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -347,32 +347,25 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&self, &self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> { callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()> {
let inner = &*self.inner; let inner = &*self.inner;
let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect(); let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect();
let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>(); let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>();
let bytes = inner inner
.database .database
.batch_get(category.key_space(), &keys) .batch_get_with(category.key_space(), &keys, |index, opt_bytes| {
callback(index, opt_bytes);
// `opt_bytes` (and any underlying ArcBytes) is dropped here before the next
// iteration, so only one decompressed block is live at a time.
Ok(())
})
.with_context(|| { .with_context(|| {
format!( format!(
"Looking up typed data for {} tasks from database failed", "Looking up typed data for {} tasks from database failed",
task_ids.len() task_ids.len()
) )
})?;
bytes
.into_iter()
.map(|opt_bytes| {
let mut storage = TaskStorage::new();
if let Some(bytes) = opt_bytes {
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))?;
}
Ok(storage)
}) })
.collect::<Result<Vec<_>>>()
} }
fn compact(&self) -> Result<bool> { fn compact(&self) -> Result<bool> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment