Commit d8ef88ee authored by Luke Sandberg's avatar Luke Sandberg

streaming reads

parent ed631bbc
......@@ -35,6 +35,43 @@ export declare class ExternalObject<T> {
[K: symbol]: T
}
}
/**
* Called by worker after WASM instantiation to register a dispatch callback.
*
* The `ops` object must have these methods:
* - transform(programPtr, programLen, unresolvedMark, commentsProxy) → number
* - getDiag() → number
* - readBuf(ptr, len) → Buffer
* - writeBuf(ptr, data: Buffer) → void
* - alloc(size) → number
* - free(ptr, size) → number
*
* A TSFN is created targeting this worker's event loop. When Rust needs to
* call a WASM export, it posts to this TSFN, which runs the appropriate
* method on `ops` and sends the result back via a sync channel.
*/
export declare function wasmWorkerRegisterCallback(
runtimeId: number,
instanceId: number,
ops: object
): void
/**
* Called by worker when WASM hits a host function import.
* Runs the Func closure synchronously on the worker thread and returns the result.
*
* The worker provides a `memoryAccessor` object with methods to read/write WASM memory:
* - readBuf(ptr, len) → Buffer
* - writeBuf(ptr, data: Buffer) → void
* - alloc(size) → number
* - free(ptr, size) → number
*/
export declare function wasmWorkerDispatchHostFn(
runtimeId: number,
instanceId: number,
fnIndex: number,
args: Array<number>,
memoryAccessor: object
): unknown
export declare function registerWorkerScheduler(
creator: (arg: NapiWorkerCreation) => any,
terminator: (arg: NapiWorkerTermination) => any
......
......@@ -1206,7 +1206,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
},
|(key, hash)| {
let result = sst
.lookup::<_, false>(hash, &key, key_block_cache, value_block_cache)
.lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
.unwrap();
black_box(result)
},
......@@ -1225,7 +1225,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
|i| keys[i as usize % keys.len()],
|(key, hash)| {
let result = sst
.lookup::<_, false>(hash, &key, key_block_cache, value_block_cache)
.lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
.unwrap();
black_box(result)
},
......@@ -1251,7 +1251,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
},
|(key, hash)| {
let result = sst
.lookup::<_, false>(hash, &key, key_block_cache, value_block_cache)
.lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
.unwrap();
black_box(result)
},
......@@ -1284,7 +1284,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
},
|(key, hash)| {
let result = sst
.lookup::<_, false>(*hash, &key, key_block_cache, value_block_cache)
.lookup::<_, _, _, false>(*hash, &key, key_block_cache, value_block_cache)
.unwrap();
black_box(result)
},
......
......@@ -106,6 +106,49 @@ struct TrackedStats {
miss_global: std::sync::atomic::AtomicU64,
}
/// A compact bitset for tracking which batch-lookup cells have been resolved.
///
/// Uses `u64` words (1 bit per cell) rather than `Vec<bool>` (1 byte per cell) for an 8x
/// reduction in memory and better cache behaviour on large batches.
pub struct FoundBitset {
words: Box<[u64]>,
len: usize,
}
impl FoundBitset {
pub(crate) fn new(len: usize) -> Self {
let words = vec![0u64; len.div_ceil(64)].into_boxed_slice();
Self { words, len }
}
#[inline]
pub(crate) fn get(&self, index: usize) -> bool {
debug_assert!(index < self.len);
(self.words[index / 64] >> (index % 64)) & 1 == 1
}
#[inline]
pub(crate) fn set(&mut self, index: usize) {
debug_assert!(index < self.len);
self.words[index / 64] |= 1u64 << (index % 64);
}
/// Returns the number of bits that are NOT set (i.e. not yet found).
pub(crate) fn count_unset(&self) -> usize {
let set: usize = self.words.iter().map(|w| w.count_ones() as usize).sum();
self.len - set
}
pub(crate) fn len(&self) -> usize {
self.len
}
/// Iterates over all indices, yielding `(index, is_set)`.
pub(crate) fn iter(&self) -> impl Iterator<Item = bool> + '_ {
(0..self.len).map(|i| self.get(i))
}
}
/// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time
/// using a single write batch. It allows for concurrent reads.
pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
......@@ -1561,15 +1604,39 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok(output)
}
/// Looks up multiple keys in batch and collects results into a `Vec`.
///
/// For large batches where memory pressure matters, prefer
/// [`batch_get_with`][Self::batch_get_with] which calls a callback per entry without
/// accumulating all decoded bytes simultaneously.
pub fn batch_get<K: QueryKey>(
&self,
family: usize,
keys: &[K],
) -> Result<Vec<Option<ArcBytes>>> {
let mut results = vec![None; keys.len()];
self.batch_get_with(family, keys, |index, opt_bytes| {
results[index] = opt_bytes.map(|b| ArcBytes::from(b.to_vec().into_boxed_slice()));
Ok(())
})?;
Ok(results)
}
/// Looks up multiple keys in batch, calling `callback(index, Option<&[u8]>)` for each entry
/// immediately after it is resolved rather than accumulating all results into a `Vec`.
///
/// This keeps at most one decompressed value block live at a time, significantly reducing
/// peak memory when the batch is large. The callback receives the 0-based key index and the
/// value bytes (`None` for not-found or deleted). Callback errors propagate immediately.
pub fn batch_get_with<K: QueryKey>(
&self,
family: usize,
keys: &[K],
mut callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<()> {
debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic
panic!("only single valued tables can be queried with `batch_get'")
panic!("only single valued tables can be queried with `batch_get_with'")
}
let span = tracing::trace_span!(
"database batch read",
......@@ -1580,22 +1647,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
result_size = tracing::field::Empty
)
.entered();
let mut cells: Vec<(u64, usize, Option<LookupValue>)> = Vec::with_capacity(keys.len());
let mut empty_cells = keys.len();
let mut cells: Vec<(u64, usize)> = Vec::with_capacity(keys.len());
for (index, key) in keys.iter().enumerate() {
let hash = hash_key(key);
cells.push((hash, index, None));
cells.push((hash, index));
}
cells.sort_by_key(|(hash, _, _)| *hash);
cells.sort_by_key(|(hash, _)| *hash);
let mut found = FoundBitset::new(cells.len());
let inner = self.inner.read();
let mut not_found = 0;
let mut deleted = 0;
let mut result_size = 0;
let mut read_blob = |seq| self.read_blob(seq);
// Wrap the callback to track stats (deleted/result_size) for found keys.
// not_found is tracked separately in the post-loop over cells.
let mut stats_callback = |index: usize, opt_bytes: Option<&[u8]>| -> Result<()> {
if let Some(bytes) = opt_bytes {
result_size += bytes.len();
} else {
deleted += 1;
}
callback(index, opt_bytes)
};
for meta in inner.meta_files.iter().rev() {
let _result = meta.batch_lookup(
let (all_found, _result) = meta.batch_lookup_with(
family as u32,
keys,
&mut cells,
&mut empty_cells,
&cells,
&mut found,
&self.key_block_cache,
&self.value_block_cache,
&mut read_blob,
&mut stats_callback,
)?;
#[cfg(feature = "stats")]
......@@ -1627,49 +1710,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
}
}
if empty_cells == 0 {
if all_found {
break;
}
}
let mut deleted = 0;
let mut not_found = 0;
let mut result_size = 0;
let mut results = vec![None; keys.len()];
for (hash, index, result) in cells {
if let Some(result) = result {
inner.accessed_key_hashes[family].insert(hash);
let result = match result {
LookupValue::Deleted => {
#[cfg(feature = "stats")]
self.stats.hits_deleted.fetch_add(1, Ordering::Relaxed);
deleted += 1;
None
}
LookupValue::Slice { value } => {
#[cfg(feature = "stats")]
self.stats.hits_small.fetch_add(1, Ordering::Relaxed);
result_size += value.len();
Some(value)
}
LookupValue::Blob { sequence_number } => {
#[cfg(feature = "stats")]
self.stats.hits_blob.fetch_add(1, Ordering::Relaxed);
let blob = self.read_blob(sequence_number)?;
result_size += blob.len();
Some(blob)
}
};
results[index] = result;
// Record accessed hashes for found keys, and fire callback for not-found keys.
for ((hash, index), is_found) in cells.iter().zip(found.iter()) {
if is_found {
inner.accessed_key_hashes[family].insert(*hash);
} else {
#[cfg(feature = "stats")]
self.stats.miss_global.fetch_add(1, Ordering::Relaxed);
not_found += 1;
callback(*index, None)?;
}
}
span.record("not_found", not_found);
span.record("deleted", deleted);
span.record("result_size", result_size);
Ok(results)
Ok(())
}
/// Returns database statistics.
......
......@@ -14,10 +14,13 @@ use smallvec::SmallVec;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be};
use crate::{
QueryKey,
ArcBytes, QueryKey,
db::FoundBitset,
lookup_entry::LookupValue,
mmap_helper::advise_mmap_for_persistence,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
static_sorted_file::{
BlockCache, LayeredBlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData,
},
};
bitfield! {
......@@ -459,7 +462,7 @@ impl MetaFile {
continue;
}
let result = entry.sst(self)?.lookup::<K, FIND_ALL>(
let result = entry.sst(self)?.lookup::<K, _, _, FIND_ALL>(
key_hash,
key,
key_block_cache,
......@@ -498,33 +501,60 @@ impl MetaFile {
Ok(miss_result)
}
pub fn batch_lookup<K: QueryKey>(
/// Looks up multiple keys in batch, calling `callback(key_index, Option<&[u8]>)` for each
/// found key immediately after resolution. This avoids keeping every decompressed value block
/// alive across all SST file iterations.
///
/// `cells` is `(hash, key_index)` sorted by hash. `found` is a parallel bitset — bit `i`
/// is set once `cells[i]` has been resolved (either by this call or a prior one on the same
/// batch). On return, bits are set for every cell resolved by this call.
///
/// Returns `true` if all cells are now resolved (caller can stop iterating meta files).
/// `read_blob` is called to obtain the bytes for large blob values.
pub fn batch_lookup_with<K: QueryKey>(
&self,
key_family: u32,
keys: &[K],
cells: &mut [(u64, usize, Option<LookupValue>)],
empty_cells: &mut usize,
cells: &[(u64, usize)],
found: &mut FoundBitset,
key_block_cache: &BlockCache,
value_block_cache: &BlockCache,
) -> Result<MetaBatchLookupResult> {
read_blob: &mut impl FnMut(u32) -> Result<ArcBytes>,
callback: &mut impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<(bool, MetaBatchLookupResult)> {
debug_assert_eq!(
cells.len(),
found.len(),
"cells and found must have the same length"
);
if key_family != self.family {
#[cfg(feature = "stats")]
return Ok(MetaBatchLookupResult {
family_miss: true,
..Default::default()
});
return Ok((
false,
MetaBatchLookupResult {
family_miss: true,
..Default::default()
},
));
#[cfg(not(feature = "stats"))]
return Ok(MetaBatchLookupResult {});
return Ok((false, MetaBatchLookupResult {}));
}
debug_assert!(
cells.is_sorted_by_key(|(hash, _, _)| *hash),
cells.is_sorted_by_key(|(hash, _)| *hash),
"Cells must be sorted by key hash"
);
#[allow(unused_mut, reason = "It's used when stats are enabled")]
let mut lookup_result = MetaBatchLookupResult::default();
let mut empty_cells = found.count_unset();
// Local single-entry caches layered on top of the global block caches.
// Sequential keys in a batch often hit the same key or value block; keeping
// the most recently used block locally avoids re-reading it if the global
// cache evicts it under concurrent pressure.
let mut local_key_cache = LayeredBlockCache::new(key_block_cache);
let mut local_value_cache = LayeredBlockCache::new(value_block_cache);
for entry in self.entries.iter().rev() {
let start_index = cells
.binary_search_by(|(hash, _, _)| hash.cmp(&entry.min_hash).then(Ordering::Greater))
.binary_search_by(|(hash, _)| hash.cmp(&entry.min_hash).then(Ordering::Greater))
.err()
.unwrap();
if start_index >= cells.len() {
......@@ -535,7 +565,7 @@ impl MetaFile {
continue;
}
let end_index = cells
.binary_search_by(|(hash, _, _)| hash.cmp(&entry.max_hash).then(Ordering::Less))
.binary_search_by(|(hash, _)| hash.cmp(&entry.max_hash).then(Ordering::Less))
.err()
.unwrap()
.checked_sub(1);
......@@ -553,12 +583,13 @@ impl MetaFile {
}
continue;
}
for (hash, index, result) in &mut cells[start_index..=end_index] {
for (cell_pos, (hash, index)) in cells[start_index..=end_index].iter().enumerate() {
let cell_pos = start_index + cell_pos;
debug_assert!(
*hash >= entry.min_hash && *hash <= entry.max_hash,
"Key hash out of range"
);
if result.is_some() {
if found.get(cell_pos) {
continue;
}
if !entry.amqf.contains_fingerprint(*hash) {
......@@ -568,11 +599,11 @@ impl MetaFile {
}
continue;
}
let sst_result = entry.sst(self)?.lookup::<_, false>(
let sst_result = entry.sst(self)?.lookup::<_, _, _, false>(
*hash,
&keys[*index],
key_block_cache,
value_block_cache,
&mut local_key_cache,
&mut local_value_cache,
)?;
if let SstLookupResult::Found(mut values) = sst_result {
// find_all=false guarantees exactly one result
......@@ -580,14 +611,39 @@ impl MetaFile {
let Some(value) = values.pop() else {
unreachable!()
};
*result = Some(value);
*empty_cells -= 1;
// Resolve and callback immediately — the ArcBytes is dropped before the
// next iteration so we never hold more than one decompressed block at a time.
let opt_bytes: Option<&[u8]> = match &value {
LookupValue::Deleted => None,
LookupValue::Slice { value } => Some(value),
LookupValue::Blob { sequence_number } => {
// Blob resolution: read_blob is called inline. The blob ArcBytes
// must outlive the callback call, so we bind it here.
let blob = read_blob(*sequence_number)?;
found.set(cell_pos);
empty_cells -= 1;
#[cfg(feature = "stats")]
{
lookup_result.hits += 1;
}
callback(*index, Some(&*blob))?;
// blob dropped here
if empty_cells == 0 {
return Ok((true, lookup_result));
}
continue;
}
};
found.set(cell_pos);
empty_cells -= 1;
#[cfg(feature = "stats")]
{
lookup_result.hits += 1;
}
if *empty_cells == 0 {
return Ok(lookup_result);
callback(*index, opt_bytes)?;
// value (ArcBytes) dropped here
if empty_cells == 0 {
return Ok((true, lookup_result));
}
} else {
#[cfg(feature = "stats")]
......@@ -597,6 +653,6 @@ impl MetaFile {
}
}
}
Ok(lookup_result)
Ok((false, lookup_result))
}
}
......@@ -1381,7 +1381,7 @@ mod tests {
kc: &TestBlockCache,
vc: &TestBlockCache,
) -> Result<()> {
let result = sst.lookup::<_, false>(entry.hash, &entry.key, kc, vc)?;
let result = sst.lookup::<_, _, _, false>(entry.hash, &entry.key, kc, vc)?;
match (&entry.value_kind, result) {
(_, SstLookupResult::Found(values))
if values.len() == 1 && matches!(values[0], LookupValue::Slice { .. }) =>
......@@ -1690,8 +1690,8 @@ mod tests {
let vc = make_cache();
for entry in &entries {
let r1 = sst1.lookup::<_, false>(entry.hash, &entry.key, &kc, &vc)?;
let r2 = sst2.lookup::<_, false>(entry.hash, &entry.key, &kc, &vc)?;
let r1 = sst1.lookup::<_, _, _, false>(entry.hash, &entry.key, &kc, &vc)?;
let r2 = sst2.lookup::<_, _, _, false>(entry.hash, &entry.key, &kc, &vc)?;
match (&r1, &r2) {
(SstLookupResult::Found(v1), SstLookupResult::Found(v2))
if v1.len() == 1 && v2.len() == 1 =>
......@@ -1816,7 +1816,7 @@ mod tests {
let sst = open_sst(dir, seq, meta).unwrap();
let kc = make_cache();
let vc = make_cache();
match sst.lookup::<_, false>(entries[0].hash, &entries[0].key, &kc, &vc) {
match sst.lookup::<_, _, _, false>(entries[0].hash, &entries[0].key, &kc, &vc) {
Err(err) => {
let msg = format!("{err}");
assert!(
......
......@@ -78,13 +78,22 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
storage: &mut TaskStorage,
) -> Result<()>;
/// Batch lookup and decode data for multiple tasks directly into TypedStorage instances.
/// Returns a vector of TypedStorage, one for each task_id in the input slice.
/// Batch lookup raw bytes for multiple tasks, calling
/// `callback(index, Option<&[u8]>)` for every task in `task_ids` immediately after its
/// entry is resolved. Index corresponds to the position in `task_ids`. `None` means the key
/// was not found in the database (no allocation is made for missing keys).
///
/// The caller is responsible for decoding the bytes. The byte slice is only valid for the
/// duration of the callback invocation and must not be stored.
///
/// This avoids allocating an intermediate `TaskStorage` per entry — callers can decode
/// directly into the live in-memory storage, reducing peak memory usage for large batches.
fn batch_lookup_data(
&self,
task_ids: &[TaskId],
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>>;
callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()>;
fn compact(&self) -> Result<bool> {
Ok(false)
......@@ -151,8 +160,9 @@ where
&self,
task_ids: &[TaskId],
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> {
either::for_both!(self, this => this.batch_lookup_data(task_ids, category))
callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()> {
either::for_both!(self, this => this.batch_lookup_data(task_ids, category, callback))
}
fn compact(&self) -> Result<bool> {
......
......@@ -75,17 +75,24 @@ pub trait KeyValueDatabase {
Ok(self.get(key_space, key)?.into_iter().collect())
}
fn batch_get(
/// Calls `callback(index, Option<&[u8]>)` for each key immediately after it is resolved,
/// rather than accumulating all results into a `Vec`. This avoids holding every decompressed
/// value simultaneously in memory.
///
/// The default implementation falls back to sequential `get` calls. Implementations backed by
/// [`TurboPersistence`] override this to use the native streaming path.
fn batch_get_with(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> {
let mut results = Vec::with_capacity(keys.len());
for key in keys {
mut callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<()> {
for (index, key) in keys.iter().enumerate() {
let value = self.get(key_space, key)?;
results.push(value);
let opt_bytes = value.as_ref().map(std::borrow::Borrow::<[u8]>::borrow);
callback(index, opt_bytes)?;
}
Ok(results)
Ok(())
}
type ConcurrentWriteBatch<'l>: ConcurrentWriteBatch<'l>
......
......@@ -96,12 +96,13 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.get(key_space as usize, &key)
}
fn batch_get(
fn batch_get_with(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> {
self.db.batch_get(key_space as usize, keys)
callback: impl FnMut(usize, Option<&[u8]>) -> Result<()>,
) -> Result<()> {
self.db.batch_get_with(key_space as usize, keys, callback)
}
fn get_multiple(
......
......@@ -347,32 +347,25 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&self,
task_ids: &[TaskId],
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> {
callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()> {
let inner = &*self.inner;
let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect();
let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>();
let bytes = inner
inner
.database
.batch_get(category.key_space(), &keys)
.batch_get_with(category.key_space(), &keys, |index, opt_bytes| {
callback(index, opt_bytes);
// `opt_bytes` (and any underlying ArcBytes) is dropped here before the next
// iteration, so only one decompressed block is live at a time.
Ok(())
})
.with_context(|| {
format!(
"Looking up typed data for {} tasks from database failed",
task_ids.len()
)
})?;
bytes
.into_iter()
.map(|opt_bytes| {
let mut storage = TaskStorage::new();
if let Some(bytes) = opt_bytes {
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))?;
}
Ok(storage)
})
.collect::<Result<Vec<_>>>()
}
fn compact(&self) -> Result<bool> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment