Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
d8ef88ee
Commit
d8ef88ee
authored
Apr 04, 2026
by
Luke Sandberg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
streaming reads
parent
ed631bbc
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
569 additions
and
279 deletions
+569
-279
generated-native.d.ts
packages/next/src/build/swc/generated-native.d.ts
+37
-0
mod.rs
turbopack/crates/turbo-persistence/benches/mod.rs
+4
-4
db.rs
turbopack/crates/turbo-persistence/src/db.rs
+99
-40
meta_file.rs
turbopack/crates/turbo-persistence/src/meta_file.rs
+81
-25
static_sorted_file.rs
turbopack/crates/turbo-persistence/src/static_sorted_file.rs
+89
-46
static_sorted_file_builder.rs
...rates/turbo-persistence/src/static_sorted_file_builder.rs
+4
-4
mod.rs
...k/crates/turbo-tasks-backend/src/backend/operation/mod.rs
+214
-130
backing_storage.rs
turbopack/crates/turbo-tasks-backend/src/backing_storage.rs
+15
-5
key_value_database.rs
...es/turbo-tasks-backend/src/database/key_value_database.rs
+13
-6
mod.rs
...pack/crates/turbo-tasks-backend/src/database/turbo/mod.rs
+4
-3
kv_backing_storage.rs
...pack/crates/turbo-tasks-backend/src/kv_backing_storage.rs
+9
-16
No files found.
packages/next/src/build/swc/generated-native.d.ts
View file @
d8ef88ee
...
@@ -35,6 +35,43 @@ export declare class ExternalObject<T> {
...
@@ -35,6 +35,43 @@ export declare class ExternalObject<T> {
[
K
:
symbol
]:
T
[
K
:
symbol
]:
T
}
}
}
}
/**
* Called by worker after WASM instantiation to register a dispatch callback.
*
* The `ops` object must have these methods:
* - transform(programPtr, programLen, unresolvedMark, commentsProxy) → number
* - getDiag() → number
* - readBuf(ptr, len) → Buffer
* - writeBuf(ptr, data: Buffer) → void
* - alloc(size) → number
* - free(ptr, size) → number
*
* A TSFN is created targeting this worker's event loop. When Rust needs to
* call a WASM export, it posts to this TSFN, which runs the appropriate
* method on `ops` and sends the result back via a sync channel.
*/
export
declare
function
wasmWorkerRegisterCallback
(
runtimeId
:
number
,
instanceId
:
number
,
ops
:
object
):
void
/**
* Called by worker when WASM hits a host function import.
* Runs the Func closure synchronously on the worker thread and returns the result.
*
* The worker provides a `memoryAccessor` object with methods to read/write WASM memory:
* - readBuf(ptr, len) → Buffer
* - writeBuf(ptr, data: Buffer) → void
* - alloc(size) → number
* - free(ptr, size) → number
*/
export
declare
function
wasmWorkerDispatchHostFn
(
runtimeId
:
number
,
instanceId
:
number
,
fnIndex
:
number
,
args
:
Array
<
number
>
,
memoryAccessor
:
object
):
unknown
export
declare
function
registerWorkerScheduler
(
export
declare
function
registerWorkerScheduler
(
creator
:
(
arg
:
NapiWorkerCreation
)
=>
any
,
creator
:
(
arg
:
NapiWorkerCreation
)
=>
any
,
terminator
:
(
arg
:
NapiWorkerTermination
)
=>
any
terminator
:
(
arg
:
NapiWorkerTermination
)
=>
any
...
...
turbopack/crates/turbo-persistence/benches/mod.rs
View file @
d8ef88ee
...
@@ -1206,7 +1206,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
...
@@ -1206,7 +1206,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
},
},
|(
key
,
hash
)|
{
|(
key
,
hash
)|
{
let
result
=
sst
let
result
=
sst
.lookup
::
<
_
,
false
>
(
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.lookup
::
<
_
,
_
,
_
,
false
>
(
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.unwrap
();
.unwrap
();
black_box
(
result
)
black_box
(
result
)
},
},
...
@@ -1225,7 +1225,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
...
@@ -1225,7 +1225,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
|
i
|
keys
[
i
as
usize
%
keys
.len
()],
|
i
|
keys
[
i
as
usize
%
keys
.len
()],
|(
key
,
hash
)|
{
|(
key
,
hash
)|
{
let
result
=
sst
let
result
=
sst
.lookup
::
<
_
,
false
>
(
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.lookup
::
<
_
,
_
,
_
,
false
>
(
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.unwrap
();
.unwrap
();
black_box
(
result
)
black_box
(
result
)
},
},
...
@@ -1251,7 +1251,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
...
@@ -1251,7 +1251,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
},
},
|(
key
,
hash
)|
{
|(
key
,
hash
)|
{
let
result
=
sst
let
result
=
sst
.lookup
::
<
_
,
false
>
(
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.lookup
::
<
_
,
_
,
_
,
false
>
(
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.unwrap
();
.unwrap
();
black_box
(
result
)
black_box
(
result
)
},
},
...
@@ -1284,7 +1284,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
...
@@ -1284,7 +1284,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
},
},
|(
key
,
hash
)|
{
|(
key
,
hash
)|
{
let
result
=
sst
let
result
=
sst
.lookup
::
<
_
,
false
>
(
*
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.lookup
::
<
_
,
_
,
_
,
false
>
(
*
hash
,
&
key
,
key_block_cache
,
value_block_cache
)
.unwrap
();
.unwrap
();
black_box
(
result
)
black_box
(
result
)
},
},
...
...
turbopack/crates/turbo-persistence/src/db.rs
View file @
d8ef88ee
...
@@ -106,6 +106,49 @@ struct TrackedStats {
...
@@ -106,6 +106,49 @@ struct TrackedStats {
miss_global
:
std
::
sync
::
atomic
::
AtomicU64
,
miss_global
:
std
::
sync
::
atomic
::
AtomicU64
,
}
}
/// A compact bitset for tracking which batch-lookup cells have been resolved.
///
/// Uses `u64` words (1 bit per cell) rather than `Vec<bool>` (1 byte per cell) for an 8x
/// reduction in memory and better cache behaviour on large batches.
pub
struct
FoundBitset
{
words
:
Box
<
[
u64
]
>
,
len
:
usize
,
}
impl
FoundBitset
{
pub
(
crate
)
fn
new
(
len
:
usize
)
->
Self
{
let
words
=
vec!
[
0u64
;
len
.div_ceil
(
64
)]
.into_boxed_slice
();
Self
{
words
,
len
}
}
#[inline]
pub
(
crate
)
fn
get
(
&
self
,
index
:
usize
)
->
bool
{
debug_assert!
(
index
<
self
.len
);
(
self
.words
[
index
/
64
]
>>
(
index
%
64
))
&
1
==
1
}
#[inline]
pub
(
crate
)
fn
set
(
&
mut
self
,
index
:
usize
)
{
debug_assert!
(
index
<
self
.len
);
self
.words
[
index
/
64
]
|
=
1u64
<<
(
index
%
64
);
}
/// Returns the number of bits that are NOT set (i.e. not yet found).
pub
(
crate
)
fn
count_unset
(
&
self
)
->
usize
{
let
set
:
usize
=
self
.words
.iter
()
.map
(|
w
|
w
.count_ones
()
as
usize
)
.sum
();
self
.len
-
set
}
pub
(
crate
)
fn
len
(
&
self
)
->
usize
{
self
.len
}
/// Iterates over all indices, yielding `(index, is_set)`.
pub
(
crate
)
fn
iter
(
&
self
)
->
impl
Iterator
<
Item
=
bool
>
+
'_
{
(
0
..
self
.len
)
.map
(|
i
|
self
.get
(
i
))
}
}
/// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time
/// TurboPersistence is a persistent key-value store. It is limited to a single writer at a time
/// using a single write batch. It allows for concurrent reads.
/// using a single write batch. It allows for concurrent reads.
pub
struct
TurboPersistence
<
S
:
ParallelScheduler
,
const
FAMILIES
:
usize
>
{
pub
struct
TurboPersistence
<
S
:
ParallelScheduler
,
const
FAMILIES
:
usize
>
{
...
@@ -1561,15 +1604,39 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
...
@@ -1561,15 +1604,39 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok
(
output
)
Ok
(
output
)
}
}
/// Looks up multiple keys in batch and collects results into a `Vec`.
///
/// For large batches where memory pressure matters, prefer
/// [`batch_get_with`][Self::batch_get_with] which calls a callback per entry without
/// accumulating all decoded bytes simultaneously.
pub
fn
batch_get
<
K
:
QueryKey
>
(
pub
fn
batch_get
<
K
:
QueryKey
>
(
&
self
,
&
self
,
family
:
usize
,
family
:
usize
,
keys
:
&
[
K
],
keys
:
&
[
K
],
)
->
Result
<
Vec
<
Option
<
ArcBytes
>>>
{
)
->
Result
<
Vec
<
Option
<
ArcBytes
>>>
{
let
mut
results
=
vec!
[
None
;
keys
.len
()];
self
.batch_get_with
(
family
,
keys
,
|
index
,
opt_bytes
|
{
results
[
index
]
=
opt_bytes
.map
(|
b
|
ArcBytes
::
from
(
b
.to_vec
()
.into_boxed_slice
()));
Ok
(())
})
?
;
Ok
(
results
)
}
/// Looks up multiple keys in batch, calling `callback(index, Option<&[u8]>)` for each entry
/// immediately after it is resolved rather than accumulating all results into a `Vec`.
///
/// This keeps at most one decompressed value block live at a time, significantly reducing
/// peak memory when the batch is large. The callback receives the 0-based key index and the
/// value bytes (`None` for not-found or deleted). Callback errors propagate immediately.
pub
fn
batch_get_with
<
K
:
QueryKey
>
(
&
self
,
family
:
usize
,
keys
:
&
[
K
],
mut
callback
:
impl
FnMut
(
usize
,
Option
<&
[
u8
]
>
)
->
Result
<
()
>
,
)
->
Result
<
()
>
{
debug_assert!
(
family
<
FAMILIES
,
"Family index out of bounds"
);
debug_assert!
(
family
<
FAMILIES
,
"Family index out of bounds"
);
if
self
.config.family_configs
[
family
]
.kind
!=
FamilyKind
::
SingleValue
{
if
self
.config.family_configs
[
family
]
.kind
!=
FamilyKind
::
SingleValue
{
// This is an error in our caller so just panic
panic!
(
"only single valued tables can be queried with `batch_get_with'"
)
panic!
(
"only single valued tables can be queried with `batch_get'"
)
}
}
let
span
=
tracing
::
trace_span!
(
let
span
=
tracing
::
trace_span!
(
"database batch read"
,
"database batch read"
,
...
@@ -1580,22 +1647,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
...
@@ -1580,22 +1647,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
result_size
=
tracing
::
field
::
Empty
result_size
=
tracing
::
field
::
Empty
)
)
.entered
();
.entered
();
let
mut
cells
:
Vec
<
(
u64
,
usize
,
Option
<
LookupValue
>
)
>
=
Vec
::
with_capacity
(
keys
.len
());
let
mut
cells
:
Vec
<
(
u64
,
usize
)
>
=
Vec
::
with_capacity
(
keys
.len
());
let
mut
empty_cells
=
keys
.len
();
for
(
index
,
key
)
in
keys
.iter
()
.enumerate
()
{
for
(
index
,
key
)
in
keys
.iter
()
.enumerate
()
{
let
hash
=
hash_key
(
key
);
let
hash
=
hash_key
(
key
);
cells
.push
((
hash
,
index
,
None
));
cells
.push
((
hash
,
index
));
}
}
cells
.sort_by_key
(|(
hash
,
_
,
_
)|
*
hash
);
cells
.sort_by_key
(|(
hash
,
_
)|
*
hash
);
let
mut
found
=
FoundBitset
::
new
(
cells
.len
());
let
inner
=
self
.inner
.read
();
let
inner
=
self
.inner
.read
();
let
mut
not_found
=
0
;
let
mut
deleted
=
0
;
let
mut
result_size
=
0
;
let
mut
read_blob
=
|
seq
|
self
.read_blob
(
seq
);
// Wrap the callback to track stats (deleted/result_size) for found keys.
// not_found is tracked separately in the post-loop over cells.
let
mut
stats_callback
=
|
index
:
usize
,
opt_bytes
:
Option
<&
[
u8
]
>
|
->
Result
<
()
>
{
if
let
Some
(
bytes
)
=
opt_bytes
{
result_size
+=
bytes
.len
();
}
else
{
deleted
+=
1
;
}
callback
(
index
,
opt_bytes
)
};
for
meta
in
inner
.meta_files
.iter
()
.rev
()
{
for
meta
in
inner
.meta_files
.iter
()
.rev
()
{
let
_result
=
meta
.batch_lookup
(
let
(
all_found
,
_result
)
=
meta
.batch_lookup_with
(
family
as
u32
,
family
as
u32
,
keys
,
keys
,
&
mut
cells
,
&
cells
,
&
mut
empty_cells
,
&
mut
found
,
&
self
.key_block_cache
,
&
self
.key_block_cache
,
&
self
.value_block_cache
,
&
self
.value_block_cache
,
&
mut
read_blob
,
&
mut
stats_callback
,
)
?
;
)
?
;
#[cfg(feature
=
"stats"
)]
#[cfg(feature
=
"stats"
)]
...
@@ -1627,49 +1710,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
...
@@ -1627,49 +1710,25 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
}
}
}
}
if
empty_cells
==
0
{
if
all_found
{
break
;
break
;
}
}
}
}
let
mut
deleted
=
0
;
// Record accessed hashes for found keys, and fire callback for not-found keys.
let
mut
not_found
=
0
;
for
((
hash
,
index
),
is_found
)
in
cells
.iter
()
.zip
(
found
.iter
())
{
let
mut
result_size
=
0
;
if
is_found
{
let
mut
results
=
vec!
[
None
;
keys
.len
()];
inner
.accessed_key_hashes
[
family
]
.insert
(
*
hash
);
for
(
hash
,
index
,
result
)
in
cells
{
if
let
Some
(
result
)
=
result
{
inner
.accessed_key_hashes
[
family
]
.insert
(
hash
);
let
result
=
match
result
{
LookupValue
::
Deleted
=>
{
#[cfg(feature
=
"stats"
)]
self
.stats.hits_deleted
.fetch_add
(
1
,
Ordering
::
Relaxed
);
deleted
+=
1
;
None
}
LookupValue
::
Slice
{
value
}
=>
{
#[cfg(feature
=
"stats"
)]
self
.stats.hits_small
.fetch_add
(
1
,
Ordering
::
Relaxed
);
result_size
+=
value
.len
();
Some
(
value
)
}
LookupValue
::
Blob
{
sequence_number
}
=>
{
#[cfg(feature
=
"stats"
)]
self
.stats.hits_blob
.fetch_add
(
1
,
Ordering
::
Relaxed
);
let
blob
=
self
.read_blob
(
sequence_number
)
?
;
result_size
+=
blob
.len
();
Some
(
blob
)
}
};
results
[
index
]
=
result
;
}
else
{
}
else
{
#[cfg(feature
=
"stats"
)]
#[cfg(feature
=
"stats"
)]
self
.stats.miss_global
.fetch_add
(
1
,
Ordering
::
Relaxed
);
self
.stats.miss_global
.fetch_add
(
1
,
Ordering
::
Relaxed
);
not_found
+=
1
;
not_found
+=
1
;
callback
(
*
index
,
None
)
?
;
}
}
}
}
span
.record
(
"not_found"
,
not_found
);
span
.record
(
"not_found"
,
not_found
);
span
.record
(
"deleted"
,
deleted
);
span
.record
(
"deleted"
,
deleted
);
span
.record
(
"result_size"
,
result_size
);
span
.record
(
"result_size"
,
result_size
);
Ok
(
results
)
Ok
(
()
)
}
}
/// Returns database statistics.
/// Returns database statistics.
...
...
turbopack/crates/turbo-persistence/src/meta_file.rs
View file @
d8ef88ee
...
@@ -14,10 +14,13 @@ use smallvec::SmallVec;
...
@@ -14,10 +14,13 @@ use smallvec::SmallVec;
use
zerocopy
::{
FromBytes
,
Immutable
,
IntoBytes
,
KnownLayout
,
Ref
,
big_endian
as
be
};
use
zerocopy
::{
FromBytes
,
Immutable
,
IntoBytes
,
KnownLayout
,
Ref
,
big_endian
as
be
};
use
crate
::{
use
crate
::{
QueryKey
,
ArcBytes
,
QueryKey
,
db
::
FoundBitset
,
lookup_entry
::
LookupValue
,
lookup_entry
::
LookupValue
,
mmap_helper
::
advise_mmap_for_persistence
,
mmap_helper
::
advise_mmap_for_persistence
,
static_sorted_file
::{
BlockCache
,
SstLookupResult
,
StaticSortedFile
,
StaticSortedFileMetaData
},
static_sorted_file
::{
BlockCache
,
LayeredBlockCache
,
SstLookupResult
,
StaticSortedFile
,
StaticSortedFileMetaData
,
},
};
};
bitfield!
{
bitfield!
{
...
@@ -459,7 +462,7 @@ impl MetaFile {
...
@@ -459,7 +462,7 @@ impl MetaFile {
continue
;
continue
;
}
}
let
result
=
entry
.sst
(
self
)
?
.lookup
::
<
K
,
FIND_ALL
>
(
let
result
=
entry
.sst
(
self
)
?
.lookup
::
<
K
,
_
,
_
,
FIND_ALL
>
(
key_hash
,
key_hash
,
key
,
key
,
key_block_cache
,
key_block_cache
,
...
@@ -498,33 +501,60 @@ impl MetaFile {
...
@@ -498,33 +501,60 @@ impl MetaFile {
Ok
(
miss_result
)
Ok
(
miss_result
)
}
}
pub
fn
batch_lookup
<
K
:
QueryKey
>
(
/// Looks up multiple keys in batch, calling `callback(key_index, Option<&[u8]>)` for each
/// found key immediately after resolution. This avoids keeping every decompressed value block
/// alive across all SST file iterations.
///
/// `cells` is `(hash, key_index)` sorted by hash. `found` is a parallel bitset — bit `i`
/// is set once `cells[i]` has been resolved (either by this call or a prior one on the same
/// batch). On return, bits are set for every cell resolved by this call.
///
/// Returns `true` if all cells are now resolved (caller can stop iterating meta files).
/// `read_blob` is called to obtain the bytes for large blob values.
pub
fn
batch_lookup_with
<
K
:
QueryKey
>
(
&
self
,
&
self
,
key_family
:
u32
,
key_family
:
u32
,
keys
:
&
[
K
],
keys
:
&
[
K
],
cells
:
&
mut
[(
u64
,
usize
,
Option
<
LookupValue
>
)],
cells
:
&
[(
u64
,
usize
)],
empty_cells
:
&
mut
usize
,
found
:
&
mut
FoundBitset
,
key_block_cache
:
&
BlockCache
,
key_block_cache
:
&
BlockCache
,
value_block_cache
:
&
BlockCache
,
value_block_cache
:
&
BlockCache
,
)
->
Result
<
MetaBatchLookupResult
>
{
read_blob
:
&
mut
impl
FnMut
(
u32
)
->
Result
<
ArcBytes
>
,
callback
:
&
mut
impl
FnMut
(
usize
,
Option
<&
[
u8
]
>
)
->
Result
<
()
>
,
)
->
Result
<
(
bool
,
MetaBatchLookupResult
)
>
{
debug_assert_eq!
(
cells
.len
(),
found
.len
(),
"cells and found must have the same length"
);
if
key_family
!=
self
.family
{
if
key_family
!=
self
.family
{
#[cfg(feature
=
"stats"
)]
#[cfg(feature
=
"stats"
)]
return
Ok
(
MetaBatchLookupResult
{
return
Ok
((
family_miss
:
true
,
false
,
..
Default
::
default
()
MetaBatchLookupResult
{
});
family_miss
:
true
,
..
Default
::
default
()
},
));
#[cfg(not(feature
=
"stats"
))]
#[cfg(not(feature
=
"stats"
))]
return
Ok
(
MetaBatchLookupResult
{}
);
return
Ok
(
(
false
,
MetaBatchLookupResult
{})
);
}
}
debug_assert!
(
debug_assert!
(
cells
.is_sorted_by_key
(|(
hash
,
_
,
_
)|
*
hash
),
cells
.is_sorted_by_key
(|(
hash
,
_
)|
*
hash
),
"Cells must be sorted by key hash"
"Cells must be sorted by key hash"
);
);
#[allow(unused_mut,
reason
=
"It's used when stats are enabled"
)]
#[allow(unused_mut,
reason
=
"It's used when stats are enabled"
)]
let
mut
lookup_result
=
MetaBatchLookupResult
::
default
();
let
mut
lookup_result
=
MetaBatchLookupResult
::
default
();
let
mut
empty_cells
=
found
.count_unset
();
// Local single-entry caches layered on top of the global block caches.
// Sequential keys in a batch often hit the same key or value block; keeping
// the most recently used block locally avoids re-reading it if the global
// cache evicts it under concurrent pressure.
let
mut
local_key_cache
=
LayeredBlockCache
::
new
(
key_block_cache
);
let
mut
local_value_cache
=
LayeredBlockCache
::
new
(
value_block_cache
);
for
entry
in
self
.entries
.iter
()
.rev
()
{
for
entry
in
self
.entries
.iter
()
.rev
()
{
let
start_index
=
cells
let
start_index
=
cells
.binary_search_by
(|(
hash
,
_
,
_
)|
hash
.cmp
(
&
entry
.min_hash
)
.then
(
Ordering
::
Greater
))
.binary_search_by
(|(
hash
,
_
)|
hash
.cmp
(
&
entry
.min_hash
)
.then
(
Ordering
::
Greater
))
.err
()
.err
()
.unwrap
();
.unwrap
();
if
start_index
>=
cells
.len
()
{
if
start_index
>=
cells
.len
()
{
...
@@ -535,7 +565,7 @@ impl MetaFile {
...
@@ -535,7 +565,7 @@ impl MetaFile {
continue
;
continue
;
}
}
let
end_index
=
cells
let
end_index
=
cells
.binary_search_by
(|(
hash
,
_
,
_
)|
hash
.cmp
(
&
entry
.max_hash
)
.then
(
Ordering
::
Less
))
.binary_search_by
(|(
hash
,
_
)|
hash
.cmp
(
&
entry
.max_hash
)
.then
(
Ordering
::
Less
))
.err
()
.err
()
.unwrap
()
.unwrap
()
.checked_sub
(
1
);
.checked_sub
(
1
);
...
@@ -553,12 +583,13 @@ impl MetaFile {
...
@@ -553,12 +583,13 @@ impl MetaFile {
}
}
continue
;
continue
;
}
}
for
(
hash
,
index
,
result
)
in
&
mut
cells
[
start_index
..=
end_index
]
{
for
(
cell_pos
,
(
hash
,
index
))
in
cells
[
start_index
..=
end_index
]
.iter
()
.enumerate
()
{
let
cell_pos
=
start_index
+
cell_pos
;
debug_assert!
(
debug_assert!
(
*
hash
>=
entry
.min_hash
&&
*
hash
<=
entry
.max_hash
,
*
hash
>=
entry
.min_hash
&&
*
hash
<=
entry
.max_hash
,
"Key hash out of range"
"Key hash out of range"
);
);
if
result
.is_some
(
)
{
if
found
.get
(
cell_pos
)
{
continue
;
continue
;
}
}
if
!
entry
.amqf
.contains_fingerprint
(
*
hash
)
{
if
!
entry
.amqf
.contains_fingerprint
(
*
hash
)
{
...
@@ -568,11 +599,11 @@ impl MetaFile {
...
@@ -568,11 +599,11 @@ impl MetaFile {
}
}
continue
;
continue
;
}
}
let
sst_result
=
entry
.sst
(
self
)
?
.lookup
::
<
_
,
false
>
(
let
sst_result
=
entry
.sst
(
self
)
?
.lookup
::
<
_
,
_
,
_
,
false
>
(
*
hash
,
*
hash
,
&
keys
[
*
index
],
&
keys
[
*
index
],
key_block
_cache
,
&
mut
local_key
_cache
,
value_block
_cache
,
&
mut
local_value
_cache
,
)
?
;
)
?
;
if
let
SstLookupResult
::
Found
(
mut
values
)
=
sst_result
{
if
let
SstLookupResult
::
Found
(
mut
values
)
=
sst_result
{
// find_all=false guarantees exactly one result
// find_all=false guarantees exactly one result
...
@@ -580,14 +611,39 @@ impl MetaFile {
...
@@ -580,14 +611,39 @@ impl MetaFile {
let
Some
(
value
)
=
values
.pop
()
else
{
let
Some
(
value
)
=
values
.pop
()
else
{
unreachable!
()
unreachable!
()
};
};
*
result
=
Some
(
value
);
// Resolve and callback immediately — the ArcBytes is dropped before the
*
empty_cells
-
=
1
;
// next iteration so we never hold more than one decompressed block at a time.
let
opt_bytes
:
Option
<&
[
u8
]
>
=
match
&
value
{
LookupValue
::
Deleted
=>
None
,
LookupValue
::
Slice
{
value
}
=>
Some
(
value
),
LookupValue
::
Blob
{
sequence_number
}
=>
{
// Blob resolution: read_blob is called inline. The blob ArcBytes
// must outlive the callback call, so we bind it here.
let
blob
=
read_blob
(
*
sequence_number
)
?
;
found
.set
(
cell_pos
);
empty_cells
-
=
1
;
#[cfg(feature
=
"stats"
)]
{
lookup_result
.hits
+=
1
;
}
callback
(
*
index
,
Some
(
&*
blob
))
?
;
// blob dropped here
if
empty_cells
==
0
{
return
Ok
((
true
,
lookup_result
));
}
continue
;
}
};
found
.set
(
cell_pos
);
empty_cells
-
=
1
;
#[cfg(feature
=
"stats"
)]
#[cfg(feature
=
"stats"
)]
{
{
lookup_result
.hits
+=
1
;
lookup_result
.hits
+=
1
;
}
}
if
*
empty_cells
==
0
{
callback
(
*
index
,
opt_bytes
)
?
;
return
Ok
(
lookup_result
);
// value (ArcBytes) dropped here
if
empty_cells
==
0
{
return
Ok
((
true
,
lookup_result
));
}
}
}
else
{
}
else
{
#[cfg(feature
=
"stats"
)]
#[cfg(feature
=
"stats"
)]
...
@@ -597,6 +653,6 @@ impl MetaFile {
...
@@ -597,6 +653,6 @@ impl MetaFile {
}
}
}
}
}
}
Ok
(
lookup_result
)
Ok
(
(
false
,
lookup_result
)
)
}
}
}
}
turbopack/crates/turbo-persistence/src/static_sorted_file.rs
View file @
d8ef88ee
This diff is collapsed.
Click to expand it.
turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs
View file @
d8ef88ee
...
@@ -1381,7 +1381,7 @@ mod tests {
...
@@ -1381,7 +1381,7 @@ mod tests {
kc
:
&
TestBlockCache
,
kc
:
&
TestBlockCache
,
vc
:
&
TestBlockCache
,
vc
:
&
TestBlockCache
,
)
->
Result
<
()
>
{
)
->
Result
<
()
>
{
let
result
=
sst
.lookup
::
<
_
,
false
>
(
entry
.hash
,
&
entry
.key
,
kc
,
vc
)
?
;
let
result
=
sst
.lookup
::
<
_
,
_
,
_
,
false
>
(
entry
.hash
,
&
entry
.key
,
kc
,
vc
)
?
;
match
(
&
entry
.value_kind
,
result
)
{
match
(
&
entry
.value_kind
,
result
)
{
(
_
,
SstLookupResult
::
Found
(
values
))
(
_
,
SstLookupResult
::
Found
(
values
))
if
values
.len
()
==
1
&&
matches!
(
values
[
0
],
LookupValue
::
Slice
{
..
})
=>
if
values
.len
()
==
1
&&
matches!
(
values
[
0
],
LookupValue
::
Slice
{
..
})
=>
...
@@ -1690,8 +1690,8 @@ mod tests {
...
@@ -1690,8 +1690,8 @@ mod tests {
let
vc
=
make_cache
();
let
vc
=
make_cache
();
for
entry
in
&
entries
{
for
entry
in
&
entries
{
let
r1
=
sst1
.lookup
::
<
_
,
false
>
(
entry
.hash
,
&
entry
.key
,
&
kc
,
&
vc
)
?
;
let
r1
=
sst1
.lookup
::
<
_
,
_
,
_
,
false
>
(
entry
.hash
,
&
entry
.key
,
&
kc
,
&
vc
)
?
;
let
r2
=
sst2
.lookup
::
<
_
,
false
>
(
entry
.hash
,
&
entry
.key
,
&
kc
,
&
vc
)
?
;
let
r2
=
sst2
.lookup
::
<
_
,
_
,
_
,
false
>
(
entry
.hash
,
&
entry
.key
,
&
kc
,
&
vc
)
?
;
match
(
&
r1
,
&
r2
)
{
match
(
&
r1
,
&
r2
)
{
(
SstLookupResult
::
Found
(
v1
),
SstLookupResult
::
Found
(
v2
))
(
SstLookupResult
::
Found
(
v1
),
SstLookupResult
::
Found
(
v2
))
if
v1
.len
()
==
1
&&
v2
.len
()
==
1
=>
if
v1
.len
()
==
1
&&
v2
.len
()
==
1
=>
...
@@ -1816,7 +1816,7 @@ mod tests {
...
@@ -1816,7 +1816,7 @@ mod tests {
let
sst
=
open_sst
(
dir
,
seq
,
meta
)
.unwrap
();
let
sst
=
open_sst
(
dir
,
seq
,
meta
)
.unwrap
();
let
kc
=
make_cache
();
let
kc
=
make_cache
();
let
vc
=
make_cache
();
let
vc
=
make_cache
();
match
sst
.lookup
::
<
_
,
false
>
(
entries
[
0
]
.hash
,
&
entries
[
0
]
.key
,
&
kc
,
&
vc
)
{
match
sst
.lookup
::
<
_
,
_
,
_
,
false
>
(
entries
[
0
]
.hash
,
&
entries
[
0
]
.key
,
&
kc
,
&
vc
)
{
Err
(
err
)
=>
{
Err
(
err
)
=>
{
let
msg
=
format!
(
"{err}"
);
let
msg
=
format!
(
"{err}"
);
assert
!
(
assert
!
(
...
...
turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs
View file @
d8ef88ee
This diff is collapsed.
Click to expand it.
turbopack/crates/turbo-tasks-backend/src/backing_storage.rs
View file @
d8ef88ee
...
@@ -78,13 +78,22 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
...
@@ -78,13 +78,22 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
storage
:
&
mut
TaskStorage
,
storage
:
&
mut
TaskStorage
,
)
->
Result
<
()
>
;
)
->
Result
<
()
>
;
/// Batch lookup and decode data for multiple tasks directly into TypedStorage instances.
/// Batch lookup raw bytes for multiple tasks, calling
/// Returns a vector of TypedStorage, one for each task_id in the input slice.
/// `callback(index, Option<&[u8]>)` for every task in `task_ids` immediately after its
/// entry is resolved. Index corresponds to the position in `task_ids`. `None` means the key
/// was not found in the database (no allocation is made for missing keys).
///
/// The caller is responsible for decoding the bytes. The byte slice is only valid for the
/// duration of the callback invocation and must not be stored.
///
/// This avoids allocating an intermediate `TaskStorage` per entry — callers can decode
/// directly into the live in-memory storage, reducing peak memory usage for large batches.
fn
batch_lookup_data
(
fn
batch_lookup_data
(
&
self
,
&
self
,
task_ids
:
&
[
TaskId
],
task_ids
:
&
[
TaskId
],
category
:
SpecificTaskDataCategory
,
category
:
SpecificTaskDataCategory
,
)
->
Result
<
Vec
<
TaskStorage
>>
;
callback
:
&
mut
dyn
FnMut
(
usize
,
Option
<&
[
u8
]
>
),
)
->
Result
<
()
>
;
fn
compact
(
&
self
)
->
Result
<
bool
>
{
fn
compact
(
&
self
)
->
Result
<
bool
>
{
Ok
(
false
)
Ok
(
false
)
...
@@ -151,8 +160,9 @@ where
...
@@ -151,8 +160,9 @@ where
&
self
,
&
self
,
task_ids
:
&
[
TaskId
],
task_ids
:
&
[
TaskId
],
category
:
SpecificTaskDataCategory
,
category
:
SpecificTaskDataCategory
,
)
->
Result
<
Vec
<
TaskStorage
>>
{
callback
:
&
mut
dyn
FnMut
(
usize
,
Option
<&
[
u8
]
>
),
either
::
for_both!
(
self
,
this
=>
this
.batch_lookup_data
(
task_ids
,
category
))
)
->
Result
<
()
>
{
either
::
for_both!
(
self
,
this
=>
this
.batch_lookup_data
(
task_ids
,
category
,
callback
))
}
}
fn
compact
(
&
self
)
->
Result
<
bool
>
{
fn
compact
(
&
self
)
->
Result
<
bool
>
{
...
...
turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs
View file @
d8ef88ee
...
@@ -75,17 +75,24 @@ pub trait KeyValueDatabase {
...
@@ -75,17 +75,24 @@ pub trait KeyValueDatabase {
Ok
(
self
.get
(
key_space
,
key
)
?
.into_iter
()
.collect
())
Ok
(
self
.get
(
key_space
,
key
)
?
.into_iter
()
.collect
())
}
}
fn
batch_get
(
/// Calls `callback(index, Option<&[u8]>)` for each key immediately after it is resolved,
/// rather than accumulating all results into a `Vec`. This avoids holding every decompressed
/// value simultaneously in memory.
///
/// The default implementation falls back to sequential `get` calls. Implementations backed by
/// [`TurboPersistence`] override this to use the native streaming path.
fn
batch_get_with
(
&
self
,
&
self
,
key_space
:
KeySpace
,
key_space
:
KeySpace
,
keys
:
&
[
&
[
u8
]],
keys
:
&
[
&
[
u8
]],
)
->
Result
<
Vec
<
Option
<
Self
::
ValueBuffer
<
'_
>>>>
{
mut
callback
:
impl
FnMut
(
usize
,
Option
<&
[
u8
]
>
)
->
Result
<
()
>
,
let
mut
results
=
Vec
::
with_capacity
(
keys
.len
());
)
->
Result
<
()
>
{
for
key
in
keys
{
for
(
index
,
key
)
in
keys
.iter
()
.enumerate
()
{
let
value
=
self
.get
(
key_space
,
key
)
?
;
let
value
=
self
.get
(
key_space
,
key
)
?
;
results
.push
(
value
);
let
opt_bytes
=
value
.as_ref
()
.map
(
std
::
borrow
::
Borrow
::
<
[
u8
]
>
::
borrow
);
callback
(
index
,
opt_bytes
)
?
;
}
}
Ok
(
results
)
Ok
(
()
)
}
}
type
ConcurrentWriteBatch
<
'l
>
:
ConcurrentWriteBatch
<
'l
>
type
ConcurrentWriteBatch
<
'l
>
:
ConcurrentWriteBatch
<
'l
>
...
...
turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs
View file @
d8ef88ee
...
@@ -96,12 +96,13 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
...
@@ -96,12 +96,13 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self
.db
.get
(
key_space
as
usize
,
&
key
)
self
.db
.get
(
key_space
as
usize
,
&
key
)
}
}
fn
batch_get
(
fn
batch_get
_with
(
&
self
,
&
self
,
key_space
:
KeySpace
,
key_space
:
KeySpace
,
keys
:
&
[
&
[
u8
]],
keys
:
&
[
&
[
u8
]],
)
->
Result
<
Vec
<
Option
<
Self
::
ValueBuffer
<
'_
>>>>
{
callback
:
impl
FnMut
(
usize
,
Option
<&
[
u8
]
>
)
->
Result
<
()
>
,
self
.db
.batch_get
(
key_space
as
usize
,
keys
)
)
->
Result
<
()
>
{
self
.db
.batch_get_with
(
key_space
as
usize
,
keys
,
callback
)
}
}
fn
get_multiple
(
fn
get_multiple
(
...
...
turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs
View file @
d8ef88ee
...
@@ -347,32 +347,25 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
...
@@ -347,32 +347,25 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&
self
,
&
self
,
task_ids
:
&
[
TaskId
],
task_ids
:
&
[
TaskId
],
category
:
SpecificTaskDataCategory
,
category
:
SpecificTaskDataCategory
,
)
->
Result
<
Vec
<
TaskStorage
>>
{
callback
:
&
mut
dyn
FnMut
(
usize
,
Option
<&
[
u8
]
>
),
)
->
Result
<
()
>
{
let
inner
=
&*
self
.inner
;
let
inner
=
&*
self
.inner
;
let
int_keys
:
Vec
<
_
>
=
task_ids
.iter
()
.map
(|
&
id
|
IntKey
::
new
(
*
id
))
.collect
();
let
int_keys
:
Vec
<
_
>
=
task_ids
.iter
()
.map
(|
&
id
|
IntKey
::
new
(
*
id
))
.collect
();
let
keys
=
int_keys
.iter
()
.map
(|
k
|
k
.as_ref
())
.collect
::
<
Vec
<
_
>>
();
let
keys
=
int_keys
.iter
()
.map
(|
k
|
k
.as_ref
())
.collect
::
<
Vec
<
_
>>
();
let
bytes
=
inner
inner
.database
.database
.batch_get
(
category
.key_space
(),
&
keys
)
.batch_get_with
(
category
.key_space
(),
&
keys
,
|
index
,
opt_bytes
|
{
callback
(
index
,
opt_bytes
);
// `opt_bytes` (and any underlying ArcBytes) is dropped here before the next
// iteration, so only one decompressed block is live at a time.
Ok
(())
})
.with_context
(||
{
.with_context
(||
{
format!
(
format!
(
"Looking up typed data for {} tasks from database failed"
,
"Looking up typed data for {} tasks from database failed"
,
task_ids
.len
()
task_ids
.len
()
)
)
})
?
;
bytes
.into_iter
()
.map
(|
opt_bytes
|
{
let
mut
storage
=
TaskStorage
::
new
();
if
let
Some
(
bytes
)
=
opt_bytes
{
let
mut
decoder
=
new_turbo_bincode_decoder
(
bytes
.borrow
());
storage
.decode
(
category
,
&
mut
decoder
)
.map_err
(|
e
|
anyhow
::
anyhow!
(
"Failed to decode {category:?}: {e:?}"
))
?
;
}
Ok
(
storage
)
})
})
.collect
::
<
Result
<
Vec
<
_
>>>
()
}
}
fn
compact
(
&
self
)
->
Result
<
bool
>
{
fn
compact
(
&
self
)
->
Result
<
bool
>
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment