Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
16a3edab
Commit
16a3edab
authored
Jan 27, 2026
by
Luke Sandberg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Create a new compaction mechanism to ensure the TaskCache functions properly
parent
49f013c7
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
280 additions
and
6 deletions
+280
-6
db.rs
turbopack/crates/turbo-persistence/src/db.rs
+14
-3
lib.rs
turbopack/crates/turbo-persistence/src/lib.rs
+15
-0
lookup_entry.rs
turbopack/crates/turbo-persistence/src/lookup_entry.rs
+26
-0
tests.rs
turbopack/crates/turbo-persistence/src/tests.rs
+220
-1
mod.rs
...pack/crates/turbo-tasks-backend/src/database/turbo/mod.rs
+5
-2
No files found.
turbopack/crates/turbo-persistence/src/db.rs
View file @
16a3edab
...
...
@@ -20,7 +20,7 @@ use smallvec::SmallVec;
pub
use
crate
::
compaction
::
selector
::
CompactConfig
;
use
crate
::{
DbConfig
,
QueryKey
,
DbConfig
,
DeduplicationMode
,
QueryKey
,
arc_slice
::
ArcSlice
,
compaction
::
selector
::{
Compactable
,
get_merge_segments
},
compression
::
decompress_into_arc
,
...
...
@@ -1081,12 +1081,23 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
}
let
mut
used_collector
=
Collector
::
default
();
let
mut
unused_collector
=
Collector
::
default
();
let
family_config
=
&
self
.config.family_configs
[
family
as
usize
];
for
entry
in
iter
{
let
entry
=
entry
?
;
// Remove duplicates
// Remove duplicates
based on family's deduplication mode
if
let
Some
(
current
)
=
current
.take
()
{
if
current
.key
!=
entry
.key
{
let
is_duplicate
=
match
family_config
.deduplication_mode
{
DeduplicationMode
::
ByKeyOnly
=>
{
current
.key
==
entry
.key
}
DeduplicationMode
::
ByKeyAndValue
=>
{
current
.key
==
entry
.key
&&
current
.value
.eq_for_dedup
(
&
entry
.value
)
}
};
if
!
is_duplicate
{
let
is_used
=
used_key_hashes
[
family
as
usize
]
.iter
()
.any
(
|
amqf
|
amqf
.contains_fingerprint
(
current
.hash
),
...
...
turbopack/crates/turbo-persistence/src/lib.rs
View file @
16a3edab
...
...
@@ -29,6 +29,17 @@ pub use arc_slice::ArcSlice;
use
constants
::{
DATA_THRESHOLD_PER_INITIAL_FILE
,
MAX_ENTRIES_PER_INITIAL_FILE
};
pub
use
db
::{
CompactConfig
,
MetaFileEntryInfo
,
MetaFileInfo
,
TurboPersistence
};
/// How to deduplicate entries during compaction.
#[derive(Clone,
Copy,
Debug,
PartialEq,
Eq)]
pub
enum
DeduplicationMode
{
/// Keep only the newest entry for each key (default LSM behavior).
/// When multiple entries have the same key, only the last one is retained.
ByKeyOnly
,
/// Keep entries that differ by key OR value (for hash-collision-tolerant keyspaces).
/// Only drops entries that are true duplicates (same key AND same value).
/// Use this when a keyspace may contain different values that hash to the same key.
ByKeyAndValue
,
}
/// Configuration for a single family's file limits during writes.
///
/// Controls when SST files are split during writes.
...
...
@@ -45,6 +56,9 @@ pub struct FamilyConfig {
/// Data size threshold for initial SST files (bytes).
/// Controls memory usage during writes.
pub
data_threshold_per_initial_file
:
usize
,
/// How to handle duplicate keys during compaction
pub
deduplication_mode
:
DeduplicationMode
,
}
impl
Default
for
FamilyConfig
{
...
...
@@ -52,6 +66,7 @@ impl Default for FamilyConfig {
Self
{
max_entries_per_initial_file
:
MAX_ENTRIES_PER_INITIAL_FILE
,
data_threshold_per_initial_file
:
DATA_THRESHOLD_PER_INITIAL_FILE
,
deduplication_mode
:
DeduplicationMode
::
ByKeyOnly
,
}
}
}
...
...
turbopack/crates/turbo-persistence/src/lookup_entry.rs
View file @
16a3edab
...
...
@@ -39,6 +39,32 @@ impl LazyLookupValue<'_> {
}
=>
*
uncompressed_size
as
usize
,
}
}
/// Returns true if two values are equal for deduplication purposes.
///
/// For `Medium` values, compares the compressed block bytes directly
/// (assumes deterministic compression).
pub
fn
eq_for_dedup
(
&
self
,
other
:
&
Self
)
->
bool
{
match
(
self
,
other
)
{
(
LazyLookupValue
::
Eager
(
LookupValue
::
Deleted
),
LazyLookupValue
::
Eager
(
LookupValue
::
Deleted
),
)
=>
true
,
(
LazyLookupValue
::
Eager
(
LookupValue
::
Slice
{
value
:
a
}),
LazyLookupValue
::
Eager
(
LookupValue
::
Slice
{
value
:
b
}),
)
=>
a
.as_ref
()
==
b
.as_ref
(),
(
LazyLookupValue
::
Eager
(
LookupValue
::
Blob
{
sequence_number
:
a
}),
LazyLookupValue
::
Eager
(
LookupValue
::
Blob
{
sequence_number
:
b
}),
)
=>
a
==
b
,
(
LazyLookupValue
::
Medium
{
block
:
a
,
..
},
LazyLookupValue
::
Medium
{
block
:
b
,
..
},
)
=>
*
a
==
*
b
,
_
=>
false
,
// Different types are not equal
}
}
}
/// An entry from a SST file lookup.
...
...
turbopack/crates/turbo-persistence/src/tests.rs
View file @
16a3edab
...
...
@@ -4,7 +4,7 @@ use anyhow::Result;
use
rayon
::
iter
::{
IntoParallelIterator
,
ParallelIterator
};
use
crate
::{
constants
::
MAX_MEDIUM_VALUE_SIZE
,
constants
::
{
DbConfig
,
DeduplicationMode
,
FamilyConfig
,
MAX_MEDIUM_VALUE_SIZE
}
,
db
::{
CompactConfig
,
TurboPersistence
},
parallel_scheduler
::
ParallelScheduler
,
write_batch
::
WriteBatch
,
...
...
@@ -1511,3 +1511,222 @@ fn get_multiple_empty_result() -> Result<()> {
Ok
(())
}
#[test]
fn
compaction_by_key_only_drops_same_key_entries
()
->
Result
<
()
>
{
let
tempdir
=
tempfile
::
tempdir
()
?
;
let
path
=
tempdir
.path
();
// Default behavior: ByKeyOnly deduplication drops entries with same key
let
key
=
vec!
[
42u8
];
{
let
db
=
TurboPersistence
::
<
_
,
1
>
::
open_with_parallel_scheduler
(
path
.to_path_buf
(),
RayonParallelScheduler
,
)
?
;
// Write same key with different values in separate batches
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
1u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
2u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
3u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
// Before compaction: all 3 values exist
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
3
,
"Should have 3 values before compaction"
);
// Compact with default ByKeyOnly mode
db
.full_compact
()
?
;
// After compaction: only newest value remains
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
1
,
"ByKeyOnly should keep only 1 value after compaction"
);
assert_eq!
(
results
[
0
]
.as_ref
(),
&
[
3u8
],
"Should be the newest value"
);
db
.shutdown
()
?
;
}
Ok
(())
}
#[test]
fn
compaction_by_key_and_value_preserves_different_values
()
->
Result
<
()
>
{
let
tempdir
=
tempfile
::
tempdir
()
?
;
let
path
=
tempdir
.path
();
// Configure family 0 to use ByKeyAndValue deduplication
let
mut
config
=
DbConfig
::
<
1
>
::
default
();
config
.family_configs
[
0
]
=
FamilyConfig
{
deduplication_mode
:
DeduplicationMode
::
ByKeyAndValue
,
..
Default
::
default
()
};
let
key
=
vec!
[
42u8
];
{
let
db
=
TurboPersistence
::
<
_
,
1
>
::
open_with_config_and_parallel_scheduler
(
path
.to_path_buf
(),
config
,
RayonParallelScheduler
,
)
?
;
// Write same key with different values in separate batches
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
1u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
2u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
3u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
// Before compaction: all 3 values exist
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
3
,
"Should have 3 values before compaction"
);
// Compact with ByKeyAndValue mode
db
.full_compact
()
?
;
// After compaction: all different values should be preserved
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
3
,
"ByKeyAndValue should preserve all different values after compaction"
);
let
mut
values
:
Vec
<
u8
>
=
results
.iter
()
.map
(|
r
|
r
[
0
])
.collect
();
values
.sort
();
assert_eq!
(
values
,
vec!
[
1
,
2
,
3
],
"All values should be preserved"
);
db
.shutdown
()
?
;
}
Ok
(())
}
#[test]
fn
compaction_by_key_and_value_drops_true_duplicates
()
->
Result
<
()
>
{
let
tempdir
=
tempfile
::
tempdir
()
?
;
let
path
=
tempdir
.path
();
// Configure family 0 to use ByKeyAndValue deduplication
let
mut
config
=
DbConfig
::
<
1
>
::
default
();
config
.family_configs
[
0
]
=
FamilyConfig
{
deduplication_mode
:
DeduplicationMode
::
ByKeyAndValue
,
..
Default
::
default
()
};
let
key
=
vec!
[
42u8
];
{
let
db
=
TurboPersistence
::
<
_
,
1
>
::
open_with_config_and_parallel_scheduler
(
path
.to_path_buf
(),
config
,
RayonParallelScheduler
,
)
?
;
// Write same key with SAME value multiple times (true duplicates)
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
100u8
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
100u8
]
.into
())
?
;
// Same value
db
.commit_write_batch
(
batch
)
?
;
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
100u8
]
.into
())
?
;
// Same value
db
.commit_write_batch
(
batch
)
?
;
// Before compaction: all 3 entries exist
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
3
,
"Should have 3 entries before compaction"
);
// Compact with ByKeyAndValue mode
db
.full_compact
()
?
;
// After compaction: true duplicates should be deduplicated
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
1
,
"ByKeyAndValue should deduplicate true duplicates (same key AND value)"
);
assert_eq!
(
results
[
0
]
.as_ref
(),
&
[
100u8
]);
db
.shutdown
()
?
;
}
Ok
(())
}
#[test]
fn
compaction_by_key_and_value_preserves_all_unique_values
()
->
Result
<
()
>
{
let
tempdir
=
tempfile
::
tempdir
()
?
;
let
path
=
tempdir
.path
();
// Configure family 0 to use ByKeyAndValue deduplication
let
mut
config
=
DbConfig
::
<
1
>
::
default
();
config
.family_configs
[
0
]
=
FamilyConfig
{
deduplication_mode
:
DeduplicationMode
::
ByKeyAndValue
,
..
Default
::
default
()
};
let
key
=
vec!
[
42u8
];
{
let
db
=
TurboPersistence
::
<
_
,
1
>
::
open_with_config_and_parallel_scheduler
(
path
.to_path_buf
(),
config
,
RayonParallelScheduler
,
)
?
;
// Write a mix of different values (simulating hash collisions)
// Each unique value should be preserved after compaction
for
value
in
[
1u8
,
2
,
3
,
4
,
5
]
{
let
batch
=
db
.write_batch
()
?
;
batch
.put
(
0
,
key
.clone
(),
vec!
[
value
]
.into
())
?
;
db
.commit_write_batch
(
batch
)
?
;
}
// Before compaction: 5 entries exist
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
5
,
"Should have 5 entries before compaction"
);
// Compact
db
.full_compact
()
?
;
// After compaction: all 5 unique values should be preserved
// (this is the key difference from ByKeyOnly which would keep only 1)
let
results
=
db
.get_multiple
(
0
,
&
key
.as_slice
())
?
;
assert_eq!
(
results
.len
(),
5
,
"ByKeyAndValue should preserve all unique values after compaction"
);
let
mut
values
:
Vec
<
u8
>
=
results
.iter
()
.map
(|
r
|
r
[
0
])
.collect
();
values
.sort
();
assert_eq!
(
values
,
vec!
[
1
,
2
,
3
,
4
,
5
],
"Should have all values 1-5"
);
db
.shutdown
()
?
;
}
Ok
(())
}
turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs
View file @
16a3edab
...
...
@@ -10,8 +10,8 @@ use anyhow::{Ok, Result};
use
parking_lot
::
Mutex
;
use
smallvec
::
SmallVec
;
use
turbo_persistence
::{
ArcSlice
,
CompactConfig
,
DbConfig
,
FamilyConfig
,
KeyBase
,
StoreKey
,
TurboPersistence
,
ValueBuffer
,
ArcSlice
,
CompactConfig
,
DbConfig
,
DeduplicationMode
,
FamilyConfig
,
KeyBase
,
StoreKey
,
TurboPersistence
,
ValueBuffer
,
};
use
turbo_tasks
::{
JoinHandle
,
message_queue
::
TimingEvent
,
spawn
,
turbo_tasks
};
...
...
@@ -69,6 +69,9 @@ impl KeySpace {
// file size as compaction
max_entries_per_initial_file
:
1024
*
1024
,
data_threshold_per_initial_file
:
256
*
1024
*
1024
,
// We need to gracefully handle hash collisions, so only deduplicate if both key and
// value are identical
deduplication_mode
:
DeduplicationMode
::
ByKeyAndValue
,
..
Default
::
default
()
},
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment