Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
49f013c7
Commit
49f013c7
authored
Jan 29, 2026
by
Luke Sandberg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
simplify the configuration
parent
881aaaaf
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
89 additions
and
83 deletions
+89
-83
collector.rs
turbopack/crates/turbo-persistence/src/collector.rs
+1
-12
constants.rs
turbopack/crates/turbo-persistence/src/constants.rs
+0
-44
db.rs
turbopack/crates/turbo-persistence/src/db.rs
+4
-6
lib.rs
turbopack/crates/turbo-persistence/src/lib.rs
+45
-1
write_batch.rs
turbopack/crates/turbo-persistence/src/write_batch.rs
+2
-2
mod.rs
...pack/crates/turbo-tasks-backend/src/database/turbo/mod.rs
+37
-18
No files found.
turbopack/crates/turbo-persistence/src/collector.rs
View file @
49f013c7
...
@@ -3,9 +3,7 @@ use std::mem::take;
...
@@ -3,9 +3,7 @@ use std::mem::take;
use
crate
::{
use
crate
::{
ValueBuffer
,
ValueBuffer
,
collector_entry
::{
CollectorEntry
,
CollectorEntryValue
,
EntryKey
,
TINY_VALUE_THRESHOLD
},
collector_entry
::{
CollectorEntry
,
CollectorEntryValue
,
EntryKey
,
TINY_VALUE_THRESHOLD
},
constants
::{
constants
::{
MAX_ENTRIES_PER_INITIAL_FILE
,
MAX_SMALL_VALUE_SIZE
},
DATA_THRESHOLD_PER_INITIAL_FILE
,
MAX_ENTRIES_PER_INITIAL_FILE
,
MAX_SMALL_VALUE_SIZE
,
},
key
::{
StoreKey
,
hash_key
},
key
::{
StoreKey
,
hash_key
},
};
};
...
@@ -22,15 +20,6 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> {
...
@@ -22,15 +20,6 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> {
}
}
impl
<
K
:
StoreKey
,
const
SIZE_SHIFT
:
usize
>
Collector
<
K
,
SIZE_SHIFT
>
{
impl
<
K
:
StoreKey
,
const
SIZE_SHIFT
:
usize
>
Collector
<
K
,
SIZE_SHIFT
>
{
/// Creates a new collector with default limits. Note that this allocates the full capacity for
/// the entries.
pub
fn
new
()
->
Self
{
Self
::
with_config
(
MAX_ENTRIES_PER_INITIAL_FILE
,
DATA_THRESHOLD_PER_INITIAL_FILE
,
)
}
/// Creates a new collector with custom limits.
/// Creates a new collector with custom limits.
///
///
/// The limits are shifted right by `SIZE_SHIFT` to support thread-local collectors
/// The limits are shifted right by `SIZE_SHIFT` to support thread-local collectors
...
...
turbopack/crates/turbo-persistence/src/constants.rs
View file @
49f013c7
...
@@ -34,47 +34,3 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
...
@@ -34,47 +34,3 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
/// Maximum RAM bytes for value block cache
/// Maximum RAM bytes for value block cache
pub
const
VALUE_BLOCK_CACHE_SIZE
:
u64
=
300
*
1024
*
1024
;
pub
const
VALUE_BLOCK_CACHE_SIZE
:
u64
=
300
*
1024
*
1024
;
pub
const
VALUE_BLOCK_AVG_SIZE
:
usize
=
132000
;
pub
const
VALUE_BLOCK_AVG_SIZE
:
usize
=
132000
;
/// Configuration for a single family's file limits.
///
/// Controls when SST files are split during writes and compaction.
/// Files are split when either the entry count OR data size threshold is reached.
#[derive(Clone,
Copy,
Debug)]
pub
struct
FamilyConfig
{
/// Maximum number of entries per initial SST file (during writes)
pub
max_entries_per_initial_file
:
usize
,
/// Data size threshold for initial SST files (bytes)
pub
data_threshold_per_initial_file
:
usize
,
/// Maximum number of entries per compacted SST file
pub
max_entries_per_compacted_file
:
usize
,
/// Data size threshold for compacted SST files (bytes)
pub
data_threshold_per_compacted_file
:
usize
,
}
impl
Default
for
FamilyConfig
{
fn
default
()
->
Self
{
Self
{
max_entries_per_initial_file
:
MAX_ENTRIES_PER_INITIAL_FILE
,
data_threshold_per_initial_file
:
DATA_THRESHOLD_PER_INITIAL_FILE
,
max_entries_per_compacted_file
:
MAX_ENTRIES_PER_COMPACTED_FILE
,
data_threshold_per_compacted_file
:
DATA_THRESHOLD_PER_COMPACTED_FILE
,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone,
Debug)]
pub
struct
DbConfig
<
const
FAMILIES
:
usize
>
{
pub
family_configs
:
[
FamilyConfig
;
FAMILIES
],
}
impl
<
const
FAMILIES
:
usize
>
Default
for
DbConfig
<
FAMILIES
>
{
fn
default
()
->
Self
{
Self
{
family_configs
:
[
FamilyConfig
::
default
();
FAMILIES
],
}
}
}
turbopack/crates/turbo-persistence/src/db.rs
View file @
49f013c7
...
@@ -20,12 +20,12 @@ use smallvec::SmallVec;
...
@@ -20,12 +20,12 @@ use smallvec::SmallVec;
pub
use
crate
::
compaction
::
selector
::
CompactConfig
;
pub
use
crate
::
compaction
::
selector
::
CompactConfig
;
use
crate
::{
use
crate
::{
QueryKey
,
DbConfig
,
QueryKey
,
arc_slice
::
ArcSlice
,
arc_slice
::
ArcSlice
,
compaction
::
selector
::{
Compactable
,
get_merge_segments
},
compaction
::
selector
::{
Compactable
,
get_merge_segments
},
compression
::
decompress_into_arc
,
compression
::
decompress_into_arc
,
constants
::{
constants
::{
DATA_THRESHOLD_PER_COMPACTED_FILE
,
DbConfig
,
KEY_BLOCK_AVG_SIZE
,
KEY_BLOCK_CACHE_SIZE
,
DATA_THRESHOLD_PER_COMPACTED_FILE
,
KEY_BLOCK_AVG_SIZE
,
KEY_BLOCK_CACHE_SIZE
,
MAX_ENTRIES_PER_COMPACTED_FILE
,
VALUE_BLOCK_AVG_SIZE
,
VALUE_BLOCK_CACHE_SIZE
,
MAX_ENTRIES_PER_COMPACTED_FILE
,
VALUE_BLOCK_AVG_SIZE
,
VALUE_BLOCK_CACHE_SIZE
,
},
},
key
::{
StoreKey
,
hash_key
},
key
::{
StoreKey
,
hash_key
},
...
@@ -1102,12 +1102,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
...
@@ -1102,12 +1102,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
collector
.total_key_size
+=
key_size
;
collector
.total_key_size
+=
key_size
;
collector
.total_value_size
+=
value_size
;
collector
.total_value_size
+=
value_size
;
let
family_config
=
&
self
.config.family_configs
[
family
as
usize
];
if
collector
.total_key_size
+
collector
.total_value_size
if
collector
.total_key_size
+
collector
.total_value_size
>
family_config
.data_threshold_per_compacted_file
>
DATA_THRESHOLD_PER_COMPACTED_FILE
||
collector
.entries
.len
()
||
collector
.entries
.len
()
>=
family_config
.max_entries_per_compacted_file
>=
MAX_ENTRIES_PER_COMPACTED_FILE
{
{
let
selected_total_key_size
=
let
selected_total_key_size
=
collector
.last_entries_total_key_size
;
collector
.last_entries_total_key_size
;
...
...
turbopack/crates/turbo-persistence/src/lib.rs
View file @
49f013c7
...
@@ -26,8 +26,52 @@ mod write_batch;
...
@@ -26,8 +26,52 @@ mod write_batch;
mod
tests
;
mod
tests
;
pub
use
arc_slice
::
ArcSlice
;
pub
use
arc_slice
::
ArcSlice
;
pub
use
constants
::{
DbConfig
,
FamilyConfig
};
use
constants
::{
DATA_THRESHOLD_PER_INITIAL_FILE
,
MAX_ENTRIES_PER_INITIAL_FILE
};
pub
use
db
::{
CompactConfig
,
MetaFileEntryInfo
,
MetaFileInfo
,
TurboPersistence
};
pub
use
db
::{
CompactConfig
,
MetaFileEntryInfo
,
MetaFileInfo
,
TurboPersistence
};
/// Configuration for a single family's file limits during writes.
///
/// Controls when SST files are split during writes.
/// Files are split when either the entry count OR data size threshold is reached.
///
/// Note: Compaction uses the global constants `MAX_ENTRIES_PER_COMPACTED_FILE` and
/// `DATA_THRESHOLD_PER_COMPACTED_FILE` for all families.
#[derive(Clone,
Copy,
Debug)]
pub
struct
FamilyConfig
{
/// Maximum number of entries per initial SST file (during writes).
/// Controls the size of the Collector datastructures when writing files,
/// as well as when we switch to a 'sharded' file writing strategy.
pub
max_entries_per_initial_file
:
usize
,
/// Data size threshold for initial SST files (bytes).
/// Controls memory usage during writes.
pub
data_threshold_per_initial_file
:
usize
,
}
impl
Default
for
FamilyConfig
{
fn
default
()
->
Self
{
Self
{
max_entries_per_initial_file
:
MAX_ENTRIES_PER_INITIAL_FILE
,
data_threshold_per_initial_file
:
DATA_THRESHOLD_PER_INITIAL_FILE
,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone,
Debug)]
pub
struct
DbConfig
<
const
FAMILIES
:
usize
>
{
pub
family_configs
:
[
FamilyConfig
;
FAMILIES
],
}
impl
<
const
FAMILIES
:
usize
>
Default
for
DbConfig
<
FAMILIES
>
{
fn
default
()
->
Self
{
Self
{
family_configs
:
[
FamilyConfig
::
default
();
FAMILIES
],
}
}
}
pub
use
key
::{
KeyBase
,
QueryKey
,
StoreKey
,
hash_key
};
pub
use
key
::{
KeyBase
,
QueryKey
,
StoreKey
,
hash_key
};
pub
use
meta_file
::
MetaEntryFlags
;
pub
use
meta_file
::
MetaEntryFlags
;
pub
use
parallel_scheduler
::{
ParallelScheduler
,
SerialScheduler
};
pub
use
parallel_scheduler
::{
ParallelScheduler
,
SerialScheduler
};
...
...
turbopack/crates/turbo-persistence/src/write_batch.rs
View file @
49f013c7
...
@@ -15,11 +15,11 @@ use smallvec::SmallVec;
...
@@ -15,11 +15,11 @@ use smallvec::SmallVec;
use
thread_local
::
ThreadLocal
;
use
thread_local
::
ThreadLocal
;
use
crate
::{
use
crate
::{
ValueBuffer
,
FamilyConfig
,
ValueBuffer
,
collector
::
Collector
,
collector
::
Collector
,
collector_entry
::
CollectorEntry
,
collector_entry
::
CollectorEntry
,
compression
::
compress_into_buffer
,
compression
::
compress_into_buffer
,
constants
::{
FamilyConfig
,
MAX_MEDIUM_VALUE_SIZE
,
THREAD_LOCAL_SIZE_SHIFT
},
constants
::{
MAX_MEDIUM_VALUE_SIZE
,
THREAD_LOCAL_SIZE_SHIFT
},
key
::
StoreKey
,
key
::
StoreKey
,
meta_file
::
MetaEntryFlags
,
meta_file
::
MetaEntryFlags
,
meta_file_builder
::
MetaFileBuilder
,
meta_file_builder
::
MetaFileBuilder
,
...
...
turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs
View file @
49f013c7
...
@@ -45,27 +45,46 @@ pub struct TurboKeyValueDatabase {
...
@@ -45,27 +45,46 @@ pub struct TurboKeyValueDatabase {
is_fresh
:
bool
,
is_fresh
:
bool
,
}
}
impl
TurboKeyValueDatabase
{
impl
KeySpace
{
pub
fn
new
(
versioned_path
:
PathBuf
,
is_ci
:
bool
,
is_short_session
:
bool
)
->
Result
<
Self
>
{
/// Returns the persistence configuration for this keyspace.
// Configure per-family limits based on keyspace characteristics
///
let
mut
config
=
DbConfig
::
<
FAMILIES
>
::
default
();
/// Configuration rationale:
/// - `max_entries_per_initial_file`: Controls SST file splitting during writes. Smaller values
// Infra (family 0): Only 2 keys ever exist, minimal allocation needed
/// create more files but reduce memory usage.
config
.family_configs
[
KeySpace
::
Infra
as
usize
]
=
FamilyConfig
{
/// - `data_threshold_per_initial_file`: Memory budget for initial file writes.
fn
family_config
(
&
self
)
->
FamilyConfig
{
match
self
{
// Infra has only 2 keys ever, so use minimal limits
KeySpace
::
Infra
=>
FamilyConfig
{
max_entries_per_initial_file
:
16
,
max_entries_per_initial_file
:
16
,
max_entries_per_compacted_file
:
16
,
..
Default
::
default
()
..
Default
::
default
()
};
},
// TaskMeta and TaskData use default limits
// TaskCache (family 3): Index with collision semantics - maximize entries per file
KeySpace
::
TaskMeta
|
KeySpace
::
TaskData
=>
FamilyConfig
::
default
(),
// to minimize files to scan during get_multiple lookups.
// TaskCache uses hash-based lookups with potential collisions.
// Maximize entries per file to minimize files scanned in get_multiple.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
config
.family_configs
[
KeySpace
::
TaskCache
as
usize
]
=
FamilyConfig
{
KeySpace
::
TaskCache
=>
FamilyConfig
{
max_entries_per_initial_file
:
usize
::
MAX
,
// The keys and values in this keyspace are very small, so we just target the same
max_entries_per_compacted_file
:
usize
::
MAX
,
// file size as compaction
max_entries_per_initial_file
:
1024
*
1024
,
data_threshold_per_initial_file
:
256
*
1024
*
1024
,
..
Default
::
default
()
..
Default
::
default
()
};
},
}
}
}
impl
TurboKeyValueDatabase
{
pub
fn
new
(
versioned_path
:
PathBuf
,
is_ci
:
bool
,
is_short_session
:
bool
)
->
Result
<
Self
>
{
let
config
=
DbConfig
{
family_configs
:
[
KeySpace
::
Infra
.family_config
(),
KeySpace
::
TaskMeta
.family_config
(),
KeySpace
::
TaskData
.family_config
(),
KeySpace
::
TaskCache
.family_config
(),
],
};
let
db
=
Arc
::
new
(
TurboPersistence
::
open_with_config
(
versioned_path
,
config
)
?
);
let
db
=
Arc
::
new
(
TurboPersistence
::
open_with_config
(
versioned_path
,
config
)
?
);
Ok
(
Self
{
Ok
(
Self
{
db
:
db
.clone
(),
db
:
db
.clone
(),
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment