Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
bbfc2280
Commit
bbfc2280
authored
Apr 04, 2026
by
Luke Sandberg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
tiny debugging hacks
parent
f65b10a5
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
70 additions
and
21 deletions
+70
-21
sst_inspect.rs
turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs
+56
-20
lib.rs
turbopack/crates/turbo-persistence/src/lib.rs
+3
-1
key_value_database.rs
...es/turbo-tasks-backend/src/database/key_value_database.rs
+11
-0
No files found.
turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs
View file @
bbfc2280
...
@@ -24,6 +24,7 @@ use turbo_persistence::{
...
@@ -24,6 +24,7 @@ use turbo_persistence::{
BLOCK_HEADER_SIZE
,
checksum_block
,
BLOCK_HEADER_SIZE
,
checksum_block
,
meta_file
::
MetaFile
,
meta_file
::
MetaFile
,
mmap_helper
::
advise_mmap_for_persistence
,
mmap_helper
::
advise_mmap_for_persistence
,
sst_filter
::
SstFilter
,
static_sorted_file
::{
static_sorted_file
::{
BLOCK_TYPE_FIXED_KEY_NO_HASH
,
BLOCK_TYPE_FIXED_KEY_WITH_HASH
,
BLOCK_TYPE_KEY_NO_HASH
,
BLOCK_TYPE_FIXED_KEY_NO_HASH
,
BLOCK_TYPE_FIXED_KEY_WITH_HASH
,
BLOCK_TYPE_KEY_NO_HASH
,
BLOCK_TYPE_KEY_WITH_HASH
,
KEY_BLOCK_ENTRY_TYPE_BLOB
,
KEY_BLOCK_ENTRY_TYPE_DELETED
,
BLOCK_TYPE_KEY_WITH_HASH
,
KEY_BLOCK_ENTRY_TYPE_BLOB
,
KEY_BLOCK_ENTRY_TYPE_DELETED
,
...
@@ -205,33 +206,68 @@ fn format_bytes(bytes: u64) -> String {
...
@@ -205,33 +206,68 @@ fn format_bytes(bytes: u64) -> String {
}
}
}
}
/// Collect SST info from all meta files in the database directory
/// Collect SST info from all active meta files in the database directory,
/// mirroring the DB's own open logic: read CURRENT, filter by .del files,
/// and apply SstFilter to skip superseded entries.
fn
collect_sst_info
(
db_path
:
&
Path
)
->
Result
<
BTreeMap
<
u32
,
Vec
<
SstInfo
>>>
{
fn
collect_sst_info
(
db_path
:
&
Path
)
->
Result
<
BTreeMap
<
u32
,
Vec
<
SstInfo
>>>
{
let
mut
meta_files
:
Vec
<
PathBuf
>
=
fs
::
read_dir
(
db_path
)
?
// Read the CURRENT sequence number — only files with seq <= current are valid.
.filter_map
(|
entry
|
entry
.ok
())
let
current
:
u32
=
File
::
open
(
db_path
.join
(
"CURRENT"
))
.map
(|
entry
|
entry
.path
())
.context
(
"Failed to open CURRENT file"
)
?
.filter
(|
path
|
path
.extension
()
.is_some_and
(|
ext
|
ext
==
"meta"
))
.read_u32
::
<
BE
>
()
.collect
();
.context
(
"Failed to read CURRENT file"
)
?
;
// Read .del files to find sequences that were deleted but not yet cleaned up.
let
mut
deleted_seqs
:
HashSet
<
u32
>
=
HashSet
::
new
();
for
entry
in
fs
::
read_dir
(
db_path
)
?
{
let
path
=
entry
?
.path
();
if
path
.extension
()
.and_then
(|
s
|
s
.to_str
())
==
Some
(
"del"
)
{
let
content
=
fs
::
read
(
&
path
)
?
;
let
mut
cursor
:
&
[
u8
]
=
&
content
;
while
!
cursor
.is_empty
()
{
deleted_seqs
.insert
(
cursor
.read_u32
::
<
BE
>
()
?
);
}
}
}
meta_files
.sort
();
// Collect valid meta sequence numbers.
let
mut
meta_seqs
:
Vec
<
u32
>
=
fs
::
read_dir
(
db_path
)
?
.filter_map
(|
e
|
e
.ok
())
.filter_map
(|
e
|
{
let
path
=
e
.path
();
if
path
.extension
()
.and_then
(|
s
|
s
.to_str
())
!=
Some
(
"meta"
)
{
return
None
;
}
let
seq
:
u32
=
path
.file_stem
()
?
.to_str
()
?
.parse
()
.ok
()
?
;
if
seq
>
current
||
deleted_seqs
.contains
(
&
seq
)
{
return
None
;
}
Some
(
seq
)
})
.collect
();
if
meta_
file
s
.is_empty
()
{
if
meta_
seq
s
.is_empty
()
{
bail!
(
"No .meta files found in {}"
,
db_path
.display
());
bail!
(
"No
active
.meta files found in {}"
,
db_path
.display
());
}
}
let
mut
family_sst_info
:
BTreeMap
<
u32
,
Vec
<
SstInfo
>>
=
BTreeMap
::
new
();
meta_seqs
.sort_unstable
();
for
meta_path
in
&
meta_files
{
// Extract sequence number from filename
let
filename
=
meta_path
.file_stem
()
.and_then
(|
s
|
s
.to_str
())
.unwrap_or
(
""
);
let
seq_num
:
u32
=
filename
.parse
()
.unwrap_or
(
0
);
let
meta_file
=
MetaFile
::
open
(
db_path
,
seq_num
)
let
mut
meta_files
:
Vec
<
MetaFile
>
=
meta_seqs
.with_context
(||
format!
(
"Failed to open {}"
,
meta_path
.display
()))
?
;
.iter
()
.map
(|
&
seq
|
{
MetaFile
::
open
(
db_path
,
seq
)
.with_context
(||
format!
(
"Failed to open {seq:08}.meta"
))
})
.collect
::
<
Result
<
_
>>
()
?
;
let
family
=
meta_file
.family
();
// Apply SstFilter (newest first) to drop entries superseded by a newer meta file.
let
mut
sst_filter
=
SstFilter
::
new
();
for
meta
in
meta_files
.iter_mut
()
.rev
()
{
sst_filter
.apply_filter
(
meta
);
}
for
entry
in
meta_file
.entries
()
{
let
mut
family_sst_info
:
BTreeMap
<
u32
,
Vec
<
SstInfo
>>
=
BTreeMap
::
new
();
for
meta
in
&
meta_files
{
let
family
=
meta
.family
();
for
entry
in
meta
.entries
()
{
family_sst_info
.entry
(
family
)
.or_default
()
.push
(
SstInfo
{
family_sst_info
.entry
(
family
)
.or_default
()
.push
(
SstInfo
{
sequence_number
:
entry
.sequence_number
(),
sequence_number
:
entry
.sequence_number
(),
block_count
:
entry
.block_count
(),
block_count
:
entry
.block_count
(),
...
@@ -317,7 +353,7 @@ fn read_block(
...
@@ -317,7 +353,7 @@ fn read_block(
///
///
/// Index block format: `[1B type][2B first_block][N * (8B hash + 2B block_index)]`.
/// Index block format: `[1B type][2B first_block][N * (8B hash + 2B block_index)]`.
fn
parse_key_block_indices
(
index_block
:
&
[
u8
])
->
HashSet
<
u16
>
{
fn
parse_key_block_indices
(
index_block
:
&
[
u8
])
->
HashSet
<
u16
>
{
assert
!
(
index_block
.len
()
>=
4
,
"Index block too small"
);
assert
!
(
index_block
.len
()
>=
3
,
"Index block too small"
);
let
mut
data
=
&
index_block
[
1
..
];
// skip block type byte
let
mut
data
=
&
index_block
[
1
..
];
// skip block type byte
let
first_block
=
data
.read_u16
::
<
BE
>
()
.unwrap
();
let
first_block
=
data
.read_u16
::
<
BE
>
()
.unwrap
();
let
mut
indices
=
HashSet
::
new
();
let
mut
indices
=
HashSet
::
new
();
...
...
turbopack/crates/turbo-persistence/src/lib.rs
View file @
bbfc2280
...
@@ -18,7 +18,7 @@ pub mod mmap_helper;
...
@@ -18,7 +18,7 @@ pub mod mmap_helper;
mod
parallel_scheduler
;
mod
parallel_scheduler
;
mod
rc_bytes
;
mod
rc_bytes
;
mod
shared_bytes
;
mod
shared_bytes
;
mod
sst_filter
;
pub
mod
sst_filter
;
pub
mod
static_sorted_file
;
pub
mod
static_sorted_file
;
mod
static_sorted_file_builder
;
mod
static_sorted_file_builder
;
mod
value_block_count_tracker
;
mod
value_block_count_tracker
;
...
@@ -49,6 +49,7 @@ pub enum FamilyKind {
...
@@ -49,6 +49,7 @@ pub enum FamilyKind {
/// Configuration for a single family to describe how the data is stored.
/// Configuration for a single family to describe how the data is stored.
#[derive(Clone,
Copy,
Debug)]
#[derive(Clone,
Copy,
Debug)]
pub
struct
FamilyConfig
{
pub
struct
FamilyConfig
{
pub
name
:
&
'static
str
,
pub
kind
:
FamilyKind
,
pub
kind
:
FamilyKind
,
}
}
...
@@ -65,6 +66,7 @@ impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
...
@@ -65,6 +66,7 @@ impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn
default
()
->
Self
{
fn
default
()
->
Self
{
Self
{
Self
{
family_configs
:
[
FamilyConfig
{
family_configs
:
[
FamilyConfig
{
name
:
"unknown"
,
kind
:
FamilyKind
::
SingleValue
,
kind
:
FamilyKind
::
SingleValue
,
};
FAMILIES
],
};
FAMILIES
],
}
}
...
...
turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs
View file @
bbfc2280
...
@@ -27,13 +27,24 @@ impl KeySpace {
...
@@ -27,13 +27,24 @@ impl KeySpace {
}
}
}
}
const
fn
name
(
&
self
)
->
&
'static
str
{
match
self
{
KeySpace
::
Infra
=>
"Infra"
,
KeySpace
::
TaskMeta
=>
"TaskMeta"
,
KeySpace
::
TaskData
=>
"TaskData"
,
KeySpace
::
TaskCache
=>
"TaskCache"
,
}
}
/// Returns the persistence configuration for this keyspace.
/// Returns the persistence configuration for this keyspace.
pub
const
fn
family_config
(
&
self
)
->
FamilyConfig
{
pub
const
fn
family_config
(
&
self
)
->
FamilyConfig
{
match
self
{
match
self
{
KeySpace
::
Infra
|
KeySpace
::
TaskMeta
|
KeySpace
::
TaskData
=>
FamilyConfig
{
KeySpace
::
Infra
|
KeySpace
::
TaskMeta
|
KeySpace
::
TaskData
=>
FamilyConfig
{
name
:
self
.name
(),
kind
:
FamilyKind
::
SingleValue
,
kind
:
FamilyKind
::
SingleValue
,
},
},
KeySpace
::
TaskCache
=>
FamilyConfig
{
KeySpace
::
TaskCache
=>
FamilyConfig
{
name
:
self
.name
(),
// TaskCache uses hash-based lookups with potential collisions.
// TaskCache uses hash-based lookups with potential collisions.
kind
:
FamilyKind
::
MultiValue
,
kind
:
FamilyKind
::
MultiValue
,
},
},
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment