Commit 0848e314 authored by Tobias Koppers's avatar Tobias Koppers

Initial database stats tool

parent f54479b9
...@@ -4411,6 +4411,18 @@ dependencies = [ ...@@ -4411,6 +4411,18 @@ dependencies = [
"urlencoding", "urlencoding",
] ]
[[package]]
name = "next-persistence-tools"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"next-api",
"rustc-hash 2.1.1",
"turbo-persistence",
"twox-hash 2.1.0",
]
[[package]] [[package]]
name = "next-swc-napi" name = "next-swc-napi"
version = "0.0.0" version = "0.0.0"
......
...@@ -10,6 +10,7 @@ members = [ ...@@ -10,6 +10,7 @@ members = [
"crates/next-build", "crates/next-build",
"crates/next-core", "crates/next-core",
"crates/next-custom-transforms", "crates/next-custom-transforms",
"crates/next-persistence-tools",
"crates/next-taskless", "crates/next-taskless",
"turbopack/crates/*", "turbopack/crates/*",
"turbopack/crates/*/fuzz", "turbopack/crates/*/fuzz",
......
...@@ -148,7 +148,7 @@ async fn pick_endpoint( ...@@ -148,7 +148,7 @@ async fn pick_endpoint(
op: OperationVc<Entrypoints>, op: OperationVc<Entrypoints>,
selector: EndpointSelector, selector: EndpointSelector,
) -> Result<Vc<OptionEndpoint>> { ) -> Result<Vc<OptionEndpoint>> {
let endpoints = op.connect().strongly_consistent().await?; let endpoints = op.read_strongly_consistent().await?;
let endpoint = match selector { let endpoint = match selector {
EndpointSelector::InstrumentationNodeJs => { EndpointSelector::InstrumentationNodeJs => {
endpoints.instrumentation.as_ref().map(|i| i.node_js) endpoints.instrumentation.as_ref().map(|i| i.node_js)
......
[package]
name = "next-persistence-tools"
version = "0.1.0"
description = "CLI tools for analyzing Next.js persistence cache"
license = "MIT"
edition = "2024"
autobenches = false
[[bin]]
name = "next-persistence-tools"
path = "src/main.rs"
bench = false
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive"] }
next-api = { workspace = true }
rustc-hash = { workspace = true }
turbo-persistence = { workspace = true }
twox-hash = { workspace = true }
This diff is collapsed.
use anyhow::Result;
use crate::{
arc_slice::ArcSlice,
compression::decompress_into_arc,
db::TurboPersistence,
lookup_entry::{LazyLookupValue, LookupValue},
merge_iter::MergeIter,
parallel_scheduler::ParallelScheduler,
static_sorted_file::StaticSortedFileIter,
};
/// Owned version of a lookup entry that can be returned from an iterator
#[derive(Clone)]
pub struct OwnedLookupEntry {
/// The hash of the key.
pub hash: u64,
/// The key.
pub key: ArcSlice<u8>,
/// The value.
pub value: ArcSlice<u8>,
}
/// Iterator that converts LookupEntry to OwnedLookupEntry
pub(crate) struct ConvertingIter<'l, S: ParallelScheduler> {
pub(crate) inner: MergeIter<'l, StaticSortedFileIter<'l>>,
pub(crate) db: &'l TurboPersistence<S>,
}
impl<'l, S: ParallelScheduler> Iterator for ConvertingIter<'l, S> {
type Item = Result<OwnedLookupEntry>;
fn next(&mut self) -> Option<Self::Item> {
loop {
let lookup_entry = match self.inner.next()? {
Ok(entry) => entry,
Err(e) => return Some(Err(e)),
};
// Extract the value based on its type
match &lookup_entry.value {
LazyLookupValue::Eager(LookupValue::Slice { value }) => {
return Some(Ok(OwnedLookupEntry {
hash: lookup_entry.hash,
key: lookup_entry.key.clone(),
value: value.clone(),
}));
}
LazyLookupValue::Eager(LookupValue::Blob { sequence_number }) => {
// Read the blob
match self.db.read_blob(*sequence_number) {
Ok(blob) => {
return Some(Ok(OwnedLookupEntry {
hash: lookup_entry.hash,
key: lookup_entry.key.clone(),
value: blob,
}));
}
Err(e) => {
return Some(Err(e));
}
}
}
LazyLookupValue::Eager(LookupValue::Deleted) => {
// Skip deleted entries
continue;
}
LazyLookupValue::Medium {
uncompressed_size,
block,
} => {
// Decompress the medium value
match decompress_into_arc(*uncompressed_size, *block, None, false) {
Ok(decompressed) => {
return Some(Ok(OwnedLookupEntry {
hash: lookup_entry.hash,
key: lookup_entry.key.clone(),
value: ArcSlice::from(decompressed),
}));
}
Err(e) => {
return Some(Err(e));
}
}
}
}
}
}
}
...@@ -11,6 +11,7 @@ use std::{ ...@@ -11,6 +11,7 @@ use std::{
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt, WriteBytesExt}; use byteorder::{BE, ReadBytesExt, WriteBytesExt};
use either::Either;
use jiff::Timestamp; use jiff::Timestamp;
use memmap2::Mmap; use memmap2::Mmap;
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
...@@ -27,6 +28,8 @@ use crate::{ ...@@ -27,6 +28,8 @@ use crate::{
KEY_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE,
VALUE_BLOCK_CACHE_SIZE, VALUE_BLOCK_CACHE_SIZE,
}, },
converting_iter::ConvertingIter,
dedupe_iter::DedupeIter,
key::{StoreKey, hash_key}, key::{StoreKey, hash_key},
lookup_entry::{LookupEntry, LookupValue}, lookup_entry::{LookupEntry, LookupValue},
merge_iter::MergeIter, merge_iter::MergeIter,
...@@ -129,7 +132,7 @@ pub struct TurboPersistence<S: ParallelScheduler> { ...@@ -129,7 +132,7 @@ pub struct TurboPersistence<S: ParallelScheduler> {
} }
/// The inner state of the database. /// The inner state of the database.
struct Inner { pub(crate) struct Inner {
/// The list of meta files in the database. This is used to derive the SST files. /// The list of meta files in the database. This is used to derive the SST files.
meta_files: Vec<MetaFile>, meta_files: Vec<MetaFile>,
/// The current sequence number for the database. /// The current sequence number for the database.
...@@ -379,7 +382,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -379,7 +382,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
/// Reads and decompresses a blob file. This is not backed by any cache. /// Reads and decompresses a blob file. This is not backed by any cache.
#[tracing::instrument(level = "info", name = "reading database blob", skip_all)] #[tracing::instrument(level = "info", name = "reading database blob", skip_all)]
fn read_blob(&self, seq: u32) -> Result<ArcSlice<u8>> { pub(crate) fn read_blob(&self, seq: u32) -> Result<ArcSlice<u8>> {
let path = self.path.join(format!("{seq:08}.blob")); let path = self.path.join(format!("{seq:08}.blob"));
let mmap = unsafe { Mmap::map(&File::open(&path)?)? }; let mmap = unsafe { Mmap::map(&File::open(&path)?)? };
#[cfg(unix)] #[cfg(unix)]
...@@ -1354,6 +1357,60 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1354,6 +1357,60 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
.collect()) .collect())
} }
/// Iterates over all key-value pairs in the specified family.
/// If `include_overwritten` is false, only the latest value for each key is returned.
/// If `include_overwritten` is true, all values including overwritten ones are returned.
pub fn iter(
&self,
family: usize,
include_overwritten: bool,
) -> Result<crate::iter::FamilyIter<'_, S>> {
let guard = self.inner.read();
let family = family as u32;
// SAFETY: We create a raw pointer to access the guard's data while we still own the guard.
// The guard will be stored in FamilyIter alongside the iterators that borrow from it,
// ensuring the data remains valid for the lifetime of the iterators.
// This is safe because:
// 1. The guard protects the data from being modified while we iterate
// 2. The guard is stored in the same struct as the iterators, so it outlives them
// 3. We never access the data through the raw pointer after the guard is dropped
let guard_ptr = &*guard as *const Inner;
let mut sst_iters = Vec::new();
for meta_file in unsafe { &(*guard_ptr).meta_files } {
if meta_file.family() != family {
continue;
}
for entry in meta_file.entries() {
let sst = entry.sst(meta_file)?;
let iter = sst.iter(&self.key_block_cache, &self.value_block_cache)?;
sst_iters.push(iter);
}
}
// Create merge iterator
let merge_iter = MergeIter::new(sst_iters.into_iter())?;
// Wrap in converting iterator
let converting_iter = ConvertingIter {
inner: merge_iter,
db: self,
};
// Conditionally wrap in DedupeIter based on include_overwritten
let inner = if include_overwritten {
Either::Left(converting_iter)
} else {
Either::Right(DedupeIter::new(converting_iter))
};
Ok(crate::iter::FamilyIter {
_guard: guard,
inner,
})
}
/// Shuts down the database. This will print statistics if the `print_stats` feature is enabled. /// Shuts down the database. This will print statistics if the `print_stats` feature is enabled.
pub fn shutdown(&self) -> Result<()> { pub fn shutdown(&self) -> Result<()> {
#[cfg(feature = "print_stats")] #[cfg(feature = "print_stats")]
......
use anyhow::Result;
use crate::converting_iter::OwnedLookupEntry;
/// Iterator that deduplicates consecutive entries with the same key.
/// Since the input iterator is sorted by (hash, key), we only need to compare consecutive entries.
pub struct DedupeIter<T: Iterator<Item = Result<OwnedLookupEntry>>> {
iter: T,
current: Option<OwnedLookupEntry>,
}
impl<T: Iterator<Item = Result<OwnedLookupEntry>>> DedupeIter<T> {
pub fn new(iter: T) -> Self {
Self {
iter,
current: None,
}
}
}
impl<T: Iterator<Item = Result<OwnedLookupEntry>>> Iterator for DedupeIter<T> {
type Item = Result<OwnedLookupEntry>;
fn next(&mut self) -> Option<Self::Item> {
loop {
// Get the next entry from the underlying iterator
match self.iter.next() {
Some(Ok(next_entry)) => {
// Check if we have a current entry
match &self.current {
Some(current_entry) => {
// Compare keys (hash and key bytes)
if current_entry.hash == next_entry.hash
&& current_entry.key.as_ref() == next_entry.key.as_ref()
{
// Same key - update current with the new value (keep latest)
self.current = Some(next_entry);
continue;
} else {
// Different key - return the current entry and store the new one
let to_return = self.current.replace(next_entry);
return to_return.map(Ok);
}
}
None => {
// No current entry - store this one and continue
self.current = Some(next_entry);
}
}
}
Some(Err(e)) => {
// Pass through errors
return Some(Err(e));
}
None => {
// Iterator exhausted - return the last current entry if any
return self.current.take().map(Ok);
}
}
}
}
}
use anyhow::Result;
use either::Either;
use parking_lot::RwLockReadGuard;
use crate::{
converting_iter::{ConvertingIter, OwnedLookupEntry},
db::Inner,
dedupe_iter::DedupeIter,
parallel_scheduler::ParallelScheduler,
};
/// Iterator over entries in a family
pub struct FamilyIter<'l, S: ParallelScheduler> {
pub(crate) inner: Either<ConvertingIter<'l, S>, DedupeIter<ConvertingIter<'l, S>>>,
/// The read guard to keep the inner data alive.
/// Must be kept last to ensure it is dropped after the iterators.
pub(crate) _guard: RwLockReadGuard<'l, Inner>,
}
impl<'l, S: ParallelScheduler> Iterator for FamilyIter<'l, S> {
type Item = Result<OwnedLookupEntry>;
fn next(&mut self) -> Option<Self::Item> {
match &mut self.inner {
Either::Left(iter) => iter.next(),
Either::Right(iter) => iter.next(),
}
}
}
...@@ -9,7 +9,10 @@ mod collector_entry; ...@@ -9,7 +9,10 @@ mod collector_entry;
mod compaction; mod compaction;
mod compression; mod compression;
mod constants; mod constants;
mod converting_iter;
mod db; mod db;
mod dedupe_iter;
mod iter;
mod key; mod key;
mod lookup_entry; mod lookup_entry;
mod merge_iter; mod merge_iter;
...@@ -26,8 +29,11 @@ mod write_batch; ...@@ -26,8 +29,11 @@ mod write_batch;
mod tests; mod tests;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use converting_iter::OwnedLookupEntry;
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use iter::FamilyIter;
pub use key::{KeyBase, QueryKey, StoreKey}; pub use key::{KeyBase, QueryKey, StoreKey};
pub use lookup_entry::LookupEntry;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler}; pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
pub use value_buf::ValueBuffer; pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch; pub use write_batch::WriteBatch;
...@@ -143,6 +143,19 @@ fn full_cycle() -> Result<()> { ...@@ -143,6 +143,19 @@ fn full_cycle() -> Result<()> {
assert_eq!(db.get(0, &[42u8, 42])?, None); assert_eq!(db.get(0, &[42u8, 42])?, None);
assert_eq!(db.get(0, &[1u8])?, None); assert_eq!(db.get(0, &[1u8])?, None);
assert_eq!(db.get(0, &[255u8])?, None); assert_eq!(db.get(0, &[255u8])?, None);
// Test iter() with deduplication - basic smoke test
let mut count = 0;
for entry_result in db.iter(0, false)? {
let entry = entry_result?;
count += 1;
// Basic validation - entries should have data
assert!(!entry.key.is_empty());
assert!(!entry.value.is_empty());
}
// Should have at least the 90 entries from this test
assert!(count >= 90, "Expected at least 90 entries, got {}", count);
Ok(()) Ok(())
}, },
); );
...@@ -165,6 +178,16 @@ fn full_cycle() -> Result<()> { ...@@ -165,6 +178,16 @@ fn full_cycle() -> Result<()> {
assert_eq!(db.get(0, &[42u8, 42])?, None); assert_eq!(db.get(0, &[42u8, 42])?, None);
assert_eq!(db.get(0, &[1u8])?, None); assert_eq!(db.get(0, &[1u8])?, None);
assert_eq!(db.get(0, &[255u8])?, None); assert_eq!(db.get(0, &[255u8])?, None);
// Test iter() - basic smoke test
let mut count = 0;
for entry_result in db.iter(0, false)? {
entry_result?;
count += 1;
}
// Should have at least 90 entries
assert!(count >= 90, "Expected at least 90 entries, got {}", count);
Ok(()) Ok(())
}, },
); );
...@@ -186,6 +209,115 @@ fn full_cycle() -> Result<()> { ...@@ -186,6 +209,115 @@ fn full_cycle() -> Result<()> {
assert!(db.get(8, &[8u8, 8])?.is_none()); assert!(db.get(8, &[8u8, 8])?.is_none());
assert!(db.get(8, &[0u8])?.is_none()); assert!(db.get(8, &[0u8])?.is_none());
assert!(db.get(8, &[255u8])?.is_none()); assert!(db.get(8, &[255u8])?.is_none());
// Test iter() on each family - basic smoke test
// In standalone run, each family should have 1 entry
// In "All" run, family 0 may have entries from other tests
for family in 0..16 {
let mut count = 0;
let mut found_expected_entry = false;
for entry_result in db.iter(family, false)? {
let entry = entry_result?;
count += 1;
// Check if this is the entry from this specific test
if entry.key.len() == 1
&& entry.key[0] == family as u8
&& entry.value.len() == 1
&& entry.value[0] == family as u8
{
found_expected_entry = true;
}
}
assert!(count >= 1, "Expected at least 1 entry in family {}", family);
assert!(
found_expected_entry,
"Expected to find entry for family {}",
family
);
}
Ok(())
},
);
test_case(
&mut test_cases,
"Overwrites with iter",
|batch| {
// Write same keys multiple times
for round in 0..5u8 {
for i in 10..20u8 {
batch.put(0, vec![i], vec![round, i].into())?;
}
unsafe { batch.flush(0)? };
}
Ok(())
},
|db| {
// Verify latest values
for i in 10..20u8 {
let Some(value) = db.get(0, &[i])? else {
panic!("Value not found for key {}", i);
};
assert_eq!(
&*value,
&[4u8, i],
"Expected latest round (4) for key {}",
i
);
}
// Test iter with deduplication - should only get latest values
// Count entries that match this test's pattern
let mut count_from_this_test = 0;
for entry_result in db.iter(0, false)? {
let entry = entry_result?;
// Only validate entries from this specific test
if entry.key.len() == 1 && entry.value.len() == 2 {
let key = entry.key[0];
if (10..20).contains(&key) {
count_from_this_test += 1;
assert_eq!(entry.value[0], 4u8, "Expected round 4 for key {}", key);
assert_eq!(entry.value[1], key);
}
}
}
assert_eq!(
count_from_this_test, 10,
"Expected 10 deduplicated entries from this test"
);
// Test iter with all entries - count entries from this test only
let mut count_from_this_test_all = 0;
let mut max_round_seen = 0u8;
for entry_result in db.iter(0, true)? {
let entry = entry_result?;
// Only count entries from this specific test
if entry.key.len() == 1 && entry.value.len() == 2 {
let key = entry.key[0];
if (10..20).contains(&key) {
count_from_this_test_all += 1;
let round = entry.value[0];
assert!(round < 5, "Round should be 0-4");
assert_eq!(entry.value[1], key);
max_round_seen = max_round_seen.max(round);
}
}
}
// We should have at least 10 entries (one per key), possibly up to 50 before compaction
assert!(
count_from_this_test_all >= 10 && count_from_this_test_all <= 50,
"Expected between 10-50 entries from this test, got {}",
count_from_this_test_all
);
// If we only have 10 entries, they should all be round 4 (after compaction)
if count_from_this_test_all == 10 {
assert_eq!(
max_round_seen, 4,
"After compaction, only latest round should remain"
);
}
Ok(()) Ok(())
}, },
); );
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment