Unverified Commit 4c683ed2 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add tool to print DB structure (#80148)

### What?

Adds a small CLI tool that prints the database structure like that:

```
META 00000417.meta: family = 4, 
  SST 00000411.sst: 000dade6cc76e41a - fffa6721f94e901f (p = 1/1)
    AQMF 4695 entries = 4 KiB
    180 KiB = 0 kiB key compression dict + 23 KiB value compression dict + 46 blocks (avg 3512 bytes/block)
META 00000416.meta: family = 3, 
  SST 00000412.sst: 0025bba83f4b6ab9 - ffde0483d58f3b5e (p = 1/1)
    AQMF 4695 entries = 4 KiB
    182 KiB = 24 kiB key compression dict + 0 KiB value compression dict + 46 blocks (avg 3522 bytes/block)
META 00000415.meta: family = 1, 
  SST 00000389.sst: 00005eac4b66cb4e - 3fffed60c1451cfe (p = 1/4)
    AQMF 299033 entries = 292 KiB
    24350 KiB = 0 kiB key compression dict + 63 KiB value compression dict + 3596 blocks (avg 6915 bytes/block)
  SST 00000391.sst: 800015a5a9666318 - bfffdd05778fc576 (p = 1/4)
    AQMF 299033 entries = 292 KiB
    24851 KiB = 0 kiB key compression dict + 63 KiB value compression dict + 3933 blocks (avg 6453 bytes/block)
  ...
...
```

The tool is only intended for internal usage and not exposed to the end user
parent 61729da4
...@@ -9513,6 +9513,7 @@ version = "0.1.0" ...@@ -9513,6 +9513,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"byteorder", "byteorder",
"either",
"jiff", "jiff",
"lzzzz", "lzzzz",
"memmap2 0.9.5", "memmap2 0.9.5",
...@@ -9531,6 +9532,14 @@ dependencies = [ ...@@ -9531,6 +9532,14 @@ dependencies = [
"zstd", "zstd",
] ]
[[package]]
name = "turbo-persistence-tools"
version = "0.1.0"
dependencies = [
"anyhow",
"turbo-persistence",
]
[[package]] [[package]]
name = "turbo-prehash" name = "turbo-prehash"
version = "0.1.0" version = "0.1.0"
......
[package]
name = "turbo-persistence-tools"
version = "0.1.0"
edition = "2024"
license = "MIT"
[dependencies]
anyhow = { workspace = true }
turbo-persistence = { workspace = true }
[lints]
workspace = true
#![feature(iter_intersperse)]
use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use turbo_persistence::{MetaFileEntryInfo, TurboPersistence};
fn main() -> Result<()> {
// Get CLI argument
let path = PathBuf::from(
std::env::args()
.nth(1)
.context("Please provide a path to the TurboPersistence directory")?,
);
if !path.exists() {
bail!("The provided path does not exist: {}", path.display());
}
let db = TurboPersistence::open_read_only(path)?;
let meta_info = db
.meta_info()
.context("Failed to retrieve meta information")?;
for meta_file in meta_info {
println!(
"META {:08}.meta: family = {}, ",
meta_file.sequence_number, meta_file.family
);
for MetaFileEntryInfo {
sequence_number,
min_hash,
max_hash,
aqmf_size,
aqmf_entries,
sst_size,
key_compression_dictionary_size,
value_compression_dictionary_size,
block_count,
} in meta_file.entries
{
println!(
" SST {sequence_number:08}.sst: {min_hash:016x} - {max_hash:016x} (p = 1/{})",
u64::MAX / (max_hash - min_hash)
);
println!(" AQMF {aqmf_entries} entries = {} KiB", aqmf_size / 1024);
println!(
" {} KiB = {} kiB key compression dict + {} KiB value compression dict + \
{block_count} blocks (avg {} bytes/block)",
sst_size / 1024,
key_compression_dictionary_size / 1024,
value_compression_dictionary_size / 1024,
(sst_size
- key_compression_dictionary_size as u64
- value_compression_dictionary_size as u64)
/ block_count as u64
);
}
if !meta_file.obsolete_sst_files.is_empty() {
println!(
" OBSOLETE SSTs {}",
meta_file
.obsolete_sst_files
.iter()
.map(|seq| format!("{seq:08}.sst"))
.intersperse(", ".to_string())
.collect::<String>()
);
}
}
Ok(())
}
...@@ -13,6 +13,7 @@ print_stats = ["stats"] ...@@ -13,6 +13,7 @@ print_stats = ["stats"]
[dependencies] [dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
either = { workspace = true}
pot = "3.0.0" pot = "3.0.0"
byteorder = "1.5.0" byteorder = "1.5.0"
jiff = "0.2.10" jiff = "0.2.10"
......
...@@ -25,7 +25,7 @@ mod tests; ...@@ -25,7 +25,7 @@ mod tests;
mod value_buf; mod value_buf;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use db::TurboPersistence; pub use db::{MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey}; pub use key::{KeyBase, QueryKey, StoreKey};
pub use value_buf::ValueBuffer; pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch; pub use write_batch::WriteBatch;
...@@ -2,12 +2,14 @@ use std::{ ...@@ -2,12 +2,14 @@ use std::{
fs::File, fs::File,
hash::BuildHasherDefault, hash::BuildHasherDefault,
io::{BufReader, Seek}, io::{BufReader, Seek},
ops::Deref,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::{Arc, OnceLock}, sync::{Arc, OnceLock},
}; };
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use either::Either;
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult; use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
...@@ -62,12 +64,54 @@ impl MetaEntry { ...@@ -62,12 +64,54 @@ impl MetaEntry {
self.size self.size
} }
pub fn aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] { pub fn aqmf_size(&self) -> u32 {
self.end_of_aqmf_data_offset - self.start_of_aqmf_data_offset
}
pub fn raw_aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] {
aqmf_data aqmf_data
.get(self.start_of_aqmf_data_offset as usize..self.end_of_aqmf_data_offset as usize) .get(self.start_of_aqmf_data_offset as usize..self.end_of_aqmf_data_offset as usize)
.expect("AQMF data out of bounds") .expect("AQMF data out of bounds")
} }
pub fn deserialize_aqmf(&self, meta: &MetaFile) -> Result<qfilter::Filter> {
let aqmf = self.raw_aqmf(meta.aqmf_data());
pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
meta.sequence_number,
self.sequence_number()
)
})
}
pub fn aqmf(
&self,
meta: &MetaFile,
aqmf_cache: &AqmfCache,
) -> Result<impl Deref<Target = qfilter::Filter>> {
let use_aqmf_cache = self.max_hash - self.min_hash < 1 << 60;
Ok(if use_aqmf_cache {
let aqmf = match aqmf_cache.get_value_or_guard(&self.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = self.deserialize_aqmf(meta)?;
let aqmf: Arc<qfilter::Filter> = Arc::new(aqmf);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
Either::Left(aqmf)
} else {
let aqmf = self.aqmf.get_or_try_init(|| {
let aqmf = self.deserialize_aqmf(meta)?;
anyhow::Ok(aqmf)
})?;
Either::Right(aqmf)
})
}
pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> { pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> {
self.sst.get_or_try_init(|| { self.sst.get_or_try_init(|| {
StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| { StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| {
...@@ -262,45 +306,13 @@ impl MetaFile { ...@@ -262,45 +306,13 @@ impl MetaFile {
if key_hash < entry.min_hash || key_hash > entry.max_hash { if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue; continue;
} }
let use_aqmf_cache = entry.max_hash - entry.min_hash < 1 << 60; {
if use_aqmf_cache { let aqmf = entry.aqmf(self, aqmf_cache)?;
let aqmf = match aqmf_cache.get_value_or_guard(&entry.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = entry.aqmf(self.aqmf_data());
let aqmf: Arc<qfilter::Filter> =
Arc::new(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
if !aqmf.contains_fingerprint(key_hash) { if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss; miss_result = MetaLookupResult::QuickFilterMiss;
continue; continue;
} }
} else { }
let aqmf = entry.aqmf.get_or_try_init(|| {
let aqmf = entry.aqmf(self.aqmf_data());
anyhow::Ok(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?)
})?;
if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
};
let result = let result =
entry entry
.sst(self)? .sst(self)?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment