Unverified Commit 4c683ed2 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add tool to print DB structure (#80148)

### What?

Adds a small CLI tool that prints the database structure like that:

```
META 00000417.meta: family = 4, 
  SST 00000411.sst: 000dade6cc76e41a - fffa6721f94e901f (p = 1/1)
    AQMF 4695 entries = 4 KiB
    180 KiB = 0 kiB key compression dict + 23 KiB value compression dict + 46 blocks (avg 3512 bytes/block)
META 00000416.meta: family = 3, 
  SST 00000412.sst: 0025bba83f4b6ab9 - ffde0483d58f3b5e (p = 1/1)
    AQMF 4695 entries = 4 KiB
    182 KiB = 24 kiB key compression dict + 0 KiB value compression dict + 46 blocks (avg 3522 bytes/block)
META 00000415.meta: family = 1, 
  SST 00000389.sst: 00005eac4b66cb4e - 3fffed60c1451cfe (p = 1/4)
    AQMF 299033 entries = 292 KiB
    24350 KiB = 0 kiB key compression dict + 63 KiB value compression dict + 3596 blocks (avg 6915 bytes/block)
  SST 00000391.sst: 800015a5a9666318 - bfffdd05778fc576 (p = 1/4)
    AQMF 299033 entries = 292 KiB
    24851 KiB = 0 kiB key compression dict + 63 KiB value compression dict + 3933 blocks (avg 6453 bytes/block)
  ...
...
```

The tool is only intended for internal usage and not exposed to the end user
parent 61729da4
...@@ -9513,6 +9513,7 @@ version = "0.1.0" ...@@ -9513,6 +9513,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"byteorder", "byteorder",
"either",
"jiff", "jiff",
"lzzzz", "lzzzz",
"memmap2 0.9.5", "memmap2 0.9.5",
...@@ -9531,6 +9532,14 @@ dependencies = [ ...@@ -9531,6 +9532,14 @@ dependencies = [
"zstd", "zstd",
] ]
[[package]]
name = "turbo-persistence-tools"
version = "0.1.0"
dependencies = [
"anyhow",
"turbo-persistence",
]
[[package]] [[package]]
name = "turbo-prehash" name = "turbo-prehash"
version = "0.1.0" version = "0.1.0"
......
[package]
name = "turbo-persistence-tools"
version = "0.1.0"
edition = "2024"
license = "MIT"
[dependencies]
anyhow = { workspace = true }
turbo-persistence = { workspace = true }
[lints]
workspace = true
#![feature(iter_intersperse)]
use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use turbo_persistence::{MetaFileEntryInfo, TurboPersistence};
fn main() -> Result<()> {
// Get CLI argument
let path = PathBuf::from(
std::env::args()
.nth(1)
.context("Please provide a path to the TurboPersistence directory")?,
);
if !path.exists() {
bail!("The provided path does not exist: {}", path.display());
}
let db = TurboPersistence::open_read_only(path)?;
let meta_info = db
.meta_info()
.context("Failed to retrieve meta information")?;
for meta_file in meta_info {
println!(
"META {:08}.meta: family = {}, ",
meta_file.sequence_number, meta_file.family
);
for MetaFileEntryInfo {
sequence_number,
min_hash,
max_hash,
aqmf_size,
aqmf_entries,
sst_size,
key_compression_dictionary_size,
value_compression_dictionary_size,
block_count,
} in meta_file.entries
{
println!(
" SST {sequence_number:08}.sst: {min_hash:016x} - {max_hash:016x} (p = 1/{})",
u64::MAX / (max_hash - min_hash)
);
println!(" AQMF {aqmf_entries} entries = {} KiB", aqmf_size / 1024);
println!(
" {} KiB = {} kiB key compression dict + {} KiB value compression dict + \
{block_count} blocks (avg {} bytes/block)",
sst_size / 1024,
key_compression_dictionary_size / 1024,
value_compression_dictionary_size / 1024,
(sst_size
- key_compression_dictionary_size as u64
- value_compression_dictionary_size as u64)
/ block_count as u64
);
}
if !meta_file.obsolete_sst_files.is_empty() {
println!(
" OBSOLETE SSTs {}",
meta_file
.obsolete_sst_files
.iter()
.map(|seq| format!("{seq:08}.sst"))
.intersperse(", ".to_string())
.collect::<String>()
);
}
}
Ok(())
}
...@@ -13,6 +13,7 @@ print_stats = ["stats"] ...@@ -13,6 +13,7 @@ print_stats = ["stats"]
[dependencies] [dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
either = { workspace = true}
pot = "3.0.0" pot = "3.0.0"
byteorder = "1.5.0" byteorder = "1.5.0"
jiff = "0.2.10" jiff = "0.2.10"
......
...@@ -111,6 +111,9 @@ struct TrackedStats { ...@@ -111,6 +111,9 @@ struct TrackedStats {
pub struct TurboPersistence { pub struct TurboPersistence {
/// The path to the directory where the database is stored /// The path to the directory where the database is stored
path: PathBuf, path: PathBuf,
/// If true, the database is opened in read-only mode. In this mode, no writes are allowed and
/// no modification on the database is performed.
read_only: bool,
/// The inner state of the database. Writing will update that. /// The inner state of the database. Writing will update that.
inner: RwLock<Inner>, inner: RwLock<Inner>,
/// A cache for the last WriteBatch. It is used to avoid reallocation of buffers for the /// A cache for the last WriteBatch. It is used to avoid reallocation of buffers for the
...@@ -149,13 +152,10 @@ pub struct CommitOptions { ...@@ -149,13 +152,10 @@ pub struct CommitOptions {
} }
impl TurboPersistence { impl TurboPersistence {
/// Open a TurboPersistence database at the given path. fn new(path: PathBuf, read_only: bool) -> Self {
/// This will read the directory and might performance cleanup when the database was not closed Self {
/// properly. Cleanup only requires to read a few bytes from a few files and to delete
/// files, so it's fast.
pub fn open(path: PathBuf) -> Result<Self> {
let mut db = Self {
path, path,
read_only,
inner: RwLock::new(Inner { inner: RwLock::new(Inner {
meta_files: Vec::new(), meta_files: Vec::new(),
current_sequence_number: 0, current_sequence_number: 0,
...@@ -185,26 +185,45 @@ impl TurboPersistence { ...@@ -185,26 +185,45 @@ impl TurboPersistence {
), ),
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
stats: TrackedStats::default(), stats: TrackedStats::default(),
}; }
db.open_directory()?; }
/// Open a TurboPersistence database at the given path.
/// This will read the directory and might performance cleanup when the database was not closed
/// properly. Cleanup only requires to read a few bytes from a few files and to delete
/// files, so it's fast.
pub fn open(path: PathBuf) -> Result<Self> {
let mut db = Self::new(path, false);
db.open_directory(false)?;
Ok(db)
}
/// Open a TurboPersistence database at the given path in read only mode.
/// This will read the directory. No Cleanup is performed.
pub fn open_read_only(path: PathBuf) -> Result<Self> {
let mut db = Self::new(path, true);
db.open_directory(false)?;
Ok(db) Ok(db)
} }
/// Performs the initial check on the database directory. /// Performs the initial check on the database directory.
fn open_directory(&mut self) -> Result<()> { fn open_directory(&mut self, read_only: bool) -> Result<()> {
match fs::read_dir(&self.path) { match fs::read_dir(&self.path) {
Ok(entries) => { Ok(entries) => {
if !self if !self
.load_directory(entries) .load_directory(entries, read_only)
.context("Loading persistence directory failed")? .context("Loading persistence directory failed")?
{ {
if read_only {
bail!("Failed to open database");
}
self.init_directory() self.init_directory()
.context("Initializing persistence directory failed")?; .context("Initializing persistence directory failed")?;
} }
Ok(()) Ok(())
} }
Err(e) => { Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound { if !read_only && e.kind() == std::io::ErrorKind::NotFound {
self.create_and_init_directory() self.create_and_init_directory()
.context("Creating and initializing persistence directory failed")?; .context("Creating and initializing persistence directory failed")?;
Ok(()) Ok(())
...@@ -230,12 +249,12 @@ impl TurboPersistence { ...@@ -230,12 +249,12 @@ impl TurboPersistence {
} }
/// Loads an existing database directory and performs cleanup if necessary. /// Loads an existing database directory and performs cleanup if necessary.
fn load_directory(&mut self, entries: ReadDir) -> Result<bool> { fn load_directory(&mut self, entries: ReadDir, read_only: bool) -> Result<bool> {
let mut meta_files = Vec::new(); let mut meta_files = Vec::new();
let mut current_file = match File::open(self.path.join("CURRENT")) { let mut current_file = match File::open(self.path.join("CURRENT")) {
Ok(file) => file, Ok(file) => file,
Err(e) => { Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound { if !read_only && e.kind() == std::io::ErrorKind::NotFound {
return Ok(false); return Ok(false);
} else { } else {
return Err(e).context("Failed to open CURRENT file"); return Err(e).context("Failed to open CURRENT file");
...@@ -260,7 +279,9 @@ impl TurboPersistence { ...@@ -260,7 +279,9 @@ impl TurboPersistence {
continue; continue;
} }
if seq > current { if seq > current {
if !read_only {
fs::remove_file(&path)?; fs::remove_file(&path)?;
}
} else { } else {
match ext { match ext {
"meta" => { "meta" => {
...@@ -272,6 +293,8 @@ impl TurboPersistence { ...@@ -272,6 +293,8 @@ impl TurboPersistence {
while !content.is_empty() { while !content.is_empty() {
let seq = content.read_u32::<BE>()?; let seq = content.read_u32::<BE>()?;
deleted_files.insert(seq); deleted_files.insert(seq);
if !read_only {
// Remove the files that are marked for deletion
let sst_file = self.path.join(format!("{seq:08}.sst")); let sst_file = self.path.join(format!("{seq:08}.sst"));
let meta_file = self.path.join(format!("{seq:08}.meta")); let meta_file = self.path.join(format!("{seq:08}.meta"));
let blob_file = self.path.join(format!("{seq:08}.blob")); let blob_file = self.path.join(format!("{seq:08}.blob"));
...@@ -282,7 +305,8 @@ impl TurboPersistence { ...@@ -282,7 +305,8 @@ impl TurboPersistence {
} }
} }
} }
if no_existing_files { }
if !read_only && no_existing_files {
fs::remove_file(&path)?; fs::remove_file(&path)?;
} }
} }
...@@ -379,6 +403,9 @@ impl TurboPersistence { ...@@ -379,6 +403,9 @@ impl TurboPersistence {
pub fn write_batch<K: StoreKey + Send + Sync + 'static, const FAMILIES: usize>( pub fn write_batch<K: StoreKey + Send + Sync + 'static, const FAMILIES: usize>(
&self, &self,
) -> Result<WriteBatch<K, FAMILIES>> { ) -> Result<WriteBatch<K, FAMILIES>> {
if self.read_only {
bail!("Cannot write to a read-only database");
}
if self if self
.active_write_operation .active_write_operation
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
...@@ -401,6 +428,9 @@ impl TurboPersistence { ...@@ -401,6 +428,9 @@ impl TurboPersistence {
} }
fn open_log(&self) -> Result<BufWriter<File>> { fn open_log(&self) -> Result<BufWriter<File>> {
if self.read_only {
unreachable!("Only write operations can open the log file");
}
let log_path = self.path.join("LOG"); let log_path = self.path.join("LOG");
let log_file = OpenOptions::new() let log_file = OpenOptions::new()
.create(true) .create(true)
...@@ -415,6 +445,9 @@ impl TurboPersistence { ...@@ -415,6 +445,9 @@ impl TurboPersistence {
&self, &self,
mut write_batch: WriteBatch<K, FAMILIES>, mut write_batch: WriteBatch<K, FAMILIES>,
) -> Result<()> { ) -> Result<()> {
if self.read_only {
unreachable!("It's not possible to create a write batch for a read-only database");
}
let FinishResult { let FinishResult {
sequence_number, sequence_number,
new_meta_files, new_meta_files,
...@@ -622,6 +655,9 @@ impl TurboPersistence { ...@@ -622,6 +655,9 @@ impl TurboPersistence {
max_merge_sequence: usize, max_merge_sequence: usize,
max_merge_size: u64, max_merge_size: u64,
) -> Result<()> { ) -> Result<()> {
if self.read_only {
bail!("Compaction is not allowed on a read only database");
}
let _span = tracing::info_span!("compact database").entered(); let _span = tracing::info_span!("compact database").entered();
if self if self
.active_write_operation .active_write_operation
...@@ -1006,7 +1042,7 @@ impl TurboPersistence { ...@@ -1006,7 +1042,7 @@ impl TurboPersistence {
let index_in_meta = ssts_with_ranges[index].index_in_meta; let index_in_meta = ssts_with_ranges[index].index_in_meta;
let meta_file = &meta_files[meta_index]; let meta_file = &meta_files[meta_index];
let entry = meta_file.entry(index_in_meta); let entry = meta_file.entry(index_in_meta);
let aqmf = entry.aqmf(meta_file.aqmf_data()).to_vec(); let aqmf = entry.raw_aqmf(meta_file.aqmf_data()).to_vec();
let meta = StaticSortedFileBuilderMeta { let meta = StaticSortedFileBuilderMeta {
min_hash: entry.min_hash(), min_hash: entry.min_hash(),
max_hash: entry.max_hash(), max_hash: entry.max_hash(),
...@@ -1169,6 +1205,44 @@ impl TurboPersistence { ...@@ -1169,6 +1205,44 @@ impl TurboPersistence {
} }
} }
pub fn meta_info(&self) -> Result<Vec<MetaFileInfo>> {
Ok(self
.inner
.read()
.meta_files
.iter()
.rev()
.map(|meta_file| {
let entries = meta_file
.entries()
.iter()
.map(|entry| {
let aqmf = entry.raw_aqmf(meta_file.aqmf_data());
MetaFileEntryInfo {
sequence_number: entry.sequence_number(),
min_hash: entry.min_hash(),
max_hash: entry.max_hash(),
sst_size: entry.size(),
aqmf_size: entry.aqmf_size(),
aqmf_entries: aqmf.len(),
key_compression_dictionary_size: entry
.key_compression_dictionary_length(),
value_compression_dictionary_size: entry
.value_compression_dictionary_length(),
block_count: entry.block_count(),
}
})
.collect();
MetaFileInfo {
sequence_number: meta_file.sequence_number(),
family: meta_file.family(),
obsolete_sst_files: meta_file.obsolete_sst_files().to_vec(),
entries,
}
})
.collect())
}
/// Shuts down the database. This will print statistics if the `print_stats` feature is enabled. /// Shuts down the database. This will print statistics if the `print_stats` feature is enabled.
pub fn shutdown(&self) -> Result<()> { pub fn shutdown(&self) -> Result<()> {
#[cfg(feature = "print_stats")] #[cfg(feature = "print_stats")]
...@@ -1176,3 +1250,22 @@ impl TurboPersistence { ...@@ -1176,3 +1250,22 @@ impl TurboPersistence {
Ok(()) Ok(())
} }
} }
pub struct MetaFileInfo {
pub sequence_number: u32,
pub family: u32,
pub obsolete_sst_files: Vec<u32>,
pub entries: Vec<MetaFileEntryInfo>,
}
pub struct MetaFileEntryInfo {
pub sequence_number: u32,
pub min_hash: u64,
pub max_hash: u64,
pub aqmf_size: u32,
pub aqmf_entries: usize,
pub sst_size: u64,
pub key_compression_dictionary_size: u16,
pub value_compression_dictionary_size: u16,
pub block_count: u16,
}
...@@ -25,7 +25,7 @@ mod tests; ...@@ -25,7 +25,7 @@ mod tests;
mod value_buf; mod value_buf;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use db::TurboPersistence; pub use db::{MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey}; pub use key::{KeyBase, QueryKey, StoreKey};
pub use value_buf::ValueBuffer; pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch; pub use write_batch::WriteBatch;
...@@ -2,12 +2,14 @@ use std::{ ...@@ -2,12 +2,14 @@ use std::{
fs::File, fs::File,
hash::BuildHasherDefault, hash::BuildHasherDefault,
io::{BufReader, Seek}, io::{BufReader, Seek},
ops::Deref,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::{Arc, OnceLock}, sync::{Arc, OnceLock},
}; };
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use either::Either;
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult; use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
...@@ -62,12 +64,54 @@ impl MetaEntry { ...@@ -62,12 +64,54 @@ impl MetaEntry {
self.size self.size
} }
pub fn aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] { pub fn aqmf_size(&self) -> u32 {
self.end_of_aqmf_data_offset - self.start_of_aqmf_data_offset
}
pub fn raw_aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] {
aqmf_data aqmf_data
.get(self.start_of_aqmf_data_offset as usize..self.end_of_aqmf_data_offset as usize) .get(self.start_of_aqmf_data_offset as usize..self.end_of_aqmf_data_offset as usize)
.expect("AQMF data out of bounds") .expect("AQMF data out of bounds")
} }
pub fn deserialize_aqmf(&self, meta: &MetaFile) -> Result<qfilter::Filter> {
let aqmf = self.raw_aqmf(meta.aqmf_data());
pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
meta.sequence_number,
self.sequence_number()
)
})
}
pub fn aqmf(
&self,
meta: &MetaFile,
aqmf_cache: &AqmfCache,
) -> Result<impl Deref<Target = qfilter::Filter>> {
let use_aqmf_cache = self.max_hash - self.min_hash < 1 << 60;
Ok(if use_aqmf_cache {
let aqmf = match aqmf_cache.get_value_or_guard(&self.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = self.deserialize_aqmf(meta)?;
let aqmf: Arc<qfilter::Filter> = Arc::new(aqmf);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
Either::Left(aqmf)
} else {
let aqmf = self.aqmf.get_or_try_init(|| {
let aqmf = self.deserialize_aqmf(meta)?;
anyhow::Ok(aqmf)
})?;
Either::Right(aqmf)
})
}
pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> { pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> {
self.sst.get_or_try_init(|| { self.sst.get_or_try_init(|| {
StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| { StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| {
...@@ -262,45 +306,13 @@ impl MetaFile { ...@@ -262,45 +306,13 @@ impl MetaFile {
if key_hash < entry.min_hash || key_hash > entry.max_hash { if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue; continue;
} }
let use_aqmf_cache = entry.max_hash - entry.min_hash < 1 << 60; {
if use_aqmf_cache { let aqmf = entry.aqmf(self, aqmf_cache)?;
let aqmf = match aqmf_cache.get_value_or_guard(&entry.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = entry.aqmf(self.aqmf_data());
let aqmf: Arc<qfilter::Filter> =
Arc::new(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
if !aqmf.contains_fingerprint(key_hash) { if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss; miss_result = MetaLookupResult::QuickFilterMiss;
continue; continue;
} }
} else {
let aqmf = entry.aqmf.get_or_try_init(|| {
let aqmf = entry.aqmf(self.aqmf_data());
anyhow::Ok(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?)
})?;
if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
} }
};
let result = let result =
entry entry
.sst(self)? .sst(self)?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment