Unverified Commit 4c683ed2 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add tool to print DB structure (#80148)

### What?

Adds a small CLI tool that prints the database structure like that:

```
META 00000417.meta: family = 4, 
  SST 00000411.sst: 000dade6cc76e41a - fffa6721f94e901f (p = 1/1)
    AQMF 4695 entries = 4 KiB
    180 KiB = 0 kiB key compression dict + 23 KiB value compression dict + 46 blocks (avg 3512 bytes/block)
META 00000416.meta: family = 3, 
  SST 00000412.sst: 0025bba83f4b6ab9 - ffde0483d58f3b5e (p = 1/1)
    AQMF 4695 entries = 4 KiB
    182 KiB = 24 kiB key compression dict + 0 KiB value compression dict + 46 blocks (avg 3522 bytes/block)
META 00000415.meta: family = 1, 
  SST 00000389.sst: 00005eac4b66cb4e - 3fffed60c1451cfe (p = 1/4)
    AQMF 299033 entries = 292 KiB
    24350 KiB = 0 kiB key compression dict + 63 KiB value compression dict + 3596 blocks (avg 6915 bytes/block)
  SST 00000391.sst: 800015a5a9666318 - bfffdd05778fc576 (p = 1/4)
    AQMF 299033 entries = 292 KiB
    24851 KiB = 0 kiB key compression dict + 63 KiB value compression dict + 3933 blocks (avg 6453 bytes/block)
  ...
...
```

The tool is only intended for internal usage and not exposed to the end user
parent 61729da4
......@@ -9513,6 +9513,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"byteorder",
"either",
"jiff",
"lzzzz",
"memmap2 0.9.5",
......@@ -9531,6 +9532,14 @@ dependencies = [
"zstd",
]
[[package]]
name = "turbo-persistence-tools"
version = "0.1.0"
dependencies = [
"anyhow",
"turbo-persistence",
]
[[package]]
name = "turbo-prehash"
version = "0.1.0"
......
[package]
name = "turbo-persistence-tools"
version = "0.1.0"
edition = "2024"
license = "MIT"
[dependencies]
anyhow = { workspace = true }
turbo-persistence = { workspace = true }
[lints]
workspace = true
#![feature(iter_intersperse)]
use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use turbo_persistence::{MetaFileEntryInfo, TurboPersistence};
fn main() -> Result<()> {
// Get CLI argument
let path = PathBuf::from(
std::env::args()
.nth(1)
.context("Please provide a path to the TurboPersistence directory")?,
);
if !path.exists() {
bail!("The provided path does not exist: {}", path.display());
}
let db = TurboPersistence::open_read_only(path)?;
let meta_info = db
.meta_info()
.context("Failed to retrieve meta information")?;
for meta_file in meta_info {
println!(
"META {:08}.meta: family = {}, ",
meta_file.sequence_number, meta_file.family
);
for MetaFileEntryInfo {
sequence_number,
min_hash,
max_hash,
aqmf_size,
aqmf_entries,
sst_size,
key_compression_dictionary_size,
value_compression_dictionary_size,
block_count,
} in meta_file.entries
{
println!(
" SST {sequence_number:08}.sst: {min_hash:016x} - {max_hash:016x} (p = 1/{})",
u64::MAX / (max_hash - min_hash)
);
println!(" AQMF {aqmf_entries} entries = {} KiB", aqmf_size / 1024);
println!(
" {} KiB = {} kiB key compression dict + {} KiB value compression dict + \
{block_count} blocks (avg {} bytes/block)",
sst_size / 1024,
key_compression_dictionary_size / 1024,
value_compression_dictionary_size / 1024,
(sst_size
- key_compression_dictionary_size as u64
- value_compression_dictionary_size as u64)
/ block_count as u64
);
}
if !meta_file.obsolete_sst_files.is_empty() {
println!(
" OBSOLETE SSTs {}",
meta_file
.obsolete_sst_files
.iter()
.map(|seq| format!("{seq:08}.sst"))
.intersperse(", ".to_string())
.collect::<String>()
);
}
}
Ok(())
}
......@@ -13,6 +13,7 @@ print_stats = ["stats"]
[dependencies]
anyhow = { workspace = true }
either = { workspace = true}
pot = "3.0.0"
byteorder = "1.5.0"
jiff = "0.2.10"
......
......@@ -111,6 +111,9 @@ struct TrackedStats {
pub struct TurboPersistence {
/// The path to the directory where the database is stored
path: PathBuf,
/// If true, the database is opened in read-only mode. In this mode, no writes are allowed and
/// no modification on the database is performed.
read_only: bool,
/// The inner state of the database. Writing will update that.
inner: RwLock<Inner>,
/// A cache for the last WriteBatch. It is used to avoid reallocation of buffers for the
......@@ -149,13 +152,10 @@ pub struct CommitOptions {
}
impl TurboPersistence {
/// Open a TurboPersistence database at the given path.
/// This will read the directory and might performance cleanup when the database was not closed
/// properly. Cleanup only requires to read a few bytes from a few files and to delete
/// files, so it's fast.
pub fn open(path: PathBuf) -> Result<Self> {
let mut db = Self {
fn new(path: PathBuf, read_only: bool) -> Self {
Self {
path,
read_only,
inner: RwLock::new(Inner {
meta_files: Vec::new(),
current_sequence_number: 0,
......@@ -185,26 +185,45 @@ impl TurboPersistence {
),
#[cfg(feature = "stats")]
stats: TrackedStats::default(),
};
db.open_directory()?;
}
}
/// Open a TurboPersistence database at the given path.
/// This will read the directory and might performance cleanup when the database was not closed
/// properly. Cleanup only requires to read a few bytes from a few files and to delete
/// files, so it's fast.
pub fn open(path: PathBuf) -> Result<Self> {
let mut db = Self::new(path, false);
db.open_directory(false)?;
Ok(db)
}
/// Open a TurboPersistence database at the given path in read only mode.
/// This will read the directory. No Cleanup is performed.
pub fn open_read_only(path: PathBuf) -> Result<Self> {
let mut db = Self::new(path, true);
db.open_directory(false)?;
Ok(db)
}
/// Performs the initial check on the database directory.
fn open_directory(&mut self) -> Result<()> {
fn open_directory(&mut self, read_only: bool) -> Result<()> {
match fs::read_dir(&self.path) {
Ok(entries) => {
if !self
.load_directory(entries)
.load_directory(entries, read_only)
.context("Loading persistence directory failed")?
{
if read_only {
bail!("Failed to open database");
}
self.init_directory()
.context("Initializing persistence directory failed")?;
}
Ok(())
}
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound {
if !read_only && e.kind() == std::io::ErrorKind::NotFound {
self.create_and_init_directory()
.context("Creating and initializing persistence directory failed")?;
Ok(())
......@@ -230,12 +249,12 @@ impl TurboPersistence {
}
/// Loads an existing database directory and performs cleanup if necessary.
fn load_directory(&mut self, entries: ReadDir) -> Result<bool> {
fn load_directory(&mut self, entries: ReadDir, read_only: bool) -> Result<bool> {
let mut meta_files = Vec::new();
let mut current_file = match File::open(self.path.join("CURRENT")) {
Ok(file) => file,
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound {
if !read_only && e.kind() == std::io::ErrorKind::NotFound {
return Ok(false);
} else {
return Err(e).context("Failed to open CURRENT file");
......@@ -260,7 +279,9 @@ impl TurboPersistence {
continue;
}
if seq > current {
fs::remove_file(&path)?;
if !read_only {
fs::remove_file(&path)?;
}
} else {
match ext {
"meta" => {
......@@ -272,17 +293,20 @@ impl TurboPersistence {
while !content.is_empty() {
let seq = content.read_u32::<BE>()?;
deleted_files.insert(seq);
let sst_file = self.path.join(format!("{seq:08}.sst"));
let meta_file = self.path.join(format!("{seq:08}.meta"));
let blob_file = self.path.join(format!("{seq:08}.blob"));
for path in [sst_file, meta_file, blob_file] {
if fs::exists(&path)? {
fs::remove_file(path)?;
no_existing_files = false;
if !read_only {
// Remove the files that are marked for deletion
let sst_file = self.path.join(format!("{seq:08}.sst"));
let meta_file = self.path.join(format!("{seq:08}.meta"));
let blob_file = self.path.join(format!("{seq:08}.blob"));
for path in [sst_file, meta_file, blob_file] {
if fs::exists(&path)? {
fs::remove_file(path)?;
no_existing_files = false;
}
}
}
}
if no_existing_files {
if !read_only && no_existing_files {
fs::remove_file(&path)?;
}
}
......@@ -379,6 +403,9 @@ impl TurboPersistence {
pub fn write_batch<K: StoreKey + Send + Sync + 'static, const FAMILIES: usize>(
&self,
) -> Result<WriteBatch<K, FAMILIES>> {
if self.read_only {
bail!("Cannot write to a read-only database");
}
if self
.active_write_operation
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
......@@ -401,6 +428,9 @@ impl TurboPersistence {
}
fn open_log(&self) -> Result<BufWriter<File>> {
if self.read_only {
unreachable!("Only write operations can open the log file");
}
let log_path = self.path.join("LOG");
let log_file = OpenOptions::new()
.create(true)
......@@ -415,6 +445,9 @@ impl TurboPersistence {
&self,
mut write_batch: WriteBatch<K, FAMILIES>,
) -> Result<()> {
if self.read_only {
unreachable!("It's not possible to create a write batch for a read-only database");
}
let FinishResult {
sequence_number,
new_meta_files,
......@@ -622,6 +655,9 @@ impl TurboPersistence {
max_merge_sequence: usize,
max_merge_size: u64,
) -> Result<()> {
if self.read_only {
bail!("Compaction is not allowed on a read only database");
}
let _span = tracing::info_span!("compact database").entered();
if self
.active_write_operation
......@@ -1006,7 +1042,7 @@ impl TurboPersistence {
let index_in_meta = ssts_with_ranges[index].index_in_meta;
let meta_file = &meta_files[meta_index];
let entry = meta_file.entry(index_in_meta);
let aqmf = entry.aqmf(meta_file.aqmf_data()).to_vec();
let aqmf = entry.raw_aqmf(meta_file.aqmf_data()).to_vec();
let meta = StaticSortedFileBuilderMeta {
min_hash: entry.min_hash(),
max_hash: entry.max_hash(),
......@@ -1169,6 +1205,44 @@ impl TurboPersistence {
}
}
pub fn meta_info(&self) -> Result<Vec<MetaFileInfo>> {
Ok(self
.inner
.read()
.meta_files
.iter()
.rev()
.map(|meta_file| {
let entries = meta_file
.entries()
.iter()
.map(|entry| {
let aqmf = entry.raw_aqmf(meta_file.aqmf_data());
MetaFileEntryInfo {
sequence_number: entry.sequence_number(),
min_hash: entry.min_hash(),
max_hash: entry.max_hash(),
sst_size: entry.size(),
aqmf_size: entry.aqmf_size(),
aqmf_entries: aqmf.len(),
key_compression_dictionary_size: entry
.key_compression_dictionary_length(),
value_compression_dictionary_size: entry
.value_compression_dictionary_length(),
block_count: entry.block_count(),
}
})
.collect();
MetaFileInfo {
sequence_number: meta_file.sequence_number(),
family: meta_file.family(),
obsolete_sst_files: meta_file.obsolete_sst_files().to_vec(),
entries,
}
})
.collect())
}
/// Shuts down the database. This will print statistics if the `print_stats` feature is enabled.
pub fn shutdown(&self) -> Result<()> {
#[cfg(feature = "print_stats")]
......@@ -1176,3 +1250,22 @@ impl TurboPersistence {
Ok(())
}
}
pub struct MetaFileInfo {
pub sequence_number: u32,
pub family: u32,
pub obsolete_sst_files: Vec<u32>,
pub entries: Vec<MetaFileEntryInfo>,
}
pub struct MetaFileEntryInfo {
pub sequence_number: u32,
pub min_hash: u64,
pub max_hash: u64,
pub aqmf_size: u32,
pub aqmf_entries: usize,
pub sst_size: u64,
pub key_compression_dictionary_size: u16,
pub value_compression_dictionary_size: u16,
pub block_count: u16,
}
......@@ -25,7 +25,7 @@ mod tests;
mod value_buf;
pub use arc_slice::ArcSlice;
pub use db::TurboPersistence;
pub use db::{MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey};
pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch;
......@@ -2,12 +2,14 @@ use std::{
fs::File,
hash::BuildHasherDefault,
io::{BufReader, Seek},
ops::Deref,
path::{Path, PathBuf},
sync::{Arc, OnceLock},
};
use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt};
use either::Either;
use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher;
......@@ -62,12 +64,54 @@ impl MetaEntry {
self.size
}
pub fn aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] {
pub fn aqmf_size(&self) -> u32 {
self.end_of_aqmf_data_offset - self.start_of_aqmf_data_offset
}
pub fn raw_aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] {
aqmf_data
.get(self.start_of_aqmf_data_offset as usize..self.end_of_aqmf_data_offset as usize)
.expect("AQMF data out of bounds")
}
pub fn deserialize_aqmf(&self, meta: &MetaFile) -> Result<qfilter::Filter> {
let aqmf = self.raw_aqmf(meta.aqmf_data());
pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
meta.sequence_number,
self.sequence_number()
)
})
}
pub fn aqmf(
&self,
meta: &MetaFile,
aqmf_cache: &AqmfCache,
) -> Result<impl Deref<Target = qfilter::Filter>> {
let use_aqmf_cache = self.max_hash - self.min_hash < 1 << 60;
Ok(if use_aqmf_cache {
let aqmf = match aqmf_cache.get_value_or_guard(&self.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = self.deserialize_aqmf(meta)?;
let aqmf: Arc<qfilter::Filter> = Arc::new(aqmf);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
Either::Left(aqmf)
} else {
let aqmf = self.aqmf.get_or_try_init(|| {
let aqmf = self.deserialize_aqmf(meta)?;
anyhow::Ok(aqmf)
})?;
Either::Right(aqmf)
})
}
pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> {
self.sst.get_or_try_init(|| {
StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| {
......@@ -262,45 +306,13 @@ impl MetaFile {
if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue;
}
let use_aqmf_cache = entry.max_hash - entry.min_hash < 1 << 60;
if use_aqmf_cache {
let aqmf = match aqmf_cache.get_value_or_guard(&entry.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = entry.aqmf(self.aqmf_data());
let aqmf: Arc<qfilter::Filter> =
Arc::new(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
{
let aqmf = entry.aqmf(self, aqmf_cache)?;
if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
} else {
let aqmf = entry.aqmf.get_or_try_init(|| {
let aqmf = entry.aqmf(self.aqmf_data());
anyhow::Ok(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?)
})?;
if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
};
}
let result =
entry
.sst(self)?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment