Unverified Commit b01037d8 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add turbo-persistence benchmark (#89446)

### What?

add benchmarks for database operations
parent 7b2a4673
...@@ -1225,40 +1225,42 @@ dependencies = [ ...@@ -1225,40 +1225,42 @@ dependencies = [
[[package]] [[package]]
name = "codspeed" name = "codspeed"
version = "3.0.5" version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35584c5fcba8059780748866387fb97c5a203bcfc563fc3d0790af406727a117" checksum = "38c2eb3388ebe26b5a0ab6bf4969d9c4840143d7f6df07caa3cc851b0606cef6"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bincode 1.3.3", "cc",
"colored", "colored",
"getrandom 0.2.15",
"glob", "glob",
"libc", "libc",
"nix 0.29.0", "nix",
"serde", "serde",
"serde_json", "serde_json",
"statrs", "statrs",
"uuid",
] ]
[[package]] [[package]]
name = "codspeed-criterion-compat" name = "codspeed-criterion-compat"
version = "3.0.5" version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78f6c1c6bed5fd84d319e8b0889da051daa361c79b7709c9394dfe1a882bba67" checksum = "e1e270597a1d1e183f86d1cc9f94f0133654ee3daf201c17903ee29363555dd7"
dependencies = [ dependencies = [
"clap",
"codspeed", "codspeed",
"codspeed-criterion-compat-walltime", "codspeed-criterion-compat-walltime",
"colored", "colored",
"futures", "futures",
"regex",
"tokio", "tokio",
] ]
[[package]] [[package]]
name = "codspeed-criterion-compat-walltime" name = "codspeed-criterion-compat-walltime"
version = "3.0.5" version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c989289ce6b1cbde72ed560496cb8fbf5aa14d5ef5666f168e7f87751038352e" checksum = "e6c2613d2fac930fe34456be76f9124ee0800bb9db2e7fd2d6c65b9ebe98a292"
dependencies = [ dependencies = [
"anes", "anes",
"cast", "cast",
...@@ -1776,7 +1778,7 @@ version = "3.4.7" ...@@ -1776,7 +1778,7 @@ version = "3.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73"
dependencies = [ dependencies = [
"nix 0.30.1", "nix",
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
...@@ -4547,18 +4549,6 @@ dependencies = [ ...@@ -4547,18 +4549,6 @@ dependencies = [
"turbo-unix-path", "turbo-unix-path",
] ]
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.9.1",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]] [[package]]
name = "nix" name = "nix"
version = "0.30.1" version = "0.30.1"
...@@ -9302,6 +9292,7 @@ dependencies = [ ...@@ -9302,6 +9292,7 @@ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"bitfield", "bitfield",
"byteorder", "byteorder",
"codspeed-criterion-compat",
"dashmap 6.1.0", "dashmap 6.1.0",
"either", "either",
"jiff", "jiff",
...@@ -9320,6 +9311,7 @@ dependencies = [ ...@@ -9320,6 +9311,7 @@ dependencies = [
"thread_local", "thread_local",
"tracing", "tracing",
"turbo-bincode", "turbo-bincode",
"turbo-tasks-malloc",
"twox-hash 2.1.0", "twox-hash 2.1.0",
"zstd", "zstd",
] ]
...@@ -9698,7 +9690,7 @@ dependencies = [ ...@@ -9698,7 +9690,7 @@ dependencies = [
"chromiumoxide", "chromiumoxide",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"futures", "futures",
"nix 0.30.1", "nix",
"once_cell", "once_cell",
"owo-colors", "owo-colors",
"parking_lot", "parking_lot",
...@@ -10536,7 +10528,6 @@ version = "1.18.1" ...@@ -10536,7 +10528,6 @@ version = "1.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
dependencies = [ dependencies = [
"getrandom 0.3.3",
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
] ]
......
...@@ -381,7 +381,7 @@ clap = { version = "4.5.2", features = ["derive"] } ...@@ -381,7 +381,7 @@ clap = { version = "4.5.2", features = ["derive"] }
concurrent-queue = "2.5.0" concurrent-queue = "2.5.0"
console-subscriber = "0.4.1" console-subscriber = "0.4.1"
const_format = "0.2.30" const_format = "0.2.30"
criterion = { package = "codspeed-criterion-compat", version = "3.0.5" } criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5.8"
dashmap = "6.1.0" dashmap = "6.1.0"
data-encoding = "2.3.3" data-encoding = "2.3.3"
......
...@@ -36,9 +36,15 @@ twox-hash = { workspace = true } ...@@ -36,9 +36,15 @@ twox-hash = { workspace = true }
zstd = { version = "0.13.2", features = ["zdict_builder"] } zstd = { version = "0.13.2", features = ["zdict_builder"] }
[dev-dependencies] [dev-dependencies]
criterion = { workspace = true }
rand = { workspace = true, features = ["small_rng"] } rand = { workspace = true, features = ["small_rng"] }
rayon = { workspace = true } rayon = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
turbo-tasks-malloc = { workspace = true }
[lints] [lints]
workspace = true workspace = true
[[bench]]
name = "mod"
harness = false
This diff is collapsed.
...@@ -414,9 +414,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -414,9 +414,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// time. The WriteBatch need to be committed with [`TurboPersistence::commit_write_batch`]. /// time. The WriteBatch need to be committed with [`TurboPersistence::commit_write_batch`].
/// Note that the WriteBatch might start writing data to disk while it's filled up with data. /// Note that the WriteBatch might start writing data to disk while it's filled up with data.
/// This data will only become visible after the WriteBatch is committed. /// This data will only become visible after the WriteBatch is committed.
pub fn write_batch<K: StoreKey + Send + Sync + 'static>( pub fn write_batch<K: StoreKey + Send + Sync>(&self) -> Result<WriteBatch<K, S, FAMILIES>> {
&self,
) -> Result<WriteBatch<K, S, FAMILIES>> {
if self.read_only { if self.read_only {
bail!("Cannot write to a read-only database"); bail!("Cannot write to a read-only database");
} }
...@@ -438,6 +436,30 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -438,6 +436,30 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
)) ))
} }
/// Clears all caches of the database.
pub fn clear_cache(&self) {
self.amqf_cache.clear();
self.key_block_cache.clear();
self.value_block_cache.clear();
for meta in self.inner.write().meta_files.iter_mut() {
meta.clear_cache();
}
}
/// Clears block caches of the database.
pub fn clear_block_caches(&self) {
self.key_block_cache.clear();
self.value_block_cache.clear();
}
/// Prefetches all SST files which are usually lazy loaded. This can be used to reduce latency
/// for the first queries after opening the database.
pub fn prepare_all_sst_caches(&self) {
for meta in self.inner.write().meta_files.iter_mut() {
meta.prepare_sst_cache(&self.amqf_cache);
}
}
fn open_log(&self) -> Result<BufWriter<File>> { fn open_log(&self) -> Result<BufWriter<File>> {
if self.read_only { if self.read_only {
unreachable!("Only write operations can open the log file"); unreachable!("Only write operations can open the log file");
...@@ -452,7 +474,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -452,7 +474,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Commits a WriteBatch to the database. This will finish writing the data to disk and make it /// Commits a WriteBatch to the database. This will finish writing the data to disk and make it
/// visible to readers. /// visible to readers.
pub fn commit_write_batch<K: StoreKey + Send + Sync + 'static>( pub fn commit_write_batch<K: StoreKey + Send + Sync>(
&self, &self,
mut write_batch: WriteBatch<K, S, FAMILIES>, mut write_batch: WriteBatch<K, S, FAMILIES>,
) -> Result<()> { ) -> Result<()> {
......
...@@ -60,6 +60,22 @@ impl KeyBase for Vec<u8> { ...@@ -60,6 +60,22 @@ impl KeyBase for Vec<u8> {
} }
} }
impl KeyBase for Box<[u8]> {
fn len(&self) -> usize {
(**self).len()
}
fn is_empty(&self) -> bool {
(**self).is_empty()
}
fn hash<H: Hasher>(&self, state: &mut H) {
for item in self {
state.write_u8(*item);
}
}
}
impl KeyBase for u8 { impl KeyBase for u8 {
fn len(&self) -> usize { fn len(&self) -> usize {
1 1
...@@ -130,6 +146,12 @@ impl QueryKey for Vec<u8> { ...@@ -130,6 +146,12 @@ impl QueryKey for Vec<u8> {
} }
} }
impl QueryKey for Box<[u8]> {
fn cmp(&self, key: &[u8]) -> std::cmp::Ordering {
Ord::cmp(&**self, key)
}
}
impl QueryKey for u8 { impl QueryKey for u8 {
fn cmp(&self, key: &[u8]) -> std::cmp::Ordering { fn cmp(&self, key: &[u8]) -> std::cmp::Ordering {
Ord::cmp(&[*self][..], key) Ord::cmp(&[*self][..], key)
...@@ -175,6 +197,12 @@ impl StoreKey for Vec<u8> { ...@@ -175,6 +197,12 @@ impl StoreKey for Vec<u8> {
} }
} }
impl StoreKey for Box<[u8]> {
fn write_to(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self);
}
}
impl StoreKey for &'_ [u8] { impl StoreKey for &'_ [u8] {
fn write_to(&self, buf: &mut Vec<u8>) { fn write_to(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self); buf.extend_from_slice(self);
......
...@@ -27,7 +27,12 @@ mod tests; ...@@ -27,7 +27,12 @@ mod tests;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey}; pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler}; pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
pub use static_sorted_file::{
BlockCache, BlockWeighter, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData,
};
pub use static_sorted_file_builder::{Entry, EntryValue, write_static_stored_file};
pub use value_buf::ValueBuffer; pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch; pub use write_batch::WriteBatch;
...@@ -335,6 +335,20 @@ impl MetaFile { ...@@ -335,6 +335,20 @@ impl MetaFile {
Ok(file) Ok(file)
} }
pub fn clear_cache(&mut self) {
for entry in self.entries.iter_mut() {
entry.amqf.take();
entry.sst.take();
}
}
pub fn prepare_sst_cache(&self, amqf_cache: &AmqfCache) {
for entry in self.entries.iter() {
let _ = entry.sst(self);
let _ = entry.amqf(self, amqf_cache);
}
}
pub fn sequence_number(&self) -> u32 { pub fn sequence_number(&self) -> u32 {
self.sequence_number self.sequence_number
} }
...@@ -488,6 +502,10 @@ impl MetaFile { ...@@ -488,6 +502,10 @@ impl MetaFile {
} }
let amqf = entry.amqf(self, amqf_cache)?; let amqf = entry.amqf(self, amqf_cache)?;
for (hash, index, result) in &mut cells[start_index..=end_index] { for (hash, index, result) in &mut cells[start_index..=end_index] {
debug_assert!(
*hash >= entry.min_hash && *hash <= entry.max_hash,
"Key hash out of range"
);
if result.is_some() { if result.is_some() {
continue; continue;
} }
......
...@@ -30,6 +30,12 @@ impl From<Vec<u8>> for ValueBuffer<'_> { ...@@ -30,6 +30,12 @@ impl From<Vec<u8>> for ValueBuffer<'_> {
} }
} }
impl From<Box<[u8]>> for ValueBuffer<'_> {
fn from(v: Box<[u8]>) -> Self {
ValueBuffer::Vec(v.into_vec())
}
}
impl From<SmallVec<[u8; 16]>> for ValueBuffer<'_> { impl From<SmallVec<[u8; 16]>> for ValueBuffer<'_> {
fn from(sv: SmallVec<[u8; 16]>) -> Self { fn from(sv: SmallVec<[u8; 16]>) -> Self {
ValueBuffer::SmallVec(sv) ValueBuffer::SmallVec(sv)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment