Unverified Commit b01037d8 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add turbo-persistence benchmark (#89446)

### What?

add benchmarks for database operations
parent 7b2a4673
......@@ -1225,40 +1225,42 @@ dependencies = [
[[package]]
name = "codspeed"
version = "3.0.5"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35584c5fcba8059780748866387fb97c5a203bcfc563fc3d0790af406727a117"
checksum = "38c2eb3388ebe26b5a0ab6bf4969d9c4840143d7f6df07caa3cc851b0606cef6"
dependencies = [
"anyhow",
"bincode 1.3.3",
"cc",
"colored",
"getrandom 0.2.15",
"glob",
"libc",
"nix 0.29.0",
"nix",
"serde",
"serde_json",
"statrs",
"uuid",
]
[[package]]
name = "codspeed-criterion-compat"
version = "3.0.5"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78f6c1c6bed5fd84d319e8b0889da051daa361c79b7709c9394dfe1a882bba67"
checksum = "e1e270597a1d1e183f86d1cc9f94f0133654ee3daf201c17903ee29363555dd7"
dependencies = [
"clap",
"codspeed",
"codspeed-criterion-compat-walltime",
"colored",
"futures",
"regex",
"tokio",
]
[[package]]
name = "codspeed-criterion-compat-walltime"
version = "3.0.5"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c989289ce6b1cbde72ed560496cb8fbf5aa14d5ef5666f168e7f87751038352e"
checksum = "e6c2613d2fac930fe34456be76f9124ee0800bb9db2e7fd2d6c65b9ebe98a292"
dependencies = [
"anes",
"cast",
......@@ -1776,7 +1778,7 @@ version = "3.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73"
dependencies = [
"nix 0.30.1",
"nix",
"windows-sys 0.59.0",
]
......@@ -4547,18 +4549,6 @@ dependencies = [
"turbo-unix-path",
]
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.9.1",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "nix"
version = "0.30.1"
......@@ -9302,6 +9292,7 @@ dependencies = [
"bincode 2.0.1",
"bitfield",
"byteorder",
"codspeed-criterion-compat",
"dashmap 6.1.0",
"either",
"jiff",
......@@ -9320,6 +9311,7 @@ dependencies = [
"thread_local",
"tracing",
"turbo-bincode",
"turbo-tasks-malloc",
"twox-hash 2.1.0",
"zstd",
]
......@@ -9698,7 +9690,7 @@ dependencies = [
"chromiumoxide",
"codspeed-criterion-compat",
"futures",
"nix 0.30.1",
"nix",
"once_cell",
"owo-colors",
"parking_lot",
......@@ -10536,7 +10528,6 @@ version = "1.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
dependencies = [
"getrandom 0.3.3",
"js-sys",
"wasm-bindgen",
]
......
......@@ -381,7 +381,7 @@ clap = { version = "4.5.2", features = ["derive"] }
concurrent-queue = "2.5.0"
console-subscriber = "0.4.1"
const_format = "0.2.30"
criterion = { package = "codspeed-criterion-compat", version = "3.0.5" }
criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
crossbeam-channel = "0.5.8"
dashmap = "6.1.0"
data-encoding = "2.3.3"
......
......@@ -36,9 +36,15 @@ twox-hash = { workspace = true }
zstd = { version = "0.13.2", features = ["zdict_builder"] }
[dev-dependencies]
criterion = { workspace = true }
rand = { workspace = true, features = ["small_rng"] }
rayon = { workspace = true }
tempfile = { workspace = true }
turbo-tasks-malloc = { workspace = true }
[lints]
workspace = true
[[bench]]
name = "mod"
harness = false
This diff is collapsed.
......@@ -414,9 +414,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// time. The WriteBatch need to be committed with [`TurboPersistence::commit_write_batch`].
/// Note that the WriteBatch might start writing data to disk while it's filled up with data.
/// This data will only become visible after the WriteBatch is committed.
pub fn write_batch<K: StoreKey + Send + Sync + 'static>(
&self,
) -> Result<WriteBatch<K, S, FAMILIES>> {
pub fn write_batch<K: StoreKey + Send + Sync>(&self) -> Result<WriteBatch<K, S, FAMILIES>> {
if self.read_only {
bail!("Cannot write to a read-only database");
}
......@@ -438,6 +436,30 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
))
}
/// Clears all caches of the database.
pub fn clear_cache(&self) {
self.amqf_cache.clear();
self.key_block_cache.clear();
self.value_block_cache.clear();
for meta in self.inner.write().meta_files.iter_mut() {
meta.clear_cache();
}
}
/// Clears block caches of the database.
pub fn clear_block_caches(&self) {
self.key_block_cache.clear();
self.value_block_cache.clear();
}
/// Prefetches all SST files which are usually lazy loaded. This can be used to reduce latency
/// for the first queries after opening the database.
pub fn prepare_all_sst_caches(&self) {
for meta in self.inner.write().meta_files.iter_mut() {
meta.prepare_sst_cache(&self.amqf_cache);
}
}
fn open_log(&self) -> Result<BufWriter<File>> {
if self.read_only {
unreachable!("Only write operations can open the log file");
......@@ -452,7 +474,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Commits a WriteBatch to the database. This will finish writing the data to disk and make it
/// visible to readers.
pub fn commit_write_batch<K: StoreKey + Send + Sync + 'static>(
pub fn commit_write_batch<K: StoreKey + Send + Sync>(
&self,
mut write_batch: WriteBatch<K, S, FAMILIES>,
) -> Result<()> {
......
......@@ -60,6 +60,22 @@ impl KeyBase for Vec<u8> {
}
}
impl KeyBase for Box<[u8]> {
fn len(&self) -> usize {
(**self).len()
}
fn is_empty(&self) -> bool {
(**self).is_empty()
}
fn hash<H: Hasher>(&self, state: &mut H) {
for item in self {
state.write_u8(*item);
}
}
}
impl KeyBase for u8 {
fn len(&self) -> usize {
1
......@@ -130,6 +146,12 @@ impl QueryKey for Vec<u8> {
}
}
impl QueryKey for Box<[u8]> {
fn cmp(&self, key: &[u8]) -> std::cmp::Ordering {
Ord::cmp(&**self, key)
}
}
impl QueryKey for u8 {
fn cmp(&self, key: &[u8]) -> std::cmp::Ordering {
Ord::cmp(&[*self][..], key)
......@@ -175,6 +197,12 @@ impl StoreKey for Vec<u8> {
}
}
impl StoreKey for Box<[u8]> {
fn write_to(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self);
}
}
impl StoreKey for &'_ [u8] {
fn write_to(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self);
......
......@@ -27,7 +27,12 @@ mod tests;
pub use arc_slice::ArcSlice;
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey};
pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
pub use static_sorted_file::{
BlockCache, BlockWeighter, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData,
};
pub use static_sorted_file_builder::{Entry, EntryValue, write_static_stored_file};
pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch;
......@@ -335,6 +335,20 @@ impl MetaFile {
Ok(file)
}
pub fn clear_cache(&mut self) {
for entry in self.entries.iter_mut() {
entry.amqf.take();
entry.sst.take();
}
}
pub fn prepare_sst_cache(&self, amqf_cache: &AmqfCache) {
for entry in self.entries.iter() {
let _ = entry.sst(self);
let _ = entry.amqf(self, amqf_cache);
}
}
pub fn sequence_number(&self) -> u32 {
self.sequence_number
}
......@@ -488,6 +502,10 @@ impl MetaFile {
}
let amqf = entry.amqf(self, amqf_cache)?;
for (hash, index, result) in &mut cells[start_index..=end_index] {
debug_assert!(
*hash >= entry.min_hash && *hash <= entry.max_hash,
"Key hash out of range"
);
if result.is_some() {
continue;
}
......
......@@ -30,6 +30,12 @@ impl From<Vec<u8>> for ValueBuffer<'_> {
}
}
impl From<Box<[u8]>> for ValueBuffer<'_> {
fn from(v: Box<[u8]>) -> Self {
ValueBuffer::Vec(v.into_vec())
}
}
impl From<SmallVec<[u8; 16]>> for ValueBuffer<'_> {
fn from(sv: SmallVec<[u8; 16]>) -> Self {
ValueBuffer::SmallVec(sv)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment