Unverified Commit 27c24bf7 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: store frequently used entries separately in database (#85826)

### What?

* capture used key hashes and store them in the meta file
* During compaction separate used and unused entries

This eventually leads to a state where frequently used entries are stored in separate files, which makes it faster to restore. This improves db restore performance after a few runs.
parent 73f50762
...@@ -4512,6 +4512,12 @@ dependencies = [ ...@@ -4512,6 +4512,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "nohash-hasher"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
[[package]] [[package]]
name = "nom" name = "nom"
version = "5.1.3" version = "5.1.3"
...@@ -9129,11 +9135,14 @@ name = "turbo-persistence" ...@@ -9129,11 +9135,14 @@ name = "turbo-persistence"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bitfield",
"byteorder", "byteorder",
"dashmap 6.1.0",
"either", "either",
"jiff", "jiff",
"lzzzz", "lzzzz",
"memmap2 0.9.5", "memmap2 0.9.5",
"nohash-hasher",
"parking_lot", "parking_lot",
"pot", "pot",
"qfilter", "qfilter",
......
...@@ -422,6 +422,7 @@ napi = { version = "2", default-features = false, features = [ ...@@ -422,6 +422,7 @@ napi = { version = "2", default-features = false, features = [
"napi5", "napi5",
"compat-mode", "compat-mode",
] } ] }
nohash-hasher = "0.2.0"
notify = "8.1.0" notify = "8.1.0"
once_cell = "1.17.1" once_cell = "1.17.1"
owo-colors = "4.2.2" owo-colors = "4.2.2"
......
...@@ -16,7 +16,7 @@ fn main() -> Result<()> { ...@@ -16,7 +16,7 @@ fn main() -> Result<()> {
bail!("The provided path does not exist: {}", path.display()); bail!("The provided path does not exist: {}", path.display());
} }
let db: TurboPersistence<SerialScheduler> = TurboPersistence::open_read_only(path)?; let db: TurboPersistence<SerialScheduler, 0> = TurboPersistence::open_read_only(path)?;
let meta_info = db let meta_info = db
.meta_info() .meta_info()
.context("Failed to retrieve meta information")?; .context("Failed to retrieve meta information")?;
...@@ -34,12 +34,14 @@ fn main() -> Result<()> { ...@@ -34,12 +34,14 @@ fn main() -> Result<()> {
amqf_size, amqf_size,
amqf_entries, amqf_entries,
sst_size, sst_size,
flags,
key_compression_dictionary_size, key_compression_dictionary_size,
block_count, block_count,
} in meta_file.entries } in meta_file.entries
{ {
println!( println!(
" SST {sequence_number:08}.sst: {min_hash:016x} - {max_hash:016x} (p = 1/{})", " SST {sequence_number:08}.sst: {flags} {min_hash:016x} - {max_hash:016x} (p = \
1/{})",
u64::MAX / (max_hash - min_hash + 1) u64::MAX / (max_hash - min_hash + 1)
); );
println!(" AMQF {amqf_entries} entries = {} KiB", amqf_size / 1024); println!(" AMQF {amqf_entries} entries = {} KiB", amqf_size / 1024);
......
...@@ -14,12 +14,15 @@ verbose_log = [] ...@@ -14,12 +14,15 @@ verbose_log = []
[dependencies] [dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
bitfield = { workspace = true }
dashmap = { workspace = true}
either = { workspace = true } either = { workspace = true }
pot = "3.0.0" pot = "3.0.0"
byteorder = { workspace = true } byteorder = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
lzzzz = "1.1.0" lzzzz = "1.1.0"
memmap2 = "0.9.5" memmap2 = "0.9.5"
nohash-hasher = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
qfilter = { version = "0.2.4", features = ["serde"] } qfilter = { version = "0.2.4", features = ["serde"] }
quick_cache = { workspace = true } quick_cache = { workspace = true }
......
...@@ -49,9 +49,14 @@ A meta file can contain metadata about multiple SST files. The metadata is store ...@@ -49,9 +49,14 @@ A meta file can contain metadata about multiple SST files. The metadata is store
- 8 bytes min hash - 8 bytes min hash
- 8 bytes max hash - 8 bytes max hash
- 8 bytes SST file size - 8 bytes SST file size
- 4 bytes flags
- bit 0: cold (compacted and not recently accessed)
- bit 1: fresh (not yet compacted)
- 4 bytes end of AMQF offset relative to start of all AMQF data - 4 bytes end of AMQF offset relative to start of all AMQF data
- 4 bytes end of AMQF offset relative to start of all AMQF data of the "used key hashes" AMQF
- foreach described SST file - foreach described SST file
- serialized AMQF - serialized AMQF
- serialized "used key hashes" AMQF
### SST file ### SST file
...@@ -169,7 +174,7 @@ Compaction chooses a few SST files and runs the merge step of merge sort on tham ...@@ -169,7 +174,7 @@ Compaction chooses a few SST files and runs the merge step of merge sort on tham
Example: Example:
``` ``` text
key hash range: | 0 ... u64::MAX | key hash range: | 0 ... u64::MAX |
SST 1: |----------------| SST 1: |----------------|
SST 2: |----------------| SST 2: |----------------|
...@@ -178,7 +183,7 @@ SST 3: |-----| ...@@ -178,7 +183,7 @@ SST 3: |-----|
can be compacted into: can be compacted into:
``` ``` text
key hash range: | 0 ... u64::MAX | key hash range: | 0 ... u64::MAX |
SST 1': |-------| SST 1': |-------|
SST 2': |------| SST 2': |------|
...@@ -206,7 +211,7 @@ Full example: ...@@ -206,7 +211,7 @@ Full example:
Example: Example:
``` ``` text
key hash range: | 0 ... u64::MAX | Family key hash range: | 0 ... u64::MAX | Family
SST 1: |-| 1 SST 1: |-| 1
SST 2: |----------------| 1 SST 2: |----------------| 1
...@@ -234,7 +239,7 @@ Then we delete SST files 2, 3, 6 and 4, 5, 8 and 7, 9. The ...@@ -234,7 +239,7 @@ Then we delete SST files 2, 3, 6 and 4, 5, 8 and 7, 9. The
SST files 1 stays unchanged. SST files 1 stays unchanged.
``` ``` text
key hash range: | 0 ... u64::MAX | Family key hash range: | 0 ... u64::MAX | Family
SST 1: |-| 1 SST 1: |-| 1
SST 10: |-----| 1 SST 10: |-----| 1
......
use std::ops::RangeInclusive; use std::ops::RangeInclusive;
use rustc_hash::FxHashMap;
use smallvec::{SmallVec, smallvec}; use smallvec::{SmallVec, smallvec};
use crate::compaction::interval_map::IntervalMap; use crate::compaction::interval_map::IntervalMap;
...@@ -12,6 +13,12 @@ pub trait Compactable { ...@@ -12,6 +13,12 @@ pub trait Compactable {
/// The size of the compactable database segment in bytes. /// The size of the compactable database segment in bytes.
fn size(&self) -> u64; fn size(&self) -> u64;
/// The category of the compactable. Overlap between different categories is not considered for
/// compaction.
fn category(&self) -> u8 {
0
}
} }
fn is_overlapping(a: &RangeInclusive<u64>, b: &RangeInclusive<u64>) -> bool { fn is_overlapping(a: &RangeInclusive<u64>, b: &RangeInclusive<u64>) -> bool {
...@@ -37,6 +44,7 @@ fn extend_range(a: &mut RangeInclusive<u64>, b: &RangeInclusive<u64>) -> bool { ...@@ -37,6 +44,7 @@ fn extend_range(a: &mut RangeInclusive<u64>, b: &RangeInclusive<u64>) -> bool {
extended extended
} }
#[cfg(test)]
#[derive(Debug)] #[derive(Debug)]
pub struct CompactableMetrics { pub struct CompactableMetrics {
/// The total coverage of the compactables. /// The total coverage of the compactables.
...@@ -53,6 +61,7 @@ pub struct CompactableMetrics { ...@@ -53,6 +61,7 @@ pub struct CompactableMetrics {
} }
/// Computes metrics about the compactables. /// Computes metrics about the compactables.
#[cfg(test)]
pub fn compute_metrics<T: Compactable>( pub fn compute_metrics<T: Compactable>(
compactables: &[T], compactables: &[T],
full_range: RangeInclusive<u64>, full_range: RangeInclusive<u64>,
...@@ -155,7 +164,7 @@ impl DuplicationInfo { ...@@ -155,7 +164,7 @@ impl DuplicationInfo {
/// Get a value in the range `0..=u64` that represents the estimated amount of duplication /// Get a value in the range `0..=u64` that represents the estimated amount of duplication
/// across the given range. The units are arbitrary, but linear. /// across the given range. The units are arbitrary, but linear.
fn duplication(&self, range: &RangeInclusive<u64>) -> u64 { fn duplication(&self, range: &RangeInclusive<u64>) -> u64 {
if self.total_size == 0 { if self.max_size == self.total_size {
return 0; return 0;
} }
// the maximum numerator value is `u64::MAX + 1` // the maximum numerator value is `u64::MAX + 1`
...@@ -167,6 +176,7 @@ impl DuplicationInfo { ...@@ -167,6 +176,7 @@ impl DuplicationInfo {
} }
/// The estimated size (in bytes) of a database segment containing `range` keys. /// The estimated size (in bytes) of a database segment containing `range` keys.
#[cfg(test)]
fn size(&self, range: &RangeInclusive<u64>) -> u64 { fn size(&self, range: &RangeInclusive<u64>) -> u64 {
if self.total_size == 0 { if self.total_size == 0 {
return 0; return 0;
...@@ -190,11 +200,14 @@ impl DuplicationInfo { ...@@ -190,11 +200,14 @@ impl DuplicationInfo {
} }
} }
fn total_duplication_size(duplication: &IntervalMap<Option<DuplicationInfo>>) -> u64 { fn total_duplication_size(duplication: &IntervalMap<FxHashMap<u8, DuplicationInfo>>) -> u64 {
duplication duplication
.iter() .iter()
.flat_map(|(range, info)| Some((range, info.as_ref()?))) .map(|(range, info)| {
.map(|(range, info)| info.duplication(&range)) info.values()
.map(|info| info.duplication(&range))
.sum::<u64>()
})
.sum() .sum()
} }
...@@ -238,16 +251,18 @@ pub fn get_merge_segments<T: Compactable>( ...@@ -238,16 +251,18 @@ pub fn get_merge_segments<T: Compactable>(
} }
let start_compactable_range = start_compactable.range(); let start_compactable_range = start_compactable.range();
let start_compactable_size = start_compactable.size(); let start_compactable_size = start_compactable.size();
let start_compactable_category = start_compactable.category();
let mut current_range = start_compactable_range.clone(); let mut current_range = start_compactable_range.clone();
// We might need to restart the search if we need to extend the range. // We might need to restart the search if we need to extend the range.
'search: loop { 'search: loop {
let mut current_set = smallvec![start_index]; let mut current_set = smallvec![start_index];
let mut current_size = start_compactable_size; let mut current_size = start_compactable_size;
let mut duplication = IntervalMap::<Option<DuplicationInfo>>::new(); let mut duplication = IntervalMap::<FxHashMap<u8, DuplicationInfo>>::new();
duplication.update(start_compactable_range.clone(), |dup_info| { duplication.update(start_compactable_range.clone(), |dup_info| {
dup_info dup_info
.get_or_insert_default() .entry(start_compactable_category)
.or_default()
.add(start_compactable_size, &start_compactable_range); .add(start_compactable_size, &start_compactable_range);
}); });
let mut current_skip = 0; let mut current_skip = 0;
...@@ -337,8 +352,9 @@ pub fn get_merge_segments<T: Compactable>( ...@@ -337,8 +352,9 @@ pub fn get_merge_segments<T: Compactable>(
// set. // set.
current_set.push(next_index); current_set.push(next_index);
current_size += size; current_size += size;
let category = compactable.category();
duplication.update(range.clone(), |dup_info| { duplication.update(range.clone(), |dup_info| {
dup_info.get_or_insert_default().add(size, &range); dup_info.entry(category).or_default().add(size, &range);
}); });
} }
} }
...@@ -633,13 +649,16 @@ mod tests { ...@@ -633,13 +649,16 @@ mod tests {
let new_metrics = compute_metrics(&containers, 0..=KEY_RANGE); let new_metrics = compute_metrics(&containers, 0..=KEY_RANGE);
println!( println!(
"Compaction done: coverage: {} ({}), overlap: {} ({}), duplication: {} ({})", "Compaction done: coverage: {} ({}), overlap: {} ({}), duplication: {} ({}), \
duplicated_size: {} ({})",
new_metrics.coverage, new_metrics.coverage,
new_metrics.coverage - metrics.coverage, new_metrics.coverage - metrics.coverage,
new_metrics.overlap, new_metrics.overlap,
new_metrics.overlap - metrics.overlap, new_metrics.overlap - metrics.overlap,
new_metrics.duplication, new_metrics.duplication,
new_metrics.duplication - metrics.duplication new_metrics.duplication - metrics.duplication,
new_metrics.duplicated_size,
(new_metrics.duplicated_size as f32) - metrics.duplicated_size as f32,
); );
} else { } else {
println!("No compaction needed"); println!("No compaction needed");
......
...@@ -3,7 +3,6 @@ use std::{mem::MaybeUninit, sync::Arc}; ...@@ -3,7 +3,6 @@ use std::{mem::MaybeUninit, sync::Arc};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use lzzzz::lz4::{ACC_LEVEL_DEFAULT, decompress, decompress_with_dict}; use lzzzz::lz4::{ACC_LEVEL_DEFAULT, decompress, decompress_with_dict};
#[tracing::instrument(level = "trace", skip_all, name = "decompress database block")]
pub fn decompress_into_arc( pub fn decompress_into_arc(
uncompressed_length: u32, uncompressed_length: u32,
block: &[u8], block: &[u8],
......
use std::{ use std::{
fmt::Display,
fs::File, fs::File,
hash::BuildHasherDefault, hash::BuildHasherDefault,
io::{BufReader, Seek}, io::{BufReader, Seek},
...@@ -8,6 +9,7 @@ use std::{ ...@@ -8,6 +9,7 @@ use std::{
}; };
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use either::Either; use either::Either;
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
...@@ -31,6 +33,35 @@ impl quick_cache::Weighter<u32, Arc<qfilter::Filter>> for AmqfWeighter { ...@@ -31,6 +33,35 @@ impl quick_cache::Weighter<u32, Arc<qfilter::Filter>> for AmqfWeighter {
pub type AmqfCache = pub type AmqfCache =
quick_cache::sync::Cache<u32, Arc<qfilter::Filter>, AmqfWeighter, BuildHasherDefault<FxHasher>>; quick_cache::sync::Cache<u32, Arc<qfilter::Filter>, AmqfWeighter, BuildHasherDefault<FxHasher>>;
bitfield! {
#[derive(Clone, Copy, Default)]
pub struct MetaEntryFlags(u32);
impl Debug;
impl From<u32>;
/// The SST file was compacted and none of the entries have been accessed recently.
pub cold, set_cold: 0;
/// The SST file was freshly written and has not been compacted yet.
pub fresh, set_fresh: 1;
}
impl MetaEntryFlags {
pub const FRESH: MetaEntryFlags = MetaEntryFlags(0b10);
pub const COLD: MetaEntryFlags = MetaEntryFlags(0b01);
pub const WARM: MetaEntryFlags = MetaEntryFlags(0b00);
}
impl Display for MetaEntryFlags {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.fresh() {
f.pad_integral(true, "", "fresh")
} else if self.cold() {
f.pad_integral(true, "", "cold")
} else {
f.pad_integral(true, "", "warm")
}
}
}
pub struct MetaEntry { pub struct MetaEntry {
/// The metadata for the static sorted file. /// The metadata for the static sorted file.
sst_data: StaticSortedFileMetaData, sst_data: StaticSortedFileMetaData,
...@@ -42,6 +73,8 @@ pub struct MetaEntry { ...@@ -42,6 +73,8 @@ pub struct MetaEntry {
max_hash: u64, max_hash: u64,
/// The size of the SST file in bytes. /// The size of the SST file in bytes.
size: u64, size: u64,
/// The status flags for this entry.
flags: MetaEntryFlags,
/// The offset of the start of the AMQF data in the meta file relative to the end of the /// The offset of the start of the AMQF data in the meta file relative to the end of the
/// header. /// header.
start_of_amqf_data_offset: u32, start_of_amqf_data_offset: u32,
...@@ -64,6 +97,10 @@ impl MetaEntry { ...@@ -64,6 +97,10 @@ impl MetaEntry {
self.size self.size
} }
pub fn flags(&self) -> MetaEntryFlags {
self.flags
}
pub fn amqf_size(&self) -> u32 { pub fn amqf_size(&self) -> u32 {
self.end_of_amqf_data_offset - self.start_of_amqf_data_offset self.end_of_amqf_data_offset - self.start_of_amqf_data_offset
} }
...@@ -183,6 +220,12 @@ pub struct MetaFile { ...@@ -183,6 +220,12 @@ pub struct MetaFile {
obsolete_entries: Vec<u32>, obsolete_entries: Vec<u32>,
/// The obsolete SST files. /// The obsolete SST files.
obsolete_sst_files: Vec<u32>, obsolete_sst_files: Vec<u32>,
/// The offset of the start of the "used keys" AMQF data in the meta file relative to the end
/// of the header.
start_of_used_keys_amqf_data_offset: u32,
/// The offset of the end of the "used keys" AMQF data in the the meta file relative to the end
/// of the header.
end_of_used_keys_amqf_data_offset: u32,
/// The memory mapped file. /// The memory mapped file.
mmap: Mmap, mmap: Mmap,
} }
...@@ -224,6 +267,7 @@ impl MetaFile { ...@@ -224,6 +267,7 @@ impl MetaFile {
min_hash: file.read_u64::<BE>()?, min_hash: file.read_u64::<BE>()?,
max_hash: file.read_u64::<BE>()?, max_hash: file.read_u64::<BE>()?,
size: file.read_u64::<BE>()?, size: file.read_u64::<BE>()?,
flags: MetaEntryFlags(file.read_u32::<BE>()?),
start_of_amqf_data_offset, start_of_amqf_data_offset,
end_of_amqf_data_offset: file.read_u32::<BE>()?, end_of_amqf_data_offset: file.read_u32::<BE>()?,
amqf: OnceLock::new(), amqf: OnceLock::new(),
...@@ -232,6 +276,9 @@ impl MetaFile { ...@@ -232,6 +276,9 @@ impl MetaFile {
start_of_amqf_data_offset = entry.end_of_amqf_data_offset; start_of_amqf_data_offset = entry.end_of_amqf_data_offset;
entries.push(entry); entries.push(entry);
} }
let start_of_used_keys_amqf_data_offset = start_of_amqf_data_offset;
let end_of_used_keys_amqf_data_offset = file.read_u32::<BE>()?;
let offset = file.stream_position()?; let offset = file.stream_position()?;
let file = file.into_inner(); let file = file.into_inner();
let mut options = MmapOptions::new(); let mut options = MmapOptions::new();
...@@ -246,6 +293,8 @@ impl MetaFile { ...@@ -246,6 +293,8 @@ impl MetaFile {
entries, entries,
obsolete_entries: Vec::new(), obsolete_entries: Vec::new(),
obsolete_sst_files, obsolete_sst_files,
start_of_used_keys_amqf_data_offset,
end_of_used_keys_amqf_data_offset,
mmap, mmap,
}; };
Ok(file) Ok(file)
...@@ -272,6 +321,20 @@ impl MetaFile { ...@@ -272,6 +321,20 @@ impl MetaFile {
&self.mmap &self.mmap
} }
pub fn deserialize_used_key_hashes_amqf(&self) -> Result<Option<qfilter::Filter>> {
if self.start_of_used_keys_amqf_data_offset == self.end_of_used_keys_amqf_data_offset {
return Ok(None);
}
let amqf = &self.amqf_data()[self.start_of_used_keys_amqf_data_offset as usize
..self.end_of_used_keys_amqf_data_offset as usize];
Ok(Some(pot::from_slice(amqf).with_context(|| {
format!(
"Failed to deserialize used key hashes AMQF from {:08}.meta",
self.sequence_number
)
})?))
}
pub fn retain_entries(&mut self, mut predicate: impl FnMut(u32) -> bool) -> bool { pub fn retain_entries(&mut self, mut predicate: impl FnMut(u32) -> bool) -> bool {
let old_len = self.entries.len(); let old_len = self.entries.len();
self.entries.retain(|entry| { self.entries.retain(|entry| {
......
...@@ -6,6 +6,7 @@ use std::{ ...@@ -6,6 +6,7 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use byteorder::{BE, WriteBytesExt}; use byteorder::{BE, WriteBytesExt};
use qfilter::Filter;
use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta; use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta;
...@@ -15,6 +16,8 @@ pub struct MetaFileBuilder<'a> { ...@@ -15,6 +16,8 @@ pub struct MetaFileBuilder<'a> {
entries: Vec<(u32, StaticSortedFileBuilderMeta<'a>)>, entries: Vec<(u32, StaticSortedFileBuilderMeta<'a>)>,
/// Obsolete SST files, represented by their sequence numbers /// Obsolete SST files, represented by their sequence numbers
obsolete_sst_files: Vec<u32>, obsolete_sst_files: Vec<u32>,
/// Optional AMQF for used key hashes
used_key_hashes_amqf: Option<Filter>,
} }
impl<'a> MetaFileBuilder<'a> { impl<'a> MetaFileBuilder<'a> {
...@@ -23,6 +26,7 @@ impl<'a> MetaFileBuilder<'a> { ...@@ -23,6 +26,7 @@ impl<'a> MetaFileBuilder<'a> {
family, family,
entries: Vec::new(), entries: Vec::new(),
obsolete_sst_files: Vec::new(), obsolete_sst_files: Vec::new(),
used_key_hashes_amqf: None,
} }
} }
...@@ -34,6 +38,10 @@ impl<'a> MetaFileBuilder<'a> { ...@@ -34,6 +38,10 @@ impl<'a> MetaFileBuilder<'a> {
self.obsolete_sst_files.push(sequence_number); self.obsolete_sst_files.push(sequence_number);
} }
pub fn set_used_key_hashes_amqf(&mut self, amqf: Filter) {
self.used_key_hashes_amqf = Some(amqf);
}
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
pub fn write(self, db_path: &Path, seq: u32) -> Result<File> { pub fn write(self, db_path: &Path, seq: u32) -> Result<File> {
let file = db_path.join(format!("{seq:08}.meta")); let file = db_path.join(format!("{seq:08}.meta"));
...@@ -62,13 +70,26 @@ impl<'a> MetaFileBuilder<'a> { ...@@ -62,13 +70,26 @@ impl<'a> MetaFileBuilder<'a> {
file.write_u64::<BE>(sst.min_hash)?; file.write_u64::<BE>(sst.min_hash)?;
file.write_u64::<BE>(sst.max_hash)?; file.write_u64::<BE>(sst.max_hash)?;
file.write_u64::<BE>(sst.size)?; file.write_u64::<BE>(sst.size)?;
file.write_u32::<BE>(sst.flags.0)?;
amqf_offset += sst.amqf.len(); amqf_offset += sst.amqf.len();
file.write_u32::<BE>(amqf_offset as u32)?; file.write_u32::<BE>(amqf_offset as u32)?;
} }
let serialized_used_key_hashes = self
.used_key_hashes_amqf
.as_ref()
.map(|f| pot::to_vec(f).expect("AMQF serialization failed"));
amqf_offset += serialized_used_key_hashes
.as_ref()
.map(|bytes| bytes.len())
.unwrap_or(0);
file.write_u32::<BE>(amqf_offset as u32)?;
for (_, sst) in &self.entries { for (_, sst) in &self.entries {
file.write_all(&sst.amqf)?; file.write_all(&sst.amqf)?;
} }
if let Some(bytes) = &serialized_used_key_hashes {
file.write_all(bytes)?;
}
Ok(file.into_inner()?) Ok(file.into_inner()?)
} }
} }
...@@ -11,6 +11,7 @@ use byteorder::{BE, ByteOrder, WriteBytesExt}; ...@@ -11,6 +11,7 @@ use byteorder::{BE, ByteOrder, WriteBytesExt};
use crate::{ use crate::{
compression::compress_into_buffer, compression::compress_into_buffer,
meta_file::MetaEntryFlags,
static_sorted_file::{ static_sorted_file::{
BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED, BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL, KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL,
...@@ -88,6 +89,8 @@ pub struct StaticSortedFileBuilderMeta<'a> { ...@@ -88,6 +89,8 @@ pub struct StaticSortedFileBuilderMeta<'a> {
pub block_count: u16, pub block_count: u16,
/// The file size of the SST file /// The file size of the SST file
pub size: u64, pub size: u64,
/// The status flags for this SST file
pub flags: MetaEntryFlags,
/// The number of entries in the SST file /// The number of entries in the SST file
pub entries: u64, pub entries: u64,
} }
...@@ -96,6 +99,7 @@ pub fn write_static_stored_file<E: Entry>( ...@@ -96,6 +99,7 @@ pub fn write_static_stored_file<E: Entry>(
entries: &[E], entries: &[E],
total_key_size: usize, total_key_size: usize,
file: &Path, file: &Path,
flags: MetaEntryFlags,
) -> Result<(StaticSortedFileBuilderMeta<'static>, File)> { ) -> Result<(StaticSortedFileBuilderMeta<'static>, File)> {
debug_assert!(entries.iter().map(|e| e.key_hash()).is_sorted()); debug_assert!(entries.iter().map(|e| e.key_hash()).is_sorted());
...@@ -141,6 +145,7 @@ pub fn write_static_stored_file<E: Entry>( ...@@ -141,6 +145,7 @@ pub fn write_static_stored_file<E: Entry>(
key_compression_dictionary_length: key_dict.len().try_into().unwrap(), key_compression_dictionary_length: key_dict.len().try_into().unwrap(),
block_count, block_count,
size: file.stream_position()?, size: file.stream_position()?,
flags,
entries: entries.len() as u64, entries: entries.len() as u64,
}; };
Ok((meta, file.into_inner()?)) Ok((meta, file.into_inner()?))
......
...@@ -109,20 +109,21 @@ fn full_cycle() -> Result<()> { ...@@ -109,20 +109,21 @@ fn full_cycle() -> Result<()> {
type TestCases = Vec<( type TestCases = Vec<(
&'static str, &'static str,
Box<dyn Fn(&mut WriteBatch<Vec<u8>, RayonParallelScheduler, 16>) -> Result<()>>, Box<dyn Fn(&mut WriteBatch<Vec<u8>, RayonParallelScheduler, 16>) -> Result<()>>,
Box<dyn Fn(&TurboPersistence<RayonParallelScheduler>) -> Result<()>>, Box<dyn Fn(&TurboPersistence<RayonParallelScheduler, 16>) -> Result<()>>,
)>; )>;
fn test_case( fn test_case(
test_cases: &mut TestCases, test_cases: &mut TestCases,
name: &'static str, name: &'static str,
write: impl Fn(&mut WriteBatch<Vec<u8>, RayonParallelScheduler, 16>) -> Result<()> + 'static, write: impl Fn(&mut WriteBatch<Vec<u8>, RayonParallelScheduler, 16>) -> Result<()> + 'static,
read: impl Fn(&TurboPersistence<RayonParallelScheduler>) -> Result<()> + 'static, read: impl Fn(&TurboPersistence<RayonParallelScheduler, 16>) -> Result<()> + 'static,
) { ) {
test_cases.push(( test_cases.push((
name, name,
Box::new(write) Box::new(write)
as Box<dyn Fn(&mut WriteBatch<Vec<u8>, RayonParallelScheduler, 16>) -> Result<()>>, as Box<dyn Fn(&mut WriteBatch<Vec<u8>, RayonParallelScheduler, 16>) -> Result<()>>,
Box::new(read) as Box<dyn Fn(&TurboPersistence<RayonParallelScheduler>) -> Result<()>>, Box::new(read)
as Box<dyn Fn(&TurboPersistence<RayonParallelScheduler, 16>) -> Result<()>>,
)); ));
} }
...@@ -506,7 +507,7 @@ fn persist_changes() -> Result<()> { ...@@ -506,7 +507,7 @@ fn persist_changes() -> Result<()> {
} }
Ok(()) Ok(())
} }
fn check(db: &TurboPersistence<RayonParallelScheduler>, key: u8, value: u8) -> Result<()> { fn check(db: &TurboPersistence<RayonParallelScheduler, 1>, key: u8, value: u8) -> Result<()> {
for i in 0..READ_COUNT { for i in 0..READ_COUNT {
// read every 10th item // read every 10th item
let i = i * 10; let i = i * 10;
...@@ -519,11 +520,11 @@ fn persist_changes() -> Result<()> { ...@@ -519,11 +520,11 @@ fn persist_changes() -> Result<()> {
} }
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
let b = db.write_batch::<_, 1>()?; let b = db.write_batch()?;
put(&b, 1, 11)?; put(&b, 1, 11)?;
put(&b, 2, 21)?; put(&b, 2, 21)?;
put(&b, 3, 31)?; put(&b, 3, 31)?;
...@@ -538,11 +539,11 @@ fn persist_changes() -> Result<()> { ...@@ -538,11 +539,11 @@ fn persist_changes() -> Result<()> {
println!("---"); println!("---");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
let b = db.write_batch::<_, 1>()?; let b = db.write_batch()?;
put(&b, 1, 12)?; put(&b, 1, 12)?;
put(&b, 2, 22)?; put(&b, 2, 22)?;
db.commit_write_batch(b)?; db.commit_write_batch(b)?;
...@@ -555,11 +556,11 @@ fn persist_changes() -> Result<()> { ...@@ -555,11 +556,11 @@ fn persist_changes() -> Result<()> {
} }
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
let b = db.write_batch::<_, 1>()?; let b = db.write_batch()?;
put(&b, 1, 13)?; put(&b, 1, 13)?;
db.commit_write_batch(b)?; db.commit_write_batch(b)?;
...@@ -638,7 +639,7 @@ fn partial_compaction() -> Result<()> { ...@@ -638,7 +639,7 @@ fn partial_compaction() -> Result<()> {
} }
Ok(()) Ok(())
} }
fn check(db: &TurboPersistence<RayonParallelScheduler>, key: u8, value: u8) -> Result<()> { fn check(db: &TurboPersistence<RayonParallelScheduler, 1>, key: u8, value: u8) -> Result<()> {
for i in 0..READ_COUNT { for i in 0..READ_COUNT {
// read every 10th item // read every 10th item
let i = i * 10; let i = i * 10;
...@@ -655,11 +656,11 @@ fn partial_compaction() -> Result<()> { ...@@ -655,11 +656,11 @@ fn partial_compaction() -> Result<()> {
println!("--- Iteration {i} ---"); println!("--- Iteration {i} ---");
println!("Add more entries"); println!("Add more entries");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
let b = db.write_batch::<_, 1>()?; let b = db.write_batch()?;
put(&b, i, i)?; put(&b, i, i)?;
put(&b, i + 1, i)?; put(&b, i + 1, i)?;
put(&b, i + 2, i)?; put(&b, i + 2, i)?;
...@@ -677,7 +678,7 @@ fn partial_compaction() -> Result<()> { ...@@ -677,7 +678,7 @@ fn partial_compaction() -> Result<()> {
println!("Compaction"); println!("Compaction");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
...@@ -701,7 +702,7 @@ fn partial_compaction() -> Result<()> { ...@@ -701,7 +702,7 @@ fn partial_compaction() -> Result<()> {
println!("Restore check"); println!("Restore check");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
...@@ -742,7 +743,7 @@ fn merge_file_removal() -> Result<()> { ...@@ -742,7 +743,7 @@ fn merge_file_removal() -> Result<()> {
} }
Ok(()) Ok(())
} }
fn check(db: &TurboPersistence<RayonParallelScheduler>, key: u8, value: u32) -> Result<()> { fn check(db: &TurboPersistence<RayonParallelScheduler, 1>, key: u8, value: u32) -> Result<()> {
for i in 0..READ_COUNT { for i in 0..READ_COUNT {
// read every 10th item // read every 10th item
let i = i * 10; let i = i * 10;
...@@ -760,11 +761,11 @@ fn merge_file_removal() -> Result<()> { ...@@ -760,11 +761,11 @@ fn merge_file_removal() -> Result<()> {
{ {
println!("--- Init ---"); println!("--- Init ---");
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
let b = db.write_batch::<_, 1>()?; let b = db.write_batch()?;
for j in 0..=255 { for j in 0..=255 {
put(&b, j, 0)?; put(&b, j, 0)?;
} }
...@@ -779,11 +780,11 @@ fn merge_file_removal() -> Result<()> { ...@@ -779,11 +780,11 @@ fn merge_file_removal() -> Result<()> {
let i = i * 37; let i = i * 37;
println!("Add more entries"); println!("Add more entries");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
let b = db.write_batch::<_, 1>()?; let b = db.write_batch()?;
for j in iter_bits(i) { for j in iter_bits(i) {
println!("Put {j} = {i}"); println!("Put {j} = {i}");
expected_values[j as usize] = i; expected_values[j as usize] = i;
...@@ -800,7 +801,7 @@ fn merge_file_removal() -> Result<()> { ...@@ -800,7 +801,7 @@ fn merge_file_removal() -> Result<()> {
println!("Compaction"); println!("Compaction");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
...@@ -821,7 +822,7 @@ fn merge_file_removal() -> Result<()> { ...@@ -821,7 +822,7 @@ fn merge_file_removal() -> Result<()> {
println!("Restore check"); println!("Restore check");
{ {
let db = TurboPersistence::open_with_parallel_scheduler( let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(), path.to_path_buf(),
RayonParallelScheduler, RayonParallelScheduler,
)?; )?;
......
...@@ -21,6 +21,7 @@ use crate::{ ...@@ -21,6 +21,7 @@ use crate::{
compression::compress_into_buffer, compression::compress_into_buffer,
constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT}, constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
key::StoreKey, key::StoreKey,
meta_file::MetaEntryFlags,
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
static_sorted_file_builder::{StaticSortedFileBuilderMeta, write_static_stored_file}, static_sorted_file_builder::{StaticSortedFileBuilderMeta, write_static_stored_file},
...@@ -267,8 +268,11 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -267,8 +268,11 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
/// Finishes the write batch by returning the new sequence number and the new SST files. This /// Finishes the write batch by returning the new sequence number and the new SST files. This
/// writes all outstanding thread local data to disk. /// writes all outstanding thread local data to disk.
#[tracing::instrument(level = "trace", skip(self))] #[tracing::instrument(level = "trace", skip_all)]
pub(crate) fn finish(&mut self) -> Result<FinishResult> { pub(crate) fn finish(
&mut self,
get_accessed_key_hashes: impl Fn(u32) -> qfilter::Filter + Send + Sync,
) -> Result<FinishResult> {
let mut new_blob_files = Vec::new(); let mut new_blob_files = Vec::new();
// First, we flush all thread local collectors to the global collectors. // First, we flush all thread local collectors to the global collectors.
...@@ -343,7 +347,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -343,7 +347,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
}, },
)?; )?;
// Not we need to write the new meta files. // Now we need to write the new meta files.
let new_meta_collectors = [(); FAMILIES].map(|_| Mutex::new(Vec::new())); let new_meta_collectors = [(); FAMILIES].map(|_| Mutex::new(Vec::new()));
let meta_collectors = replace(&mut self.meta_collectors, new_meta_collectors); let meta_collectors = replace(&mut self.meta_collectors, new_meta_collectors);
let keys_written = AtomicU64::new(0); let keys_written = AtomicU64::new(0);
...@@ -366,6 +370,8 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -366,6 +370,8 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
builder.add(seq, sst); builder.add(seq, sst);
} }
keys_written.fetch_add(entries, Ordering::Relaxed); keys_written.fetch_add(entries, Ordering::Relaxed);
let accessed_key_hashes = get_accessed_key_hashes(family);
builder.set_used_key_hashes_amqf(accessed_key_hashes);
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1; let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let file = builder.write(&self.db_path, seq)?; let file = builder.write(&self.db_path, seq)?;
Ok((seq, file)) Ok((seq, file))
...@@ -415,7 +421,9 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -415,7 +421,9 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
let path = self.db_path.join(format!("{seq:08}.sst")); let path = self.db_path.join(format!("{seq:08}.sst"));
let (meta, file) = self let (meta, file) = self
.parallel_scheduler .parallel_scheduler
.block_in_place(|| write_static_stored_file(entries, total_key_size, &path)) .block_in_place(|| {
write_static_stored_file(entries, total_key_size, &path, MetaEntryFlags::FRESH)
})
.with_context(|| format!("Unable to write SST file {seq:08}.sst"))?; .with_context(|| format!("Unable to write SST file {seq:08}.sst"))?;
#[cfg(feature = "verify_sst_content")] #[cfg(feature = "verify_sst_content")]
......
...@@ -21,6 +21,9 @@ use crate::database::{ ...@@ -21,6 +21,9 @@ use crate::database::{
mod parallel_scheduler; mod parallel_scheduler;
/// Number of key families, see KeySpace enum for their numbers.
const FAMILIES: usize = 5;
const MB: u64 = 1024 * 1024; const MB: u64 = 1024 * 1024;
const COMPACT_CONFIG: CompactConfig = CompactConfig { const COMPACT_CONFIG: CompactConfig = CompactConfig {
min_merge_count: 3, min_merge_count: 3,
...@@ -33,10 +36,11 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig { ...@@ -33,10 +36,11 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig {
}; };
pub struct TurboKeyValueDatabase { pub struct TurboKeyValueDatabase {
db: Arc<TurboPersistence<TurboTasksParallelScheduler>>, db: Arc<TurboPersistence<TurboTasksParallelScheduler, FAMILIES>>,
compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>, compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>,
is_ci: bool, is_ci: bool,
is_short_session: bool, is_short_session: bool,
is_fresh: bool,
} }
impl TurboKeyValueDatabase { impl TurboKeyValueDatabase {
...@@ -47,6 +51,7 @@ impl TurboKeyValueDatabase { ...@@ -47,6 +51,7 @@ impl TurboKeyValueDatabase {
compact_join_handle: Mutex::new(None), compact_join_handle: Mutex::new(None),
is_ci, is_ci,
is_short_session, is_short_session,
is_fresh: db.is_empty(),
}) })
} }
} }
...@@ -108,6 +113,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -108,6 +113,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
join_handle.join()?; join_handle.join()?;
} }
// Compact the database on shutdown // Compact the database on shutdown
// (Avoid compacting a fresh database since we don't have any usage info yet)
if !self.is_fresh {
if self.is_ci { if self.is_ci {
// Fully compact in CI to reduce cache size // Fully compact in CI to reduce cache size
do_compact( do_compact(
...@@ -123,13 +130,14 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -123,13 +130,14 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
available_parallelism().map_or(4, |c| max(4, c.get())), available_parallelism().map_or(4, |c| max(4, c.get())),
)?; )?;
} }
}
// Shutdown the database // Shutdown the database
self.db.shutdown() self.db.shutdown()
} }
} }
fn do_compact( fn do_compact(
db: &TurboPersistence<TurboTasksParallelScheduler>, db: &TurboPersistence<TurboTasksParallelScheduler, FAMILIES>,
message: &'static str, message: &'static str,
max_merge_segment_count: usize, max_merge_segment_count: usize,
) -> Result<()> { ) -> Result<()> {
...@@ -151,8 +159,9 @@ fn do_compact( ...@@ -151,8 +159,9 @@ fn do_compact(
} }
pub struct TurboWriteBatch<'a> { pub struct TurboWriteBatch<'a> {
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, TurboTasksParallelScheduler, 5>, batch:
db: &'a Arc<TurboPersistence<TurboTasksParallelScheduler>>, turbo_persistence::WriteBatch<WriteBuffer<'static>, TurboTasksParallelScheduler, FAMILIES>,
db: &'a Arc<TurboPersistence<TurboTasksParallelScheduler, FAMILIES>>,
compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>, compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>,
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment