Unverified Commit f3a6889e authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: split meta data and AQMF into separate files (#79659)

### What?

Split the SST meta data and AQMF into a separate `*.meta` file. A single meta file can contain meta data and AQMFs of multiple SST files.

On database startup only the meta data files need to be read and SST files are lazily mmapped and read only when the AQM-Filter is successful. This ensures that only SST files with a high likelyhood need to be read at all.

Currently this has not much benefit as keys are randomly distributed in SST files, so all files need to be read anyway.
There is a small benefit after a bunch of incremental builds, when some SST files become unused as they have been overwritten.

In future we want to split keys into recently used and not recently used keys and place them in separate SST files. This ensures that less SST files need to be read. This change is a pre-requirement to that change.
parent df846a29
......@@ -9492,6 +9492,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"byteorder",
"either",
"jiff",
"lzzzz",
"memmap2 0.9.5",
......
......@@ -5,6 +5,7 @@ edition = "2024"
license = "MIT"
[features]
default = []
verify_sst_content = []
strict_checks = []
stats = ["quick_cache/stats"]
......@@ -12,6 +13,7 @@ print_stats = ["stats"]
[dependencies]
anyhow = { workspace = true }
either = { workspace = true }
pot = "3.0.0"
byteorder = "1.5.0"
jiff = "0.2.10"
......
......@@ -16,10 +16,12 @@ There is a single `CURRENT` file which stores the latest committed sequence numb
All other files have a sequence number as file name, e. g. `0000123.sst`. All files are immutable once there sequence number is <= the committed sequence number. But they might be deleted when they are superseeded by other committed files.
There are two different file types:
There are four different file types:
- Static Sorted Table (SST, `*.sst`): These files contain key value pairs.
- Blob files (`*.blob`): These files contain large values.
- Delete files (`*.del`): These files contain a list of sequence numbers of files that should be considered as deleted.
- Meta files (`*.meta`): These files contain metadata about the SST files. They contains the hash range and a AQMF for quick filtering.
Therefore there are there value types:
......@@ -29,18 +31,33 @@ Therefore there are there value types:
- Future:
- MERGE: An application specific update operation that is applied on the old value.
### SST file
### Meta file
A meta file can contain metadata about multiple SST files. The metadata is stored in a single file to avoid having too many small files.
- Headers
- 4 bytes magic number and version
- Header
- 4 bytes magic number (0xFE4ADA4A)
- 4 bytes key family
- 8 bytes min hash
- 8 bytes max hash
- 3 bytes AQMF length
- 2 bytes key Compression Dictionary length
- 2 bytes value Compression Dictionary length
- 2 bytes block count
- serialized AQMF
- 4 bytes count of obsolete SST files
- foreach obsolete SST file
- 4 bytes sequence number of the obsolete SST file
- 4 bytes count of described SST files
- foreach described SST file
- 4 bytes sequence number of the SST file
- 2 bytes key Compression Dictionary length
- 2 bytes value Compression Dictionary length
- 2 bytes block count
- 8 bytes min hash
- 8 bytes max hash
- 8 bytes SST file size
- 4 bytes end of AQMF offset relative to start of all AQMF data
- foreach described SST file
- serialized AQMF
### SST file
The SST file contains only data without any header.
- serialized key Compression Dictionary
- serialized value Compression Dictionary
- foreach block
......
......@@ -21,7 +21,7 @@ pub trait Compactable {
fn range(&self) -> Range;
/// Returns the size of the compactable.
fn size(&self) -> usize;
fn size(&self) -> u64;
}
fn is_overlapping(a: &Range, b: &Range) -> bool {
......@@ -65,7 +65,7 @@ pub struct CompactConfig {
pub max_merge: usize,
/// The maximum size of all files to merge at once.
pub max_merge_size: usize,
pub max_merge_size: u64,
}
/// For a list of compactables, computes merge and move jobs that are expected to perform best.
......@@ -228,7 +228,7 @@ mod tests {
struct TestCompactable {
range: Range,
size: usize,
size: u64,
}
impl Compactable for TestCompactable {
......@@ -236,7 +236,7 @@ mod tests {
self.range
}
fn size(&self) -> usize {
fn size(&self) -> u64 {
self.size
}
}
......@@ -244,7 +244,7 @@ mod tests {
fn compact<const N: usize>(
ranges: [(u64, u64); N],
max_merge: usize,
max_merge_size: usize,
max_merge_size: u64,
) -> CompactionJobs {
let compactables = ranges
.iter()
......@@ -277,7 +277,7 @@ mod tests {
(30, 40),
],
3,
usize::MAX,
u64::MAX,
);
assert_eq!(merge_jobs, vec![vec![0, 1, 2], vec![4, 5, 6]]);
assert_eq!(move_jobs, vec![3, 8]);
......@@ -341,7 +341,7 @@ mod tests {
let config = CompactConfig {
max_merge: 4,
min_merge: 2,
max_merge_size: usize::MAX,
max_merge_size: u64::MAX,
};
let jobs = get_compaction_jobs(&containers, &config);
if !jobs.is_empty() {
......@@ -387,8 +387,8 @@ mod tests {
(self.keys[0], *self.keys.last().unwrap())
}
fn size(&self) -> usize {
self.keys.len()
fn size(&self) -> u64 {
self.keys.len() as u64
}
}
......
......@@ -2,6 +2,7 @@
#![feature(new_zeroed_alloc)]
#![feature(get_mut_unchecked)]
#![feature(sync_unsafe_cell)]
#![feature(iter_collect_into)]
mod arc_slice;
mod collector;
......@@ -16,6 +17,9 @@ mod static_sorted_file;
mod static_sorted_file_builder;
mod write_batch;
mod meta_file;
mod meta_file_builder;
mod sst_filter;
#[cfg(test)]
mod tests;
mod value_buf;
......
use std::{
fs::File,
hash::BuildHasherDefault,
io::{BufReader, Seek},
path::{Path, PathBuf},
sync::{Arc, OnceLock},
};
use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt};
use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher;
use crate::{
QueryKey,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
};
#[derive(Clone, Default)]
pub struct AqmfWeighter;
impl quick_cache::Weighter<u32, Arc<qfilter::Filter>> for AqmfWeighter {
fn weight(&self, _key: &u32, filter: &Arc<qfilter::Filter>) -> u64 {
filter.capacity() + 1
}
}
pub type AqmfCache =
quick_cache::sync::Cache<u32, Arc<qfilter::Filter>, AqmfWeighter, BuildHasherDefault<FxHasher>>;
pub struct MetaEntry {
/// The metadata for the static sorted file.
sst_data: StaticSortedFileMetaData,
/// The key family of the SST file.
family: u32,
/// The minimum hash value of the keys in the SST file.
min_hash: u64,
/// The maximum hash value of the keys in the SST file.
max_hash: u64,
/// The size of the SST file in bytes.
size: u64,
/// The offset of the start of the AQMF data in the meta file relative to the end of the
/// header.
start_of_aqmf_data_offset: u32,
/// The offset of the end of the AQMF data in the the meta file relative to the end of the
/// header.
end_of_aqmf_data_offset: u32,
/// The AQMF filter of this file. This is only used if the range is very large. Smaller ranges
/// use the AQMF cache instead.
aqmf: OnceLock<qfilter::Filter>,
/// The static sorted file that is lazily loaded
sst: OnceLock<StaticSortedFile>,
}
impl MetaEntry {
pub fn sequence_number(&self) -> u32 {
self.sst_data.sequence_number
}
pub fn size(&self) -> u64 {
self.size
}
pub fn aqmf<'l>(&self, aqmf_data: &'l [u8]) -> &'l [u8] {
aqmf_data
.get(self.start_of_aqmf_data_offset as usize..self.end_of_aqmf_data_offset as usize)
.expect("AQMF data out of bounds")
}
pub fn sst(&self, db_path: &Path) -> Result<&StaticSortedFile> {
self.sst
.get_or_try_init(|| StaticSortedFile::open(db_path, self.sst_data.clone()))
}
/// Returns the key family and hash range of this file.
pub fn range(&self) -> StaticSortedFileRange {
StaticSortedFileRange {
family: self.family,
min_hash: self.min_hash,
max_hash: self.max_hash,
}
}
pub fn min_hash(&self) -> u64 {
self.min_hash
}
pub fn max_hash(&self) -> u64 {
self.max_hash
}
pub fn key_compression_dictionary_length(&self) -> u16 {
self.sst_data.key_compression_dictionary_length
}
pub fn value_compression_dictionary_length(&self) -> u16 {
self.sst_data.value_compression_dictionary_length
}
pub fn block_count(&self) -> u16 {
self.sst_data.block_count
}
}
/// The result of a lookup operation.
pub enum MetaLookupResult {
/// The key was not found because it is from a different key family.
FamilyMiss,
/// The key was not found because it is out of the range of this SST file. But it was the
/// correct key family.
RangeMiss,
/// The key was not found because it was not in the AQMF filter. But it was in the range.
QuickFilterMiss,
/// The key was looked up in the SST file. It was in the AQMF filter.
SstLookup(SstLookupResult),
}
/// The key family and hash range of an SST file.
#[derive(Clone, Copy)]
pub struct StaticSortedFileRange {
pub family: u32,
pub min_hash: u64,
pub max_hash: u64,
}
pub struct MetaFile {
/// The database path
db_path: PathBuf,
/// The sequence number of this file.
sequence_number: u32,
/// The key family of the SST files in this meta file.
family: u32,
/// The entries of the file.
entries: Vec<MetaEntry>,
/// The obsolete SST files.
obsolete_sst_files: Vec<u32>,
/// The memory mapped file.
mmap: Mmap,
}
impl MetaFile {
/// Opens a meta file at the given path. This memory maps the file, but does not read it yet.
/// It's lazy read on demand.
pub fn open(db_path: &Path, sequence_number: u32) -> Result<Self> {
let filename = format!("{sequence_number:08}.meta");
let path = db_path.join(&filename);
Self::open_internal(db_path.to_path_buf(), sequence_number, &path)
.with_context(|| format!("Unable to open meta file {filename}"))
}
fn open_internal(db_path: PathBuf, sequence_number: u32, path: &Path) -> Result<Self> {
let mut file = BufReader::new(File::open(path)?);
let magic = file.read_u32::<BE>()?;
if magic != 0xFE4ADA4A {
bail!("Invalid magic number");
}
let family = file.read_u32::<BE>()?;
let obsolete_count = file.read_u32::<BE>()?;
let mut obsolete_sst_files = Vec::with_capacity(obsolete_count as usize);
for _ in 0..obsolete_count {
let obsolete_sst = file.read_u32::<BE>()?;
obsolete_sst_files.push(obsolete_sst);
}
let count = file.read_u32::<BE>()?;
let mut entries = Vec::with_capacity(count as usize);
let mut start_of_aqmf_data_offset = 0;
for _ in 0..count {
let entry = MetaEntry {
sst_data: StaticSortedFileMetaData {
sequence_number: file.read_u32::<BE>()?,
key_compression_dictionary_length: file.read_u16::<BE>()?,
value_compression_dictionary_length: file.read_u16::<BE>()?,
block_count: file.read_u16::<BE>()?,
},
family,
min_hash: file.read_u64::<BE>()?,
max_hash: file.read_u64::<BE>()?,
size: file.read_u64::<BE>()?,
start_of_aqmf_data_offset,
end_of_aqmf_data_offset: file.read_u32::<BE>()?,
aqmf: OnceLock::new(),
sst: OnceLock::new(),
};
start_of_aqmf_data_offset = entry.end_of_aqmf_data_offset;
entries.push(entry);
}
let offset = file.stream_position()?;
let file = file.into_inner();
let mut options = MmapOptions::new();
options.offset(offset);
let mmap = unsafe { options.map(&file)? };
#[cfg(unix)]
mmap.advise(memmap2::Advice::Random)?;
let file = Self {
db_path,
sequence_number,
family,
entries,
obsolete_sst_files,
mmap,
};
Ok(file)
}
pub fn sequence_number(&self) -> u32 {
self.sequence_number
}
pub fn family(&self) -> u32 {
self.family
}
pub fn entries(&self) -> &[MetaEntry] {
&self.entries
}
pub fn entry(&self, index: u32) -> &MetaEntry {
let index = index as usize;
&self.entries[index]
}
pub fn aqmf_data(&self) -> &[u8] {
&self.mmap
}
pub fn retain_entries(&mut self, mut predicate: impl FnMut(u32) -> bool) -> bool {
let old_len = self.entries.len();
self.entries
.retain(|entry| predicate(entry.sst_data.sequence_number));
old_len != self.entries.len()
}
pub fn has_active_entries(&self) -> bool {
!self.entries.is_empty()
}
pub fn obsolete_sst_files(&self) -> &[u32] {
&self.obsolete_sst_files
}
pub fn lookup<K: QueryKey>(
&self,
key_family: u32,
key_hash: u64,
key: &K,
aqmf_cache: &AqmfCache,
key_block_cache: &BlockCache,
value_block_cache: &BlockCache,
) -> Result<MetaLookupResult> {
if key_family != self.family {
return Ok(MetaLookupResult::FamilyMiss);
}
let mut miss_result = MetaLookupResult::RangeMiss;
for entry in self.entries.iter() {
if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue;
}
let use_aqmf_cache = entry.max_hash - entry.min_hash < 1 << 60;
if use_aqmf_cache {
let aqmf = match aqmf_cache.get_value_or_guard(&entry.sequence_number(), None) {
GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => {
let aqmf = entry.aqmf(self.aqmf_data());
let aqmf: Arc<qfilter::Filter> = Arc::new(pot::from_slice(aqmf)?);
let _ = guard.insert(aqmf.clone());
aqmf
}
GuardResult::Timeout => unreachable!(),
};
if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
} else {
let aqmf = entry.aqmf.get_or_try_init(|| {
let aqmf = entry.aqmf(self.aqmf_data());
anyhow::Ok(pot::from_slice(aqmf)?)
})?;
if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
};
let result = entry.sst(&self.db_path)?.lookup(
key_hash,
key,
key_block_cache,
value_block_cache,
)?;
if !matches!(result, SstLookupResult::NotFound) {
return Ok(MetaLookupResult::SstLookup(result));
}
}
Ok(miss_result)
}
}
use std::{
fs::File,
io::{self, BufWriter, Write},
path::Path,
};
use byteorder::{BE, WriteBytesExt};
use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta;
pub struct MetaFileBuilder {
family: u32,
/// Entries in the meta file, tuples of (sequence_number, StaticSortedFileBuilderMetaResult)
entries: Vec<(u32, StaticSortedFileBuilderMeta)>,
/// Obsolete SST files, represented by their sequence numbers
obsolete_sst_files: Vec<u32>,
}
impl MetaFileBuilder {
pub fn new(family: u32) -> Self {
Self {
family,
entries: Vec::new(),
obsolete_sst_files: Vec::new(),
}
}
pub fn add(&mut self, sequence_number: u32, sst: StaticSortedFileBuilderMeta) {
self.entries.push((sequence_number, sst));
}
pub fn add_obsolete_sst_file(&mut self, sequence_number: u32) {
self.obsolete_sst_files.push(sequence_number);
}
#[tracing::instrument(level = "trace", skip_all)]
pub fn write(&self, file: &Path) -> io::Result<File> {
let mut file = BufWriter::new(File::create(file)?);
file.write_u32::<BE>(0xFE4ADA4A)?; // Magic number
file.write_u32::<BE>(self.family)?;
file.write_u32::<BE>(self.obsolete_sst_files.len() as u32)?;
for obsolete_sst in &self.obsolete_sst_files {
file.write_u32::<BE>(*obsolete_sst)?;
}
file.write_u32::<BE>(self.entries.len() as u32)?;
let mut aqmf_offset = 0;
for (sequence_number, sst) in &self.entries {
file.write_u32::<BE>(*sequence_number)?;
file.write_u16::<BE>(sst.key_compression_dictionary_length)?;
file.write_u16::<BE>(sst.value_compression_dictionary_length)?;
file.write_u16::<BE>(sst.block_count)?;
file.write_u64::<BE>(sst.min_hash)?;
file.write_u64::<BE>(sst.max_hash)?;
file.write_u64::<BE>(sst.size)?;
aqmf_offset += sst.aqmf.len();
file.write_u32::<BE>(aqmf_offset as u32)?;
}
for (_, sst) in &self.entries {
file.write_all(&sst.aqmf)?;
}
Ok(file.into_inner()?)
}
}
use std::collections::hash_map::Entry;
use rustc_hash::FxHashMap;
use crate::meta_file::MetaFile;
enum SstState {
Active,
UnusedObsolete,
Obsolete,
}
pub struct SstFilter(FxHashMap<u32, SstState>);
impl SstFilter {
pub fn new() -> Self {
Self(FxHashMap::default())
}
/// Phase 1: Apply the filter to the meta file and update the state in the filter.
pub fn apply_filter(&mut self, meta: &mut MetaFile) {
meta.retain_entries(|seq| match self.0.entry(seq) {
Entry::Occupied(mut e) => {
let state = e.get_mut();
match state {
SstState::Active => false,
SstState::UnusedObsolete => {
*state = SstState::Obsolete;
false
}
SstState::Obsolete => false,
}
}
Entry::Vacant(e) => {
e.insert(SstState::Active);
true
}
});
for seq in meta.obsolete_sst_files() {
self.0.entry(*seq).or_insert(SstState::UnusedObsolete);
}
}
/// Phase 2: Check if the meta file can be removed based on the filter state after phase 1.
/// Updates the filter state for the next meta file. Returns true if the meta file can be
/// removed.
pub fn apply_and_get_remove(&mut self, meta: &MetaFile) -> bool {
let mut used = false;
for seq in meta.obsolete_sst_files() {
match self.0.entry(*seq) {
Entry::Occupied(e) => match e.get() {
SstState::Active => {}
SstState::UnusedObsolete => {
e.remove();
}
SstState::Obsolete => {
e.remove();
used = true;
}
},
Entry::Vacant(e) => {
e.insert(SstState::UnusedObsolete);
}
}
}
!used && !meta.has_active_entries()
}
}
use std::{
cmp::min,
fs::File,
io::{self, BufWriter, Write},
io::{self, BufWriter, Seek, Write},
path::Path,
};
......@@ -71,29 +71,48 @@ pub enum EntryValue<'l> {
Deleted,
}
#[derive(Debug, Clone)]
pub struct StaticSortedFileBuilderMeta {
/// The minimum hash of the keys in the SST file
pub min_hash: u64,
/// The maximum hash of the keys in the SST file
pub max_hash: u64,
/// The AQMF data
pub aqmf: Vec<u8>,
/// The key compression dictionary
pub key_compression_dictionary_length: u16,
/// The value compression dictionary
pub value_compression_dictionary_length: u16,
/// The number of blocks in the SST file
pub block_count: u16,
/// The file size of the SST file
pub size: u64,
/// The number of entries in the SST file
pub entries: u64,
}
#[derive(Debug, Default)]
pub struct StaticSortedFileBuilder {
family: u32,
aqmf: Vec<u8>,
key_compression_dictionary: Vec<u8>,
value_compression_dictionary: Vec<u8>,
blocks: Vec<(u32, Vec<u8>)>,
min_hash: u64,
max_hash: u64,
entries: u64,
}
impl StaticSortedFileBuilder {
pub fn new<E: Entry>(
family: u32,
entries: &[E],
total_key_size: usize,
total_value_size: usize,
) -> Result<Self> {
debug_assert!(entries.iter().map(|e| e.key_hash()).is_sorted());
let mut builder = Self {
family,
min_hash: entries.first().map(|e| e.key_hash()).unwrap_or(u64::MAX),
max_hash: entries.last().map(|e| e.key_hash()).unwrap_or(0),
entries: entries.len() as u64,
..Default::default()
};
builder.compute_aqmf(entries);
......@@ -114,6 +133,9 @@ impl StaticSortedFileBuilder {
// This can't fail as we allocated enough capacity
.expect("AQMF insert failed");
}
for entry in entries {
debug_assert!(filter.contains_fingerprint(entry.key_hash()));
}
self.aqmf = pot::to_vec(&filter).expect("AQMF serialization failed");
}
......@@ -368,27 +390,8 @@ impl StaticSortedFileBuilder {
/// Writes the SST file.
#[tracing::instrument(level = "trace", skip_all)]
pub fn write(&self, file: &Path) -> io::Result<File> {
pub fn write(self, file: &Path) -> io::Result<(StaticSortedFileBuilderMeta, File)> {
let mut file = BufWriter::new(File::create(file)?);
// magic number and version
file.write_u32::<BE>(0x53535401)?;
// family
file.write_u32::<BE>(self.family)?;
// min hash
file.write_u64::<BE>(self.min_hash)?;
// max hash
file.write_u64::<BE>(self.max_hash)?;
// AQMF length
file.write_u24::<BE>(self.aqmf.len().try_into().unwrap())?;
// Key compression dictionary length
file.write_u16::<BE>(self.key_compression_dictionary.len().try_into().unwrap())?;
// Value compression dictionary length
file.write_u16::<BE>(self.value_compression_dictionary.len().try_into().unwrap())?;
// Number of blocks
file.write_u16::<BE>(self.blocks.len().try_into().unwrap())?;
// Write the AQMF
file.write_all(&self.aqmf)?;
// Write the key compression dictionary
file.write_all(&self.key_compression_dictionary)?;
// Write the value compression dictionary
......@@ -402,13 +405,32 @@ impl StaticSortedFileBuilder {
offset += len;
file.write_u32::<BE>(offset.try_into().unwrap())?;
}
for (uncompressed_size, block) in &self.blocks {
let block_count = self.blocks.len().try_into().unwrap();
for (uncompressed_size, block) in self.blocks {
// Uncompressed size
file.write_u32::<BE>(*uncompressed_size)?;
file.write_u32::<BE>(uncompressed_size)?;
// Compressed block
file.write_all(block)?;
file.write_all(&block)?;
}
Ok(file.into_inner()?)
let meta = StaticSortedFileBuilderMeta {
min_hash: self.min_hash,
max_hash: self.max_hash,
aqmf: self.aqmf,
key_compression_dictionary_length: self
.key_compression_dictionary
.len()
.try_into()
.unwrap(),
value_compression_dictionary_length: self
.value_compression_dictionary
.len()
.try_into()
.unwrap(),
block_count,
size: file.stream_position()?,
entries: self.entries,
};
Ok((meta, file.into_inner()?))
}
}
......
......@@ -455,7 +455,7 @@ fn persist_changes() -> Result<()> {
{
let db = TurboPersistence::open(path.to_path_buf())?;
db.compact(1.0, 3, usize::MAX)?;
db.compact(1.0, 3, u64::MAX)?;
check(&db, 1, 13)?;
check(&db, 2, 22)?;
......
......@@ -4,7 +4,7 @@ use std::{
io::Write,
mem::{replace, take},
path::PathBuf,
sync::atomic::{AtomicU32, Ordering},
sync::atomic::{AtomicU32, AtomicU64, Ordering},
};
use anyhow::{Context, Result};
......@@ -25,7 +25,8 @@ use crate::{
collector_entry::CollectorEntry,
constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
key::StoreKey,
static_sorted_file_builder::StaticSortedFileBuilder,
meta_file_builder::MetaFileBuilder,
static_sorted_file_builder::{StaticSortedFileBuilder, StaticSortedFileBuilderMeta},
};
/// The thread local state of a `WriteBatch`. `FAMILIES` should fit within a `u32`.
......@@ -49,9 +50,13 @@ const COLLECTOR_SHARD_SHIFT: usize =
pub(crate) struct FinishResult {
pub(crate) sequence_number: u32,
/// Tuple of (sequence number, file).
pub(crate) new_meta_files: Vec<(u32, File)>,
/// Tuple of (sequence number, file).
pub(crate) new_sst_files: Vec<(u32, File)>,
/// Tuple of (sequence number, file).
pub(crate) new_blob_files: Vec<(u32, File)>,
/// Number of keys written in this batch.
pub(crate) keys_written: u64,
}
enum GlobalCollectorState<K: StoreKey + Send> {
......@@ -65,13 +70,15 @@ enum GlobalCollectorState<K: StoreKey + Send> {
/// A write batch.
pub struct WriteBatch<K: StoreKey + Send, const FAMILIES: usize> {
/// The database path
path: PathBuf,
db_path: PathBuf,
/// The current sequence number counter. Increased for every new SST file or blob file.
current_sequence_number: AtomicU32,
/// The thread local state.
thread_locals: ThreadLocal<SyncUnsafeCell<ThreadLocalState<K, FAMILIES>>>,
/// Collectors in use. The thread local collectors flush into these when they are full.
collectors: [Mutex<GlobalCollectorState<K>>; FAMILIES],
/// Meta file builders for each family.
meta_collectors: [Mutex<Vec<(u32, StaticSortedFileBuilderMeta)>>; FAMILIES],
/// The list of new SST files that have been created.
/// Tuple of (sequence number, file).
new_sst_files: Mutex<Vec<(u32, File)>>,
......@@ -88,11 +95,12 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
assert!(FAMILIES <= usize_from_u32(u32::MAX));
};
Self {
path,
db_path: path,
current_sequence_number: AtomicU32::new(current),
thread_locals: ThreadLocal::new(),
collectors: [(); FAMILIES]
.map(|_| Mutex::new(GlobalCollectorState::Unsharded(Collector::new()))),
meta_collectors: [(); FAMILIES].map(|_| Mutex::new(Vec::new())),
new_sst_files: Mutex::new(Vec::new()),
idle_collectors: Mutex::new(Vec::new()),
idle_thread_local_collectors: Mutex::new(Vec::new()),
......@@ -369,12 +377,42 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
})?;
shared_error.into_inner()?;
// Not we need to write the new meta files.
let new_meta_collectors = [(); FAMILIES].map(|_| Mutex::new(Vec::new()));
let meta_collectors = replace(&mut self.meta_collectors, new_meta_collectors);
let keys_written = AtomicU64::new(0);
let new_meta_files = meta_collectors
.into_par_iter()
.map(|mutex| mutex.into_inner())
.enumerate()
.filter(|(_, sst_files)| !sst_files.is_empty())
.map(|(family, sst_files)| {
let family = family as u32;
let mut entries = 0;
let mut builder = MetaFileBuilder::new(family);
for (seq, sst) in sst_files {
entries += sst.entries;
builder.add(seq, sst);
}
keys_written.fetch_add(entries, Ordering::Relaxed);
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let file = self.db_path.join(format!("{seq:08}.meta"));
let file = builder
.write(&file)
.with_context(|| format!("Unable to write meta file {seq:08}.meta"))?;
Ok((seq, file))
})
.collect::<Result<Vec<_>>>()?;
// Finally we return the new files and sequence number.
let seq = self.current_sequence_number.load(Ordering::SeqCst);
new_sst_files.sort_unstable_by_key(|(seq, _)| *seq);
Ok(FinishResult {
sequence_number: seq,
new_meta_files,
new_sst_files,
new_blob_files,
keys_written: keys_written.into_inner(),
})
}
......@@ -388,7 +426,7 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
lz4::compress_to_vec(value, &mut buffer, ACC_LEVEL_DEFAULT)
.context("Compression of value for blob file failed")?;
let file = self.path.join(format!("{seq:08}.blob"));
let file = self.db_path.join(format!("{seq:08}.blob"));
let mut file = File::create(&file).context("Unable to create blob file")?;
file.write_all(&buffer)
.context("Unable to write blob file")?;
......@@ -407,11 +445,10 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
let (entries, total_key_size, total_value_size) = collector_data;
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let builder =
StaticSortedFileBuilder::new(family, entries, total_key_size, total_value_size)?;
let builder = StaticSortedFileBuilder::new(entries, total_key_size, total_value_size)?;
let path = self.path.join(format!("{seq:08}.sst"));
let file = builder
let path = self.db_path.join(format!("{seq:08}.sst"));
let (meta, file) = builder
.write(&path)
.with_context(|| format!("Unable to write SST file {seq:08}.sst"))?;
......@@ -422,19 +459,23 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
use crate::{
collector_entry::CollectorEntryValue,
key::hash_key,
static_sorted_file::{AqmfCache, BlockCache, LookupResult, StaticSortedFile},
lookup_entry::LookupValue,
static_sorted_file::{
BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData,
},
static_sorted_file_builder::Entry,
};
file.sync_all()?;
let sst = StaticSortedFile::open(seq, path)?;
let cache1 = AqmfCache::with(
10,
u64::MAX,
Default::default(),
Default::default(),
Default::default(),
);
let sst = StaticSortedFile::open(
&self.db_path,
StaticSortedFileMetaData {
sequence_number: seq,
key_compression_dictionary_length: meta.key_compression_dictionary_length,
value_compression_dictionary_length: meta.value_compression_dictionary_length,
block_count: meta.block_count,
},
)?;
let cache2 = BlockCache::with(
10,
u64::MAX,
......@@ -453,21 +494,14 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
for entry in entries {
entry.write_key_to(&mut key_buf);
let result = sst
.lookup(
family,
hash_key(&key_buf),
&key_buf,
&cache1,
&cache2,
&cache3,
)
.lookup(hash_key(&key_buf), &key_buf, &cache2, &cache3)
.expect("key found");
key_buf.clear();
match result {
LookupResult::Deleted => {}
LookupResult::Slice {
SstLookupResult::Found(LookupValue::Deleted) => {}
SstLookupResult::Found(LookupValue::Slice {
value: lookup_value,
} => {
}) => {
let expected_value_slice = match &entry.value {
CollectorEntryValue::Small { value } => &**value,
CollectorEntryValue::Medium { value } => &**value,
......@@ -475,14 +509,16 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
};
assert_eq!(*lookup_value, *expected_value_slice);
}
LookupResult::Blob { sequence_number: _ } => {}
LookupResult::QuickFilterMiss => panic!("aqmf must include"),
LookupResult::RangeMiss => panic!("Index must cover"),
LookupResult::KeyMiss => panic!("All keys must exist"),
SstLookupResult::Found(LookupValue::Blob { sequence_number: _ }) => {}
SstLookupResult::NotFound => panic!("All keys must exist"),
}
}
}
self.meta_collectors[usize_from_u32(family)]
.lock()
.push((seq, meta));
Ok((seq, file))
}
}
......
......@@ -15,7 +15,7 @@ use crate::database::{
const COMPACT_MAX_COVERAGE: f32 = 20.0;
const COMPACT_MAX_MERGE_SEQUENCE: usize = 64;
const COMPACT_MAX_MERGE_SIZE: usize = 512 * 1024 * 1024; // 512 MiB
const COMPACT_MAX_MERGE_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB
pub struct TurboKeyValueDatabase {
db: Arc<TurboPersistence>,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment