Unverified Commit 22fffea9 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

perf(turbo-persistence): shard meta files by family (#97703)

## Summary

- Store live meta files in primary per-family shards so queries avoid
unrelated-family metadata
- Consolidate live metadata during compaction, retire subsumed meta
files in the same commit, and retain merged used-key-hash AMQFs.
- Add debug assertions for shard ownership/order and compaction
precedence invariants.

## Performance

Paired `LARGE_DB=1` Criterion binaries were run interleaved on the same
machine to reduce host drift.

| Scenario | Canary | PR | Result |
| --- | ---: | ---: | --- |
| Four-family cached `get` miss (100 metas/family) | 2.40–2.55 µs |
2.11–2.21 µs | **~10–12% faster** |
| Existing single-family cached `get` miss | 1.93–2.12 µs | 2.00–2.04 µs
| Neutral within run-to-run noise (~+0.5% on means) |
| Existing full compaction (4 Mi entries, 8 commits) | ~624 ms | ~624 ms
| Neutral in the stable paired run; repeated samples stayed within
normal noise |

Read hits should see a small improvement but it isn't measurable, misses
are the major benefit.

## Verification

- `cargo test -p turbo-persistence`
- `cargo test -p turbo-persistence --features stats`
- `cargo clippy -p turbo-persistence --all-targets --features stats --
-D warnings`
- `LARGE_DB=1` interleaved Criterion A/B runs

<!-- NEXT_JS_LLM -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 0f69132c
...@@ -254,10 +254,11 @@ The checksum is verified on the compressed data **before** decompression when th ...@@ -254,10 +254,11 @@ The checksum is verified on the compressed data **before** decompression when th
## Reading ## Reading
Reading start from the current sequence number and goes downwards. Opened meta files are stored in per-family shards. A lookup scans only the requested family's meta
files, from newest to oldest; there is no ordering dependency between families.
- We have all SST files memory mapped - We have all SST files memory mapped
- for i = CURRENT sequence number .. 0 - for each meta file of the queried key family, newest first
- Check AMQF from SST file for key existence -> if not continue - Check AMQF from SST file for key existence -> if not continue
- let block = 0 - let block = 0
- loop - loop
...@@ -325,6 +326,10 @@ Since the process might exit unexpectedly, to avoid "forgetting" to delete the S ...@@ -325,6 +326,10 @@ Since the process might exit unexpectedly, to avoid "forgetting" to delete the S
We limit the number of SST files that are merged at once to avoid long compactions. We limit the number of SST files that are merged at once to avoid long compactions.
Compaction keeps meta files incremental: a new meta file only describes SST files that were merged
or moved. Metadata for untouched SST files stays in its existing meta file. When every active entry
in an old meta file is superseded, that meta file is retired in the same compaction commit.
Full example: Full example:
Example: Example:
......
...@@ -840,6 +840,83 @@ fn bench_read_get_multiple(c: &mut Criterion) { ...@@ -840,6 +840,83 @@ fn bench_read_get_multiple(c: &mut Criterion) {
// Compaction Benchmarks // Compaction Benchmarks
// ============================================================================= // =============================================================================
fn family_benchmark_key(family: u32, commit: u32, item: u32) -> [u8; 12] {
let mut key = [0; 12];
key[..4].copy_from_slice(&family.to_be_bytes());
key[4..8].copy_from_slice(&commit.to_be_bytes());
key[8..].copy_from_slice(&item.to_be_bytes());
key
}
fn bench_family_sharding(c: &mut Criterion) {
const FAMILIES: usize = 4;
const COMMITS: u32 = 100;
let entries_per_commit = scaled(1_000);
let db = LazyLock::new(|| {
let tempdir = tempfile::tempdir().unwrap();
let config = TpDbConfig {
family_configs: ["family-0", "family-1", "family-2", "family-3"].map(|name| {
FamilyConfig {
name,
kind: FamilyKind::SingleValue,
compression: Compression::Lz4,
}
}),
..TpDbConfig::new()
};
let db = TurboPersistence::<SerialScheduler, FAMILIES>::open_with_config(
tempdir.path().to_path_buf(),
config,
)
.unwrap();
for commit in 0..COMMITS {
for family in 0..FAMILIES as u32 {
let batch = db.write_batch().unwrap();
for item in 0..entries_per_commit as u32 {
batch
.put(
family,
family_benchmark_key(family, commit, item),
item.to_be_bytes().to_vec().into(),
)
.unwrap();
}
db.commit_write_batch(batch).unwrap();
}
}
(tempdir, db)
});
let mut group = c.benchmark_group("read/family_sharding");
group.measurement_time(Duration::from_secs(5));
let hit = family_benchmark_key(2, COMMITS - 1, 0);
let miss = family_benchmark_key(2, COMMITS, 0);
let batch_hits = (0..64)
.map(|item| family_benchmark_key(2, COMMITS - 1, item))
.collect::<Vec<_>>();
let batch_misses = (0..64)
.map(|item| family_benchmark_key(2, COMMITS, item))
.collect::<Vec<_>>();
group.bench_function("get/hit", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.get(2, black_box(&hit)).unwrap()))
});
group.bench_function("get/miss", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.get(2, black_box(&miss)).unwrap()))
});
group.bench_function("batch_get/hit_64", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.batch_get(2, black_box(&batch_hits)).unwrap()))
});
group.bench_function("batch_get/miss_64", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.batch_get(2, black_box(&batch_misses)).unwrap()))
});
group.finish();
}
fn bench_compaction(c: &mut Criterion) { fn bench_compaction(c: &mut Criterion) {
let mut group = c.benchmark_group("compaction"); let mut group = c.benchmark_group("compaction");
// Compaction is expensive, reduce sample size // Compaction is expensive, reduce sample size
...@@ -1499,6 +1576,6 @@ fn bench_block_cache(c: &mut Criterion) { ...@@ -1499,6 +1576,6 @@ fn bench_block_cache(c: &mut Criterion) {
criterion_group!( criterion_group!(
name = benches; name = benches;
config = Criterion::default(); config = Criterion::default();
targets = bench_write, bench_write_multi_value, bench_read_get, bench_read_batch_get, bench_read_get_multiple, bench_compaction, bench_compaction_multi_value, bench_qfilter, bench_static_sorted_file_lookup, bench_block_cache targets = bench_write, bench_write_multi_value, bench_read_get, bench_read_batch_get, bench_read_get_multiple, bench_family_sharding, bench_compaction, bench_compaction_multi_value, bench_qfilter, bench_static_sorted_file_lookup, bench_block_cache
); );
criterion_main!(benches); criterion_main!(benches);
...@@ -309,7 +309,7 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> { ...@@ -309,7 +309,7 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
/// The inner state of the database. Writing will update that. /// The inner state of the database. Writing will update that.
inner: RwLock<Inner<FAMILIES>>, inner: RwLock<Inner<FAMILIES>>,
/// A flag to indicate if the database is empty (no meta files). This is an atomic mirror of /// A flag to indicate if the database is empty (no meta files). This is an atomic mirror of
/// `inner.meta_files.is_empty()` to avoid taking a lock on the hot path. /// `inner.is_empty()` to avoid taking a lock on the hot path.
is_empty: AtomicBool, is_empty: AtomicBool,
/// Tracks whether a write operation is in progress or has permanently failed. /// Tracks whether a write operation is in progress or has permanently failed.
/// `None` = idle, `Some(Active)` = in progress, `Some(Error)` = permanently disabled. /// `None` = idle, `Some(Active)` = in progress, `Some(Error)` = permanently disabled.
...@@ -334,8 +334,10 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> { ...@@ -334,8 +334,10 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
/// The inner state of the database. /// The inner state of the database.
struct Inner<const FAMILIES: usize> { struct Inner<const FAMILIES: usize> {
/// The list of meta files in the database. This is used to derive the SST files. /// The list of meta files in the database sharded by family. This is used to derive the SST
meta_files: Vec<MetaFile>, /// files. Each family's files are in ascending sequence order; there are no ordering
/// constraints across families.
meta_files_by_family: [Vec<MetaFile>; FAMILIES],
/// The current sequence number for the database. /// The current sequence number for the database.
current_sequence_number: u32, current_sequence_number: u32,
/// The in progress set of hashes of keys that have been accessed. /// The in progress set of hashes of keys that have been accessed.
...@@ -344,6 +346,43 @@ struct Inner<const FAMILIES: usize> { ...@@ -344,6 +346,43 @@ struct Inner<const FAMILIES: usize> {
accessed_key_hashes: [DashSet<u64, BuildNoHashHasher<u64>>; FAMILIES], accessed_key_hashes: [DashSet<u64, BuildNoHashHasher<u64>>; FAMILIES],
} }
impl<const FAMILIES: usize> Inner<FAMILIES> {
fn is_empty(&self) -> bool {
self.meta_files_by_family.iter().all(Vec::is_empty)
}
fn push_meta_file(&mut self, meta_file: MetaFile) {
let family = meta_file.family() as usize;
debug_assert!(family < FAMILIES, "meta file family is out of bounds");
let shard = &mut self.meta_files_by_family[family];
debug_assert!(
shard.last().is_none_or(|previous| {
previous.sequence_number() < meta_file.sequence_number()
}),
"meta file appended out of sequence order for family {family}"
);
shard.push(meta_file);
}
#[cfg(debug_assertions)]
fn debug_assert_meta_invariants(&self) {
for (family, meta_files) in self.meta_files_by_family.iter().enumerate() {
debug_assert!(
meta_files
.iter()
.all(|meta| meta.family() as usize == family),
"meta file stored in the wrong family shard"
);
debug_assert!(
meta_files
.windows(2)
.all(|pair| pair[0].sequence_number() < pair[1].sequence_number()),
"meta files in family {family} are not in ascending sequence order"
);
}
}
}
pub struct CommitOptions { pub struct CommitOptions {
new_meta_files: Vec<NewFile>, new_meta_files: Vec<NewFile>,
new_sst_files: Vec<NewFile>, new_sst_files: Vec<NewFile>,
...@@ -442,7 +481,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -442,7 +481,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
path, path,
read_only, read_only,
inner: RwLock::new(Inner { inner: RwLock::new(Inner {
meta_files: Vec::new(), meta_files_by_family: [(); FAMILIES].map(|_| Vec::new()),
current_sequence_number: 0, current_sequence_number: 0,
accessed_key_hashes: [(); FAMILIES] accessed_key_hashes: [(); FAMILIES]
.map(|_| DashSet::with_hasher(BuildNoHashHasher::default())), .map(|_| DashSet::with_hasher(BuildNoHashHasher::default())),
...@@ -650,10 +689,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -650,10 +689,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
let inner = self.inner.get_mut(); let inner = self.inner.get_mut();
self.is_empty for meta_file in meta_files {
.store(meta_files.is_empty(), Ordering::Relaxed); inner.push_meta_file(meta_file);
inner.meta_files = meta_files; }
#[cfg(debug_assertions)]
inner.debug_assert_meta_invariants();
inner.current_sequence_number = current; inner.current_sequence_number = current;
self.is_empty.store(inner.is_empty(), Ordering::Relaxed);
Ok(true) Ok(true)
} }
...@@ -797,7 +839,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -797,7 +839,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Clears all caches of the database. /// Clears all caches of the database.
pub fn clear_cache(&self) { pub fn clear_cache(&self) {
self.clear_block_caches(); self.clear_block_caches();
for meta in self.inner.write().meta_files.iter_mut() { for meta in self.inner.write().meta_files_by_family.iter_mut().flatten() {
meta.clear_cache(); meta.clear_cache();
} }
} }
...@@ -816,7 +858,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -816,7 +858,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Prefetches all SST files which are usually lazy loaded. This can be used to reduce latency /// Prefetches all SST files which are usually lazy loaded. This can be used to reduce latency
/// for the first queries after opening the database. /// for the first queries after opening the database.
pub fn prepare_all_sst_caches(&self) { pub fn prepare_all_sst_caches(&self) {
for meta in self.inner.write().meta_files.iter_mut() { for meta in self.inner.write().meta_files_by_family.iter_mut().flatten() {
meta.prepare_sst_cache(); meta.prepare_sst_cache();
} }
} }
...@@ -1012,8 +1054,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1012,8 +1054,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// in-memory mutations. The MetaFile in-memory optimization // in-memory mutations. The MetaFile in-memory optimization
// (retain_entries) is deferred to Phase C. // (retain_entries) is deferred to Phase C.
let has_delete_file; let has_delete_file;
let mut meta_seq_numbers_to_delete = Vec::new(); let mut meta_seq_numbers_to_delete = [(); FAMILIES].map(|_| Vec::new());
let entries_to_remove; let mut entries_to_remove = [(); FAMILIES].map(|_| Vec::new());
// Deleted SST bytes: the caller knows each deleted SST's size when it decides to delete it, // Deleted SST bytes: the caller knows each deleted SST's size when it decides to delete it,
// so it's carried on `DeletedFile` and summed here (no scan, no stat). // so it's carried on `DeletedFile` and summed here (no scan, no stat).
stats.bytes_deleted += sst_files_to_delete.iter().map(|f| f.size).sum::<u64>(); stats.bytes_deleted += sst_files_to_delete.iter().map(|f| f.size).sum::<u64>();
...@@ -1026,17 +1068,17 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1026,17 +1068,17 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
{ {
let inner = self.inner.read(); let inner = self.inner.read();
// (A1) Run the SST filter on existing meta files. This only // (A1) Run the SST filter on existing meta files. This only updates filter state; the
// updates the SstFilter state — the MetaFile in-memory layout is // MetaFile mutation is deferred to Phase C. Each family's removal list is newest-first,
// not modified yet (that happens in Phase C via retain_entries). // matching the filter's required recency order.
// Collects the set of SST entry sequence numbers to remove from for (family, meta_files) in inner.meta_files_by_family.iter().enumerate() {
// each meta file, keyed by position in `inner.meta_files`. entries_to_remove[family].extend(
entries_to_remove = inner meta_files
.meta_files .iter()
.iter() .rev()
.rev() .map(|meta_file| sst_filter.apply_filter_collect(meta_file)),
.map(|meta_file| sst_filter.apply_filter_collect(meta_file)) );
.collect::<Vec<_>>(); }
// (A2) Determine which meta files are fully obsolete by running // (A2) Determine which meta files are fully obsolete by running
// `apply_and_get_remove` in newest-first order. Process new metas // `apply_and_get_remove` in newest-first order. Process new metas
...@@ -1051,11 +1093,15 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1051,11 +1093,15 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
"newly created meta file should never be a candidate for removal" "newly created meta file should never be a candidate for removal"
); );
} }
for i in (0..inner.meta_files.len()).rev() { for (family, meta_files) in inner.meta_files_by_family.iter().enumerate() {
if sst_filter.apply_and_get_remove(&inner.meta_files[i]) { for i in (0..meta_files.len()).rev() {
meta_seq_numbers_to_delete.push(inner.meta_files[i].sequence_number()); // Removal lists are newest-first while each shard is oldest-first.
// Deleted meta bytes, read from the `MetaFile`'s mmap length (no stat). let to_remove = &entries_to_remove[family][meta_files.len() - 1 - i];
stats.bytes_deleted += inner.meta_files[i].byte_size(); if sst_filter.apply_and_get_remove_after_removing(&meta_files[i], to_remove) {
meta_seq_numbers_to_delete[family].push(meta_files[i].sequence_number());
// Deleted meta bytes, read from the `MetaFile`'s mmap length (no stat).
stats.bytes_deleted += meta_files[i].byte_size();
}
} }
} }
...@@ -1064,7 +1110,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1064,7 +1110,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// delete, which consumes one extra sequence number. // delete, which consumes one extra sequence number.
has_delete_file = !sst_files_to_delete.is_empty() has_delete_file = !sst_files_to_delete.is_empty()
|| !blob_seq_numbers_to_delete.is_empty() || !blob_seq_numbers_to_delete.is_empty()
|| !meta_seq_numbers_to_delete.is_empty(); || meta_seq_numbers_to_delete
.iter()
.any(|seqs| !seqs.is_empty());
} }
// Deleted blob bytes. Unlike SST/meta sizes (both already in memory), blob sizes aren't // Deleted blob bytes. Unlike SST/meta sizes (both already in memory), blob sizes aren't
...@@ -1088,19 +1136,24 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1088,19 +1136,24 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
self.parallel_scheduler.block_in_place(|| { self.parallel_scheduler.block_in_place(|| {
if has_delete_file { if has_delete_file {
sst_seq_numbers_to_delete.sort_unstable(); sst_seq_numbers_to_delete.sort_unstable();
meta_seq_numbers_to_delete.sort_unstable(); for seqs in &mut meta_seq_numbers_to_delete {
seqs.sort_unstable();
}
blob_seq_numbers_to_delete.sort_unstable(); blob_seq_numbers_to_delete.sort_unstable();
// Write *.del file, marking the selected files as to delete // Write *.del file, marking the selected files as to delete
let mut buf = Vec::with_capacity( let mut buf = Vec::with_capacity(
(sst_seq_numbers_to_delete.len() (sst_seq_numbers_to_delete.len()
+ meta_seq_numbers_to_delete.len() + meta_seq_numbers_to_delete
.iter()
.map(Vec::len)
.sum::<usize>()
+ blob_seq_numbers_to_delete.len()) + blob_seq_numbers_to_delete.len())
* size_of::<u32>(), * size_of::<u32>(),
); );
for seq in sst_seq_numbers_to_delete.iter() { for seq in sst_seq_numbers_to_delete.iter() {
buf.write_u32::<BE>(*seq)?; buf.write_u32::<BE>(*seq)?;
} }
for seq in meta_seq_numbers_to_delete.iter() { for seq in meta_seq_numbers_to_delete.iter().flatten() {
buf.write_u32::<BE>(*seq)?; buf.write_u32::<BE>(*seq)?;
} }
for seq in blob_seq_numbers_to_delete.iter() { for seq in blob_seq_numbers_to_delete.iter() {
...@@ -1186,12 +1239,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1186,12 +1239,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
"SST DELETED", "SST DELETED",
|&seq| seq, |&seq| seq,
)?; )?;
write_seq_numbers( for seqs in &meta_seq_numbers_to_delete {
&mut log, write_seq_numbers(&mut log, seqs, "META DELETED", |&seq| seq)?;
&meta_seq_numbers_to_delete, }
"META DELETED",
|&seq| seq,
)?;
anyhow::Ok(()) anyhow::Ok(())
})() { })() {
eprintln!("turbo-persistence: failed to write LOG after commit {seq:08}: {e:#}"); eprintln!("turbo-persistence: failed to write LOG after commit {seq:08}: {e:#}");
...@@ -1208,31 +1258,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1208,31 +1258,38 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
{ {
let mut inner = self.inner.write(); let mut inner = self.inner.write();
// Apply the deferred MetaFile mutations from Phase A1. apply_filter // Apply the deferred removals oldest-first within each family.
// was called read-only earlier; now we actually move superseded for (meta_files, family_removals) in
// entries from active to obsolete inside each MetaFile. inner.meta_files_by_family.iter_mut().zip(entries_to_remove)
// entries_to_remove was collected in reverse order, so iterate it
// in reverse to match the forward order of inner.meta_files.
for (meta_file, to_remove) in inner
.meta_files
.iter_mut()
.zip(entries_to_remove.into_iter().rev())
{ {
if !to_remove.is_empty() { for (meta_file, to_remove) in
meta_file.retain_entries(|seq| !to_remove.contains(&seq)); meta_files.iter_mut().zip(family_removals.into_iter().rev())
{
if !to_remove.is_empty() {
meta_file.retain_entries(|seq| !to_remove.contains(&seq));
}
} }
} }
inner.meta_files.append(&mut new_meta_files); for meta_file in new_meta_files.drain(..) {
if !meta_seq_numbers_to_delete.is_empty() { inner.push_meta_file(meta_file);
let to_delete: HashSet<u32> = meta_seq_numbers_to_delete.iter().copied().collect();
inner
.meta_files
.retain(|meta| !to_delete.contains(&meta.sequence_number()));
} }
for (meta_files, seqs_to_delete) in inner
.meta_files_by_family
.iter_mut()
.zip(&meta_seq_numbers_to_delete)
{
if !seqs_to_delete.is_empty() {
let to_delete: HashSet<u32> = seqs_to_delete.iter().copied().collect();
meta_files.retain(|meta| !to_delete.contains(&meta.sequence_number()));
}
}
#[cfg(debug_assertions)]
inner.debug_assert_meta_invariants();
inner.current_sequence_number = seq; inner.current_sequence_number = seq;
self.is_empty self.is_empty.store(inner.is_empty(), Ordering::Relaxed);
.store(inner.meta_files.is_empty(), Ordering::Relaxed); // The write guard must be released after publishing the matching empty state.
drop(inner);
} }
// Try to delete superseded files immediately. On Linux/macOS this always // Try to delete superseded files immediately. On Linux/macOS this always
...@@ -1243,7 +1300,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1243,7 +1300,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Self::try_delete_files(&self.path, &sst_seq_numbers_to_delete, "sst") Self::try_delete_files(&self.path, &sst_seq_numbers_to_delete, "sst")
.map(DeferredDeletion::Sst) .map(DeferredDeletion::Sst)
.chain( .chain(
Self::try_delete_files(&self.path, &meta_seq_numbers_to_delete, "meta") meta_seq_numbers_to_delete
.iter()
.flat_map(|seqs| Self::try_delete_files(&self.path, seqs, "meta"))
.map(DeferredDeletion::Meta), .map(DeferredDeletion::Meta),
) )
.chain( .chain(
...@@ -1263,15 +1322,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1263,15 +1322,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
writeln!(log, "New database state:")?; writeln!(log, "New database state:")?;
writeln!(log, "FAM | META SEQ | SST SEQ FLAGS | RANGE")?; writeln!(log, "FAM | META SEQ | SST SEQ FLAGS | RANGE")?;
let inner = self.inner.read(); let inner = self.inner.read();
let families = inner.meta_files.iter().map(|meta| meta.family()).filter({ for (family, meta_files) in inner.meta_files_by_family.iter().enumerate() {
let mut set = HashSet::new(); for meta in meta_files {
move |family| set.insert(*family)
});
for family in families {
for meta in inner.meta_files.iter() {
if meta.family() != family {
continue;
}
let meta_seq = meta.sequence_number(); let meta_seq = meta.sequence_number();
for (entry, range) in meta.entries().iter().zip(meta.hash_ranges()) { for (entry, range) in meta.entries().iter().zip(meta.hash_ranges()) {
let seq = entry.sequence_number(); let seq = entry.sequence_number();
...@@ -1333,7 +1385,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1333,7 +1385,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
let inner = self.inner.read(); let inner = self.inner.read();
sequence_number = AtomicU32::new(inner.current_sequence_number); sequence_number = AtomicU32::new(inner.current_sequence_number);
self.compact_internal( self.compact_internal(
&inner.meta_files, &inner.meta_files_by_family,
&sequence_number, &sequence_number,
&mut new_meta_files, &mut new_meta_files,
&mut new_sst_files, &mut new_sst_files,
...@@ -1370,7 +1422,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1370,7 +1422,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Internal function to perform a compaction. /// Internal function to perform a compaction.
fn compact_internal( fn compact_internal(
&self, &self,
meta_files: &[MetaFile], meta_files_by_family: &[Vec<MetaFile>; FAMILIES],
sequence_number: &AtomicU32, sequence_number: &AtomicU32,
new_meta_files: &mut Vec<NewFile>, new_meta_files: &mut Vec<NewFile>,
new_sst_files: &mut Vec<NewFile>, new_sst_files: &mut Vec<NewFile>,
...@@ -1379,13 +1431,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1379,13 +1431,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
keys_written: &mut u64, keys_written: &mut u64,
compact_config: &CompactConfig, compact_config: &CompactConfig,
) -> Result<()> { ) -> Result<()> {
if meta_files.is_empty() { if meta_files_by_family.iter().all(Vec::is_empty) {
return Ok(()); return Ok(());
} }
struct SstWithRange { struct SstWithRange {
/// Index in the current family's `meta_files_by_family` shard.
meta_index: usize, meta_index: usize,
family: u32,
index_in_meta: u32, index_in_meta: u32,
seq: u32, seq: u32,
range: StaticSortedFileRange, range: StaticSortedFileRange,
...@@ -1409,31 +1461,35 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1409,31 +1461,35 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
} }
let ssts_with_ranges = meta_files let sst_by_family = meta_files_by_family
.iter() .iter()
.enumerate() .enumerate()
.flat_map(|(meta_index, meta)| { .map(|(family, meta_files)| {
meta.entries() meta_files
.iter() .iter()
.enumerate() .enumerate()
.map(move |(index_in_meta, entry)| SstWithRange { .flat_map(|(meta_index, meta)| {
meta_index, debug_assert_eq!(
family: meta.family(), meta.family() as usize,
index_in_meta: index_in_meta as u32, family,
seq: entry.sequence_number(), "meta file stored in the wrong family shard during compaction"
range: meta.range(index_in_meta as u32), );
size: entry.size(), meta.entries()
flags: entry.flags(), .iter()
.enumerate()
.map(move |(index_in_meta, entry)| SstWithRange {
meta_index,
index_in_meta: index_in_meta as u32,
seq: entry.sequence_number(),
range: meta.range(index_in_meta as u32),
size: entry.size(),
flags: entry.flags(),
})
}) })
.collect::<Vec<_>>()
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut sst_by_family = [(); FAMILIES].map(|_| Vec::new());
for sst in ssts_with_ranges {
sst_by_family[sst.family as usize].push(sst);
}
let path = &self.path; let path = &self.path;
let log_mutex = Mutex::new(()); let log_mutex = Mutex::new(());
...@@ -1466,7 +1522,12 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1466,7 +1522,12 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
.parallel_map_collect_owned::<_, _, Result<Vec<_>>>( .parallel_map_collect_owned::<_, _, Result<Vec<_>>>(
merge_jobs, merge_jobs,
|(family, ssts_with_ranges, merge_jobs)| { |(family, ssts_with_ranges, merge_jobs)| {
let meta_files = &meta_files_by_family[family];
let family = family as u32; let family = family as u32;
debug_assert!(
meta_files.iter().all(|meta| meta.family() == family),
"compaction received a meta file from the wrong family shard"
);
if merge_jobs.is_empty() { if merge_jobs.is_empty() {
return Ok(PartialResultPerFamily { return Ok(PartialResultPerFamily {
...@@ -1485,7 +1546,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1485,7 +1546,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
let used_key_hashes: Option<qfilter::Filter> = { let used_key_hashes: Option<qfilter::Filter> = {
let filters: Vec<qfilter::FilterRef<'_>> = meta_files let filters: Vec<qfilter::FilterRef<'_>> = meta_files
.iter() .iter()
.filter(|m| m.family() == family)
.filter_map(|meta_file| { .filter_map(|meta_file| {
meta_file.deserialize_used_key_hashes_amqf().transpose() meta_file.deserialize_used_key_hashes_amqf().transpose()
}) })
...@@ -1880,6 +1940,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1880,6 +1940,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
let guard = log_mutex.lock(); let guard = log_mutex.lock();
let mut log = self.open_log()?; let mut log = self.open_log()?;
writeln!(log, "{family:3} | {meta_seq:08} | Compaction:",)?; writeln!(log, "{family:3} | {meta_seq:08} | Compaction:",)?;
for result in merge_result { for result in merge_result {
match result { match result {
PartialMergeResult::Merged { PartialMergeResult::Merged {
...@@ -1943,6 +2004,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1943,6 +2004,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
for f in sst_files_to_delete.iter() { for f in sst_files_to_delete.iter() {
meta_file_builder.add_obsolete_sst_file(f.seq); meta_file_builder.add_obsolete_sst_file(f.seq);
} }
// Do not copy `used_key_hashes` into the new meta file. Those marks must expire
// as their source meta files are retired; persisting the merged filter here
// would make keys that were used once stay marked as used forever.
let new_meta_file = { let new_meta_file = {
let _span = tracing::trace_span!("write meta file").entered(); let _span = tracing::trace_span!("write meta file").entered();
...@@ -2063,7 +2127,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2063,7 +2127,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
let key_block_cache = self.key_block_cache(); let key_block_cache = self.key_block_cache();
let value_block_cache = self.value_block_cache(); let value_block_cache = self.value_block_cache();
for meta in inner.meta_files.iter().rev() { debug_assert!(
inner.meta_files_by_family[family]
.iter()
.all(|meta| meta.family() as usize == family),
"meta file stored in the wrong family shard while querying family {family}"
);
for meta in inner.meta_files_by_family[family].iter().rev() {
match meta.lookup::<K, FIND_ALL>( match meta.lookup::<K, FIND_ALL>(
family as u32, family as u32,
hash, hash,
...@@ -2202,7 +2272,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2202,7 +2272,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
let inner = self.inner.read(); let inner = self.inner.read();
let key_block_cache = self.key_block_cache(); let key_block_cache = self.key_block_cache();
let value_block_cache = self.value_block_cache(); let value_block_cache = self.value_block_cache();
for meta in inner.meta_files.iter().rev() { debug_assert!(
inner.meta_files_by_family[family]
.iter()
.all(|meta| meta.family() as usize == family),
"meta file stored in the wrong family shard while querying family {family}"
);
for meta in inner.meta_files_by_family[family].iter().rev() {
let _result = meta.batch_lookup( let _result = meta.batch_lookup(
family as u32, family as u32,
keys, keys,
...@@ -2302,8 +2378,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2302,8 +2378,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
pub fn statistics(&self) -> Statistics { pub fn statistics(&self) -> Statistics {
let inner = self.inner.read(); let inner = self.inner.read();
Statistics { Statistics {
meta_files: inner.meta_files.len(), meta_files: inner.meta_files_by_family.iter().map(Vec::len).sum(),
sst_files: inner.meta_files.iter().map(|m| m.entries().len()).sum(), sst_files: inner
.meta_files_by_family
.iter()
.flatten()
.map(|meta| meta.entries().len())
.sum(),
key_block_cache: CacheStatistics::new(self.key_block_cache()), key_block_cache: CacheStatistics::new(self.key_block_cache()),
value_block_cache: CacheStatistics::new(self.value_block_cache()), value_block_cache: CacheStatistics::new(self.value_block_cache()),
hits: self.stats.hits_deleted.load(Ordering::Relaxed) hits: self.stats.hits_deleted.load(Ordering::Relaxed)
...@@ -2321,9 +2402,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2321,9 +2402,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok(self Ok(self
.inner .inner
.read() .read()
.meta_files .meta_files_by_family
.iter() .iter()
.rev() .flat_map(|meta_files| meta_files.iter().rev())
.map(|meta_file| { .map(|meta_file| {
let entries = meta_file let entries = meta_file
.entries() .entries()
......
...@@ -531,10 +531,6 @@ impl MetaFile { ...@@ -531,10 +531,6 @@ impl MetaFile {
&self.obsolete_entries &self.obsolete_entries
} }
pub fn has_active_entries(&self) -> bool {
!self.entries.is_empty()
}
pub fn obsolete_sst_files(&self) -> &[u32] { pub fn obsolete_sst_files(&self) -> &[u32] {
&self.obsolete_sst_files &self.obsolete_sst_files
} }
......
...@@ -87,6 +87,18 @@ impl SstFilter { ...@@ -87,6 +87,18 @@ impl SstFilter {
/// Updates the filter state for the next meta file. Returns true if the meta file can be /// Updates the filter state for the next meta file. Returns true if the meta file can be
/// removed. /// removed.
pub fn apply_and_get_remove(&mut self, meta: &MetaFile) -> bool { pub fn apply_and_get_remove(&mut self, meta: &MetaFile) -> bool {
self.apply_and_get_remove_after_removing(meta, &FxHashSet::default())
}
/// Like [`apply_and_get_remove`](Self::apply_and_get_remove), but treats entries in
/// `entries_to_remove` as already removed. Commit uses this while its in-memory mutations are
/// deferred until after `CURRENT` is durable, so a newly-subsumed meta file can be retired in
/// the same commit.
pub fn apply_and_get_remove_after_removing(
&mut self,
meta: &MetaFile,
entries_to_remove: &FxHashSet<u32>,
) -> bool {
let mut used = false; let mut used = false;
for seq in meta.obsolete_sst_files() { for seq in meta.obsolete_sst_files() {
if let Entry::Occupied(e) = self.0.entry(*seq) { if let Entry::Occupied(e) = self.0.entry(*seq) {
...@@ -100,7 +112,11 @@ impl SstFilter { ...@@ -100,7 +112,11 @@ impl SstFilter {
} }
} }
!used && !meta.has_active_entries() !used
&& !meta
.entries()
.iter()
.any(|entry| !entries_to_remove.contains(&entry.sequence_number()))
} }
} }
......
...@@ -9,6 +9,7 @@ use crate::{ ...@@ -9,6 +9,7 @@ use crate::{
constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
db::{CompactConfig, TurboPersistence, read_current_version}, db::{CompactConfig, TurboPersistence, read_current_version},
lookup_entry::IterValue, lookup_entry::IterValue,
meta_file::MetaFile,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData}, static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData},
write_batch::WriteBatch, write_batch::WriteBatch,
...@@ -2812,3 +2813,99 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> { ...@@ -2812,3 +2813,99 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> {
db.shutdown()?; db.shutdown()?;
Ok(()) Ok(())
} }
#[rstest]
#[case(true)]
#[case(false)]
fn partial_compaction_retires_fully_consumed_meta_files(#[case] mmap: bool) -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
let access_mode = if mmap {
AccessMode::Mmap
} else {
AccessMode::File
};
let db = open_db::<1>(path, mmap)?;
const KEYS: u32 = 2_000;
for generation in 0..4u32 {
let batch = db.write_batch()?;
for key in 0..KEYS {
batch.put(
0,
key.to_be_bytes().to_vec(),
generation.to_be_bytes().to_vec().into(),
)?;
}
db.commit_write_batch(batch)?;
if generation == 0 {
// Flush this access into the following commit's used-key-hash AMQF.
assert!(db.get(0, &0u32.to_be_bytes())?.is_some());
}
}
let before_meta_sequences = db
.meta_info()?
.into_iter()
.map(|meta| meta.sequence_number)
.collect::<Vec<_>>();
assert_eq!(before_meta_sequences.len(), 4);
assert!(before_meta_sequences.iter().any(|&seq| {
MetaFile::open(path, seq, None, access_mode)
.unwrap()
.deserialize_used_key_hashes_amqf()
.unwrap()
.is_some()
}));
let partial = CompactConfig {
min_merge_count: 2,
optimal_merge_count: 2,
max_merge_count: 2,
max_merge_bytes: u64::MAX,
min_merge_duplication_bytes: 0,
optimal_merge_duplication_bytes: 0,
max_merge_segment_count: 1,
};
assert!(db.compact(&partial)?.is_some());
let after_partial = db.meta_info()?;
assert_eq!(
after_partial.len(),
3,
"two fully consumed meta files should retire while two untouched metas remain"
);
assert_eq!(
after_partial
.iter()
.filter(|meta| before_meta_sequences.contains(&meta.sequence_number))
.count(),
2,
"untouched SST metadata should stay in its two existing meta files"
);
for key in 0..KEYS {
assert_eq!(
&*db.get(0, &key.to_be_bytes())?.unwrap(),
&3u32.to_be_bytes()
);
}
db.full_compact()?;
let fully_compacted = db.meta_info()?;
assert_eq!(fully_compacted.len(), 1);
let compacted_meta =
MetaFile::open(path, fully_compacted[0].sequence_number, None, access_mode)?;
assert!(
compacted_meta.deserialize_used_key_hashes_amqf()?.is_none(),
"used-key marks should expire instead of being copied into compaction output"
);
drop(db);
let reopened = open_db::<1>(path, mmap)?;
assert_eq!(reopened.meta_info()?.len(), 1);
for key in 0..KEYS {
assert_eq!(
&*reopened.get(0, &key.to_be_bytes())?.unwrap(),
&3u32.to_be_bytes()
);
}
Ok(())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment