Unverified Commit 22fffea9 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

perf(turbo-persistence): shard meta files by family (#97703)

## Summary

- Store live meta files in primary per-family shards so queries avoid
unrelated-family metadata
- Consolidate live metadata during compaction, retire subsumed meta
files in the same commit, and retain merged used-key-hash AMQFs.
- Add debug assertions for shard ownership/order and compaction
precedence invariants.

## Performance

Paired `LARGE_DB=1` Criterion binaries were run interleaved on the same
machine to reduce host drift.

| Scenario | Canary | PR | Result |
| --- | ---: | ---: | --- |
| Four-family cached `get` miss (100 metas/family) | 2.40–2.55 µs |
2.11–2.21 µs | **~10–12% faster** |
| Existing single-family cached `get` miss | 1.93–2.12 µs | 2.00–2.04 µs
| Neutral within run-to-run noise (~+0.5% on means) |
| Existing full compaction (4 Mi entries, 8 commits) | ~624 ms | ~624 ms
| Neutral in the stable paired run; repeated samples stayed within
normal noise |

Read hits should see a small improvement but it isn't measurable, misses
are the major benefit.

## Verification

- `cargo test -p turbo-persistence`
- `cargo test -p turbo-persistence --features stats`
- `cargo clippy -p turbo-persistence --all-targets --features stats --
-D warnings`
- `LARGE_DB=1` interleaved Criterion A/B runs

<!-- NEXT_JS_LLM -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 0f69132c
...@@ -254,10 +254,11 @@ The checksum is verified on the compressed data **before** decompression when th ...@@ -254,10 +254,11 @@ The checksum is verified on the compressed data **before** decompression when th
## Reading ## Reading
Reading start from the current sequence number and goes downwards. Opened meta files are stored in per-family shards. A lookup scans only the requested family's meta
files, from newest to oldest; there is no ordering dependency between families.
- We have all SST files memory mapped - We have all SST files memory mapped
- for i = CURRENT sequence number .. 0 - for each meta file of the queried key family, newest first
- Check AMQF from SST file for key existence -> if not continue - Check AMQF from SST file for key existence -> if not continue
- let block = 0 - let block = 0
- loop - loop
...@@ -325,6 +326,10 @@ Since the process might exit unexpectedly, to avoid "forgetting" to delete the S ...@@ -325,6 +326,10 @@ Since the process might exit unexpectedly, to avoid "forgetting" to delete the S
We limit the number of SST files that are merged at once to avoid long compactions. We limit the number of SST files that are merged at once to avoid long compactions.
Compaction keeps meta files incremental: a new meta file only describes SST files that were merged
or moved. Metadata for untouched SST files stays in its existing meta file. When every active entry
in an old meta file is superseded, that meta file is retired in the same compaction commit.
Full example: Full example:
Example: Example:
......
...@@ -840,6 +840,83 @@ fn bench_read_get_multiple(c: &mut Criterion) { ...@@ -840,6 +840,83 @@ fn bench_read_get_multiple(c: &mut Criterion) {
// Compaction Benchmarks // Compaction Benchmarks
// ============================================================================= // =============================================================================
fn family_benchmark_key(family: u32, commit: u32, item: u32) -> [u8; 12] {
let mut key = [0; 12];
key[..4].copy_from_slice(&family.to_be_bytes());
key[4..8].copy_from_slice(&commit.to_be_bytes());
key[8..].copy_from_slice(&item.to_be_bytes());
key
}
fn bench_family_sharding(c: &mut Criterion) {
const FAMILIES: usize = 4;
const COMMITS: u32 = 100;
let entries_per_commit = scaled(1_000);
let db = LazyLock::new(|| {
let tempdir = tempfile::tempdir().unwrap();
let config = TpDbConfig {
family_configs: ["family-0", "family-1", "family-2", "family-3"].map(|name| {
FamilyConfig {
name,
kind: FamilyKind::SingleValue,
compression: Compression::Lz4,
}
}),
..TpDbConfig::new()
};
let db = TurboPersistence::<SerialScheduler, FAMILIES>::open_with_config(
tempdir.path().to_path_buf(),
config,
)
.unwrap();
for commit in 0..COMMITS {
for family in 0..FAMILIES as u32 {
let batch = db.write_batch().unwrap();
for item in 0..entries_per_commit as u32 {
batch
.put(
family,
family_benchmark_key(family, commit, item),
item.to_be_bytes().to_vec().into(),
)
.unwrap();
}
db.commit_write_batch(batch).unwrap();
}
}
(tempdir, db)
});
let mut group = c.benchmark_group("read/family_sharding");
group.measurement_time(Duration::from_secs(5));
let hit = family_benchmark_key(2, COMMITS - 1, 0);
let miss = family_benchmark_key(2, COMMITS, 0);
let batch_hits = (0..64)
.map(|item| family_benchmark_key(2, COMMITS - 1, item))
.collect::<Vec<_>>();
let batch_misses = (0..64)
.map(|item| family_benchmark_key(2, COMMITS, item))
.collect::<Vec<_>>();
group.bench_function("get/hit", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.get(2, black_box(&hit)).unwrap()))
});
group.bench_function("get/miss", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.get(2, black_box(&miss)).unwrap()))
});
group.bench_function("batch_get/hit_64", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.batch_get(2, black_box(&batch_hits)).unwrap()))
});
group.bench_function("batch_get/miss_64", |b| {
let (_, db) = &*db;
b.iter(|| black_box(db.batch_get(2, black_box(&batch_misses)).unwrap()))
});
group.finish();
}
fn bench_compaction(c: &mut Criterion) { fn bench_compaction(c: &mut Criterion) {
let mut group = c.benchmark_group("compaction"); let mut group = c.benchmark_group("compaction");
// Compaction is expensive, reduce sample size // Compaction is expensive, reduce sample size
...@@ -1499,6 +1576,6 @@ fn bench_block_cache(c: &mut Criterion) { ...@@ -1499,6 +1576,6 @@ fn bench_block_cache(c: &mut Criterion) {
criterion_group!( criterion_group!(
name = benches; name = benches;
config = Criterion::default(); config = Criterion::default();
targets = bench_write, bench_write_multi_value, bench_read_get, bench_read_batch_get, bench_read_get_multiple, bench_compaction, bench_compaction_multi_value, bench_qfilter, bench_static_sorted_file_lookup, bench_block_cache targets = bench_write, bench_write_multi_value, bench_read_get, bench_read_batch_get, bench_read_get_multiple, bench_family_sharding, bench_compaction, bench_compaction_multi_value, bench_qfilter, bench_static_sorted_file_lookup, bench_block_cache
); );
criterion_main!(benches); criterion_main!(benches);
...@@ -531,10 +531,6 @@ impl MetaFile { ...@@ -531,10 +531,6 @@ impl MetaFile {
&self.obsolete_entries &self.obsolete_entries
} }
pub fn has_active_entries(&self) -> bool {
!self.entries.is_empty()
}
pub fn obsolete_sst_files(&self) -> &[u32] { pub fn obsolete_sst_files(&self) -> &[u32] {
&self.obsolete_sst_files &self.obsolete_sst_files
} }
......
...@@ -87,6 +87,18 @@ impl SstFilter { ...@@ -87,6 +87,18 @@ impl SstFilter {
/// Updates the filter state for the next meta file. Returns true if the meta file can be /// Updates the filter state for the next meta file. Returns true if the meta file can be
/// removed. /// removed.
pub fn apply_and_get_remove(&mut self, meta: &MetaFile) -> bool { pub fn apply_and_get_remove(&mut self, meta: &MetaFile) -> bool {
self.apply_and_get_remove_after_removing(meta, &FxHashSet::default())
}
/// Like [`apply_and_get_remove`](Self::apply_and_get_remove), but treats entries in
/// `entries_to_remove` as already removed. Commit uses this while its in-memory mutations are
/// deferred until after `CURRENT` is durable, so a newly-subsumed meta file can be retired in
/// the same commit.
pub fn apply_and_get_remove_after_removing(
&mut self,
meta: &MetaFile,
entries_to_remove: &FxHashSet<u32>,
) -> bool {
let mut used = false; let mut used = false;
for seq in meta.obsolete_sst_files() { for seq in meta.obsolete_sst_files() {
if let Entry::Occupied(e) = self.0.entry(*seq) { if let Entry::Occupied(e) = self.0.entry(*seq) {
...@@ -100,7 +112,11 @@ impl SstFilter { ...@@ -100,7 +112,11 @@ impl SstFilter {
} }
} }
!used && !meta.has_active_entries() !used
&& !meta
.entries()
.iter()
.any(|entry| !entries_to_remove.contains(&entry.sequence_number()))
} }
} }
......
...@@ -9,6 +9,7 @@ use crate::{ ...@@ -9,6 +9,7 @@ use crate::{
constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
db::{CompactConfig, TurboPersistence, read_current_version}, db::{CompactConfig, TurboPersistence, read_current_version},
lookup_entry::IterValue, lookup_entry::IterValue,
meta_file::MetaFile,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData}, static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData},
write_batch::WriteBatch, write_batch::WriteBatch,
...@@ -2812,3 +2813,99 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> { ...@@ -2812,3 +2813,99 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> {
db.shutdown()?; db.shutdown()?;
Ok(()) Ok(())
} }
#[rstest]
#[case(true)]
#[case(false)]
fn partial_compaction_retires_fully_consumed_meta_files(#[case] mmap: bool) -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
let access_mode = if mmap {
AccessMode::Mmap
} else {
AccessMode::File
};
let db = open_db::<1>(path, mmap)?;
const KEYS: u32 = 2_000;
for generation in 0..4u32 {
let batch = db.write_batch()?;
for key in 0..KEYS {
batch.put(
0,
key.to_be_bytes().to_vec(),
generation.to_be_bytes().to_vec().into(),
)?;
}
db.commit_write_batch(batch)?;
if generation == 0 {
// Flush this access into the following commit's used-key-hash AMQF.
assert!(db.get(0, &0u32.to_be_bytes())?.is_some());
}
}
let before_meta_sequences = db
.meta_info()?
.into_iter()
.map(|meta| meta.sequence_number)
.collect::<Vec<_>>();
assert_eq!(before_meta_sequences.len(), 4);
assert!(before_meta_sequences.iter().any(|&seq| {
MetaFile::open(path, seq, None, access_mode)
.unwrap()
.deserialize_used_key_hashes_amqf()
.unwrap()
.is_some()
}));
let partial = CompactConfig {
min_merge_count: 2,
optimal_merge_count: 2,
max_merge_count: 2,
max_merge_bytes: u64::MAX,
min_merge_duplication_bytes: 0,
optimal_merge_duplication_bytes: 0,
max_merge_segment_count: 1,
};
assert!(db.compact(&partial)?.is_some());
let after_partial = db.meta_info()?;
assert_eq!(
after_partial.len(),
3,
"two fully consumed meta files should retire while two untouched metas remain"
);
assert_eq!(
after_partial
.iter()
.filter(|meta| before_meta_sequences.contains(&meta.sequence_number))
.count(),
2,
"untouched SST metadata should stay in its two existing meta files"
);
for key in 0..KEYS {
assert_eq!(
&*db.get(0, &key.to_be_bytes())?.unwrap(),
&3u32.to_be_bytes()
);
}
db.full_compact()?;
let fully_compacted = db.meta_info()?;
assert_eq!(fully_compacted.len(), 1);
let compacted_meta =
MetaFile::open(path, fully_compacted[0].sequence_number, None, access_mode)?;
assert!(
compacted_meta.deserialize_used_key_hashes_amqf()?.is_none(),
"used-key marks should expire instead of being copied into compaction output"
);
drop(db);
let reopened = open_db::<1>(path, mmap)?;
assert_eq!(reopened.meta_info()?.len(), 1);
for key in 0..KEYS {
assert_eq!(
&*reopened.get(0, &key.to_be_bytes())?.unwrap(),
&3u32.to_be_bytes()
);
}
Ok(())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment