Unverified Commit fc13ca90 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack Persistence: Remove amqf cache, store all amqfs in memory (#89562)

### What?

When AMQF is not available performance will degrade significately.
So we want to make sure that AMQF is always cached.
This removes the AMQF cache, which tends to overflow with very large databases.

### read, get, key = 8, value = 4, commits = 1, compacted, hit, uncached

| entries  |      | Baseline  |      | This PR   |      | Change             |
| -------- | ---- | --------- | ---- | --------- | ---- | ------------------ |
| 10.67Mi  | 1.0x | 20.55 µs  | 1.0x | 20.55 µs  | 1.0x | ~same              |
| 21.33Mi  | 2.0x | 23.30 µs  | 1.1x | 23.30 µs  | 1.1x | ~same              |
| 42.67Mi  | 4.0x | 37.18 µs  | 1.8x | 37.18 µs  | 1.8x | ~same              |
| 85.33Mi  | 8.0x | 27.80 µs  | 1.4x | 27.80 µs  | 1.4x | ~same              |
| 128.00Mi | 12x  | 33.07 µs  | 1.6x | 86.24 µs  | 4.2x | 2.4x - 2.8x slower |
| 170.67Mi | 16x  | 807.11 µs | 39x  | 180.47 µs | 8.8x | 4.1x - 4.8x faster |
| 341.33Mi | 32x  | 992.15 µs | 48x  | 283.60 µs | 14x  | 3.2x - 3.8x faster |

### read, get, key = 8, value = 4, commits = 1, compacted, miss, uncached

| entries  |      | Baseline  |      | This PR   |      | Change               |
| -------- | ---- | --------- | ---- | --------- | ---- | -------------------- |
| 10.67Mi  | 1.0x | 302.07 ns | 1.0x | 302.07 ns | 1.0x | ~same                |
| 21.33Mi  | 2.0x | 465.93 ns | 1.5x | 465.93 ns | 1.5x | ~same                |
| 42.67Mi  | 4.0x | 757.96 ns | 2.5x | 757.96 ns | 2.5x | ~same                |
| 85.33Mi  | 8.0x | 1.47 µs   | 4.9x | 1.47 µs   | 4.9x | ~same                |
| 128.00Mi | 12x  | 4.30 µs   | 14x  | 367.02 ns | 1.2x | 5.9x - 23.0x faster  |
| 170.67Mi | 16x  | 21.89 µs  | 72x  | 731.82 ns | 2.4x | 19.4x - 43.3x faster |
| 341.33Mi | 32x  | 39.28 µs  | 130x | 972.98 ns | 3.2x | 31.5x - 52.7x faster |
parent 46ce14c8
...@@ -21,10 +21,6 @@ pub const DATA_THRESHOLD_PER_COMPACTED_FILE: usize = 256 * 1024 * 1024; ...@@ -21,10 +21,6 @@ pub const DATA_THRESHOLD_PER_COMPACTED_FILE: usize = 256 * 1024 * 1024;
/// MAX_ENTRIES_PER_INITIAL_FILE and DATA_THRESHOLD_PER_INITIAL_FILE. /// MAX_ENTRIES_PER_INITIAL_FILE and DATA_THRESHOLD_PER_INITIAL_FILE.
pub const THREAD_LOCAL_SIZE_SHIFT: usize = 7; pub const THREAD_LOCAL_SIZE_SHIFT: usize = 7;
/// Maximum RAM bytes for AMQF cache
pub const AMQF_CACHE_SIZE: u64 = 300 * 1024 * 1024;
pub const AMQF_AVG_SIZE: usize = 37399;
/// Maximum RAM bytes for key block cache /// Maximum RAM bytes for key block cache
pub const KEY_BLOCK_CACHE_SIZE: u64 = 400 * 1024 * 1024; pub const KEY_BLOCK_CACHE_SIZE: u64 = 400 * 1024 * 1024;
pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024; pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
......
...@@ -25,14 +25,13 @@ use crate::{ ...@@ -25,14 +25,13 @@ use crate::{
compaction::selector::{Compactable, get_merge_segments}, compaction::selector::{Compactable, get_merge_segments},
compression::decompress_into_arc, compression::decompress_into_arc,
constants::{ constants::{
AMQF_AVG_SIZE, AMQF_CACHE_SIZE, DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE, DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE,
KEY_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE,
VALUE_BLOCK_CACHE_SIZE,
}, },
key::{StoreKey, hash_key}, key::{StoreKey, hash_key},
lookup_entry::{LookupEntry, LookupValue}, lookup_entry::{LookupEntry, LookupValue},
merge_iter::MergeIter, merge_iter::MergeIter,
meta_file::{AmqfCache, MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange}, meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange},
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
sst_filter::SstFilter, sst_filter::SstFilter,
...@@ -83,7 +82,6 @@ pub struct Statistics { ...@@ -83,7 +82,6 @@ pub struct Statistics {
pub sst_files: usize, pub sst_files: usize,
pub key_block_cache: CacheStatistics, pub key_block_cache: CacheStatistics,
pub value_block_cache: CacheStatistics, pub value_block_cache: CacheStatistics,
pub amqf_cache: CacheStatistics,
pub hits: u64, pub hits: u64,
pub misses: u64, pub misses: u64,
pub miss_family: u64, pub miss_family: u64,
...@@ -119,8 +117,6 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> { ...@@ -119,8 +117,6 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
/// A flag to indicate if a write operation is currently active. Prevents multiple concurrent /// A flag to indicate if a write operation is currently active. Prevents multiple concurrent
/// write operations. /// write operations.
active_write_operation: AtomicBool, active_write_operation: AtomicBool,
/// A cache for deserialized AMQF filters.
amqf_cache: AmqfCache,
/// A cache for decompressed key blocks. /// A cache for decompressed key blocks.
key_block_cache: BlockCache, key_block_cache: BlockCache,
/// A cache for decompressed value blocks. /// A cache for decompressed value blocks.
...@@ -181,13 +177,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -181,13 +177,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
.map(|_| DashSet::with_hasher(BuildNoHashHasher::default())), .map(|_| DashSet::with_hasher(BuildNoHashHasher::default())),
}), }),
active_write_operation: AtomicBool::new(false), active_write_operation: AtomicBool::new(false),
amqf_cache: AmqfCache::with(
AMQF_CACHE_SIZE as usize / AMQF_AVG_SIZE,
AMQF_CACHE_SIZE,
Default::default(),
Default::default(),
Default::default(),
),
key_block_cache: BlockCache::with( key_block_cache: BlockCache::with(
KEY_BLOCK_CACHE_SIZE as usize / KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE as usize / KEY_BLOCK_AVG_SIZE,
KEY_BLOCK_CACHE_SIZE, KEY_BLOCK_CACHE_SIZE,
...@@ -438,7 +427,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -438,7 +427,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Clears all caches of the database. /// Clears all caches of the database.
pub fn clear_cache(&self) { pub fn clear_cache(&self) {
self.amqf_cache.clear();
self.key_block_cache.clear(); self.key_block_cache.clear();
self.value_block_cache.clear(); self.value_block_cache.clear();
for meta in self.inner.write().meta_files.iter_mut() { for meta in self.inner.write().meta_files.iter_mut() {
...@@ -456,7 +444,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -456,7 +444,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// for the first queries after opening the database. /// for the first queries after opening the database.
pub fn prepare_all_sst_caches(&self) { pub fn prepare_all_sst_caches(&self) {
for meta in self.inner.write().meta_files.iter_mut() { for meta in self.inner.write().meta_files.iter_mut() {
meta.prepare_sst_cache(&self.amqf_cache); meta.prepare_sst_cache();
} }
} }
...@@ -1373,7 +1361,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1373,7 +1361,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
family as u32, family as u32,
hash, hash,
key, key,
&self.amqf_cache,
&self.key_block_cache, &self.key_block_cache,
&self.value_block_cache, &self.value_block_cache,
)? { )? {
...@@ -1456,7 +1443,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1456,7 +1443,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
keys, keys,
&mut cells, &mut cells,
&mut empty_cells, &mut empty_cells,
&self.amqf_cache,
&self.key_block_cache, &self.key_block_cache,
&self.value_block_cache, &self.value_block_cache,
)?; )?;
...@@ -1544,7 +1530,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1544,7 +1530,6 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
sst_files: inner.meta_files.iter().map(|m| m.entries().len()).sum(), sst_files: inner.meta_files.iter().map(|m| m.entries().len()).sum(),
key_block_cache: CacheStatistics::new(&self.key_block_cache), key_block_cache: CacheStatistics::new(&self.key_block_cache),
value_block_cache: CacheStatistics::new(&self.value_block_cache), value_block_cache: CacheStatistics::new(&self.value_block_cache),
amqf_cache: CacheStatistics::new(&self.amqf_cache),
hits: self.stats.hits_deleted.load(Ordering::Relaxed) hits: self.stats.hits_deleted.load(Ordering::Relaxed)
+ self.stats.hits_small.load(Ordering::Relaxed) + self.stats.hits_small.load(Ordering::Relaxed)
+ self.stats.hits_blob.load(Ordering::Relaxed), + self.stats.hits_blob.load(Ordering::Relaxed),
......
...@@ -2,21 +2,17 @@ use std::{ ...@@ -2,21 +2,17 @@ use std::{
cmp::Ordering, cmp::Ordering,
fmt::Display, fmt::Display,
fs::File, fs::File,
hash::BuildHasherDefault,
io::{BufReader, Seek}, io::{BufReader, Seek},
ops::Deref, ops::Deref,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::{Arc, OnceLock}, sync::OnceLock,
}; };
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use bitfield::bitfield; use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use either::Either;
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher;
use turbo_bincode::turbo_bincode_decode; use turbo_bincode::turbo_bincode_decode;
use crate::{ use crate::{
...@@ -25,18 +21,6 @@ use crate::{ ...@@ -25,18 +21,6 @@ use crate::{
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
}; };
#[derive(Clone, Default)]
pub struct AmqfWeighter;
impl quick_cache::Weighter<u32, Arc<qfilter::Filter>> for AmqfWeighter {
fn weight(&self, _key: &u32, filter: &Arc<qfilter::Filter>) -> u64 {
filter.capacity() + 1
}
}
pub type AmqfCache =
quick_cache::sync::Cache<u32, Arc<qfilter::Filter>, AmqfWeighter, BuildHasherDefault<FxHasher>>;
bitfield! { bitfield! {
#[derive(Clone, Copy, Default)] #[derive(Clone, Copy, Default)]
pub struct MetaEntryFlags(u32); pub struct MetaEntryFlags(u32);
...@@ -136,30 +120,10 @@ impl MetaEntry { ...@@ -136,30 +120,10 @@ impl MetaEntry {
.0) .0)
} }
pub fn amqf( pub fn amqf(&self, meta: &MetaFile) -> Result<impl Deref<Target = qfilter::Filter>> {
&self, self.amqf.get_or_try_init(|| {
meta: &MetaFile,
amqf_cache: &AmqfCache,
) -> Result<impl Deref<Target = qfilter::Filter>> {
let use_amqf_cache = self.max_hash - self.min_hash < 1 << 60;
Ok(if use_amqf_cache {
let amqf = match amqf_cache.get_value_or_guard(&self.sequence_number(), None) {
GuardResult::Value(amqf) => amqf,
GuardResult::Guard(guard) => {
let amqf = self.deserialize_amqf(meta)?;
let amqf: Arc<qfilter::Filter> = Arc::new(amqf);
let _ = guard.insert(amqf.clone());
amqf
}
GuardResult::Timeout => unreachable!(),
};
Either::Left(amqf)
} else {
let amqf = self.amqf.get_or_try_init(|| {
let amqf = self.deserialize_amqf(meta)?; let amqf = self.deserialize_amqf(meta)?;
anyhow::Ok(amqf) anyhow::Ok(amqf)
})?;
Either::Right(amqf)
}) })
} }
...@@ -342,10 +306,10 @@ impl MetaFile { ...@@ -342,10 +306,10 @@ impl MetaFile {
} }
} }
pub fn prepare_sst_cache(&self, amqf_cache: &AmqfCache) { pub fn prepare_sst_cache(&self) {
for entry in self.entries.iter() { for entry in self.entries.iter() {
let _ = entry.sst(self); let _ = entry.sst(self);
let _ = entry.amqf(self, amqf_cache); let _ = entry.amqf(self);
} }
} }
...@@ -414,7 +378,6 @@ impl MetaFile { ...@@ -414,7 +378,6 @@ impl MetaFile {
key_family: u32, key_family: u32,
key_hash: u64, key_hash: u64,
key: &K, key: &K,
amqf_cache: &AmqfCache,
key_block_cache: &BlockCache, key_block_cache: &BlockCache,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
) -> Result<MetaLookupResult> { ) -> Result<MetaLookupResult> {
...@@ -427,7 +390,7 @@ impl MetaFile { ...@@ -427,7 +390,7 @@ impl MetaFile {
continue; continue;
} }
{ {
let amqf = entry.amqf(self, amqf_cache)?; let amqf = entry.amqf(self)?;
if !amqf.contains_fingerprint(key_hash) { if !amqf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss; miss_result = MetaLookupResult::QuickFilterMiss;
continue; continue;
...@@ -450,7 +413,6 @@ impl MetaFile { ...@@ -450,7 +413,6 @@ impl MetaFile {
keys: &[K], keys: &[K],
cells: &mut [(u64, usize, Option<LookupValue>)], cells: &mut [(u64, usize, Option<LookupValue>)],
empty_cells: &mut usize, empty_cells: &mut usize,
amqf_cache: &AmqfCache,
key_block_cache: &BlockCache, key_block_cache: &BlockCache,
value_block_cache: &BlockCache, value_block_cache: &BlockCache,
) -> Result<MetaBatchLookupResult> { ) -> Result<MetaBatchLookupResult> {
...@@ -500,7 +462,7 @@ impl MetaFile { ...@@ -500,7 +462,7 @@ impl MetaFile {
} }
continue; continue;
} }
let amqf = entry.amqf(self, amqf_cache)?; let amqf = entry.amqf(self)?;
for (hash, index, result) in &mut cells[start_index..=end_index] { for (hash, index, result) in &mut cells[start_index..=end_index] {
debug_assert!( debug_assert!(
*hash >= entry.min_hash && *hash <= entry.max_hash, *hash >= entry.min_hash && *hash <= entry.max_hash,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment