Commit 881aaaaf authored by Luke Sandberg's avatar Luke Sandberg

Create a per family configuration that will allow us to optimize the file sizes…

Create a per family configuration that will allow us to optimize the file sizes for our different file types.
parent 439f2abe
...@@ -15,15 +15,37 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> { ...@@ -15,15 +15,37 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> {
total_key_size: usize, total_key_size: usize,
total_value_size: usize, total_value_size: usize,
entries: Vec<CollectorEntry<K>>, entries: Vec<CollectorEntry<K>>,
/// Maximum number of entries before the collector is considered full
max_entries: usize,
/// Maximum total data size (keys + values) before the collector is considered full
data_threshold: usize,
} }
impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> { impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
/// Creates a new collector. Note that this allocates the full capacity for the entries. /// Creates a new collector with default limits. Note that this allocates the full capacity for
/// the entries.
pub fn new() -> Self { pub fn new() -> Self {
Self::with_config(
MAX_ENTRIES_PER_INITIAL_FILE,
DATA_THRESHOLD_PER_INITIAL_FILE,
)
}
/// Creates a new collector with custom limits.
///
/// The limits are shifted right by `SIZE_SHIFT` to support thread-local collectors
/// that use smaller buffers.
pub fn with_config(max_entries: usize, data_threshold: usize) -> Self {
let max_entries = max_entries >> SIZE_SHIFT;
let data_threshold = data_threshold >> SIZE_SHIFT;
Self { Self {
total_key_size: 0, total_key_size: 0,
total_value_size: 0, total_value_size: 0,
entries: Vec::with_capacity(MAX_ENTRIES_PER_INITIAL_FILE >> SIZE_SHIFT), entries: Vec::with_capacity(
max_entries.min(MAX_ENTRIES_PER_INITIAL_FILE >> SIZE_SHIFT),
),
max_entries,
data_threshold,
} }
} }
...@@ -34,9 +56,8 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> { ...@@ -34,9 +56,8 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
/// Returns true if the collector is full. /// Returns true if the collector is full.
pub fn is_full(&self) -> bool { pub fn is_full(&self) -> bool {
self.entries.len() >= MAX_ENTRIES_PER_INITIAL_FILE >> SIZE_SHIFT self.entries.len() >= self.max_entries
|| self.total_key_size + self.total_value_size || self.total_key_size + self.total_value_size > self.data_threshold
> DATA_THRESHOLD_PER_INITIAL_FILE >> SIZE_SHIFT
} }
/// Adds a normal key-value pair to the collector. /// Adds a normal key-value pair to the collector.
......
...@@ -34,3 +34,47 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024; ...@@ -34,3 +34,47 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
/// Maximum RAM bytes for value block cache /// Maximum RAM bytes for value block cache
pub const VALUE_BLOCK_CACHE_SIZE: u64 = 300 * 1024 * 1024; pub const VALUE_BLOCK_CACHE_SIZE: u64 = 300 * 1024 * 1024;
pub const VALUE_BLOCK_AVG_SIZE: usize = 132000; pub const VALUE_BLOCK_AVG_SIZE: usize = 132000;
/// Configuration for a single family's file limits.
///
/// Controls when SST files are split during writes and compaction.
/// Files are split when either the entry count OR data size threshold is reached.
#[derive(Clone, Copy, Debug)]
pub struct FamilyConfig {
/// Maximum number of entries per initial SST file (during writes)
pub max_entries_per_initial_file: usize,
/// Data size threshold for initial SST files (bytes)
pub data_threshold_per_initial_file: usize,
/// Maximum number of entries per compacted SST file
pub max_entries_per_compacted_file: usize,
/// Data size threshold for compacted SST files (bytes)
pub data_threshold_per_compacted_file: usize,
}
impl Default for FamilyConfig {
fn default() -> Self {
Self {
max_entries_per_initial_file: MAX_ENTRIES_PER_INITIAL_FILE,
data_threshold_per_initial_file: DATA_THRESHOLD_PER_INITIAL_FILE,
max_entries_per_compacted_file: MAX_ENTRIES_PER_COMPACTED_FILE,
data_threshold_per_compacted_file: DATA_THRESHOLD_PER_COMPACTED_FILE,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone, Debug)]
pub struct DbConfig<const FAMILIES: usize> {
pub family_configs: [FamilyConfig; FAMILIES],
}
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn default() -> Self {
Self {
family_configs: [FamilyConfig::default(); FAMILIES],
}
}
}
...@@ -25,7 +25,7 @@ use crate::{ ...@@ -25,7 +25,7 @@ use crate::{
compaction::selector::{Compactable, get_merge_segments}, compaction::selector::{Compactable, get_merge_segments},
compression::decompress_into_arc, compression::decompress_into_arc,
constants::{ constants::{
DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE, DATA_THRESHOLD_PER_COMPACTED_FILE, DbConfig, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE,
MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE,
}, },
key::{StoreKey, hash_key}, key::{StoreKey, hash_key},
...@@ -121,6 +121,8 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> { ...@@ -121,6 +121,8 @@ pub struct TurboPersistence<S: ParallelScheduler, const FAMILIES: usize> {
key_block_cache: BlockCache, key_block_cache: BlockCache,
/// A cache for decompressed value blocks. /// A cache for decompressed value blocks.
value_block_cache: BlockCache, value_block_cache: BlockCache,
/// Per-family configuration for file limits.
config: DbConfig<FAMILIES>,
/// Statistics for the database. /// Statistics for the database.
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
stats: TrackedStats, stats: TrackedStats,
...@@ -157,6 +159,11 @@ impl<S: ParallelScheduler + Default, const FAMILIES: usize> TurboPersistence<S, ...@@ -157,6 +159,11 @@ impl<S: ParallelScheduler + Default, const FAMILIES: usize> TurboPersistence<S,
Self::open_with_parallel_scheduler(path, Default::default()) Self::open_with_parallel_scheduler(path, Default::default())
} }
/// Open a TurboPersistence database at the given path with custom per-family configuration.
pub fn open_with_config(path: PathBuf, config: DbConfig<FAMILIES>) -> Result<Self> {
Self::open_with_config_and_parallel_scheduler(path, config, Default::default())
}
/// Open a TurboPersistence database at the given path in read only mode. /// Open a TurboPersistence database at the given path in read only mode.
/// This will read the directory. No Cleanup is performed. /// This will read the directory. No Cleanup is performed.
pub fn open_read_only(path: PathBuf) -> Result<Self> { pub fn open_read_only(path: PathBuf) -> Result<Self> {
...@@ -165,7 +172,12 @@ impl<S: ParallelScheduler + Default, const FAMILIES: usize> TurboPersistence<S, ...@@ -165,7 +172,12 @@ impl<S: ParallelScheduler + Default, const FAMILIES: usize> TurboPersistence<S,
} }
impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> { impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> {
fn new(path: PathBuf, read_only: bool, parallel_scheduler: S) -> Self { fn new(
path: PathBuf,
read_only: bool,
parallel_scheduler: S,
config: DbConfig<FAMILIES>,
) -> Self {
Self { Self {
parallel_scheduler, parallel_scheduler,
path, path,
...@@ -191,6 +203,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -191,6 +203,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Default::default(), Default::default(),
Default::default(), Default::default(),
), ),
config,
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
stats: TrackedStats::default(), stats: TrackedStats::default(),
} }
...@@ -201,7 +214,16 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -201,7 +214,16 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// properly. Cleanup only requires to read a few bytes from a few files and to delete /// properly. Cleanup only requires to read a few bytes from a few files and to delete
/// files, so it's fast. /// files, so it's fast.
pub fn open_with_parallel_scheduler(path: PathBuf, parallel_scheduler: S) -> Result<Self> { pub fn open_with_parallel_scheduler(path: PathBuf, parallel_scheduler: S) -> Result<Self> {
let mut db = Self::new(path, false, parallel_scheduler); Self::open_with_config_and_parallel_scheduler(path, DbConfig::default(), parallel_scheduler)
}
/// Open a TurboPersistence database at the given path with custom per-family configuration.
pub fn open_with_config_and_parallel_scheduler(
path: PathBuf,
config: DbConfig<FAMILIES>,
parallel_scheduler: S,
) -> Result<Self> {
let mut db = Self::new(path, false, parallel_scheduler, config);
db.open_directory(false)?; db.open_directory(false)?;
Ok(db) Ok(db)
} }
...@@ -212,7 +234,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -212,7 +234,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
path: PathBuf, path: PathBuf,
parallel_scheduler: S, parallel_scheduler: S,
) -> Result<Self> { ) -> Result<Self> {
let mut db = Self::new(path, true, parallel_scheduler); let mut db = Self::new(path, true, parallel_scheduler, DbConfig::default());
db.open_directory(false)?; db.open_directory(false)?;
Ok(db) Ok(db)
} }
...@@ -422,6 +444,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -422,6 +444,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
self.path.clone(), self.path.clone(),
current, current,
self.parallel_scheduler.clone(), self.parallel_scheduler.clone(),
self.config.family_configs,
)) ))
} }
...@@ -1079,10 +1102,12 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1079,10 +1102,12 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
collector.total_key_size += key_size; collector.total_key_size += key_size;
collector.total_value_size += value_size; collector.total_value_size += value_size;
let family_config =
&self.config.family_configs[family as usize];
if collector.total_key_size + collector.total_value_size if collector.total_key_size + collector.total_value_size
> DATA_THRESHOLD_PER_COMPACTED_FILE > family_config.data_threshold_per_compacted_file
|| collector.entries.len() || collector.entries.len()
>= MAX_ENTRIES_PER_COMPACTED_FILE >= family_config.max_entries_per_compacted_file
{ {
let selected_total_key_size = let selected_total_key_size =
collector.last_entries_total_key_size; collector.last_entries_total_key_size;
......
...@@ -26,6 +26,7 @@ mod write_batch; ...@@ -26,6 +26,7 @@ mod write_batch;
mod tests; mod tests;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use constants::{DbConfig, FamilyConfig};
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey, hash_key}; pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags; pub use meta_file::MetaEntryFlags;
......
...@@ -19,7 +19,7 @@ use crate::{ ...@@ -19,7 +19,7 @@ use crate::{
collector::Collector, collector::Collector,
collector_entry::CollectorEntry, collector_entry::CollectorEntry,
compression::compress_into_buffer, compression::compress_into_buffer,
constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT}, constants::{FamilyConfig, MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
key::StoreKey, key::StoreKey,
meta_file::MetaEntryFlags, meta_file::MetaEntryFlags,
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
...@@ -82,25 +82,39 @@ pub struct WriteBatch<K: StoreKey + Send, S: ParallelScheduler, const FAMILIES: ...@@ -82,25 +82,39 @@ pub struct WriteBatch<K: StoreKey + Send, S: ParallelScheduler, const FAMILIES:
/// The list of new SST files that have been created. /// The list of new SST files that have been created.
/// Tuple of (sequence number, file). /// Tuple of (sequence number, file).
new_sst_files: Mutex<Vec<(u32, File)>>, new_sst_files: Mutex<Vec<(u32, File)>>,
/// Per-family configuration for file limits.
family_configs: [FamilyConfig; FAMILIES],
} }
impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
WriteBatch<K, S, FAMILIES> WriteBatch<K, S, FAMILIES>
{ {
/// Creates a new write batch for a database. /// Creates a new write batch for a database with per-family configuration.
pub(crate) fn new(path: PathBuf, current: u32, parallel_scheduler: S) -> Self { pub(crate) fn new(
path: PathBuf,
current: u32,
parallel_scheduler: S,
family_configs: [FamilyConfig; FAMILIES],
) -> Self {
const { const {
assert!(FAMILIES <= usize_from_u32(u32::MAX)); assert!(FAMILIES <= usize_from_u32(u32::MAX));
}; };
let collectors = std::array::from_fn(|i| {
let config = &family_configs[i];
Mutex::new(GlobalCollectorState::Unsharded(Collector::with_config(
config.max_entries_per_initial_file,
config.data_threshold_per_initial_file,
)))
});
Self { Self {
parallel_scheduler, parallel_scheduler,
db_path: path, db_path: path,
current_sequence_number: AtomicU32::new(current), current_sequence_number: AtomicU32::new(current),
thread_locals: ThreadLocal::new(), thread_locals: ThreadLocal::new(),
collectors: [(); FAMILIES] collectors,
.map(|_| Mutex::new(GlobalCollectorState::Unsharded(Collector::new()))),
meta_collectors: [(); FAMILIES].map(|_| Mutex::new(Vec::new())), meta_collectors: [(); FAMILIES].map(|_| Mutex::new(Vec::new())),
new_sst_files: Mutex::new(Vec::new()), new_sst_files: Mutex::new(Vec::new()),
family_configs,
} }
} }
...@@ -124,8 +138,14 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -124,8 +138,14 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
family: u32, family: u32,
) -> Result<&'l mut Collector<K, THREAD_LOCAL_SIZE_SHIFT>> { ) -> Result<&'l mut Collector<K, THREAD_LOCAL_SIZE_SHIFT>> {
debug_assert!(usize_from_u32(family) < FAMILIES); debug_assert!(usize_from_u32(family) < FAMILIES);
let collector = let family_idx = usize_from_u32(family);
state.collectors[usize_from_u32(family)].get_or_insert_with(|| Collector::new()); let config = &self.family_configs[family_idx];
let collector = state.collectors[family_idx].get_or_insert_with(|| {
Collector::with_config(
config.max_entries_per_initial_file,
config.data_threshold_per_initial_file,
)
});
if collector.is_full() { if collector.is_full() {
self.flush_thread_local_collector(family, collector)?; self.flush_thread_local_collector(family, collector)?;
} }
...@@ -138,9 +158,18 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -138,9 +158,18 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
family: u32, family: u32,
collector: &mut Collector<K, THREAD_LOCAL_SIZE_SHIFT>, collector: &mut Collector<K, THREAD_LOCAL_SIZE_SHIFT>,
) -> Result<()> { ) -> Result<()> {
let family_idx = usize_from_u32(family);
let config = &self.family_configs[family_idx];
let new_collector = || {
Collector::with_config(
config.max_entries_per_initial_file,
config.data_threshold_per_initial_file,
)
};
let mut full_collectors = SmallVec::<[_; 2]>::new(); let mut full_collectors = SmallVec::<[_; 2]>::new();
{ {
let mut global_collector_state = self.collectors[usize_from_u32(family)].lock(); let mut global_collector_state = self.collectors[family_idx].lock();
for entry in collector.drain() { for entry in collector.drain() {
match &mut *global_collector_state { match &mut *global_collector_state {
GlobalCollectorState::Unsharded(collector) => { GlobalCollectorState::Unsharded(collector) => {
...@@ -148,7 +177,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -148,7 +177,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
if collector.is_full() { if collector.is_full() {
// When full, split the entries into shards. // When full, split the entries into shards.
let mut shards: [Collector<K>; 4] = let mut shards: [Collector<K>; 4] =
[(); COLLECTOR_SHARDS].map(|_| Collector::new()); [(); COLLECTOR_SHARDS].map(|_| new_collector());
for entry in collector.drain() { for entry in collector.drain() {
let shard = (entry.key.hash >> COLLECTOR_SHARD_SHIFT) as usize; let shard = (entry.key.hash >> COLLECTOR_SHARD_SHIFT) as usize;
shards[shard].add_entry(entry); shards[shard].add_entry(entry);
...@@ -157,8 +186,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -157,8 +186,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
// and the collector is full after the split. // and the collector is full after the split.
for collector in shards.iter_mut() { for collector in shards.iter_mut() {
if collector.is_full() { if collector.is_full() {
full_collectors full_collectors.push(replace(&mut *collector, new_collector()));
.push(replace(&mut *collector, Collector::new()));
} }
} }
*global_collector_state = GlobalCollectorState::Sharded(shards); *global_collector_state = GlobalCollectorState::Sharded(shards);
...@@ -169,7 +197,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -169,7 +197,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
let collector = &mut shards[shard]; let collector = &mut shards[shard];
collector.add_entry(entry); collector.add_entry(entry);
if collector.is_full() { if collector.is_full() {
full_collectors.push(replace(&mut *collector, Collector::new())); full_collectors.push(replace(&mut *collector, new_collector()));
} }
} }
} }
...@@ -252,7 +280,9 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -252,7 +280,9 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
})?; })?;
// Now we flush the global collector(s). // Now we flush the global collector(s).
let mut collector_state = self.collectors[usize_from_u32(family)].lock(); let family_idx = usize_from_u32(family);
let config = &self.family_configs[family_idx];
let mut collector_state = self.collectors[family_idx].lock();
match &mut *collector_state { match &mut *collector_state {
GlobalCollectorState::Unsharded(collector) => { GlobalCollectorState::Unsharded(collector) => {
if !collector.is_empty() { if !collector.is_empty() {
...@@ -264,7 +294,10 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -264,7 +294,10 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
GlobalCollectorState::Sharded(_) => { GlobalCollectorState::Sharded(_) => {
let GlobalCollectorState::Sharded(mut shards) = replace( let GlobalCollectorState::Sharded(mut shards) = replace(
&mut *collector_state, &mut *collector_state,
GlobalCollectorState::Unsharded(Collector::new()), GlobalCollectorState::Unsharded(Collector::with_config(
config.max_entries_per_initial_file,
config.data_threshold_per_initial_file,
)),
) else { ) else {
unreachable!(); unreachable!();
}; };
...@@ -333,8 +366,13 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -333,8 +366,13 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
let mut new_sst_files = take(self.new_sst_files.get_mut()); let mut new_sst_files = take(self.new_sst_files.get_mut());
let shared_new_sst_files = Mutex::new(&mut new_sst_files); let shared_new_sst_files = Mutex::new(&mut new_sst_files);
let new_collectors = let new_collectors = std::array::from_fn(|i| {
[(); FAMILIES].map(|_| Mutex::new(GlobalCollectorState::Unsharded(Collector::new()))); let config = &self.family_configs[i];
Mutex::new(GlobalCollectorState::Unsharded(Collector::with_config(
config.max_entries_per_initial_file,
config.data_threshold_per_initial_file,
)))
});
let collectors = replace(&mut self.collectors, new_collectors); let collectors = replace(&mut self.collectors, new_collectors);
let collectors = collectors let collectors = collectors
.into_iter() .into_iter()
......
...@@ -10,7 +10,8 @@ use anyhow::{Ok, Result}; ...@@ -10,7 +10,8 @@ use anyhow::{Ok, Result};
use parking_lot::Mutex; use parking_lot::Mutex;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_persistence::{ use turbo_persistence::{
ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer, ArcSlice, CompactConfig, DbConfig, FamilyConfig, KeyBase, StoreKey, TurboPersistence,
ValueBuffer,
}; };
use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks}; use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks};
...@@ -46,7 +47,26 @@ pub struct TurboKeyValueDatabase { ...@@ -46,7 +47,26 @@ pub struct TurboKeyValueDatabase {
impl TurboKeyValueDatabase { impl TurboKeyValueDatabase {
pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> { pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
let db = Arc::new(TurboPersistence::open(versioned_path)?); // Configure per-family limits based on keyspace characteristics
let mut config = DbConfig::<FAMILIES>::default();
// Infra (family 0): Only 2 keys ever exist, minimal allocation needed
config.family_configs[KeySpace::Infra as usize] = FamilyConfig {
max_entries_per_initial_file: 16,
max_entries_per_compacted_file: 16,
..Default::default()
};
// TaskCache (family 3): Index with collision semantics - maximize entries per file
// to minimize files to scan during get_multiple lookups.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
config.family_configs[KeySpace::TaskCache as usize] = FamilyConfig {
max_entries_per_initial_file: usize::MAX,
max_entries_per_compacted_file: usize::MAX,
..Default::default()
};
let db = Arc::new(TurboPersistence::open_with_config(versioned_path, config)?);
Ok(Self { Ok(Self {
db: db.clone(), db: db.clone(),
compact_join_handle: Mutex::new(None), compact_join_handle: Mutex::new(None),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment