Commit 49f013c7 authored by Luke Sandberg's avatar Luke Sandberg

simplify the configuration

parent 881aaaaf
...@@ -3,9 +3,7 @@ use std::mem::take; ...@@ -3,9 +3,7 @@ use std::mem::take;
use crate::{ use crate::{
ValueBuffer, ValueBuffer,
collector_entry::{CollectorEntry, CollectorEntryValue, EntryKey, TINY_VALUE_THRESHOLD}, collector_entry::{CollectorEntry, CollectorEntryValue, EntryKey, TINY_VALUE_THRESHOLD},
constants::{ constants::{MAX_ENTRIES_PER_INITIAL_FILE, MAX_SMALL_VALUE_SIZE},
DATA_THRESHOLD_PER_INITIAL_FILE, MAX_ENTRIES_PER_INITIAL_FILE, MAX_SMALL_VALUE_SIZE,
},
key::{StoreKey, hash_key}, key::{StoreKey, hash_key},
}; };
...@@ -22,15 +20,6 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> { ...@@ -22,15 +20,6 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> {
} }
impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> { impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
/// Creates a new collector with default limits. Note that this allocates the full capacity for
/// the entries.
pub fn new() -> Self {
Self::with_config(
MAX_ENTRIES_PER_INITIAL_FILE,
DATA_THRESHOLD_PER_INITIAL_FILE,
)
}
/// Creates a new collector with custom limits. /// Creates a new collector with custom limits.
/// ///
/// The limits are shifted right by `SIZE_SHIFT` to support thread-local collectors /// The limits are shifted right by `SIZE_SHIFT` to support thread-local collectors
......
...@@ -34,47 +34,3 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024; ...@@ -34,47 +34,3 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
/// Maximum RAM bytes for value block cache /// Maximum RAM bytes for value block cache
pub const VALUE_BLOCK_CACHE_SIZE: u64 = 300 * 1024 * 1024; pub const VALUE_BLOCK_CACHE_SIZE: u64 = 300 * 1024 * 1024;
pub const VALUE_BLOCK_AVG_SIZE: usize = 132000; pub const VALUE_BLOCK_AVG_SIZE: usize = 132000;
/// Configuration for a single family's file limits.
///
/// Controls when SST files are split during writes and compaction.
/// Files are split when either the entry count OR data size threshold is reached.
#[derive(Clone, Copy, Debug)]
pub struct FamilyConfig {
/// Maximum number of entries per initial SST file (during writes)
pub max_entries_per_initial_file: usize,
/// Data size threshold for initial SST files (bytes)
pub data_threshold_per_initial_file: usize,
/// Maximum number of entries per compacted SST file
pub max_entries_per_compacted_file: usize,
/// Data size threshold for compacted SST files (bytes)
pub data_threshold_per_compacted_file: usize,
}
impl Default for FamilyConfig {
fn default() -> Self {
Self {
max_entries_per_initial_file: MAX_ENTRIES_PER_INITIAL_FILE,
data_threshold_per_initial_file: DATA_THRESHOLD_PER_INITIAL_FILE,
max_entries_per_compacted_file: MAX_ENTRIES_PER_COMPACTED_FILE,
data_threshold_per_compacted_file: DATA_THRESHOLD_PER_COMPACTED_FILE,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone, Debug)]
pub struct DbConfig<const FAMILIES: usize> {
pub family_configs: [FamilyConfig; FAMILIES],
}
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn default() -> Self {
Self {
family_configs: [FamilyConfig::default(); FAMILIES],
}
}
}
...@@ -20,12 +20,12 @@ use smallvec::SmallVec; ...@@ -20,12 +20,12 @@ use smallvec::SmallVec;
pub use crate::compaction::selector::CompactConfig; pub use crate::compaction::selector::CompactConfig;
use crate::{ use crate::{
QueryKey, DbConfig, QueryKey,
arc_slice::ArcSlice, arc_slice::ArcSlice,
compaction::selector::{Compactable, get_merge_segments}, compaction::selector::{Compactable, get_merge_segments},
compression::decompress_into_arc, compression::decompress_into_arc,
constants::{ constants::{
DATA_THRESHOLD_PER_COMPACTED_FILE, DbConfig, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE, DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE,
MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE,
}, },
key::{StoreKey, hash_key}, key::{StoreKey, hash_key},
...@@ -1102,12 +1102,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1102,12 +1102,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
collector.total_key_size += key_size; collector.total_key_size += key_size;
collector.total_value_size += value_size; collector.total_value_size += value_size;
let family_config =
&self.config.family_configs[family as usize];
if collector.total_key_size + collector.total_value_size if collector.total_key_size + collector.total_value_size
> family_config.data_threshold_per_compacted_file > DATA_THRESHOLD_PER_COMPACTED_FILE
|| collector.entries.len() || collector.entries.len()
>= family_config.max_entries_per_compacted_file >= MAX_ENTRIES_PER_COMPACTED_FILE
{ {
let selected_total_key_size = let selected_total_key_size =
collector.last_entries_total_key_size; collector.last_entries_total_key_size;
......
...@@ -26,8 +26,52 @@ mod write_batch; ...@@ -26,8 +26,52 @@ mod write_batch;
mod tests; mod tests;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use constants::{DbConfig, FamilyConfig}; use constants::{DATA_THRESHOLD_PER_INITIAL_FILE, MAX_ENTRIES_PER_INITIAL_FILE};
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
/// Configuration for a single family's file limits during writes.
///
/// Controls when SST files are split during writes.
/// Files are split when either the entry count OR data size threshold is reached.
///
/// Note: Compaction uses the global constants `MAX_ENTRIES_PER_COMPACTED_FILE` and
/// `DATA_THRESHOLD_PER_COMPACTED_FILE` for all families.
#[derive(Clone, Copy, Debug)]
pub struct FamilyConfig {
/// Maximum number of entries per initial SST file (during writes).
/// Controls the size of the Collector datastructures when writing files,
/// as well as when we switch to a 'sharded' file writing strategy.
pub max_entries_per_initial_file: usize,
/// Data size threshold for initial SST files (bytes).
/// Controls memory usage during writes.
pub data_threshold_per_initial_file: usize,
}
impl Default for FamilyConfig {
fn default() -> Self {
Self {
max_entries_per_initial_file: MAX_ENTRIES_PER_INITIAL_FILE,
data_threshold_per_initial_file: DATA_THRESHOLD_PER_INITIAL_FILE,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone, Debug)]
pub struct DbConfig<const FAMILIES: usize> {
pub family_configs: [FamilyConfig; FAMILIES],
}
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn default() -> Self {
Self {
family_configs: [FamilyConfig::default(); FAMILIES],
}
}
}
pub use key::{KeyBase, QueryKey, StoreKey, hash_key}; pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags; pub use meta_file::MetaEntryFlags;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler}; pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
......
...@@ -15,11 +15,11 @@ use smallvec::SmallVec; ...@@ -15,11 +15,11 @@ use smallvec::SmallVec;
use thread_local::ThreadLocal; use thread_local::ThreadLocal;
use crate::{ use crate::{
ValueBuffer, FamilyConfig, ValueBuffer,
collector::Collector, collector::Collector,
collector_entry::CollectorEntry, collector_entry::CollectorEntry,
compression::compress_into_buffer, compression::compress_into_buffer,
constants::{FamilyConfig, MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT}, constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
key::StoreKey, key::StoreKey,
meta_file::MetaEntryFlags, meta_file::MetaEntryFlags,
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
......
...@@ -45,27 +45,46 @@ pub struct TurboKeyValueDatabase { ...@@ -45,27 +45,46 @@ pub struct TurboKeyValueDatabase {
is_fresh: bool, is_fresh: bool,
} }
impl KeySpace {
/// Returns the persistence configuration for this keyspace.
///
/// Configuration rationale:
/// - `max_entries_per_initial_file`: Controls SST file splitting during writes. Smaller values
/// create more files but reduce memory usage.
/// - `data_threshold_per_initial_file`: Memory budget for initial file writes.
fn family_config(&self) -> FamilyConfig {
match self {
// Infra has only 2 keys ever, so use minimal limits
KeySpace::Infra => FamilyConfig {
max_entries_per_initial_file: 16,
..Default::default()
},
// TaskMeta and TaskData use default limits
KeySpace::TaskMeta | KeySpace::TaskData => FamilyConfig::default(),
// TaskCache uses hash-based lookups with potential collisions.
// Maximize entries per file to minimize files scanned in get_multiple.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
KeySpace::TaskCache => FamilyConfig {
// The keys and values in this keyspace are very small, so we just target the same
// file size as compaction
max_entries_per_initial_file: 1024 * 1024,
data_threshold_per_initial_file: 256 * 1024 * 1024,
..Default::default()
},
}
}
}
impl TurboKeyValueDatabase { impl TurboKeyValueDatabase {
pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> { pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
// Configure per-family limits based on keyspace characteristics let config = DbConfig {
let mut config = DbConfig::<FAMILIES>::default(); family_configs: [
KeySpace::Infra.family_config(),
// Infra (family 0): Only 2 keys ever exist, minimal allocation needed KeySpace::TaskMeta.family_config(),
config.family_configs[KeySpace::Infra as usize] = FamilyConfig { KeySpace::TaskData.family_config(),
max_entries_per_initial_file: 16, KeySpace::TaskCache.family_config(),
max_entries_per_compacted_file: 16, ],
..Default::default()
}; };
// TaskCache (family 3): Index with collision semantics - maximize entries per file
// to minimize files to scan during get_multiple lookups.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
config.family_configs[KeySpace::TaskCache as usize] = FamilyConfig {
max_entries_per_initial_file: usize::MAX,
max_entries_per_compacted_file: usize::MAX,
..Default::default()
};
let db = Arc::new(TurboPersistence::open_with_config(versioned_path, config)?); let db = Arc::new(TurboPersistence::open_with_config(versioned_path, config)?);
Ok(Self { Ok(Self {
db: db.clone(), db: db.clone(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment