Commit 49f013c7 authored by Luke Sandberg's avatar Luke Sandberg

simplify the configuration

parent 881aaaaf
......@@ -3,9 +3,7 @@ use std::mem::take;
use crate::{
ValueBuffer,
collector_entry::{CollectorEntry, CollectorEntryValue, EntryKey, TINY_VALUE_THRESHOLD},
constants::{
DATA_THRESHOLD_PER_INITIAL_FILE, MAX_ENTRIES_PER_INITIAL_FILE, MAX_SMALL_VALUE_SIZE,
},
constants::{MAX_ENTRIES_PER_INITIAL_FILE, MAX_SMALL_VALUE_SIZE},
key::{StoreKey, hash_key},
};
......@@ -22,15 +20,6 @@ pub struct Collector<K: StoreKey, const SIZE_SHIFT: usize = 0> {
}
impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
/// Creates a new collector with default limits. Note that this allocates the full capacity for
/// the entries.
pub fn new() -> Self {
Self::with_config(
MAX_ENTRIES_PER_INITIAL_FILE,
DATA_THRESHOLD_PER_INITIAL_FILE,
)
}
/// Creates a new collector with custom limits.
///
/// The limits are shifted right by `SIZE_SHIFT` to support thread-local collectors
......
......@@ -34,47 +34,3 @@ pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
/// Maximum RAM bytes for value block cache
pub const VALUE_BLOCK_CACHE_SIZE: u64 = 300 * 1024 * 1024;
pub const VALUE_BLOCK_AVG_SIZE: usize = 132000;
/// Configuration for a single family's file limits.
///
/// Controls when SST files are split during writes and compaction.
/// Files are split when either the entry count OR data size threshold is reached.
#[derive(Clone, Copy, Debug)]
pub struct FamilyConfig {
/// Maximum number of entries per initial SST file (during writes)
pub max_entries_per_initial_file: usize,
/// Data size threshold for initial SST files (bytes)
pub data_threshold_per_initial_file: usize,
/// Maximum number of entries per compacted SST file
pub max_entries_per_compacted_file: usize,
/// Data size threshold for compacted SST files (bytes)
pub data_threshold_per_compacted_file: usize,
}
impl Default for FamilyConfig {
fn default() -> Self {
Self {
max_entries_per_initial_file: MAX_ENTRIES_PER_INITIAL_FILE,
data_threshold_per_initial_file: DATA_THRESHOLD_PER_INITIAL_FILE,
max_entries_per_compacted_file: MAX_ENTRIES_PER_COMPACTED_FILE,
data_threshold_per_compacted_file: DATA_THRESHOLD_PER_COMPACTED_FILE,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone, Debug)]
pub struct DbConfig<const FAMILIES: usize> {
pub family_configs: [FamilyConfig; FAMILIES],
}
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn default() -> Self {
Self {
family_configs: [FamilyConfig::default(); FAMILIES],
}
}
}
......@@ -20,12 +20,12 @@ use smallvec::SmallVec;
pub use crate::compaction::selector::CompactConfig;
use crate::{
QueryKey,
DbConfig, QueryKey,
arc_slice::ArcSlice,
compaction::selector::{Compactable, get_merge_segments},
compression::decompress_into_arc,
constants::{
DATA_THRESHOLD_PER_COMPACTED_FILE, DbConfig, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE,
DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE,
MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE,
},
key::{StoreKey, hash_key},
......@@ -1102,12 +1102,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
collector.total_key_size += key_size;
collector.total_value_size += value_size;
let family_config =
&self.config.family_configs[family as usize];
if collector.total_key_size + collector.total_value_size
> family_config.data_threshold_per_compacted_file
> DATA_THRESHOLD_PER_COMPACTED_FILE
|| collector.entries.len()
>= family_config.max_entries_per_compacted_file
>= MAX_ENTRIES_PER_COMPACTED_FILE
{
let selected_total_key_size =
collector.last_entries_total_key_size;
......
......@@ -26,8 +26,52 @@ mod write_batch;
mod tests;
pub use arc_slice::ArcSlice;
pub use constants::{DbConfig, FamilyConfig};
use constants::{DATA_THRESHOLD_PER_INITIAL_FILE, MAX_ENTRIES_PER_INITIAL_FILE};
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
/// Configuration for a single family's file limits during writes.
///
/// Controls when SST files are split during writes.
/// Files are split when either the entry count OR data size threshold is reached.
///
/// Note: Compaction uses the global constants `MAX_ENTRIES_PER_COMPACTED_FILE` and
/// `DATA_THRESHOLD_PER_COMPACTED_FILE` for all families.
#[derive(Clone, Copy, Debug)]
pub struct FamilyConfig {
/// Maximum number of entries per initial SST file (during writes).
/// Controls the size of the Collector datastructures when writing files,
/// as well as when we switch to a 'sharded' file writing strategy.
pub max_entries_per_initial_file: usize,
/// Data size threshold for initial SST files (bytes).
/// Controls memory usage during writes.
pub data_threshold_per_initial_file: usize,
}
impl Default for FamilyConfig {
fn default() -> Self {
Self {
max_entries_per_initial_file: MAX_ENTRIES_PER_INITIAL_FILE,
data_threshold_per_initial_file: DATA_THRESHOLD_PER_INITIAL_FILE,
}
}
}
/// Database-wide configuration with per-family settings.
///
/// Each family (keyspace) can have different file size limits to optimize
/// for its specific access patterns and data characteristics.
#[derive(Clone, Debug)]
pub struct DbConfig<const FAMILIES: usize> {
pub family_configs: [FamilyConfig; FAMILIES],
}
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn default() -> Self {
Self {
family_configs: [FamilyConfig::default(); FAMILIES],
}
}
}
pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
......
......@@ -15,11 +15,11 @@ use smallvec::SmallVec;
use thread_local::ThreadLocal;
use crate::{
ValueBuffer,
FamilyConfig, ValueBuffer,
collector::Collector,
collector_entry::CollectorEntry,
compression::compress_into_buffer,
constants::{FamilyConfig, MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
key::StoreKey,
meta_file::MetaEntryFlags,
meta_file_builder::MetaFileBuilder,
......
......@@ -45,27 +45,46 @@ pub struct TurboKeyValueDatabase {
is_fresh: bool,
}
impl KeySpace {
/// Returns the persistence configuration for this keyspace.
///
/// Configuration rationale:
/// - `max_entries_per_initial_file`: Controls SST file splitting during writes. Smaller values
/// create more files but reduce memory usage.
/// - `data_threshold_per_initial_file`: Memory budget for initial file writes.
fn family_config(&self) -> FamilyConfig {
match self {
// Infra has only 2 keys ever, so use minimal limits
KeySpace::Infra => FamilyConfig {
max_entries_per_initial_file: 16,
..Default::default()
},
// TaskMeta and TaskData use default limits
KeySpace::TaskMeta | KeySpace::TaskData => FamilyConfig::default(),
// TaskCache uses hash-based lookups with potential collisions.
// Maximize entries per file to minimize files scanned in get_multiple.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
KeySpace::TaskCache => FamilyConfig {
// The keys and values in this keyspace are very small, so we just target the same
// file size as compaction
max_entries_per_initial_file: 1024 * 1024,
data_threshold_per_initial_file: 256 * 1024 * 1024,
..Default::default()
},
}
}
}
impl TurboKeyValueDatabase {
pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
// Configure per-family limits based on keyspace characteristics
let mut config = DbConfig::<FAMILIES>::default();
// Infra (family 0): Only 2 keys ever exist, minimal allocation needed
config.family_configs[KeySpace::Infra as usize] = FamilyConfig {
max_entries_per_initial_file: 16,
max_entries_per_compacted_file: 16,
..Default::default()
let config = DbConfig {
family_configs: [
KeySpace::Infra.family_config(),
KeySpace::TaskMeta.family_config(),
KeySpace::TaskData.family_config(),
KeySpace::TaskCache.family_config(),
],
};
// TaskCache (family 3): Index with collision semantics - maximize entries per file
// to minimize files to scan during get_multiple lookups.
// Use usize::MAX to effectively disable entry limit; data threshold controls file size.
config.family_configs[KeySpace::TaskCache as usize] = FamilyConfig {
max_entries_per_initial_file: usize::MAX,
max_entries_per_compacted_file: usize::MAX,
..Default::default()
};
let db = Arc::new(TurboPersistence::open_with_config(versioned_path, config)?);
Ok(Self {
db: db.clone(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment