Commit 16a3edab authored by Luke Sandberg's avatar Luke Sandberg

Create a new compaction mechanism to ensure the TaskCache functions properly

parent 49f013c7
......@@ -20,7 +20,7 @@ use smallvec::SmallVec;
pub use crate::compaction::selector::CompactConfig;
use crate::{
DbConfig, QueryKey,
DbConfig, DeduplicationMode, QueryKey,
arc_slice::ArcSlice,
compaction::selector::{Compactable, get_merge_segments},
compression::decompress_into_arc,
......@@ -1081,12 +1081,23 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
}
let mut used_collector = Collector::default();
let mut unused_collector = Collector::default();
let family_config = &self.config.family_configs[family as usize];
for entry in iter {
let entry = entry?;
// Remove duplicates
// Remove duplicates based on family's deduplication mode
if let Some(current) = current.take() {
if current.key != entry.key {
let is_duplicate = match family_config.deduplication_mode {
DeduplicationMode::ByKeyOnly => {
current.key == entry.key
}
DeduplicationMode::ByKeyAndValue => {
current.key == entry.key
&& current.value.eq_for_dedup(&entry.value)
}
};
if !is_duplicate {
let is_used =
used_key_hashes[family as usize].iter().any(
|amqf| amqf.contains_fingerprint(current.hash),
......
......@@ -29,6 +29,17 @@ pub use arc_slice::ArcSlice;
use constants::{DATA_THRESHOLD_PER_INITIAL_FILE, MAX_ENTRIES_PER_INITIAL_FILE};
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
/// How to deduplicate entries during compaction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DeduplicationMode {
/// Keep only the newest entry for each key (default LSM behavior).
/// When multiple entries have the same key, only the last one is retained.
ByKeyOnly,
/// Keep entries that differ by key OR value (for hash-collision-tolerant keyspaces).
/// Only drops entries that are true duplicates (same key AND same value).
/// Use this when a keyspace may contain different values that hash to the same key.
ByKeyAndValue,
}
/// Configuration for a single family's file limits during writes.
///
/// Controls when SST files are split during writes.
......@@ -45,6 +56,9 @@ pub struct FamilyConfig {
/// Data size threshold for initial SST files (bytes).
/// Controls memory usage during writes.
pub data_threshold_per_initial_file: usize,
/// How to handle duplicate keys during compaction
pub deduplication_mode: DeduplicationMode,
}
impl Default for FamilyConfig {
......@@ -52,6 +66,7 @@ impl Default for FamilyConfig {
Self {
max_entries_per_initial_file: MAX_ENTRIES_PER_INITIAL_FILE,
data_threshold_per_initial_file: DATA_THRESHOLD_PER_INITIAL_FILE,
deduplication_mode: DeduplicationMode::ByKeyOnly,
}
}
}
......
......@@ -39,6 +39,32 @@ impl LazyLookupValue<'_> {
} => *uncompressed_size as usize,
}
}
/// Returns true if two values are equal for deduplication purposes.
///
/// For `Medium` values, compares the compressed block bytes directly
/// (assumes deterministic compression).
pub fn eq_for_dedup(&self, other: &Self) -> bool {
match (self, other) {
(
LazyLookupValue::Eager(LookupValue::Deleted),
LazyLookupValue::Eager(LookupValue::Deleted),
) => true,
(
LazyLookupValue::Eager(LookupValue::Slice { value: a }),
LazyLookupValue::Eager(LookupValue::Slice { value: b }),
) => a.as_ref() == b.as_ref(),
(
LazyLookupValue::Eager(LookupValue::Blob { sequence_number: a }),
LazyLookupValue::Eager(LookupValue::Blob { sequence_number: b }),
) => a == b,
(
LazyLookupValue::Medium { block: a, .. },
LazyLookupValue::Medium { block: b, .. },
) => *a == *b,
_ => false, // Different types are not equal
}
}
}
/// An entry from a SST file lookup.
......
......@@ -4,7 +4,7 @@ use anyhow::Result;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use crate::{
constants::MAX_MEDIUM_VALUE_SIZE,
constants::{DbConfig, DeduplicationMode, FamilyConfig, MAX_MEDIUM_VALUE_SIZE},
db::{CompactConfig, TurboPersistence},
parallel_scheduler::ParallelScheduler,
write_batch::WriteBatch,
......@@ -1511,3 +1511,222 @@ fn get_multiple_empty_result() -> Result<()> {
Ok(())
}
#[test]
fn compaction_by_key_only_drops_same_key_entries() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Default behavior: ByKeyOnly deduplication drops entries with same key
let key = vec![42u8];
{
let db = TurboPersistence::<_, 1>::open_with_parallel_scheduler(
path.to_path_buf(),
RayonParallelScheduler,
)?;
// Write same key with different values in separate batches
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![1u8].into())?;
db.commit_write_batch(batch)?;
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![2u8].into())?;
db.commit_write_batch(batch)?;
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![3u8].into())?;
db.commit_write_batch(batch)?;
// Before compaction: all 3 values exist
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(results.len(), 3, "Should have 3 values before compaction");
// Compact with default ByKeyOnly mode
db.full_compact()?;
// After compaction: only newest value remains
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(
results.len(),
1,
"ByKeyOnly should keep only 1 value after compaction"
);
assert_eq!(results[0].as_ref(), &[3u8], "Should be the newest value");
db.shutdown()?;
}
Ok(())
}
#[test]
fn compaction_by_key_and_value_preserves_different_values() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Configure family 0 to use ByKeyAndValue deduplication
let mut config = DbConfig::<1>::default();
config.family_configs[0] = FamilyConfig {
deduplication_mode: DeduplicationMode::ByKeyAndValue,
..Default::default()
};
let key = vec![42u8];
{
let db = TurboPersistence::<_, 1>::open_with_config_and_parallel_scheduler(
path.to_path_buf(),
config,
RayonParallelScheduler,
)?;
// Write same key with different values in separate batches
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![1u8].into())?;
db.commit_write_batch(batch)?;
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![2u8].into())?;
db.commit_write_batch(batch)?;
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![3u8].into())?;
db.commit_write_batch(batch)?;
// Before compaction: all 3 values exist
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(results.len(), 3, "Should have 3 values before compaction");
// Compact with ByKeyAndValue mode
db.full_compact()?;
// After compaction: all different values should be preserved
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(
results.len(),
3,
"ByKeyAndValue should preserve all different values after compaction"
);
let mut values: Vec<u8> = results.iter().map(|r| r[0]).collect();
values.sort();
assert_eq!(values, vec![1, 2, 3], "All values should be preserved");
db.shutdown()?;
}
Ok(())
}
#[test]
fn compaction_by_key_and_value_drops_true_duplicates() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Configure family 0 to use ByKeyAndValue deduplication
let mut config = DbConfig::<1>::default();
config.family_configs[0] = FamilyConfig {
deduplication_mode: DeduplicationMode::ByKeyAndValue,
..Default::default()
};
let key = vec![42u8];
{
let db = TurboPersistence::<_, 1>::open_with_config_and_parallel_scheduler(
path.to_path_buf(),
config,
RayonParallelScheduler,
)?;
// Write same key with SAME value multiple times (true duplicates)
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![100u8].into())?;
db.commit_write_batch(batch)?;
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![100u8].into())?; // Same value
db.commit_write_batch(batch)?;
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![100u8].into())?; // Same value
db.commit_write_batch(batch)?;
// Before compaction: all 3 entries exist
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(results.len(), 3, "Should have 3 entries before compaction");
// Compact with ByKeyAndValue mode
db.full_compact()?;
// After compaction: true duplicates should be deduplicated
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(
results.len(),
1,
"ByKeyAndValue should deduplicate true duplicates (same key AND value)"
);
assert_eq!(results[0].as_ref(), &[100u8]);
db.shutdown()?;
}
Ok(())
}
#[test]
fn compaction_by_key_and_value_preserves_all_unique_values() -> Result<()> {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Configure family 0 to use ByKeyAndValue deduplication
let mut config = DbConfig::<1>::default();
config.family_configs[0] = FamilyConfig {
deduplication_mode: DeduplicationMode::ByKeyAndValue,
..Default::default()
};
let key = vec![42u8];
{
let db = TurboPersistence::<_, 1>::open_with_config_and_parallel_scheduler(
path.to_path_buf(),
config,
RayonParallelScheduler,
)?;
// Write a mix of different values (simulating hash collisions)
// Each unique value should be preserved after compaction
for value in [1u8, 2, 3, 4, 5] {
let batch = db.write_batch()?;
batch.put(0, key.clone(), vec![value].into())?;
db.commit_write_batch(batch)?;
}
// Before compaction: 5 entries exist
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(results.len(), 5, "Should have 5 entries before compaction");
// Compact
db.full_compact()?;
// After compaction: all 5 unique values should be preserved
// (this is the key difference from ByKeyOnly which would keep only 1)
let results = db.get_multiple(0, &key.as_slice())?;
assert_eq!(
results.len(),
5,
"ByKeyAndValue should preserve all unique values after compaction"
);
let mut values: Vec<u8> = results.iter().map(|r| r[0]).collect();
values.sort();
assert_eq!(values, vec![1, 2, 3, 4, 5], "Should have all values 1-5");
db.shutdown()?;
}
Ok(())
}
......@@ -10,8 +10,8 @@ use anyhow::{Ok, Result};
use parking_lot::Mutex;
use smallvec::SmallVec;
use turbo_persistence::{
ArcSlice, CompactConfig, DbConfig, FamilyConfig, KeyBase, StoreKey, TurboPersistence,
ValueBuffer,
ArcSlice, CompactConfig, DbConfig, DeduplicationMode, FamilyConfig, KeyBase, StoreKey,
TurboPersistence, ValueBuffer,
};
use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks};
......@@ -69,6 +69,9 @@ impl KeySpace {
// file size as compaction
max_entries_per_initial_file: 1024 * 1024,
data_threshold_per_initial_file: 256 * 1024 * 1024,
// We need to gracefully handle hash collisions, so only deduplicate if both key and
// value are identical
deduplication_mode: DeduplicationMode::ByKeyAndValue,
..Default::default()
},
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment