Commit 6a73cf81 authored by Luke Sandberg's avatar Luke Sandberg

Add direct-key SST format support for optimized integer-keyed families

This adds a new storage format for families with fixed-size integer keys:
- FamilyFormat trait for per-family storage configuration
- Direct-key fixed-value format: inline storage without compression
- Direct-key variable-value format: direct keys with compressed values
- MetaEntryFlags extension for format metadata
- New SST writers: write_direct_fixed_sst(), write_direct_variable_sst()
- New SST readers: lookup_direct_fixed(), lookup_direct_variable()

This enables ~40% size reduction for TaskIdToTaskTypeHash and
~8 bytes/entry savings for TaskMeta/TaskData families.
parent d1616083
//! Family format configuration for per-family storage optimization.
//!
//! This module provides traits and types for configuring how different key-value
//! families are stored in the database. The key insight is that different families
//! have different characteristics that can be exploited for optimization:
//!
//! - **Direct keys**: Integer keys (like TaskId) can be stored without hashing
//! - **Fixed values**: Known-size values can be stored inline without offset tables
//! - **Skip compression**: High-entropy data (like hashes) doesn't compress well
/// Describes the storage format for a key-value family.
///
/// Implement this trait to define custom storage characteristics for a family.
pub trait FamilyFormat: 'static + Send + Sync {
/// Key size if fixed (enables direct key storage without hashing).
/// `None` = variable-size keys (requires hashing).
const KEY_SIZE: Option<usize> = None;
/// Value size if fixed (enables inline storage).
/// `None` = variable-size values (stored in separate blocks).
const VALUE_SIZE: Option<usize> = None;
/// Whether to skip compression (for high-entropy data).
const SKIP_COMPRESSION: bool = false;
/// Helper: does this format use direct keys?
fn uses_direct_keys() -> bool {
Self::KEY_SIZE.is_some()
}
/// Helper: does this format use fixed-size entries?
fn uses_fixed_layout() -> bool {
Self::KEY_SIZE.is_some() && Self::VALUE_SIZE.is_some()
}
}
/// Runtime-inspectable format configuration.
///
/// This struct provides runtime access to format properties for storage decisions.
#[derive(Clone, Copy, Debug, Default)]
pub struct FormatConfig {
/// Key size if fixed.
pub key_size: Option<usize>,
/// Value size if fixed.
pub value_size: Option<usize>,
/// Whether to skip compression.
pub skip_compression: bool,
}
impl FormatConfig {
/// Creates a FormatConfig from a FamilyFormat type.
pub fn from_format<F: FamilyFormat>() -> Self {
Self {
key_size: F::KEY_SIZE,
value_size: F::VALUE_SIZE,
skip_compression: F::SKIP_COMPRESSION,
}
}
/// Returns true if this format uses direct keys.
pub fn uses_direct_keys(&self) -> bool {
self.key_size.is_some()
}
/// Returns true if this format uses fixed-size entries.
pub fn uses_fixed_layout(&self) -> bool {
self.key_size.is_some() && self.value_size.is_some()
}
}
/// Marker trait for a tuple of FamilyFormat types.
///
/// This trait allows specifying per-family formats at the type level.
pub trait FamilyFormats: 'static + Send + Sync {
/// Number of families.
const COUNT: usize;
/// Get the runtime config for a specific family.
fn config(family: usize) -> FormatConfig;
}
// === Concrete Format Types ===
/// Default format: variable keys (hashed), variable values, compressed.
///
/// Use this for families with variable-size keys or where hashing is desired.
pub struct DefaultFormat;
impl FamilyFormat for DefaultFormat {}
/// Direct key with variable values: fixed-size integer keys, variable values, compressed.
///
/// Use this for families with fixed-size integer keys (like TaskId) but variable-size values.
/// Saves 8 bytes per entry by storing keys directly instead of with a hash prefix.
pub struct DirectKeyVariableValue<const KEY_SIZE: usize>;
impl<const K: usize> FamilyFormat for DirectKeyVariableValue<K> {
const KEY_SIZE: Option<usize> = Some(K);
}
/// Direct key with fixed values: fixed-size keys + values, no compression.
///
/// Use this for families with fixed-size integer keys and fixed-size values.
/// The most optimized format - stores entries inline without offset tables or compression.
pub struct DirectKeyFixedValue<const KEY_SIZE: usize, const VALUE_SIZE: usize>;
impl<const K: usize, const V: usize> FamilyFormat for DirectKeyFixedValue<K, V> {
const KEY_SIZE: Option<usize> = Some(K);
const VALUE_SIZE: Option<usize> = Some(V);
const SKIP_COMPRESSION: bool = true;
}
// === Convenience Aliases ===
/// TaskId-keyed with variable values (for TaskMeta, TaskData).
pub type TaskIdKeyedFormat = DirectKeyVariableValue<4>;
/// TaskId-keyed with u64 hash value (for TaskIdToTaskTypeHash).
pub type TaskIdToHashFormat = DirectKeyFixedValue<4, 8>;
// === FamilyFormats implementations for tuples ===
macro_rules! impl_family_formats {
($count:expr, $($idx:tt: $ty:ident),+) => {
impl<$($ty: FamilyFormat),+> FamilyFormats for ($($ty,)+) {
const COUNT: usize = $count;
fn config(family: usize) -> FormatConfig {
match family {
$($idx => FormatConfig::from_format::<$ty>(),)+
_ => panic!("Invalid family index: {}", family),
}
}
}
};
}
impl_family_formats!(1, 0: F0);
impl_family_formats!(2, 0: F0, 1: F1);
impl_family_formats!(3, 0: F0, 1: F1, 2: F2);
impl_family_formats!(4, 0: F0, 1: F1, 2: F2, 3: F3);
impl_family_formats!(5, 0: F0, 1: F1, 2: F2, 3: F3, 4: F4);
impl_family_formats!(6, 0: F0, 1: F1, 2: F2, 3: F3, 4: F4, 5: F5);
impl_family_formats!(7, 0: F0, 1: F1, 2: F2, 3: F3, 4: F4, 5: F5, 6: F6);
impl_family_formats!(8, 0: F0, 1: F1, 2: F2, 3: F3, 4: F4, 5: F5, 6: F6, 7: F7);
...@@ -10,6 +10,7 @@ mod compaction; ...@@ -10,6 +10,7 @@ mod compaction;
mod compression; mod compression;
mod constants; mod constants;
mod db; mod db;
mod family_format;
mod key; mod key;
mod lookup_entry; mod lookup_entry;
mod merge_iter; mod merge_iter;
...@@ -27,7 +28,15 @@ mod tests; ...@@ -27,7 +28,15 @@ mod tests;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use key::{KeyBase, QueryKey, StoreKey}; pub use family_format::{
DefaultFormat, DirectKeyFixedValue, DirectKeyVariableValue, FamilyFormat, FamilyFormats,
FormatConfig, TaskIdKeyedFormat, TaskIdToHashFormat,
};
pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler}; pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
pub use static_sorted_file_builder::{
DirectFixedEntry, DirectVariableEntry, write_direct_fixed_sst, write_direct_variable_sst,
};
pub use value_buf::ValueBuffer; pub use value_buf::ValueBuffer;
pub use write_batch::WriteBatch; pub use write_batch::WriteBatch;
...@@ -20,7 +20,7 @@ use rustc_hash::FxHasher; ...@@ -20,7 +20,7 @@ use rustc_hash::FxHasher;
use turbo_bincode::turbo_bincode_decode; use turbo_bincode::turbo_bincode_decode;
use crate::{ use crate::{
QueryKey, ArcSlice, QueryKey,
lookup_entry::LookupValue, lookup_entry::LookupValue,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
}; };
...@@ -46,23 +46,96 @@ bitfield! { ...@@ -46,23 +46,96 @@ bitfield! {
pub cold, set_cold: 0; pub cold, set_cold: 0;
/// The SST file was freshly written and has not been compacted yet. /// The SST file was freshly written and has not been compacted yet.
pub fresh, set_fresh: 1; pub fresh, set_fresh: 1;
/// The SST file uses direct key storage (no hashing). Keys are stored as raw bytes.
pub direct_key, set_direct_key: 2;
/// The SST file uses fixed-size value storage (inline with keys).
pub fixed_value, set_fixed_value: 3;
/// The SST file is stored uncompressed (for high-entropy data).
pub uncompressed, set_uncompressed: 4;
/// Fixed key size in bytes (bits 8-15). Only valid if direct_key is set.
pub u8, key_size, set_key_size: 15, 8;
/// Fixed value size in bytes (bits 16-23). Only valid if fixed_value is set.
pub u8, value_size, set_value_size: 23, 16;
} }
impl MetaEntryFlags { impl MetaEntryFlags {
pub const FRESH: MetaEntryFlags = MetaEntryFlags(0b10); pub const FRESH: MetaEntryFlags = MetaEntryFlags(0b10);
pub const COLD: MetaEntryFlags = MetaEntryFlags(0b01); pub const COLD: MetaEntryFlags = MetaEntryFlags(0b01);
pub const WARM: MetaEntryFlags = MetaEntryFlags(0b00); pub const WARM: MetaEntryFlags = MetaEntryFlags(0b00);
/// Create flags for a direct-key SST with variable values.
pub fn direct_key_variable(key_size: u8) -> Self {
let mut flags = Self::FRESH;
flags.set_direct_key(true);
flags.set_key_size(key_size);
flags
}
/// Create flags for a direct-key SST with fixed values and no compression.
pub fn direct_key_fixed(key_size: u8, value_size: u8) -> Self {
let mut flags = Self::FRESH;
flags.set_direct_key(true);
flags.set_fixed_value(true);
flags.set_uncompressed(true);
flags.set_key_size(key_size);
flags.set_value_size(value_size);
flags
}
/// Returns true if this SST uses direct key storage (no hashing).
#[inline]
pub fn uses_direct_keys(&self) -> bool {
self.direct_key()
}
/// Returns true if this SST uses fixed-size inline value storage.
#[inline]
pub fn uses_fixed_values(&self) -> bool {
self.fixed_value()
}
/// Returns true if this SST is stored uncompressed.
#[inline]
pub fn is_uncompressed(&self) -> bool {
self.uncompressed()
}
/// Returns the entry size for fully fixed-layout SSTs, or None if not applicable.
#[inline]
pub fn entry_size(&self) -> Option<usize> {
if self.direct_key() && self.fixed_value() {
Some(self.key_size() as usize + self.value_size() as usize)
} else {
None
}
}
} }
impl Display for MetaEntryFlags { impl Display for MetaEntryFlags {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut parts = Vec::new();
// Compaction status
if self.fresh() { if self.fresh() {
f.pad_integral(true, "", "fresh") parts.push("fresh".to_string());
} else if self.cold() { } else if self.cold() {
f.pad_integral(true, "", "cold") parts.push("cold".to_string());
} else { } else {
f.pad_integral(true, "", "warm") parts.push("warm".to_string());
} }
// Format flags
if self.direct_key() {
parts.push(format!("direct_key({})", self.key_size()));
}
if self.fixed_value() {
parts.push(format!("fixed_value({})", self.value_size()));
}
if self.uncompressed() {
parts.push("uncompressed".to_string());
}
write!(f, "{}", parts.join(","))
} }
} }
...@@ -430,6 +503,106 @@ impl MetaFile { ...@@ -430,6 +503,106 @@ impl MetaFile {
Ok(miss_result) Ok(miss_result)
} }
/// Lookup for direct-key fixed-value format.
///
/// This is optimized for families with fixed-size integer keys and fixed-size values
/// stored inline without compression (e.g., TaskIdToTaskTypeHash).
///
/// The key should be the raw key bytes (e.g., TaskId as 4 bytes).
pub fn lookup_direct_fixed<const KEY_SIZE: usize, const VALUE_SIZE: usize>(
&self,
key_family: u32,
key: &[u8; KEY_SIZE],
amqf_cache: &AmqfCache,
) -> Result<MetaLookupResult> {
if key_family != self.family {
return Ok(MetaLookupResult::FamilyMiss);
}
// Convert key to hash for AMQF filtering
let key_hash = key_bytes_to_hash::<KEY_SIZE>(key);
let mut miss_result = MetaLookupResult::RangeMiss;
for entry in self.entries.iter().rev() {
debug_assert!(
entry.flags.uses_direct_keys() && entry.flags.uses_fixed_values(),
"lookup_direct_fixed called on non-direct-fixed SST"
);
if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue;
}
{
let amqf = entry.amqf(self, amqf_cache)?;
if !amqf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
}
let result = entry
.sst(self)?
.lookup_direct_fixed::<KEY_SIZE, VALUE_SIZE>(key)?;
if let Some(value) = result {
return Ok(MetaLookupResult::SstLookup(SstLookupResult::Found(
LookupValue::Slice {
value: ArcSlice::from(value.to_vec().into_boxed_slice()),
},
)));
}
}
Ok(miss_result)
}
/// Lookup for direct-key variable-value format.
///
/// This is optimized for families with fixed-size integer keys but variable-size
/// values stored in compressed blocks (e.g., TaskMeta, TaskData).
///
/// The key should be the raw key bytes (e.g., TaskId as 4 bytes).
pub fn lookup_direct_variable<const KEY_SIZE: usize>(
&self,
key_family: u32,
key: &[u8; KEY_SIZE],
amqf_cache: &AmqfCache,
key_block_cache: &BlockCache,
value_block_cache: &BlockCache,
) -> Result<MetaLookupResult> {
if key_family != self.family {
return Ok(MetaLookupResult::FamilyMiss);
}
// Convert key to hash for AMQF filtering
let key_hash = key_bytes_to_hash::<KEY_SIZE>(key);
let mut miss_result = MetaLookupResult::RangeMiss;
for entry in self.entries.iter().rev() {
debug_assert!(
entry.flags.uses_direct_keys() && !entry.flags.uses_fixed_values(),
"lookup_direct_variable called on non-direct-variable SST"
);
if key_hash < entry.min_hash || key_hash > entry.max_hash {
continue;
}
{
let amqf = entry.amqf(self, amqf_cache)?;
if !amqf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss;
continue;
}
}
let result = entry.sst(self)?.lookup_direct_variable::<KEY_SIZE>(
key,
key_block_cache,
value_block_cache,
)?;
if !matches!(result, SstLookupResult::NotFound) {
return Ok(MetaLookupResult::SstLookup(result));
}
}
Ok(miss_result)
}
pub fn batch_lookup<K: QueryKey>( pub fn batch_lookup<K: QueryKey>(
&self, &self,
key_family: u32, key_family: u32,
...@@ -525,3 +698,15 @@ impl MetaFile { ...@@ -525,3 +698,15 @@ impl MetaFile {
Ok(lookup_result) Ok(lookup_result)
} }
} }
/// Converts key bytes to a u64 hash for AMQF filter and range queries.
///
/// For keys smaller than 8 bytes, pads with zeros on the right.
/// For keys larger than 8 bytes, uses only the first 8 bytes.
#[inline]
fn key_bytes_to_hash<const KEY_SIZE: usize>(key: &[u8]) -> u64 {
let mut hash_bytes = [0u8; 8];
let copy_len = KEY_SIZE.min(8);
hash_bytes[..copy_len].copy_from_slice(&key[..copy_len]);
u64::from_be_bytes(hash_bytes)
}
...@@ -23,6 +23,10 @@ use crate::{ ...@@ -23,6 +23,10 @@ use crate::{
pub const BLOCK_TYPE_INDEX: u8 = 0; pub const BLOCK_TYPE_INDEX: u8 = 0;
/// The block header for a key block. /// The block header for a key block.
pub const BLOCK_TYPE_KEY: u8 = 1; pub const BLOCK_TYPE_KEY: u8 = 1;
/// The block header for a direct-key fixed-value block (no compression, no offset table).
pub const BLOCK_TYPE_DIRECT_FIXED: u8 = 2;
/// The block header for a direct-key variable-value block (direct keys, variable values).
pub const BLOCK_TYPE_DIRECT_VARIABLE: u8 = 3;
/// The tag for a small-sized value. /// The tag for a small-sized value.
pub const KEY_BLOCK_ENTRY_TYPE_SMALL: u8 = 0; pub const KEY_BLOCK_ENTRY_TYPE_SMALL: u8 = 0;
...@@ -135,6 +139,137 @@ impl StaticSortedFile { ...@@ -135,6 +139,137 @@ impl StaticSortedFile {
Ok(iter) Ok(iter)
} }
/// Looks up a key in a direct-fixed SST file.
///
/// This is optimized for files with fixed-size keys and values stored inline
/// without compression. Uses binary search directly on the mmap'd data.
///
/// Returns the value bytes if found, or None if not found.
pub fn lookup_direct_fixed<const KEY_SIZE: usize, const VALUE_SIZE: usize>(
&self,
key: &[u8],
) -> Result<Option<[u8; VALUE_SIZE]>> {
debug_assert_eq!(key.len(), KEY_SIZE);
let entry_size = KEY_SIZE + VALUE_SIZE;
// The file layout is: [entries][single block_offset u32]
// So entry data occupies: file_size - 4 bytes
let data_end = self.mmap.len().saturating_sub(4);
let entry_count = data_end / entry_size;
if entry_count == 0 {
return Ok(None);
}
// Binary search for the key
let mut l = 0;
let mut r = entry_count;
while l < r {
let m = (l + r) / 2;
let entry_start = m * entry_size;
let mid_key = &self.mmap[entry_start..entry_start + KEY_SIZE];
match key.cmp(mid_key) {
Ordering::Less => {
r = m;
}
Ordering::Equal => {
// Found it - extract the value
let value_start = entry_start + KEY_SIZE;
let mut value = [0u8; VALUE_SIZE];
value.copy_from_slice(&self.mmap[value_start..value_start + VALUE_SIZE]);
return Ok(Some(value));
}
Ordering::Greater => {
l = m + 1;
}
}
}
Ok(None)
}
/// Looks up a key in a direct-variable SST file.
///
/// This is for files with fixed-size keys stored directly (no hash prefix)
/// but variable-size values stored in separate compressed blocks.
pub fn lookup_direct_variable<const KEY_SIZE: usize>(
&self,
key: &[u8],
key_block_cache: &BlockCache,
value_block_cache: &BlockCache,
) -> Result<SstLookupResult> {
debug_assert_eq!(key.len(), KEY_SIZE);
// Convert key to hash for AMQF and index lookups
let key_hash = key_bytes_to_hash::<KEY_SIZE>(key);
let mut current_block = self.meta.block_count - 1;
loop {
let block = self.get_key_block(current_block, key_block_cache)?;
let mut block_slice = &block[..];
let block_type = block_slice.read_u8()?;
match block_type {
BLOCK_TYPE_INDEX => {
current_block = self.lookup_index_block(block_slice, key_hash)?;
}
BLOCK_TYPE_DIRECT_VARIABLE => {
return self.lookup_direct_key_block::<KEY_SIZE>(
block_slice,
key,
value_block_cache,
);
}
BLOCK_TYPE_KEY => {
// Fall back to regular key block lookup (shouldn't happen for direct files)
bail!("Unexpected regular key block in direct-variable SST file");
}
_ => {
bail!("Invalid block type");
}
}
}
}
/// Looks up a key in a direct-key block (no hash prefix).
fn lookup_direct_key_block<const KEY_SIZE: usize>(
&self,
mut block: &[u8],
key: &[u8],
value_block_cache: &BlockCache,
) -> Result<SstLookupResult> {
let entry_count = block.read_u24::<BE>()? as usize;
let offsets = &block[..entry_count * 4];
let entries = &block[entry_count * 4..];
let mut l = 0;
let mut r = entry_count;
// binary search for the key
while l < r {
let m = (l + r) / 2;
let GetDirectKeyEntryResult {
key: mid_key,
ty,
val: mid_val,
} = get_direct_key_entry::<KEY_SIZE>(offsets, entries, entry_count, m)?;
match key.cmp(mid_key) {
Ordering::Less => {
r = m;
}
Ordering::Equal => {
return Ok(self
.handle_key_match(ty, mid_val, value_block_cache)?
.into());
}
Ordering::Greater => {
l = m + 1;
}
}
}
Ok(SstLookupResult::NotFound)
}
/// Looks up a key in this file. /// Looks up a key in this file.
pub fn lookup<K: QueryKey>( pub fn lookup<K: QueryKey>(
&self, &self,
...@@ -598,3 +733,70 @@ fn get_key_entry<'l>( ...@@ -598,3 +733,70 @@ fn get_key_entry<'l>(
} }
}) })
} }
// =============================================================================
// Direct Key Format Support
// =============================================================================
/// Converts key bytes to a u64 hash for AMQF filter and index lookups.
///
/// For keys smaller than 8 bytes, pads with zeros on the right.
/// For keys larger than 8 bytes, uses only the first 8 bytes.
#[inline]
fn key_bytes_to_hash<const KEY_SIZE: usize>(key: &[u8]) -> u64 {
let mut hash_bytes = [0u8; 8];
let copy_len = KEY_SIZE.min(8);
hash_bytes[..copy_len].copy_from_slice(&key[..copy_len]);
u64::from_be_bytes(hash_bytes)
}
struct GetDirectKeyEntryResult<'l> {
key: &'l [u8],
ty: u8,
val: &'l [u8],
}
/// Reads a direct-key entry from a direct-key block.
/// Direct-key blocks store keys without the 8-byte hash prefix.
fn get_direct_key_entry<'l, const KEY_SIZE: usize>(
offsets: &[u8],
entries: &'l [u8],
entry_count: usize,
index: usize,
) -> Result<GetDirectKeyEntryResult<'l>> {
let mut offset = &offsets[index * 4..];
let ty = offset.read_u8()?;
let start = offset.read_u24::<BE>()? as usize;
let end = if index == entry_count - 1 {
entries.len()
} else {
(&offsets[(index + 1) * 4 + 1..]).read_u24::<BE>()? as usize
};
// Direct keys: no hash prefix, key is at the start
Ok(match ty {
KEY_BLOCK_ENTRY_TYPE_SMALL => GetDirectKeyEntryResult {
key: &entries[start..start + KEY_SIZE],
ty,
val: &entries[end - 8..end], // value_block(2) + value_size(2) + value_offset(4)
},
KEY_BLOCK_ENTRY_TYPE_MEDIUM => GetDirectKeyEntryResult {
key: &entries[start..start + KEY_SIZE],
ty,
val: &entries[end - 2..end], // value_block(2)
},
KEY_BLOCK_ENTRY_TYPE_BLOB => GetDirectKeyEntryResult {
key: &entries[start..start + KEY_SIZE],
ty,
val: &entries[end - 4..end], // blob(4)
},
KEY_BLOCK_ENTRY_TYPE_DELETED => GetDirectKeyEntryResult {
key: &entries[start..start + KEY_SIZE],
ty,
val: &[],
},
_ => {
bail!("Invalid key block entry type");
}
})
}
...@@ -1303,3 +1303,133 @@ fn batch_get_after_restore() -> Result<()> { ...@@ -1303,3 +1303,133 @@ fn batch_get_after_restore() -> Result<()> {
Ok(()) Ok(())
} }
#[test]
fn test_direct_fixed_sst() -> Result<()> {
use crate::{
meta_file::MetaEntryFlags,
static_sorted_file::{StaticSortedFile, StaticSortedFileMetaData},
static_sorted_file_builder::{DirectFixedEntry, write_direct_fixed_sst},
};
struct TestEntry {
key: [u8; 4],
value: [u8; 8],
}
impl DirectFixedEntry for TestEntry {
fn key_bytes(&self) -> &[u8] {
&self.key
}
fn value_bytes(&self) -> &[u8] {
&self.value
}
}
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
let sst_path = path.join("00000001.sst");
// Create test entries (sorted by key)
let mut entries: Vec<TestEntry> = (0..100u32)
.map(|i| TestEntry {
key: i.to_be_bytes(),
value: (i as u64 * 2).to_be_bytes(),
})
.collect();
entries.sort_by_key(|e| e.key);
// Create flags for direct-key fixed-value format
let flags = MetaEntryFlags::direct_key_fixed(4, 8);
// Write the SST file
let (meta, file) = write_direct_fixed_sst::<4, 8>(&entries, &sst_path, flags)?;
file.sync_all()?;
// Verify metadata
assert_eq!(meta.entries, 100);
assert_eq!(meta.key_compression_dictionary_length, 0);
assert!(meta.flags.uses_direct_keys());
assert!(meta.flags.uses_fixed_values());
// Open and read back
let sst = StaticSortedFile::open(
path,
StaticSortedFileMetaData {
sequence_number: 1,
key_compression_dictionary_length: meta.key_compression_dictionary_length,
block_count: meta.block_count,
},
)?;
// Test lookup for various keys
for i in [0u32, 42, 50, 99] {
let key = i.to_be_bytes();
let expected_value = (i as u64 * 2).to_be_bytes();
let result = sst.lookup_direct_fixed::<4, 8>(&key)?;
assert_eq!(result, Some(expected_value), "Failed to lookup key {i}");
}
// Test lookup for non-existent keys
for i in [100u32, 200, 1000] {
let key = i.to_be_bytes();
let result = sst.lookup_direct_fixed::<4, 8>(&key)?;
assert_eq!(result, None, "Should not find non-existent key {i}");
}
Ok(())
}
#[test]
fn test_meta_entry_flags() {
use crate::meta_file::MetaEntryFlags;
// Test default flags
let flags = MetaEntryFlags::default();
assert!(!flags.cold());
assert!(!flags.fresh());
assert!(!flags.direct_key());
assert!(!flags.fixed_value());
assert!(!flags.uncompressed());
assert_eq!(flags.key_size(), 0);
assert_eq!(flags.value_size(), 0);
// Test FRESH constant
let flags = MetaEntryFlags::FRESH;
assert!(!flags.cold());
assert!(flags.fresh());
assert!(!flags.direct_key());
// Test direct_key_variable
let flags = MetaEntryFlags::direct_key_variable(4);
assert!(flags.fresh());
assert!(flags.direct_key());
assert!(!flags.fixed_value());
assert!(!flags.uncompressed());
assert_eq!(flags.key_size(), 4);
assert_eq!(flags.value_size(), 0);
assert!(flags.uses_direct_keys());
assert!(!flags.uses_fixed_values());
assert_eq!(flags.entry_size(), None);
// Test direct_key_fixed
let flags = MetaEntryFlags::direct_key_fixed(4, 8);
assert!(flags.fresh());
assert!(flags.direct_key());
assert!(flags.fixed_value());
assert!(flags.uncompressed());
assert_eq!(flags.key_size(), 4);
assert_eq!(flags.value_size(), 8);
assert!(flags.uses_direct_keys());
assert!(flags.uses_fixed_values());
assert!(flags.is_uncompressed());
assert_eq!(flags.entry_size(), Some(12));
// Test Display
let flags = MetaEntryFlags::direct_key_fixed(4, 8);
let display = format!("{}", flags);
assert!(display.contains("fresh"));
assert!(display.contains("direct_key(4)"));
assert!(display.contains("fixed_value(8)"));
assert!(display.contains("uncompressed"));
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment