Unverified Commit 30072af8 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-persistence: add TURBO_PERSISTENCE_MMAP=0 to disable mmap

Adds support for disabling memory-mapped file I/O in turbo-persistence via
the TURBO_PERSISTENCE_MMAP=0 environment variable. This is useful in environments
where mmap is not supported or causes issues.

Key changes:
- Add `AccessMode` enum (Mmap / File) to control how SST and meta files are read
- Add `access_mode` field to `DbConfig`, read from TURBO_PERSISTENCE_MMAP env var
- Add `ArcBytes<'l>` with lifetime parameter and `MmapRef` backing variant to
  avoid cloning `Arc<Mmap>` in hot paths
- Add `StaticSortedFileBacking` enum to support both mmap and file-backed reads
- Add `pread`-based file reading path in `StaticSortedFile` for no-mmap mode
- Make `get_raw_block_slice` return `ArcBytes<'_>` to avoid Arc::clone
- Add `ArcBytes::into_static()` for owned promotion and `from_mmap_ref` for borrowing
- Cache the env-var lookup in a `LazyLock`
- `DbConfig::new()` and `Default` both read the env var (via shared helper)
- Remove `RcBytes`/`SharedBytes` compaction path; use `StaticSortedFile::try_into_iter()`
  with backing-enum dispatch instead
- Restore blob sequence number deletion during compaction for dropped entries
Co-Authored-By: 's avatarClaude <noreply@anthropic.com>
Co-authored-by: 's avatarTobias Koppers <sokra@users.noreply.github.com>
parent 605425e1
...@@ -10020,6 +10020,7 @@ dependencies = [ ...@@ -10020,6 +10020,7 @@ dependencies = [
"quick_cache", "quick_cache",
"rand 0.10.1", "rand 0.10.1",
"rayon", "rayon",
"rstest",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"serde_json", "serde_json",
......
...@@ -43,6 +43,7 @@ xxhash-rust = { workspace = true } ...@@ -43,6 +43,7 @@ xxhash-rust = { workspace = true }
criterion = { workspace = true } criterion = { workspace = true }
rand = { workspace = true } rand = { workspace = true }
rayon = { workspace = true } rayon = { workspace = true }
rstest = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
turbo-tasks-malloc = { workspace = true, features = ["custom_allocator"] } turbo-tasks-malloc = { workspace = true, features = ["custom_allocator"] }
......
...@@ -623,6 +623,7 @@ fn prefill_multi_value_database( ...@@ -623,6 +623,7 @@ fn prefill_multi_value_database(
name: "test", name: "test",
kind: FamilyKind::MultiValue, kind: FamilyKind::MultiValue,
}], }],
..TpDbConfig::new()
}; };
let db = let db =
TurboPersistence::<SerialScheduler, 1>::open_with_config(path.to_path_buf(), db_config)?; TurboPersistence::<SerialScheduler, 1>::open_with_config(path.to_path_buf(), db_config)?;
...@@ -697,6 +698,7 @@ fn open_multi_value_db(path: &Path) -> TurboPersistence<SerialScheduler, 1> { ...@@ -697,6 +698,7 @@ fn open_multi_value_db(path: &Path) -> TurboPersistence<SerialScheduler, 1> {
name: "test", name: "test",
kind: FamilyKind::MultiValue, kind: FamilyKind::MultiValue,
}], }],
..TpDbConfig::new()
}; };
TurboPersistence::<SerialScheduler, 1>::open_with_config(path.to_path_buf(), db_config).unwrap() TurboPersistence::<SerialScheduler, 1>::open_with_config(path.to_path_buf(), db_config).unwrap()
} }
...@@ -965,6 +967,7 @@ fn bench_write_multi_value(c: &mut Criterion) { ...@@ -965,6 +967,7 @@ fn bench_write_multi_value(c: &mut Criterion) {
name: "test", name: "test",
kind: FamilyKind::MultiValue, kind: FamilyKind::MultiValue,
}], }],
..TpDbConfig::new()
}; };
let db = TurboPersistence::<SerialScheduler, 1>::open_with_config( let db = TurboPersistence::<SerialScheduler, 1>::open_with_config(
tempdir.path().to_path_buf(), tempdir.path().to_path_buf(),
...@@ -1209,7 +1212,12 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { ...@@ -1209,7 +1212,12 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
sequence_number: 1, sequence_number: 1,
block_count: meta.block_count, block_count: meta.block_count,
}; };
let sst = StaticSortedFile::open(tempdir.path(), sst_meta).unwrap(); let sst = StaticSortedFile::open(
tempdir.path(),
sst_meta,
turbo_persistence::AccessMode::Mmap,
)
.unwrap();
// Create block caches // Create block caches
let key_block_cache: BlockCache = BlockCache::with( let key_block_cache: BlockCache = BlockCache::with(
......
...@@ -2,40 +2,51 @@ use std::{ ...@@ -2,40 +2,51 @@ use std::{
borrow::Borrow, borrow::Borrow,
fmt::{self, Debug, Formatter}, fmt::{self, Debug, Formatter},
hash::{Hash, Hasher}, hash::{Hash, Hasher},
io::{self, Read},
ops::{Deref, Range}, ops::{Deref, Range},
sync::Arc, sync::Arc,
}; };
use memmap2::Mmap; use memmap2::Mmap;
use crate::{
compression::decompress_into_arc,
shared_bytes::{SharedBytes, is_subslice_of},
};
/// The backing storage for an `ArcBytes`. /// The backing storage for an `ArcBytes`.
/// ///
/// The inner values are never read directly — they exist solely to keep the /// The inner values are never read directly — they exist solely to keep the
/// backing memory alive while the raw `data` pointer in `ArcBytes` references it. /// backing memory alive while the raw `data` pointer in `ArcBytes` references it.
#[derive(Clone)] #[derive(Clone)]
enum Backing { enum Backing<'l> {
Arc { _backing: Arc<[u8]> }, Arc {
Mmap { _backing: Arc<Mmap> }, _backing: Arc<[u8]>,
},
Mmap {
_backing: Arc<Mmap>,
},
/// A borrowed reference to an `Arc<Mmap>`, avoiding an `Arc::clone` when the
/// caller only needs short-lived access to the mapped data.
MmapRef {
_backing: &'l Arc<Mmap>,
},
} }
/// An owned byte slice backed by either an `Arc<[u8]>` or a memory-mapped file. /// A byte slice backed by either an `Arc<[u8]>`, a memory-mapped file, or a borrowed
/// reference to a memory-mapped file.
///
/// Most code uses `ArcBytes<'static>` which owns its backing storage.
/// A shorter lifetime is used when the backing is borrowed (e.g. `MmapRef`),
/// and can be promoted to `'static` via [`ArcBytes::into_static`].
#[derive(Clone)] #[derive(Clone)]
pub struct ArcBytes { pub struct ArcBytes<'l> {
data: *const [u8], data: *const [u8],
// Safety: Backing should come last so that it is dropped after the data pointer so we don't // Safety: Backing should come last so that it is dropped after the data pointer so we don't
// create a dangling pointer. This isn't really a problem since it is technically ok to have // create a dangling pointer. This isn't really a problem since it is technically okay to have
// dangling _pointers_. // dangling pointers.
backing: Backing, backing: Backing<'l>,
} }
unsafe impl Send for ArcBytes {} unsafe impl Send for ArcBytes<'_> {}
unsafe impl Sync for ArcBytes {} unsafe impl Sync for ArcBytes<'_> {}
impl From<Arc<[u8]>> for ArcBytes { impl From<Arc<[u8]>> for ArcBytes<'_> {
fn from(arc: Arc<[u8]>) -> Self { fn from(arc: Arc<[u8]>) -> Self {
Self { Self {
data: &*arc as *const [u8], data: &*arc as *const [u8],
...@@ -44,13 +55,13 @@ impl From<Arc<[u8]>> for ArcBytes { ...@@ -44,13 +55,13 @@ impl From<Arc<[u8]>> for ArcBytes {
} }
} }
impl From<Box<[u8]>> for ArcBytes { impl From<Box<[u8]>> for ArcBytes<'_> {
fn from(b: Box<[u8]>) -> Self { fn from(b: Box<[u8]>) -> Self {
Self::from(Arc::from(b)) Self::from(Arc::from(b))
} }
} }
impl Deref for ArcBytes { impl Deref for ArcBytes<'_> {
type Target = [u8]; type Target = [u8];
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
...@@ -58,38 +69,44 @@ impl Deref for ArcBytes { ...@@ -58,38 +69,44 @@ impl Deref for ArcBytes {
} }
} }
impl Borrow<[u8]> for ArcBytes { impl Borrow<[u8]> for ArcBytes<'_> {
fn borrow(&self) -> &[u8] { fn borrow(&self) -> &[u8] {
self self
} }
} }
impl Hash for ArcBytes { impl Hash for ArcBytes<'_> {
fn hash<H: Hasher>(&self, state: &mut H) { fn hash<H: Hasher>(&self, state: &mut H) {
self.deref().hash(state) self.deref().hash(state)
} }
} }
impl PartialEq for ArcBytes { impl PartialEq for ArcBytes<'_> {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
self.deref().eq(other.deref()) self.deref().eq(other.deref())
} }
} }
impl Debug for ArcBytes { impl Debug for ArcBytes<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
Debug::fmt(&**self, f) Debug::fmt(&**self, f)
} }
} }
impl Eq for ArcBytes {} impl Eq for ArcBytes<'_> {}
impl ArcBytes { impl Read for ArcBytes<'_> {
/// Returns `true` if this `ArcBytes` is backed by a memory-mapped file. fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
pub fn is_mmap_backed(&self) -> bool { let available = &**self;
matches!(self.backing, Backing::Mmap { .. }) let len = std::cmp::min(buf.len(), available.len());
buf[..len].copy_from_slice(&available[..len]);
// Advance the slice view
self.data = &available[len..] as *const [u8];
Ok(len)
} }
}
impl ArcBytes<'_> {
/// Returns `true` if the backing `Arc` allocation is shared (i.e., there /// Returns `true` if the backing `Arc` allocation is shared (i.e., there
/// are other `Arc` clones referencing the same data outside the cache). /// are other `Arc` clones referencing the same data outside the cache).
/// Always returns `false` for mmap-backed bytes, since the mmap `Arc` is /// Always returns `false` for mmap-backed bytes, since the mmap `Arc` is
...@@ -97,15 +114,29 @@ impl ArcBytes { ...@@ -97,15 +114,29 @@ impl ArcBytes {
pub fn is_shared_arc(&self) -> bool { pub fn is_shared_arc(&self) -> bool {
match &self.backing { match &self.backing {
Backing::Arc { _backing } => Arc::strong_count(_backing) > 1, Backing::Arc { _backing } => Arc::strong_count(_backing) > 1,
Backing::Mmap { .. } => false, Backing::Mmap { .. } | Backing::MmapRef { .. } => false,
} }
} }
} }
impl SharedBytes for ArcBytes { /// Returns `true` if `subslice` lies entirely within `backing`.
type MmapHandle = Arc<Mmap>; fn is_subslice_of(subslice: &[u8], backing: &[u8]) -> bool {
let backing = backing.as_ptr_range();
let sub = subslice.as_ptr_range();
sub.start >= backing.start && sub.end <= backing.end
}
fn slice(self, range: Range<usize>) -> Self { fn backing_as_slice<'a>(backing: &'a Backing<'a>) -> &'a [u8] {
match backing {
Backing::Arc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
Backing::MmapRef { _backing } => _backing,
}
}
impl<'l> ArcBytes<'l> {
/// Returns a new `ArcBytes` that points to a sub-range of the current slice.
pub fn slice(self, range: Range<usize>) -> ArcBytes<'l> {
let data = &*self; let data = &*self;
let data = &data[range] as *const [u8]; let data = &data[range] as *const [u8];
Self { Self {
...@@ -114,15 +145,16 @@ impl SharedBytes for ArcBytes { ...@@ -114,15 +145,16 @@ impl SharedBytes for ArcBytes {
} }
} }
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self { /// Creates a sub-slice from a slice reference that points into this ArcBytes' backing data.
///
/// # Safety
///
/// The caller must ensure that `subslice` points to memory within this ArcBytes'
/// backing storage (not just within the current slice view, but anywhere in the original
/// backing data).
pub unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> ArcBytes<'l> {
debug_assert!( debug_assert!(
is_subslice_of( is_subslice_of(subslice, backing_as_slice(&self.backing)),
subslice,
match &self.backing {
Backing::Arc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
}
),
"slice_from_subslice: subslice is not within the backing storage" "slice_from_subslice: subslice is not within the backing storage"
); );
Self { Self {
...@@ -131,23 +163,58 @@ impl SharedBytes for ArcBytes { ...@@ -131,23 +163,58 @@ impl SharedBytes for ArcBytes {
} }
} }
unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self { /// Converts this `ArcBytes` into one with a `'static` lifetime.
///
/// For `Arc` and `Mmap` backings this is a no-op. For `MmapRef` backings this
/// clones the inner `Arc<Mmap>` to produce an owned backing.
pub fn into_static(self) -> ArcBytes<'static> {
ArcBytes {
data: self.data,
backing: match self.backing {
Backing::Arc { _backing } => Backing::Arc { _backing },
Backing::Mmap { _backing } => Backing::Mmap { _backing },
Backing::MmapRef { _backing } => Backing::Mmap {
_backing: _backing.clone(),
},
},
}
}
/// Returns `true` if this `ArcBytes` is backed by a memory-mapped file.
pub fn is_mmap_backed(&self) -> bool {
matches!(self.backing, Backing::Mmap { .. } | Backing::MmapRef { .. })
}
/// Creates an `ArcBytes` backed by a memory-mapped file.
///
/// # Safety
///
/// The caller must ensure that `subslice` points to memory within the given `mmap`.
pub unsafe fn from_mmap(mmap: Arc<Mmap>, subslice: &[u8]) -> ArcBytes<'static> {
debug_assert!( debug_assert!(
is_subslice_of(subslice, mmap), is_subslice_of(subslice, &mmap),
"from_mmap: subslice is not within the mmap" "from_mmap: subslice is not within the mmap"
); );
ArcBytes { ArcBytes {
data: subslice as *const [u8], data: subslice as *const [u8],
backing: Backing::Mmap { backing: Backing::Mmap { _backing: mmap },
_backing: mmap.clone(),
},
} }
} }
fn from_decompressed(uncompressed_length: u32, block: &[u8]) -> anyhow::Result<Self> { /// Creates an `ArcBytes` that borrows a reference to an `Arc<Mmap>`, avoiding an
Ok(ArcBytes::from(decompress_into_arc( /// `Arc::clone`. The returned `ArcBytes` is only valid for the lifetime of the reference.
uncompressed_length, ///
block, /// # Safety
)?)) ///
/// The caller must ensure that `subslice` points to memory within the given `mmap`.
pub unsafe fn from_mmap_ref(mmap: &'l Arc<Mmap>, subslice: &[u8]) -> ArcBytes<'l> {
debug_assert!(
is_subslice_of(subslice, mmap),
"from_mmap_ref: subslice is not within the mmap"
);
ArcBytes {
data: subslice as *const [u8],
backing: Backing::MmapRef { _backing: mmap },
}
} }
} }
//! Big-endian integer parsing helpers.
//!
//! Each function reads a fixed-width big-endian integer from the prefix of
//! a byte slice. Panics if the slice is too short (single bounds check via
//! `first_chunk`).
#[inline(always)]
pub fn read_u8(s: &[u8]) -> u8 {
s[0]
}
#[inline(always)]
pub fn read_u16(s: &[u8]) -> u16 {
u16::from_be_bytes(*s.first_chunk().unwrap())
}
#[inline(always)]
pub fn read_u24(s: &[u8]) -> u32 {
let &[a, b, c] = s.first_chunk().unwrap();
u32::from_be_bytes([0, a, b, c])
}
#[inline(always)]
pub fn read_u32(s: &[u8]) -> u32 {
u32::from_be_bytes(*s.first_chunk().unwrap())
}
#[inline(always)]
pub fn read_u64(s: &[u8]) -> u64 {
u64::from_be_bytes(*s.first_chunk().unwrap())
}
...@@ -266,7 +266,8 @@ fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> { ...@@ -266,7 +266,8 @@ fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
let mut meta_files: Vec<MetaFile> = meta_seqs let mut meta_files: Vec<MetaFile> = meta_seqs
.iter() .iter()
.map(|&seq| { .map(|&seq| {
MetaFile::open(db_path, seq).with_context(|| format!("Failed to open {seq:08}.meta")) MetaFile::open(db_path, seq, turbo_persistence::AccessMode::Mmap)
.with_context(|| format!("Failed to open {seq:08}.meta"))
}) })
.collect::<Result<_>>()?; .collect::<Result<_>>()?;
......
use std::{mem::MaybeUninit, rc::Rc, sync::Arc}; use std::{mem::MaybeUninit, sync::Arc};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use lzzzz::lz4::{self, decompress}; use lzzzz::lz4::{self, decompress};
...@@ -41,17 +41,6 @@ pub fn decompress_into_arc(uncompressed_length: u32, block: &[u8]) -> Result<Arc ...@@ -41,17 +41,6 @@ pub fn decompress_into_arc(uncompressed_length: u32, block: &[u8]) -> Result<Arc
Ok(buffer) Ok(buffer)
} }
/// Like [`decompress_into_arc`] but returns an `Rc<[u8]>` for thread-local use.
pub fn decompress_into_rc(uncompressed_length: u32, block: &[u8]) -> Result<Rc<[u8]>> {
let buffer: Rc<[MaybeUninit<u8>]> = Rc::new_uninit_slice(uncompressed_length as usize);
// Safety: decompression will fully initialize the buffer (verified by the assert in
// decompress_block).
let mut buffer = unsafe { buffer.assume_init() };
let dest = Rc::get_mut(&mut buffer).expect("Rc refcount should be 1");
decompress_block(block, dest, uncompressed_length)?;
Ok(buffer)
}
/// Computes a CRC32 checksum of a byte slice. /// Computes a CRC32 checksum of a byte slice.
pub fn checksum_block(data: &[u8]) -> u32 { pub fn checksum_block(data: &[u8]) -> u32 {
crc32fast::hash(data) crc32fast::hash(data)
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
#![feature(sync_unsafe_cell)] #![feature(sync_unsafe_cell)]
mod arc_bytes; mod arc_bytes;
pub(crate) mod be;
mod collector; mod collector;
mod collector_entry; mod collector_entry;
mod compaction; mod compaction;
...@@ -16,8 +15,6 @@ pub mod meta_file; ...@@ -16,8 +15,6 @@ pub mod meta_file;
mod meta_file_builder; mod meta_file_builder;
pub mod mmap_helper; pub mod mmap_helper;
mod parallel_scheduler; mod parallel_scheduler;
mod rc_bytes;
mod shared_bytes;
pub mod sst_filter; pub mod sst_filter;
pub mod static_sorted_file; pub mod static_sorted_file;
mod static_sorted_file_builder; mod static_sorted_file_builder;
...@@ -35,6 +32,15 @@ pub use db::{ ...@@ -35,6 +32,15 @@ pub use db::{
TurboPersistence, read_current_version, TurboPersistence, read_current_version,
}; };
/// Controls how SST and meta files are read from disk.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AccessMode {
/// Memory-map the file and access blocks via the mapped region.
Mmap,
/// Read blocks directly from the file via pread (no mmap).
File,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FamilyKind { pub enum FamilyKind {
/// Each key maps to a single value (default LSM behavior). /// Each key maps to a single value (default LSM behavior).
...@@ -63,21 +69,48 @@ pub struct FamilyConfig { ...@@ -63,21 +69,48 @@ pub struct FamilyConfig {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct DbConfig<const FAMILIES: usize> { pub struct DbConfig<const FAMILIES: usize> {
pub family_configs: [FamilyConfig; FAMILIES], pub family_configs: [FamilyConfig; FAMILIES],
/// How SST and meta files are read from disk.
pub access_mode: AccessMode,
} }
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> { /// Reads the `TURBO_PERSISTENCE_MMAP` env var (cached). Returns `AccessMode::File` when the var
fn default() -> Self { /// is set to `"0"`, `AccessMode::Mmap` otherwise.
fn access_mode_env_var() -> AccessMode {
static ACCESS_MODE_ENV: std::sync::LazyLock<AccessMode> = std::sync::LazyLock::new(|| {
if std::env::var("TURBO_PERSISTENCE_MMAP")
.map(|v| v == "0")
.unwrap_or(false)
{
AccessMode::File
} else {
AccessMode::Mmap
}
});
*ACCESS_MODE_ENV
}
impl<const FAMILIES: usize> DbConfig<FAMILIES> {
/// Returns a config with all defaults, reading the `TURBO_PERSISTENCE_MMAP` env var
/// to determine the access mode.
pub fn new() -> Self {
Self { Self {
family_configs: [FamilyConfig { family_configs: [FamilyConfig {
name: "unknown", name: "unknown",
kind: FamilyKind::SingleValue, kind: FamilyKind::SingleValue,
}; FAMILIES], }; FAMILIES],
access_mode: access_mode_env_var(),
} }
} }
} }
/// The largest value that [`WriteBatch::delete_value`] can delete, since the tombstone stores /// The largest value that [`WriteBatch::delete_value`] can delete, since the tombstone stores
/// a copy of the value inline. /// a copy of the value inline.
pub use constants::MAX_INLINE_VALUE_SIZE; pub use constants::MAX_INLINE_VALUE_SIZE;
impl<const FAMILIES: usize> Default for DbConfig<FAMILIES> {
fn default() -> Self {
Self::new()
}
}
pub use key::{KeyBase, QueryKey, StoreKey, hash_key}; pub use key::{KeyBase, QueryKey, StoreKey, hash_key};
pub use meta_file::MetaEntryFlags; pub use meta_file::MetaEntryFlags;
pub use parallel_scheduler::{ParallelScheduler, SerialScheduler}; pub use parallel_scheduler::{ParallelScheduler, SerialScheduler};
......
use crate::{ use crate::{
ArcBytes, ArcBytes,
constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
rc_bytes::RcBytes,
static_sorted_file_builder::{Entry, EntryValue}, static_sorted_file_builder::{Entry, EntryValue},
}; };
/// A value from a SST file. Generic over the byte representation, defaulting to /// A value from an SST file lookup.
/// `ArcBytes` for the lookup path. The compaction/iteration path uses
/// `LookupValue<RcBytes>` which is convertible to `IterValue`.
#[derive(PartialEq)] #[derive(PartialEq)]
pub enum LookupValue<B = ArcBytes> { pub enum LookupValue {
/// The value was deleted. /// The key and all of its values were deleted.
KeyDeleted, KeyDeleted,
/// A single value was deleted from this key's group (MultiValue families only). Other values /// A single value was deleted from this key's group (MultiValue families only).
/// for the same key are unaffected. The bytes are the deleted value. KeyValueDeleted { value: ArcBytes<'static> },
KeyValueDeleted { value: B },
/// The value is stored in the SST file. /// The value is stored in the SST file.
/// Slice { value: ArcBytes<'static> },
/// The bytes will be pointing either at a keyblock or a value block in the SST
Slice { value: B },
/// The value is stored in a blob file. /// The value is stored in a blob file.
Blob { sequence_number: u32 }, Blob { sequence_number: u32 },
} }
/// A value from SST file iteration (compaction path, uses RcBytes for /// A value from an SST file lookup that may still be compressed.
/// non-atomic refcounting). pub enum LazyLookupValue {
pub enum IterValue { Eager(LookupValue),
/// The value was deleted. /// A medium-sized value that is still compressed.
KeyDeleted,
/// A single value was deleted from this key's group (MultiValue families only).
KeyValueDeleted { value: RcBytes },
/// The value is stored in the SST file.
Slice { value: RcBytes },
/// The value is stored in a blob file.
Blob { sequence_number: u32 },
/// A medium sized value that is still compressed.
Medium { Medium {
uncompressed_size: u32, uncompressed_size: u32,
checksum: u32, checksum: u32,
block: RcBytes, block: ArcBytes<'static>,
}, },
} }
impl From<LookupValue<RcBytes>> for IterValue {
fn from(v: LookupValue<RcBytes>) -> Self { impl LazyLookupValue {
match v { /// Returns the size of the value in the SST file.
LookupValue::KeyDeleted => IterValue::KeyDeleted, pub fn uncompressed_size_in_sst(&self) -> usize {
LookupValue::KeyValueDeleted { value } => IterValue::KeyValueDeleted { value }, match self {
LookupValue::Slice { value } => IterValue::Slice { value }, LazyLookupValue::Eager(LookupValue::Slice { value }) => value.len(),
LookupValue::Blob { sequence_number } => IterValue::Blob { sequence_number }, LazyLookupValue::Eager(
LookupValue::KeyDeleted
| LookupValue::KeyValueDeleted { .. }
| LookupValue::Blob { .. },
) => 0,
LazyLookupValue::Medium {
uncompressed_size,
block,
..
} => {
if *uncompressed_size == 0 {
block.len()
} else {
*uncompressed_size as usize
}
}
}
}
/// Returns true if this value gets its own dedicated value block.
pub fn is_medium_value(&self) -> bool {
match self {
LazyLookupValue::Eager(LookupValue::Slice { value })
if value.len() > MAX_SMALL_VALUE_SIZE =>
{
true
}
LazyLookupValue::Medium { .. } => true,
_ => false,
}
}
/// Returns the value size if it will be packed into a small value block, or 0 otherwise.
pub fn small_value_size(&self) -> usize {
match self {
LazyLookupValue::Eager(LookupValue::Slice { value })
if value.len() > MAX_INLINE_VALUE_SIZE && value.len() <= MAX_SMALL_VALUE_SIZE =>
{
value.len()
}
_ => 0,
} }
} }
} }
/// An entry from SST file iteration (compaction path, uses RcBytes).
/// An entry from an SST file lookup.
pub struct LookupEntry { pub struct LookupEntry {
/// The hash of the key.
pub hash: u64, pub hash: u64,
/// The key. pub key: ArcBytes<'static>,
pub key: RcBytes, pub value: LazyLookupValue,
/// The value.
pub value: IterValue,
} }
impl Entry for LookupEntry { impl Entry for LookupEntry {
...@@ -76,9 +100,11 @@ impl Entry for LookupEntry { ...@@ -76,9 +100,11 @@ impl Entry for LookupEntry {
fn value(&self) -> EntryValue<'_> { fn value(&self) -> EntryValue<'_> {
match &self.value { match &self.value {
IterValue::KeyDeleted => EntryValue::KeyDeleted, LazyLookupValue::Eager(LookupValue::KeyDeleted) => EntryValue::KeyDeleted,
IterValue::KeyValueDeleted { value } => EntryValue::KeyValueDeleted { value }, LazyLookupValue::Eager(LookupValue::KeyValueDeleted { value }) => {
IterValue::Slice { value } => { EntryValue::KeyValueDeleted { value }
}
LazyLookupValue::Eager(LookupValue::Slice { value }) => {
if value.len() <= MAX_INLINE_VALUE_SIZE { if value.len() <= MAX_INLINE_VALUE_SIZE {
EntryValue::Inline { value } EntryValue::Inline { value }
} else if value.len() > MAX_SMALL_VALUE_SIZE { } else if value.len() > MAX_SMALL_VALUE_SIZE {
...@@ -87,10 +113,10 @@ impl Entry for LookupEntry { ...@@ -87,10 +113,10 @@ impl Entry for LookupEntry {
EntryValue::Small { value } EntryValue::Small { value }
} }
} }
IterValue::Blob { sequence_number } => EntryValue::Large { LazyLookupValue::Eager(LookupValue::Blob { sequence_number }) => EntryValue::Large {
blob: *sequence_number, blob: *sequence_number,
}, },
IterValue::Medium { LazyLookupValue::Medium {
uncompressed_size, uncompressed_size,
checksum, checksum,
block, block,
......
...@@ -14,17 +14,17 @@ use crate::{ ...@@ -14,17 +14,17 @@ use crate::{
static_sorted_file_builder::StaticSortedFileBuilderMeta, static_sorted_file_builder::StaticSortedFileBuilderMeta,
}; };
pub struct MetaFileBuilder<'a> { pub struct MetaFileBuilder {
family: u32, family: u32,
/// Entries in the meta file, tuples of (sequence_number, StaticSortedFileBuilderMetaResult) /// Entries in the meta file, tuples of (sequence_number, StaticSortedFileBuilderMetaResult)
entries: Vec<(u32, StaticSortedFileBuilderMeta<'a>)>, entries: Vec<(u32, StaticSortedFileBuilderMeta)>,
/// Obsolete SST files, represented by their sequence numbers /// Obsolete SST files, represented by their sequence numbers
obsolete_sst_files: Vec<u32>, obsolete_sst_files: Vec<u32>,
/// Optional AMQF for used key hashes /// Optional AMQF for used key hashes
used_key_hashes_amqf: Option<Filter>, used_key_hashes_amqf: Option<Filter>,
} }
impl<'a> MetaFileBuilder<'a> { impl MetaFileBuilder {
pub fn new(family: u32) -> Self { pub fn new(family: u32) -> Self {
Self { Self {
family, family,
...@@ -34,7 +34,7 @@ impl<'a> MetaFileBuilder<'a> { ...@@ -34,7 +34,7 @@ impl<'a> MetaFileBuilder<'a> {
} }
} }
pub fn add(&mut self, sequence_number: u32, sst: StaticSortedFileBuilderMeta<'a>) { pub fn add(&mut self, sequence_number: u32, sst: StaticSortedFileBuilderMeta) {
self.entries.push((sequence_number, sst)); self.entries.push((sequence_number, sst));
} }
......
use std::{
borrow::Borrow,
fmt::{self, Debug, Formatter},
hash::{Hash, Hasher},
ops::{Deref, Range},
rc::Rc,
};
use memmap2::Mmap;
use crate::{
compression::decompress_into_rc,
shared_bytes::{SharedBytes, is_subslice_of},
};
/// The backing storage for an `RcBytes`.
///
/// Uses `Rc` for all refcounting, eliminating atomic operations.
#[derive(Clone)]
enum Backing {
Rc { _backing: Rc<[u8]> },
Mmap { _backing: Rc<Mmap> },
}
/// An owned byte slice backed by either an `Rc<[u8]>` or a memory-mapped file.
///
/// Identical to `ArcBytes` but uses `Rc` instead of `Arc`, eliminating atomic
/// refcount overhead. Use this in single-threaded contexts like SST iteration
/// during compaction.
#[derive(Clone)]
pub struct RcBytes {
data: *const [u8],
backing: Backing,
}
impl From<Rc<[u8]>> for RcBytes {
fn from(rc: Rc<[u8]>) -> Self {
Self {
data: &*rc as *const [u8],
backing: Backing::Rc { _backing: rc },
}
}
}
impl From<Box<[u8]>> for RcBytes {
fn from(b: Box<[u8]>) -> Self {
Self::from(Rc::from(b))
}
}
impl Deref for RcBytes {
type Target = [u8];
fn deref(&self) -> &Self::Target {
unsafe { &*self.data }
}
}
impl Borrow<[u8]> for RcBytes {
fn borrow(&self) -> &[u8] {
self
}
}
impl PartialEq for RcBytes {
fn eq(&self, other: &Self) -> bool {
self.deref().eq(other.deref())
}
}
impl Debug for RcBytes {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
Debug::fmt(&**self, f)
}
}
impl Eq for RcBytes {}
impl Hash for RcBytes {
fn hash<H: Hasher>(&self, state: &mut H) {
Hash::hash(self.deref(), state);
}
}
impl SharedBytes for RcBytes {
type MmapHandle = Rc<Mmap>;
fn slice(self, range: Range<usize>) -> Self {
let data = &*self;
let data = &data[range] as *const [u8];
Self {
data,
backing: self.backing,
}
}
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
debug_assert!(
is_subslice_of(
subslice,
match &self.backing {
Backing::Rc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
}
),
"slice_from_subslice: subslice is not within the backing storage"
);
Self {
data: subslice as *const [u8],
backing: self.backing.clone(),
}
}
unsafe fn from_mmap(mmap: &Rc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!(
is_subslice_of(subslice, mmap),
"from_mmap: subslice is not within the mmap"
);
RcBytes {
data: subslice as *const [u8],
backing: Backing::Mmap {
_backing: mmap.clone(),
},
}
}
fn from_decompressed(uncompressed_length: u32, block: &[u8]) -> anyhow::Result<Self> {
Ok(RcBytes::from(decompress_into_rc(
uncompressed_length,
block,
)?))
}
}
use std::ops::{Deref, Range};
use memmap2::Mmap;
/// Trait abstracting over `ArcBytes` and `RcBytes`.
///
/// Both types are owned byte slices backed by either a ref-counted heap
/// allocation or a memory-mapped file. The only difference is the refcount
/// flavor: `Arc` (atomic, thread-safe) vs `Rc` (non-atomic, single-threaded).
///
/// This trait captures the shared interface so that code in
/// `static_sorted_file.rs` (block reading, key matching, etc.) can be written
/// once and used for both lookup (ArcBytes) and iteration (RcBytes) paths.
pub trait SharedBytes: Clone + Deref<Target = [u8]> + Sized {
/// The ref-counted handle to the memory-mapped file.
type MmapHandle: Deref<Target = Mmap>;
/// Returns a new instance that points to a sub-range of the current slice.
fn slice(self, range: Range<usize>) -> Self;
/// Creates a sub-slice from a reference that points into this instance's
/// backing data.
///
/// # Safety
///
/// The caller must ensure that `subslice` points to memory within this
/// instance's backing storage.
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self;
/// Creates an instance backed by a memory-mapped file.
///
/// # Safety
///
/// The caller must ensure that `subslice` points to memory within the
/// given `mmap`.
unsafe fn from_mmap(mmap: &Self::MmapHandle, subslice: &[u8]) -> Self;
/// Creates an instance from a decompressed block.
fn from_decompressed(uncompressed_length: u32, block: &[u8]) -> anyhow::Result<Self>;
}
/// Returns `true` if `subslice` lies entirely within `backing`.
pub(crate) fn is_subslice_of(subslice: &[u8], backing: &[u8]) -> bool {
let backing = backing.as_ptr_range();
let sub = subslice.as_ptr_range();
sub.start >= backing.start && sub.end <= backing.end
}
use std::{ use std::{
borrow::Cow,
collections::VecDeque, collections::VecDeque,
fmt,
io::{BufWriter, Write}, io::{BufWriter, Write},
ops::Deref,
path::Path, path::Path,
}; };
...@@ -10,6 +11,7 @@ use byteorder::{BE, ByteOrder, WriteBytesExt}; ...@@ -10,6 +11,7 @@ use byteorder::{BE, ByteOrder, WriteBytesExt};
use fs_err::File; use fs_err::File;
use crate::{ use crate::{
arc_bytes::ArcBytes,
compression::{checksum_block, compress_into_buffer}, compression::{checksum_block, compress_into_buffer},
constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE},
meta_file::MetaEntryFlags, meta_file::MetaEntryFlags,
...@@ -281,14 +283,51 @@ pub enum EntryValue<'l> { ...@@ -281,14 +283,51 @@ pub enum EntryValue<'l> {
KeyValueDeleted { value: &'l [u8] }, KeyValueDeleted { value: &'l [u8] },
} }
/// Owned byte data for an AMQF filter, backed by either an `ArcBytes<'static>` (zero-copy from
/// mmap / pread) or a `Vec<u8>` (freshly serialized).
#[derive(Clone)]
pub enum AmqfData {
ArcBytes(ArcBytes<'static>),
Vec(Vec<u8>),
}
impl Deref for AmqfData {
type Target = [u8];
fn deref(&self) -> &[u8] {
match self {
AmqfData::ArcBytes(a) => a,
AmqfData::Vec(v) => v,
}
}
}
impl fmt::Debug for AmqfData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_tuple("AmqfData").field(&self.len()).finish()
}
}
impl From<ArcBytes<'static>> for AmqfData {
fn from(a: ArcBytes<'static>) -> Self {
AmqfData::ArcBytes(a)
}
}
impl From<Vec<u8>> for AmqfData {
fn from(v: Vec<u8>) -> Self {
AmqfData::Vec(v)
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct StaticSortedFileBuilderMeta<'a> { pub struct StaticSortedFileBuilderMeta {
/// The minimum hash of the keys in the SST file /// The minimum hash of the keys in the SST file
pub min_hash: u64, pub min_hash: u64,
/// The maximum hash of the keys in the SST file /// The maximum hash of the keys in the SST file
pub max_hash: u64, pub max_hash: u64,
/// The AMQF data /// The AMQF data
pub amqf: Cow<'a, [u8]>, pub amqf: AmqfData,
/// The number of blocks in the SST file /// The number of blocks in the SST file
pub block_count: u16, pub block_count: u16,
/// The file size of the SST file /// The file size of the SST file
...@@ -309,7 +348,7 @@ pub fn write_static_stored_file<E: Entry>( ...@@ -309,7 +348,7 @@ pub fn write_static_stored_file<E: Entry>(
entries: &[E], entries: &[E],
file: &Path, file: &Path,
flags: MetaEntryFlags, flags: MetaEntryFlags,
) -> Result<(StaticSortedFileBuilderMeta<'static>, File)> { ) -> Result<(StaticSortedFileBuilderMeta, File)> {
debug_assert!(entries.iter().map(|e| e.key_hash()).is_sorted()); debug_assert!(entries.iter().map(|e| e.key_hash()).is_sorted());
let mut writer = StreamingSstWriter::new(file, flags, entries.len() as u64)?; let mut writer = StreamingSstWriter::new(file, flags, entries.len() as u64)?;
for entry in entries { for entry in entries {
...@@ -939,7 +978,7 @@ impl<E: Entry> StreamingSstWriter<E> { ...@@ -939,7 +978,7 @@ impl<E: Entry> StreamingSstWriter<E> {
/// Finishes writing the SST file. Flushes remaining blocks, writes the index, and returns /// Finishes writing the SST file. Flushes remaining blocks, writes the index, and returns
/// metadata. /// metadata.
pub fn close(mut self) -> Result<(StaticSortedFileBuilderMeta<'static>, File)> { pub fn close(mut self) -> Result<(StaticSortedFileBuilderMeta, File)> {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
{ {
self.finished = true; self.finished = true;
...@@ -1029,7 +1068,7 @@ impl<E: Entry> StreamingSstWriter<E> { ...@@ -1029,7 +1068,7 @@ impl<E: Entry> StreamingSstWriter<E> {
let meta = StaticSortedFileBuilderMeta { let meta = StaticSortedFileBuilderMeta {
min_hash: self.min_hash, min_hash: self.min_hash,
max_hash: self.max_hash, max_hash: self.max_hash,
amqf: Cow::Owned(amqf), amqf: amqf.into(),
block_count, block_count,
size: file_size, size: file_size,
flags: self.flags, flags: self.flags,
...@@ -1424,7 +1463,7 @@ mod tests { ...@@ -1424,7 +1463,7 @@ mod tests {
fn open_sst( fn open_sst(
dir: &Path, dir: &Path,
seq: u32, seq: u32,
meta: &StaticSortedFileBuilderMeta<'_>, meta: &StaticSortedFileBuilderMeta,
) -> Result<StaticSortedFile> { ) -> Result<StaticSortedFile> {
StaticSortedFile::open( StaticSortedFile::open(
dir, dir,
...@@ -1432,6 +1471,7 @@ mod tests { ...@@ -1432,6 +1471,7 @@ mod tests {
sequence_number: seq, sequence_number: seq,
block_count: meta.block_count, block_count: meta.block_count,
}, },
crate::AccessMode::Mmap,
) )
} }
...@@ -1441,7 +1481,7 @@ mod tests { ...@@ -1441,7 +1481,7 @@ mod tests {
seq: u32, seq: u32,
entries: &[TestEntry], entries: &[TestEntry],
flags: MetaEntryFlags, flags: MetaEntryFlags,
) -> Result<StaticSortedFileBuilderMeta<'static>> { ) -> Result<StaticSortedFileBuilderMeta> {
let sst_path = dir.join(format!("{seq:08}.sst")); let sst_path = dir.join(format!("{seq:08}.sst"));
let mut writer = StreamingSstWriter::new(&sst_path, flags, entries.len() as u64)?; let mut writer = StreamingSstWriter::new(&sst_path, flags, entries.len() as u64)?;
for entry in entries { for entry in entries {
...@@ -1769,6 +1809,7 @@ mod tests { ...@@ -1769,6 +1809,7 @@ mod tests {
sequence_number: 1, sequence_number: 1,
block_count: meta1.block_count, block_count: meta1.block_count,
}, },
crate::AccessMode::Mmap,
)?; )?;
let sst2 = StaticSortedFile::open( let sst2 = StaticSortedFile::open(
dir.path(), dir.path(),
...@@ -1776,6 +1817,7 @@ mod tests { ...@@ -1776,6 +1817,7 @@ mod tests {
sequence_number: 2, sequence_number: 2,
block_count: meta2.block_count, block_count: meta2.block_count,
}, },
crate::AccessMode::Mmap,
)?; )?;
let kc = make_cache(); let kc = make_cache();
let vc = make_cache(); let vc = make_cache();
...@@ -2003,7 +2045,7 @@ mod tests { ...@@ -2003,7 +2045,7 @@ mod tests {
fn assert_corruption_detected( fn assert_corruption_detected(
dir: &Path, dir: &Path,
seq: u32, seq: u32,
meta: &StaticSortedFileBuilderMeta<'_>, meta: &StaticSortedFileBuilderMeta,
entries: &[TestEntry], entries: &[TestEntry],
) { ) {
let sst = open_sst(dir, seq, meta).unwrap(); let sst = open_sst(dir, seq, meta).unwrap();
......
...@@ -89,7 +89,7 @@ pub struct WriteBatch<'db, K: StoreKey + Send, S: ParallelScheduler, const FAMIL ...@@ -89,7 +89,7 @@ pub struct WriteBatch<'db, K: StoreKey + Send, S: ParallelScheduler, const FAMIL
/// Collectors in use. The thread local collectors flush into these when they are full. /// Collectors in use. The thread local collectors flush into these when they are full.
collectors: [Mutex<GlobalCollectorState<K>>; FAMILIES], collectors: [Mutex<GlobalCollectorState<K>>; FAMILIES],
/// Meta file builders for each family. /// Meta file builders for each family.
meta_collectors: [Mutex<Vec<(u32, StaticSortedFileBuilderMeta<'static>)>>; FAMILIES], meta_collectors: [Mutex<Vec<(u32, StaticSortedFileBuilderMeta)>>; FAMILIES],
/// The list of new SST files that have been created. /// The list of new SST files that have been created.
new_sst_files: Mutex<Vec<NewFile>>, new_sst_files: Mutex<Vec<NewFile>>,
} }
...@@ -540,6 +540,7 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize ...@@ -540,6 +540,7 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
sequence_number: seq, sequence_number: seq,
block_count: meta.block_count, block_count: meta.block_count,
}, },
true,
)?; )?;
let cache2 = BlockCache::with( let cache2 = BlockCache::with(
10, 10,
......
...@@ -34,6 +34,7 @@ const MB: u64 = 1024 * 1024; ...@@ -34,6 +34,7 @@ const MB: u64 = 1024 * 1024;
pub fn db_config() -> DbConfig<FAMILIES> { pub fn db_config() -> DbConfig<FAMILIES> {
DbConfig { DbConfig {
family_configs: std::array::from_fn(|i| KeySpace::from_index(i).family_config()), family_configs: std::array::from_fn(|i| KeySpace::from_index(i).family_config()),
..DbConfig::new()
} }
} }
...@@ -94,18 +95,26 @@ impl TurboKeyValueDatabase { ...@@ -94,18 +95,26 @@ impl TurboKeyValueDatabase {
self.db.is_empty() self.db.is_empty()
} }
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> { pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes<'static>>> {
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
pub fn batch_get(&self, key_space: KeySpace, keys: &[&[u8]]) -> Result<Vec<Option<ArcBytes>>> { pub fn batch_get(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<ArcBytes<'static>>>> {
self.db.batch_get(key_space as usize, keys) self.db.batch_get(key_space as usize, keys)
} }
/// Looks up a key and returns all matching values. /// Looks up a key and returns all matching values.
/// ///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache). /// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
pub fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> { pub fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
self.db.get_multiple(key_space as usize, &key) self.db.get_multiple(key_space as usize, &key)
} }
...@@ -216,7 +225,7 @@ pub struct TurboWriteBatch<'a> { ...@@ -216,7 +225,7 @@ pub struct TurboWriteBatch<'a> {
} }
impl<'a> TurboWriteBatch<'a> { impl<'a> TurboWriteBatch<'a> {
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> { pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes<'static>>> {
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment