Unverified Commit cb3f3ac4 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-persistence: simplify compaction byte storage

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent dcb0d012
...@@ -8,11 +8,7 @@ use std::{ ...@@ -8,11 +8,7 @@ use std::{
use memmap2::Mmap; use memmap2::Mmap;
use crate::{ use crate::compression::{Compression, decompress_into_arc};
Compression,
compression::decompress_into_arc,
shared_bytes::{SharedBytes, is_subslice_of},
};
/// The backing storage for an `ArcBytes`. /// The backing storage for an `ArcBytes`.
#[derive(Clone)] #[derive(Clone)]
...@@ -89,6 +85,12 @@ impl Debug for ArcBytes<'_> { ...@@ -89,6 +85,12 @@ impl Debug for ArcBytes<'_> {
impl Eq for ArcBytes<'_> {} impl Eq for ArcBytes<'_> {}
fn is_subslice_of(subslice: &[u8], backing: &[u8]) -> bool {
let backing = backing.as_ptr_range();
let subslice = subslice.as_ptr_range();
subslice.start >= backing.start && subslice.end <= backing.end
}
fn backing_as_slice<'a>(backing: &'a Backing<'a>) -> &'a [u8] { fn backing_as_slice<'a>(backing: &'a Backing<'a>) -> &'a [u8] {
match backing { match backing {
Backing::Arc { _backing } => _backing, Backing::Arc { _backing } => _backing,
...@@ -160,28 +162,11 @@ impl<'l> ArcBytes<'l> { ...@@ -160,28 +162,11 @@ impl<'l> ArcBytes<'l> {
backing: Backing::MmapRef { _backing: mmap }, backing: Backing::MmapRef { _backing: mmap },
} }
} }
} pub(crate) fn from_decompressed(
impl SharedBytes for ArcBytes<'static> {
type MmapHandle = Arc<Mmap>;
fn slice(self, range: Range<usize>) -> Self {
ArcBytes::slice(self, range)
}
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
unsafe { ArcBytes::slice_from_subslice(self, subslice) }
}
unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self {
unsafe { ArcBytes::from_mmap(mmap.clone(), subslice) }
}
fn from_decompressed(
compression: Compression, compression: Compression,
uncompressed_length: u32, uncompressed_length: u32,
block: &[u8], block: &[u8],
) -> anyhow::Result<Self> { ) -> anyhow::Result<ArcBytes<'static>> {
Ok(ArcBytes::from(decompress_into_arc( Ok(ArcBytes::from(decompress_into_arc(
compression, compression,
uncompressed_length, uncompressed_length,
......
use std::{cell::RefCell, mem::MaybeUninit, rc::Rc, sync::Arc}; use std::{cell::RefCell, mem::MaybeUninit, sync::Arc};
use anyhow::{Context, Result, ensure}; use anyhow::{Context, Result, ensure};
use lzzzz::lz4::{self, decompress}; use lzzzz::lz4::{self, decompress};
...@@ -79,21 +79,6 @@ pub(crate) fn decompress_into_arc( ...@@ -79,21 +79,6 @@ pub(crate) fn decompress_into_arc(
Ok(buffer) Ok(buffer)
} }
/// Like [`decompress_into_arc`] but returns an `Rc<[u8]>` for thread-local use.
pub(crate) fn decompress_into_rc(
compression: Compression,
uncompressed_length: u32,
block: &[u8],
) -> Result<Rc<[u8]>> {
let buffer: Rc<[MaybeUninit<u8>]> = Rc::new_uninit_slice(uncompressed_length as usize);
// Safety: decompression will fully initialize the buffer (verified by the length check in
// decompress_block).
let mut buffer = unsafe { buffer.assume_init() };
let dest = Rc::get_mut(&mut buffer).expect("Rc refcount should be 1");
decompress_block(compression, block, dest, uncompressed_length)?;
Ok(buffer)
}
/// Computes a CRC32 checksum of a byte slice. /// Computes a CRC32 checksum of a byte slice.
pub fn checksum_block(data: &[u8]) -> u32 { pub fn checksum_block(data: &[u8]) -> u32 {
crc32fast::hash(data) crc32fast::hash(data)
......
...@@ -39,13 +39,12 @@ use crate::{ ...@@ -39,13 +39,12 @@ use crate::{
MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE,
}, },
key::{StoreKey, hash_key}, key::{StoreKey, hash_key},
lookup_entry::{IterValue, LookupEntry, LookupValue}, lookup_entry::{LazyLookupValue, LookupEntry, LookupValue},
merge_iter::MergeIter, merge_iter::MergeIter,
meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange}, meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange},
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
mmap_helper::advise_mmap_for_persistence, mmap_helper::advise_mmap_for_persistence,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
rc_bytes::RcBytes,
sst_filter::SstFilter, sst_filter::SstFilter,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFileIter}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFileIter},
static_sorted_file_builder::{StaticSortedFileBuilderMeta, StreamingSstWriter}, static_sorted_file_builder::{StaticSortedFileBuilderMeta, StreamingSstWriter},
...@@ -1735,7 +1734,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1735,7 +1734,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Collector::new(MetaEntryFlags::WARM, compression); Collector::new(MetaEntryFlags::WARM, compression);
let mut unused_collector = let mut unused_collector =
Collector::new(MetaEntryFlags::COLD, compression); Collector::new(MetaEntryFlags::COLD, compression);
let mut current_key: Option<RcBytes> = None; let mut current_key: Option<ArcBytes<'static>> = None;
let mut keys_written = 0; let mut keys_written = 0;
// MergeIter yields entries from newer SSTs first (by SST sequence // MergeIter yields entries from newer SSTs first (by SST sequence
...@@ -1748,7 +1747,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1748,7 +1747,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// Values deleted by key-value tombstones in the current key group. // Values deleted by key-value tombstones in the current key group.
// Reset at each key boundary. // Reset at each key boundary.
let mut deleted_values_for_this_key: AutoSet< let mut deleted_values_for_this_key: AutoSet<
RcBytes, ArcBytes<'static>,
BuildHasherDefault<FxHasher>, BuildHasherDefault<FxHasher>,
1, 1,
> = AutoSet::default(); > = AutoSet::default();
...@@ -1765,7 +1764,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1765,7 +1764,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// Key-value tombstones sort first within a group, so each is // Key-value tombstones sort first within a group, so each is
// recorded before the values it might delete. // recorded before the values it might delete.
// See: `crate::collector_entry::sort_rank` // See: `crate::collector_entry::sort_rank`
if let IterValue::KeyValueDeleted { value } = &entry.value { if let LazyLookupValue::Eager(LookupValue::KeyValueDeleted { value }) =
&entry.value {
deleted_values_for_this_key.insert(value.clone()); deleted_values_for_this_key.insert(value.clone());
// Applied to this job's values above; keep it only if an SST // Applied to this job's values above; keep it only if an SST
// outside the job could still hold a matching key. // outside the job could still hold a matching key.
...@@ -1774,7 +1774,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1774,7 +1774,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
} else if !deleted_values_for_this_key.is_empty() } else if !deleted_values_for_this_key.is_empty()
// Deleted values cannot match blobs, just normal payloads. // Deleted values cannot match blobs, just normal payloads.
&& let IterValue::Slice { value } = &entry.value && let LazyLookupValue::Eager(LookupValue::Slice { value }) = &entry.value
&& deleted_values_for_this_key.contains(value) && deleted_values_for_this_key.contains(value)
{ {
// Deleted by a key-value tombstone seen earlier in this group. // Deleted by a key-value tombstone seen earlier in this group.
...@@ -1794,7 +1794,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1794,7 +1794,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// For MultiValue families we only skip remaining if we // For MultiValue families we only skip remaining if we
// see a key tombstone. Key-value tombstones are // see a key tombstone. Key-value tombstones are
// handled above and never reach here. // handled above and never reach here.
if matches!(entry.value, IterValue::KeyDeleted) { if matches!(
entry.value,
LazyLookupValue::Eager(LookupValue::KeyDeleted)
) {
skip_remaining_for_this_key = true; skip_remaining_for_this_key = true;
} }
} }
...@@ -1805,7 +1808,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1805,7 +1808,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
} }
// If this is a tombstone, see if we need to retain it or not. // If this is a tombstone, see if we need to retain it or not.
if matches!(entry.value, IterValue::KeyDeleted) if matches!(
entry.value,
LazyLookupValue::Eager(LookupValue::KeyDeleted)
)
&& tombstone_is_dead(entry.hash) && tombstone_is_dead(entry.hash)
{ {
continue; continue;
...@@ -1820,7 +1826,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1820,7 +1826,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// Entry is being dropped (superseded by newer entry or // Entry is being dropped (superseded by newer entry or
// pruned by tombstone). If it references a blob file, // pruned by tombstone). If it references a blob file,
// mark that blob for deletion. // mark that blob for deletion.
if let IterValue::Blob { sequence_number } = &entry.value { if let LazyLookupValue::Eager(LookupValue::Blob { sequence_number }) =
&entry.value {
blob_seq_numbers_to_delete.push(*sequence_number); blob_seq_numbers_to_delete.push(*sequence_number);
} }
} }
......
...@@ -17,8 +17,6 @@ pub mod meta_file; ...@@ -17,8 +17,6 @@ pub mod meta_file;
mod meta_file_builder; mod meta_file_builder;
pub mod mmap_helper; pub mod mmap_helper;
mod parallel_scheduler; mod parallel_scheduler;
mod rc_bytes;
mod shared_bytes;
pub mod sst_filter; pub mod sst_filter;
pub mod static_sorted_file; pub mod static_sorted_file;
mod static_sorted_file_builder; mod static_sorted_file_builder;
......
use crate::{ use crate::{
ArcBytes, ArcBytes,
constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
rc_bytes::RcBytes,
static_sorted_file_builder::{Entry, EntryValue}, static_sorted_file_builder::{Entry, EntryValue},
}; };
/// A value from a SST file. Generic over the byte representation, defaulting to /// A value from an SST file lookup.
/// `ArcBytes` for the lookup path. The compaction/iteration path uses
/// `LookupValue<RcBytes>` which is convertible to `IterValue`.
#[derive(PartialEq)] #[derive(PartialEq)]
pub enum LookupValue<B = ArcBytes<'static>> { pub enum LookupValue {
/// The value was deleted. /// The key and all of its values were deleted.
KeyDeleted, KeyDeleted,
/// A single value was deleted from this key's group (MultiValue families only). Other values /// A single value was deleted from this key's group (MultiValue families only).
/// for the same key are unaffected. The bytes are the deleted value. KeyValueDeleted { value: ArcBytes<'static> },
KeyValueDeleted { value: B },
/// The value is stored in the SST file. /// The value is stored in the SST file.
/// Slice { value: ArcBytes<'static> },
/// The bytes will be pointing either at a keyblock or a value block in the SST
Slice { value: B },
/// The value is stored in a blob file. /// The value is stored in a blob file.
Blob { sequence_number: u32 }, Blob { sequence_number: u32 },
} }
/// A value from SST file iteration (compaction path, uses RcBytes for /// A value from an SST file lookup that may still be compressed.
/// non-atomic refcounting). pub enum LazyLookupValue {
pub enum IterValue { Eager(LookupValue),
/// The value was deleted. /// A medium-sized value that is still compressed.
KeyDeleted,
/// A single value was deleted from this key's group (MultiValue families only).
KeyValueDeleted { value: RcBytes },
/// The value is stored in the SST file.
Slice { value: RcBytes },
/// The value is stored in a blob file.
Blob { sequence_number: u32 },
/// A medium sized value that is still compressed.
Medium { Medium {
uncompressed_size: u32, uncompressed_size: u32,
checksum: u32, checksum: u32,
block: RcBytes, block: ArcBytes<'static>,
}, },
} }
impl From<LookupValue<RcBytes>> for IterValue {
fn from(v: LookupValue<RcBytes>) -> Self { /// An entry from an SST file lookup.
match v {
LookupValue::KeyDeleted => IterValue::KeyDeleted,
LookupValue::KeyValueDeleted { value } => IterValue::KeyValueDeleted { value },
LookupValue::Slice { value } => IterValue::Slice { value },
LookupValue::Blob { sequence_number } => IterValue::Blob { sequence_number },
}
}
}
/// An entry from SST file iteration (compaction path, uses RcBytes).
pub struct LookupEntry { pub struct LookupEntry {
/// The hash of the key.
pub hash: u64, pub hash: u64,
/// The key. pub key: ArcBytes<'static>,
pub key: RcBytes, pub value: LazyLookupValue,
/// The value.
pub value: IterValue,
} }
impl Entry for LookupEntry { impl Entry for LookupEntry {
...@@ -76,9 +50,11 @@ impl Entry for LookupEntry { ...@@ -76,9 +50,11 @@ impl Entry for LookupEntry {
fn value(&self) -> EntryValue<'_> { fn value(&self) -> EntryValue<'_> {
match &self.value { match &self.value {
IterValue::KeyDeleted => EntryValue::KeyDeleted, LazyLookupValue::Eager(LookupValue::KeyDeleted) => EntryValue::KeyDeleted,
IterValue::KeyValueDeleted { value } => EntryValue::KeyValueDeleted { value }, LazyLookupValue::Eager(LookupValue::KeyValueDeleted { value }) => {
IterValue::Slice { value } => { EntryValue::KeyValueDeleted { value }
}
LazyLookupValue::Eager(LookupValue::Slice { value }) => {
if value.len() <= MAX_INLINE_VALUE_SIZE { if value.len() <= MAX_INLINE_VALUE_SIZE {
EntryValue::Inline { value } EntryValue::Inline { value }
} else if value.len() > MAX_SMALL_VALUE_SIZE { } else if value.len() > MAX_SMALL_VALUE_SIZE {
...@@ -87,10 +63,10 @@ impl Entry for LookupEntry { ...@@ -87,10 +63,10 @@ impl Entry for LookupEntry {
EntryValue::Small { value } EntryValue::Small { value }
} }
} }
IterValue::Blob { sequence_number } => EntryValue::Large { LazyLookupValue::Eager(LookupValue::Blob { sequence_number }) => EntryValue::Large {
blob: *sequence_number, blob: *sequence_number,
}, },
IterValue::Medium { LazyLookupValue::Medium {
uncompressed_size, uncompressed_size,
checksum, checksum,
block, block,
......
use std::{
borrow::Borrow,
fmt::{self, Debug, Formatter},
hash::{Hash, Hasher},
ops::{Deref, Range},
rc::Rc,
};
use memmap2::Mmap;
use crate::{
Compression,
compression::decompress_into_rc,
shared_bytes::{SharedBytes, is_subslice_of},
};
/// The backing storage for an `RcBytes`.
///
/// Uses `Rc` for all refcounting, eliminating atomic operations.
#[derive(Clone)]
enum Backing {
Rc { _backing: Rc<[u8]> },
Mmap { _backing: Rc<Mmap> },
}
/// An owned byte slice backed by either an `Rc<[u8]>` or a memory-mapped file.
///
/// Identical to `ArcBytes` but uses `Rc` instead of `Arc`, eliminating atomic
/// refcount overhead. Use this in single-threaded contexts like SST iteration
/// during compaction.
#[derive(Clone)]
pub struct RcBytes {
data: *const [u8],
backing: Backing,
}
impl From<Rc<[u8]>> for RcBytes {
fn from(rc: Rc<[u8]>) -> Self {
Self {
data: &*rc as *const [u8],
backing: Backing::Rc { _backing: rc },
}
}
}
impl From<Box<[u8]>> for RcBytes {
fn from(b: Box<[u8]>) -> Self {
Self::from(Rc::from(b))
}
}
impl Deref for RcBytes {
type Target = [u8];
fn deref(&self) -> &Self::Target {
unsafe { &*self.data }
}
}
impl Borrow<[u8]> for RcBytes {
fn borrow(&self) -> &[u8] {
self
}
}
impl PartialEq for RcBytes {
fn eq(&self, other: &Self) -> bool {
self.deref().eq(other.deref())
}
}
impl Debug for RcBytes {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
Debug::fmt(&**self, f)
}
}
impl Eq for RcBytes {}
impl Hash for RcBytes {
fn hash<H: Hasher>(&self, state: &mut H) {
Hash::hash(self.deref(), state);
}
}
impl SharedBytes for RcBytes {
type MmapHandle = Rc<Mmap>;
fn slice(self, range: Range<usize>) -> Self {
let data = &*self;
let data = &data[range] as *const [u8];
Self {
data,
backing: self.backing,
}
}
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
debug_assert!(
is_subslice_of(
subslice,
match &self.backing {
Backing::Rc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
}
),
"slice_from_subslice: subslice is not within the backing storage"
);
Self {
data: subslice as *const [u8],
backing: self.backing.clone(),
}
}
unsafe fn from_mmap(mmap: &Rc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!(
is_subslice_of(subslice, mmap),
"from_mmap: subslice is not within the mmap"
);
RcBytes {
data: subslice as *const [u8],
backing: Backing::Mmap {
_backing: mmap.clone(),
},
}
}
fn from_decompressed(
compression: Compression,
uncompressed_length: u32,
block: &[u8],
) -> anyhow::Result<Self> {
Ok(RcBytes::from(decompress_into_rc(
compression,
uncompressed_length,
block,
)?))
}
}
use std::ops::{Deref, Range};
use memmap2::Mmap;
use crate::Compression;
/// Trait abstracting over `ArcBytes` and `RcBytes`.
///
/// Both types are owned byte slices backed by either a ref-counted heap
/// allocation or a memory-mapped file. The only difference is the refcount
/// flavor: `Arc` (atomic, thread-safe) vs `Rc` (non-atomic, single-threaded).
///
/// This trait captures the shared interface so that code in
/// `static_sorted_file.rs` (block reading, key matching, etc.) can be written
/// once and used for both lookup (ArcBytes) and iteration (RcBytes) paths.
pub trait SharedBytes: Clone + Deref<Target = [u8]> + Sized {
/// The ref-counted handle to the memory-mapped file.
type MmapHandle: Deref<Target = Mmap>;
/// Returns a new instance that points to a sub-range of the current slice.
fn slice(self, range: Range<usize>) -> Self;
/// Creates a sub-slice from a reference that points into this instance's
/// backing data.
///
/// # Safety
///
/// The caller must ensure that `subslice` points to memory within this
/// instance's backing storage.
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self;
/// Creates an instance backed by a memory-mapped file.
///
/// # Safety
///
/// The caller must ensure that `subslice` points to memory within the
/// given `mmap`.
unsafe fn from_mmap(mmap: &Self::MmapHandle, subslice: &[u8]) -> Self;
/// Creates an instance from a decompressed block.
fn from_decompressed(
compression: Compression,
uncompressed_length: u32,
block: &[u8],
) -> anyhow::Result<Self>;
}
/// Returns `true` if `subslice` lies entirely within `backing`.
pub(crate) fn is_subslice_of(subslice: &[u8], backing: &[u8]) -> bool {
let backing = backing.as_ptr_range();
let sub = subslice.as_ptr_range();
sub.start >= backing.start && sub.end <= backing.end
}
...@@ -3,7 +3,6 @@ use std::{ ...@@ -3,7 +3,6 @@ use std::{
hash::BuildHasherDefault, hash::BuildHasherDefault,
io, io,
path::Path, path::Path,
rc::Rc,
sync::{ sync::{
Arc, Arc,
atomic::{AtomicU64, Ordering as AtomicOrdering}, atomic::{AtomicU64, Ordering as AtomicOrdering},
...@@ -23,10 +22,8 @@ use crate::{ ...@@ -23,10 +22,8 @@ use crate::{
be, be,
compression::checksum_block, compression::checksum_block,
constants::MAX_INLINE_VALUE_SIZE, constants::MAX_INLINE_VALUE_SIZE,
lookup_entry::{IterValue, LookupEntry, LookupValue}, lookup_entry::{LazyLookupValue, LookupEntry, LookupValue},
mmap_helper::advise_mmap_for_persistence, mmap_helper::advise_mmap_for_persistence,
rc_bytes::RcBytes,
shared_bytes::SharedBytes,
static_sorted_file_builder::{BLOCK_HEADER_SIZE, INDEX_BLOCK_ENTRY_SIZE}, static_sorted_file_builder::{BLOCK_HEADER_SIZE, INDEX_BLOCK_ENTRY_SIZE},
}; };
...@@ -190,19 +187,19 @@ pub type BlockCache = quick_cache::sync::Cache< ...@@ -190,19 +187,19 @@ pub type BlockCache = quick_cache::sync::Cache<
>; >;
/// Trait abstracting value block reading for `handle_key_match_generic`. /// Trait abstracting value block reading for `handle_key_match_generic`.
trait ValueBlockCache<B: SharedBytes> { trait ValueBlockCache {
fn get_or_read( fn get_or_read(
self, self,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<B>; ) -> Result<ArcBytes<'static>>;
fn read_uncached( fn read_uncached(
self, self,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<B>; ) -> Result<ArcBytes<'static>>;
} }
/// Bundles the lookup backing with the shared block cache and per-file CRC bitmap. /// Bundles the lookup backing with the shared block cache and per-file CRC bitmap.
...@@ -213,7 +210,7 @@ struct ArcBlockCacheReader<'a> { ...@@ -213,7 +210,7 @@ struct ArcBlockCacheReader<'a> {
verified_blocks: &'a [AtomicU64], verified_blocks: &'a [AtomicU64],
} }
impl ValueBlockCache<ArcBytes<'static>> for ArcBlockCacheReader<'_> { impl ValueBlockCache for ArcBlockCacheReader<'_> {
fn get_or_read( fn get_or_read(
self, self,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
...@@ -241,18 +238,18 @@ impl ValueBlockCache<ArcBytes<'static>> for ArcBlockCacheReader<'_> { ...@@ -241,18 +238,18 @@ impl ValueBlockCache<ArcBytes<'static>> for ArcBlockCacheReader<'_> {
} }
/// Iteration-path reader with a lightweight single-entry cache for sequential reads. /// Iteration-path reader with a lightweight single-entry cache for sequential reads.
struct RcBlockCacheReader<'a> { struct IterBlockCacheReader<'a> {
backing: &'a StaticSortedFileIterBacking, backing: &'a StaticSortedFileIterBacking,
cache: &'a mut Option<(u16, RcBytes)>, cache: &'a mut Option<(u16, ArcBytes<'static>)>,
} }
impl ValueBlockCache<RcBytes> for RcBlockCacheReader<'_> { impl ValueBlockCache for IterBlockCacheReader<'_> {
fn get_or_read( fn get_or_read(
self, self,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<RcBytes> { ) -> Result<ArcBytes<'static>> {
if let Some((idx, block)) = self.cache.as_ref() if let Some((idx, block)) = self.cache.as_ref()
&& *idx == block_index && *idx == block_index
{ {
...@@ -268,7 +265,7 @@ impl ValueBlockCache<RcBytes> for RcBlockCacheReader<'_> { ...@@ -268,7 +265,7 @@ impl ValueBlockCache<RcBytes> for RcBlockCacheReader<'_> {
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<RcBytes> { ) -> Result<ArcBytes<'static>> {
read_block_iter(self.backing, meta, block_index, compression) read_block_iter(self.backing, meta, block_index, compression)
} }
} }
...@@ -901,19 +898,19 @@ fn read_block_lookup( ...@@ -901,19 +898,19 @@ fn read_block_lookup(
} }
/// Returns `(uncompressed_length, checksum, block)` wrapping the raw on-disk data as /// Returns `(uncompressed_length, checksum, block)` wrapping the raw on-disk data as
/// `RcBytes` for the single-threaded iteration path. /// `ArcBytes` for the iteration path.
fn get_raw_block_iter( fn get_raw_block_iter(
backing: &StaticSortedFileIterBacking, backing: &StaticSortedFileIterBacking,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
) -> Result<(u32, u32, RcBytes)> { ) -> Result<(u32, u32, ArcBytes<'static>)> {
match backing { match backing {
StaticSortedFileIterBacking::Mmap(mmap) => { StaticSortedFileIterBacking::Mmap(mmap) => {
let (uncompressed_length, checksum, block) = let (uncompressed_length, checksum, block) =
get_raw_block_slice(mmap, meta, block_index)?; get_raw_block_slice(mmap, meta, block_index)?;
// SAFETY: block points into mmap. // SAFETY: block points into mmap.
Ok((uncompressed_length, checksum, unsafe { Ok((uncompressed_length, checksum, unsafe {
RcBytes::from_mmap(mmap, block) ArcBytes::from_mmap(mmap.clone(), block)
})) }))
} }
StaticSortedFileIterBacking::File { StaticSortedFileIterBacking::File {
...@@ -947,7 +944,7 @@ fn get_raw_block_iter( ...@@ -947,7 +944,7 @@ fn get_raw_block_iter(
Ok(( Ok((
uncompressed_length, uncompressed_length,
checksum, checksum,
RcBytes::from(block.into_boxed_slice()), ArcBytes::from(block.into_boxed_slice()),
)) ))
} }
} }
...@@ -959,13 +956,13 @@ fn read_block_iter( ...@@ -959,13 +956,13 @@ fn read_block_iter(
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<RcBytes> { ) -> Result<ArcBytes<'static>> {
let (uncompressed_length, checksum, block) = get_raw_block_iter(backing, meta, block_index)?; let (uncompressed_length, checksum, block) = get_raw_block_iter(backing, meta, block_index)?;
verify_checksum(meta, &block, checksum, block_index)?; verify_checksum(meta, &block, checksum, block_index)?;
if uncompressed_length == 0 { if uncompressed_length == 0 {
return Ok(block); return Ok(block);
} }
RcBytes::from_decompressed(compression, uncompressed_length, &block).with_context(|| { ArcBytes::from_decompressed(compression, uncompressed_length, &block).with_context(|| {
format!( format!(
"Failed to decompress block {} from {:08}.sst ({} bytes uncompressed)", "Failed to decompress block {} from {:08}.sst ({} bytes uncompressed)",
block_index, meta.sequence_number, uncompressed_length block_index, meta.sequence_number, uncompressed_length
...@@ -973,15 +970,15 @@ fn read_block_iter( ...@@ -973,15 +970,15 @@ fn read_block_iter(
}) })
} }
/// Handles a key match by resolving the value reference. Generic over byte type. /// Handles a key match by resolving the value reference.
fn handle_key_match_generic<B: SharedBytes>( fn handle_key_match_generic(
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
ty: u8, ty: u8,
val: &[u8], val: &[u8],
key_block: &B, key_block: &ArcBytes<'static>,
compression: Compression, compression: Compression,
reader: impl ValueBlockCache<B>, reader: impl ValueBlockCache,
) -> Result<LookupValue<B>> { ) -> Result<LookupValue> {
Ok(match ty { Ok(match ty {
KEY_BLOCK_ENTRY_TYPE_SMALL => { KEY_BLOCK_ENTRY_TYPE_SMALL => {
let block = be::read_u16(val); let block = be::read_u16(val);
...@@ -1019,11 +1016,11 @@ fn handle_key_match_generic<B: SharedBytes>( ...@@ -1019,11 +1016,11 @@ fn handle_key_match_generic<B: SharedBytes>(
} }
enum StaticSortedFileIterBacking { enum StaticSortedFileIterBacking {
Mmap(Rc<Mmap>), Mmap(Arc<Mmap>),
File { File {
file: Rc<File>, file: Arc<File>,
file_len: usize, file_len: usize,
block_offsets: Rc<[u32]>, block_offsets: Arc<[u32]>,
}, },
} }
...@@ -1035,7 +1032,7 @@ pub struct StaticSortedFileIter { ...@@ -1035,7 +1032,7 @@ pub struct StaticSortedFileIter {
/// The root index block entries (body bytes starting after the type byte). /// The root index block entries (body bytes starting after the type byte).
/// SST files have exactly one index level. /// SST files have exactly one index level.
index_entries: RcBytes, index_entries: ArcBytes<'static>,
/// Total key block references in the index block (first_child + boundary entries). /// Total key block references in the index block (first_child + boundary entries).
num_index_entries: usize, num_index_entries: usize,
/// Next index entry to read from the index block. /// Next index entry to read from the index block.
...@@ -1044,13 +1041,13 @@ pub struct StaticSortedFileIter { ...@@ -1044,13 +1041,13 @@ pub struct StaticSortedFileIter {
/// Single-entry value block cache. Within a key block, entries reference /// Single-entry value block cache. Within a key block, entries reference
/// value blocks sequentially and don't revisit earlier blocks, so caching /// value blocks sequentially and don't revisit earlier blocks, so caching
/// just the current one avoids redundant decompression. /// just the current one avoids redundant decompression.
value_block_cache: Option<(u16, RcBytes)>, value_block_cache: Option<(u16, ArcBytes<'static>)>,
compression: Compression, compression: Compression,
} }
enum CurrentKeyBlockKind { enum CurrentKeyBlockKind {
/// Variable-size entries with an offset table for random access. /// Variable-size entries with an offset table for random access.
Variable { offsets: RcBytes }, Variable { offsets: ArcBytes<'static> },
/// Fixed-size entries with uniform key size and value size (no offset table). /// Fixed-size entries with uniform key size and value size (no offset table).
Fixed { Fixed {
key_size: usize, key_size: usize,
...@@ -1093,7 +1090,7 @@ struct CurrentKeyBlock { ...@@ -1093,7 +1090,7 @@ struct CurrentKeyBlock {
kind: CurrentKeyBlockKind, kind: CurrentKeyBlockKind,
/// Whether entries carry a hash, and so what order they are stored in. /// Whether entries carry a hash, and so what order they are stored in.
layout: KeyBlockLayout, layout: KeyBlockLayout,
entries: RcBytes, entries: ArcBytes<'static>,
/// Number of entries in this key block (max ~819 per 16 KiB block). /// Number of entries in this key block (max ~819 per 16 KiB block).
entry_count: u32, entry_count: u32,
/// Current iteration position. Indexes `hash_order` when that is present, and the block's /// Current iteration position. Indexes `hash_order` when that is present, and the block's
...@@ -1135,7 +1132,7 @@ impl StaticSortedFileIter { ...@@ -1135,7 +1132,7 @@ impl StaticSortedFileIter {
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::Sequential)?; mmap.advise(memmap2::Advice::Sequential)?;
advise_mmap_for_persistence(&mmap)?; advise_mmap_for_persistence(&mmap)?;
StaticSortedFileIterBacking::Mmap(Rc::new(mmap)) StaticSortedFileIterBacking::Mmap(Arc::new(mmap))
} }
AccessMode::File => { AccessMode::File => {
let file_len: usize = file.metadata()?.len().try_into()?; let file_len: usize = file.metadata()?.len().try_into()?;
...@@ -1150,7 +1147,7 @@ impl StaticSortedFileIter { ...@@ -1150,7 +1147,7 @@ impl StaticSortedFileIter {
.collect::<Vec<_>>() .collect::<Vec<_>>()
.into(); .into();
StaticSortedFileIterBacking::File { StaticSortedFileIterBacking::File {
file: Rc::new(file), file: Arc::new(file),
file_len, file_len,
block_offsets, block_offsets,
} }
...@@ -1198,7 +1195,7 @@ impl StaticSortedFileIter { ...@@ -1198,7 +1195,7 @@ impl StaticSortedFileIter {
}) })
} }
/// Parses a key block at the given index, returning `RcBytes`-backed data. /// Parses a key block at the given index, returning `ArcBytes<'static>`-backed data.
fn parse_key_block( fn parse_key_block(
backing: &StaticSortedFileIterBacking, backing: &StaticSortedFileIterBacking,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
...@@ -1286,24 +1283,23 @@ impl StaticSortedFileIter { ...@@ -1286,24 +1283,23 @@ impl StaticSortedFileIter {
let block = be::read_u16(val); let block = be::read_u16(val);
let (uncompressed_size, checksum, block) = let (uncompressed_size, checksum, block) =
get_raw_block_iter(&self.backing, &self.meta, block)?; get_raw_block_iter(&self.backing, &self.meta, block)?;
IterValue::Medium { LazyLookupValue::Medium {
uncompressed_size, uncompressed_size,
checksum, checksum,
block, block,
} }
} else { } else {
handle_key_match_generic( LazyLookupValue::Eager(handle_key_match_generic(
&self.meta, &self.meta,
ty, ty,
val, val,
&kb.entries, &kb.entries,
self.compression, self.compression,
RcBlockCacheReader { IterBlockCacheReader {
backing: &self.backing, backing: &self.backing,
cache: &mut self.value_block_cache, cache: &mut self.value_block_cache,
}, },
)? )?)
.into()
}; };
let entry = LookupEntry { let entry = LookupEntry {
hash: full_hash, hash: full_hash,
......
...@@ -8,7 +8,7 @@ use crate::{ ...@@ -8,7 +8,7 @@ use crate::{
AccessMode, Compression, DbConfig, FamilyConfig, FamilyKind, AccessMode, Compression, DbConfig, FamilyConfig, FamilyKind,
constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
db::{CompactConfig, TurboPersistence, read_current_version}, db::{CompactConfig, TurboPersistence, read_current_version},
lookup_entry::IterValue, lookup_entry::{LazyLookupValue, LookupValue},
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData}, static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData},
write_batch::WriteBatch, write_batch::WriteBatch,
...@@ -2459,7 +2459,9 @@ fn count_tombstones( ...@@ -2459,7 +2459,9 @@ fn count_tombstones(
for item in StaticSortedFileIter::open(path, sst, Compression::Lz4, AccessMode::Mmap)? { for item in StaticSortedFileIter::open(path, sst, Compression::Lz4, AccessMode::Mmap)? {
if matches!( if matches!(
item?.value, item?.value,
IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } LazyLookupValue::Eager(
LookupValue::KeyDeleted | LookupValue::KeyValueDeleted { .. }
)
) { ) {
count += 1; count += 1;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment