Unverified Commit dcb0d012 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-persistence: add borrowed mmap byte slices

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 4d621240
...@@ -13,30 +13,34 @@ use crate::{ ...@@ -13,30 +13,34 @@ use crate::{
compression::decompress_into_arc, compression::decompress_into_arc,
shared_bytes::{SharedBytes, is_subslice_of}, shared_bytes::{SharedBytes, is_subslice_of},
}; };
/// The backing storage for an `ArcBytes`. /// The backing storage for an `ArcBytes`.
///
/// The inner values are never read directly — they exist solely to keep the
/// backing memory alive while the raw `data` pointer in `ArcBytes` references it.
#[derive(Clone)] #[derive(Clone)]
enum Backing { enum Backing<'l> {
Arc { _backing: Arc<[u8]> }, Arc {
Mmap { _backing: Arc<Mmap> }, _backing: Arc<[u8]>,
},
Mmap {
_backing: Arc<Mmap>,
},
/// Borrows an mmap handle when the byte view cannot escape the caller's scope.
MmapRef {
_backing: &'l Arc<Mmap>,
},
} }
/// An owned byte slice backed by either an `Arc<[u8]>` or a memory-mapped file. /// A byte slice backed by owned bytes, an owned mmap handle, or a borrowed mmap handle.
#[derive(Clone)] #[derive(Clone)]
pub struct ArcBytes { pub struct ArcBytes<'l> {
data: *const [u8], data: *const [u8],
// Safety: Backing should come last so that it is dropped after the data pointer so we don't // Keep the backing after the raw pointer so it is dropped last.
// create a dangling pointer. This isn't really a problem since it is technically ok to have backing: Backing<'l>,
// dangling _pointers_.
backing: Backing,
} }
unsafe impl Send for ArcBytes {} unsafe impl Send for ArcBytes<'_> {}
unsafe impl Sync for ArcBytes {} unsafe impl Sync for ArcBytes<'_> {}
impl From<Arc<[u8]>> for ArcBytes { impl From<Arc<[u8]>> for ArcBytes<'static> {
fn from(arc: Arc<[u8]>) -> Self { fn from(arc: Arc<[u8]>) -> Self {
Self { Self {
data: &*arc as *const [u8], data: &*arc as *const [u8],
...@@ -45,13 +49,13 @@ impl From<Arc<[u8]>> for ArcBytes { ...@@ -45,13 +49,13 @@ impl From<Arc<[u8]>> for ArcBytes {
} }
} }
impl From<Box<[u8]>> for ArcBytes { impl From<Box<[u8]>> for ArcBytes<'static> {
fn from(b: Box<[u8]>) -> Self { fn from(bytes: Box<[u8]>) -> Self {
Self::from(Arc::from(b)) Self::from(Arc::from(bytes))
} }
} }
impl Deref for ArcBytes { impl Deref for ArcBytes<'_> {
type Target = [u8]; type Target = [u8];
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
...@@ -59,90 +63,118 @@ impl Deref for ArcBytes { ...@@ -59,90 +63,118 @@ impl Deref for ArcBytes {
} }
} }
impl Borrow<[u8]> for ArcBytes { impl Borrow<[u8]> for ArcBytes<'_> {
fn borrow(&self) -> &[u8] { fn borrow(&self) -> &[u8] {
self self
} }
} }
impl Hash for ArcBytes { impl Hash for ArcBytes<'_> {
fn hash<H: Hasher>(&self, state: &mut H) { fn hash<H: Hasher>(&self, state: &mut H) {
self.deref().hash(state) self.deref().hash(state)
} }
} }
impl PartialEq for ArcBytes { impl PartialEq for ArcBytes<'_> {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
self.deref().eq(other.deref()) self.deref().eq(other.deref())
} }
} }
impl Debug for ArcBytes { impl Debug for ArcBytes<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
Debug::fmt(&**self, f) Debug::fmt(&**self, f)
} }
} }
impl Eq for ArcBytes {} impl Eq for ArcBytes<'_> {}
fn backing_as_slice<'a>(backing: &'a Backing<'a>) -> &'a [u8] {
match backing {
Backing::Arc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
Backing::MmapRef { _backing } => _backing,
}
}
impl<'l> ArcBytes<'l> {
pub fn slice(self, range: Range<usize>) -> Self {
let data = &self[range] as *const [u8];
Self {
data,
backing: self.backing,
}
}
/// # Safety
/// `subslice` must point into this value's backing storage.
pub unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
debug_assert!(is_subslice_of(subslice, backing_as_slice(&self.backing)));
Self {
data: subslice as *const [u8],
backing: self.backing.clone(),
}
}
/// Promotes a borrowed mmap view by cloning its mmap handle.
pub fn into_static(self) -> ArcBytes<'static> {
ArcBytes {
data: self.data,
backing: match self.backing {
Backing::Arc { _backing } => Backing::Arc { _backing },
Backing::Mmap { _backing } => Backing::Mmap { _backing },
Backing::MmapRef { _backing } => Backing::Mmap {
_backing: _backing.clone(),
},
},
}
}
impl ArcBytes {
/// Returns `true` if this `ArcBytes` is backed by a memory-mapped file.
pub fn is_mmap_backed(&self) -> bool { pub fn is_mmap_backed(&self) -> bool {
matches!(self.backing, Backing::Mmap { .. }) matches!(self.backing, Backing::Mmap { .. } | Backing::MmapRef { .. })
} }
/// Returns `true` if the backing `Arc` allocation is shared (i.e., there
/// are other `Arc` clones referencing the same data outside the cache).
/// Always returns `false` for mmap-backed bytes, since the mmap `Arc` is
/// shared across all slices from the same file and is not a useful signal.
pub fn is_shared_arc(&self) -> bool { pub fn is_shared_arc(&self) -> bool {
match &self.backing { match &self.backing {
Backing::Arc { _backing } => Arc::strong_count(_backing) > 1, Backing::Arc { _backing } => Arc::strong_count(_backing) > 1,
Backing::Mmap { .. } => false, Backing::Mmap { .. } | Backing::MmapRef { .. } => false,
}
}
/// # Safety
/// `subslice` must point into `mmap`.
pub unsafe fn from_mmap(mmap: Arc<Mmap>, subslice: &[u8]) -> ArcBytes<'static> {
debug_assert!(is_subslice_of(subslice, &mmap));
ArcBytes {
data: subslice as *const [u8],
backing: Backing::Mmap { _backing: mmap },
}
}
/// # Safety
/// `subslice` must point into `mmap`.
pub unsafe fn from_mmap_ref(mmap: &'l Arc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!(is_subslice_of(subslice, mmap));
Self {
data: subslice as *const [u8],
backing: Backing::MmapRef { _backing: mmap },
} }
} }
} }
impl SharedBytes for ArcBytes { impl SharedBytes for ArcBytes<'static> {
type MmapHandle = Arc<Mmap>; type MmapHandle = Arc<Mmap>;
fn slice(self, range: Range<usize>) -> Self { fn slice(self, range: Range<usize>) -> Self {
let data = &*self; ArcBytes::slice(self, range)
let data = &data[range] as *const [u8];
Self {
data,
backing: self.backing,
}
} }
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self { unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
debug_assert!( unsafe { ArcBytes::slice_from_subslice(self, subslice) }
is_subslice_of(
subslice,
match &self.backing {
Backing::Arc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
}
),
"slice_from_subslice: subslice is not within the backing storage"
);
Self {
data: subslice as *const [u8],
backing: self.backing.clone(),
}
} }
unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self { unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!( unsafe { ArcBytes::from_mmap(mmap.clone(), subslice) }
is_subslice_of(subslice, mmap),
"from_mmap: subslice is not within the mmap"
);
ArcBytes {
data: subslice as *const [u8],
backing: Backing::Mmap {
_backing: mmap.clone(),
},
}
} }
fn from_decompressed( fn from_decompressed(
...@@ -157,3 +189,22 @@ impl SharedBytes for ArcBytes { ...@@ -157,3 +189,22 @@ impl SharedBytes for ArcBytes {
)?)) )?))
} }
} }
#[cfg(test)]
mod tests {
use std::io::Write;
use super::*;
#[test]
fn borrowed_mmap_can_be_sliced_and_promoted() -> anyhow::Result<()> {
let mut file = tempfile::tempfile()?;
file.write_all(b"0123456789")?;
let mmap = Arc::new(unsafe { Mmap::map(&file)? });
let borrowed = unsafe { ArcBytes::from_mmap_ref(&mmap, &mmap[2..8]) };
let promoted = borrowed.slice(1..5).into_static();
drop(mmap);
assert_eq!(&*promoted, b"3456");
Ok(())
}
}
...@@ -659,7 +659,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -659,7 +659,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Reads and decompresses a blob file. This is not backed by any cache. /// Reads and decompresses a blob file. This is not backed by any cache.
#[tracing::instrument(level = "info", name = "reading database blob", skip_all)] #[tracing::instrument(level = "info", name = "reading database blob", skip_all)]
fn read_blob(&self, seq: u32, compression: Compression) -> Result<ArcBytes> { fn read_blob(&self, seq: u32, compression: Compression) -> Result<ArcBytes<'static>> {
let path = self.path.join(format!("{seq:08}.blob")); let path = self.path.join(format!("{seq:08}.blob"));
let file = File::open(&path)?; let file = File::open(&path)?;
let data: Either<Mmap, Vec<u8>> = match self.config.access_mode { let data: Either<Mmap, Vec<u8>> = match self.config.access_mode {
...@@ -1983,7 +1983,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1983,7 +1983,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Get a value from the database. Returns None if the key is not found. The returned value /// Get a value from the database. Returns None if the key is not found. The returned value
/// might hold onto a block of the database and it should not be hold long-term. /// might hold onto a block of the database and it should not be hold long-term.
pub fn get<K: QueryKey>(&self, family: usize, key: &K) -> Result<Option<ArcBytes>> { pub fn get<K: QueryKey>(&self, family: usize, key: &K) -> Result<Option<ArcBytes<'static>>> {
debug_assert!(family < FAMILIES, "Family index out of bounds"); debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue { if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic // This is an error in our caller so just panic
...@@ -2015,7 +2015,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2015,7 +2015,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
&self, &self,
family: usize, family: usize,
key: &K, key: &K,
) -> Result<SmallVec<[ArcBytes; 1]>> { ) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
debug_assert!(family < FAMILIES, "Family index out of bounds"); debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::MultiValue { if self.config.family_configs[family].kind != FamilyKind::MultiValue {
// This is an error in our caller so just panic // This is an error in our caller so just panic
...@@ -2041,10 +2041,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2041,10 +2041,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
family: usize, family: usize,
key: &K, key: &K,
span: &EnteredSpan, span: &EnteredSpan,
) -> Result<SmallVec<[ArcBytes; 1]>> { ) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
let hash = hash_key(key); let hash = hash_key(key);
let inner = self.inner.read(); let inner = self.inner.read();
let mut output: SmallVec<[ArcBytes; 1]> = SmallVec::new(); let mut output: SmallVec<[ArcBytes<'static>; 1]> = SmallVec::new();
// Track whether we found the key in any SST (even if deleted). // Track whether we found the key in any SST (even if deleted).
// Used for miss_global stat: only fires if key was never found anywhere. // Used for miss_global stat: only fires if key was never found anywhere.
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
...@@ -2053,7 +2053,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2053,7 +2053,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// Values deleted by key-value tombstones seen so far. Because we walk meta files newest // Values deleted by key-value tombstones seen so far. Because we walk meta files newest
// first, and tombstones sort first within a key group, every tombstone that could apply to // first, and tombstones sort first within a key group, every tombstone that could apply to
// a value has already been seen by the time we reach that value. // a value has already been seen by the time we reach that value.
let mut deleted_values: AutoSet<ArcBytes, BuildHasherDefault<FxHasher>, 1> = let mut deleted_values: AutoSet<ArcBytes<'static>, BuildHasherDefault<FxHasher>, 1> =
AutoSet::default(); AutoSet::default();
let mut size = 0; let mut size = 0;
...@@ -2172,7 +2172,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2172,7 +2172,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
&self, &self,
family: usize, family: usize,
keys: &[K], keys: &[K],
) -> Result<Vec<Option<ArcBytes>>> { ) -> Result<Vec<Option<ArcBytes<'static>>>> {
debug_assert!(family < FAMILIES, "Family index out of bounds"); debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue { if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic // This is an error in our caller so just panic
......
...@@ -9,7 +9,7 @@ use crate::{ ...@@ -9,7 +9,7 @@ use crate::{
/// `ArcBytes` for the lookup path. The compaction/iteration path uses /// `ArcBytes` for the lookup path. The compaction/iteration path uses
/// `LookupValue<RcBytes>` which is convertible to `IterValue`. /// `LookupValue<RcBytes>` which is convertible to `IterValue`.
#[derive(PartialEq)] #[derive(PartialEq)]
pub enum LookupValue<B = ArcBytes> { pub enum LookupValue<B = ArcBytes<'static>> {
/// The value was deleted. /// The value was deleted.
KeyDeleted, KeyDeleted,
/// A single value was deleted from this key's group (MultiValue families only). Other values /// A single value was deleted from this key's group (MultiValue families only). Other values
......
use std::{ use std::{
borrow::Cow,
cmp::Ordering, cmp::Ordering,
hash::BuildHasherDefault, hash::BuildHasherDefault,
io, io,
...@@ -145,8 +144,8 @@ impl From<LookupValue> for SstLookupResult { ...@@ -145,8 +144,8 @@ impl From<LookupValue> for SstLookupResult {
#[derive(Clone, Default)] #[derive(Clone, Default)]
pub struct BlockWeighter; pub struct BlockWeighter;
impl quick_cache::Weighter<(u32, u16), ArcBytes> for BlockWeighter { impl quick_cache::Weighter<(u32, u16), ArcBytes<'static>> for BlockWeighter {
fn weight(&self, _key: &(u32, u16), val: &ArcBytes) -> u64 { fn weight(&self, _key: &(u32, u16), val: &ArcBytes<'static>) -> u64 {
if val.is_mmap_backed() { if val.is_mmap_backed() {
// Mmap-backed blocks bypass the cache (served directly from mmap), // Mmap-backed blocks bypass the cache (served directly from mmap),
// so this branch should never be reached. // so this branch should never be reached.
...@@ -166,11 +165,11 @@ impl quick_cache::Weighter<(u32, u16), ArcBytes> for BlockWeighter { ...@@ -166,11 +165,11 @@ impl quick_cache::Weighter<(u32, u16), ArcBytes> for BlockWeighter {
#[derive(Clone, Default)] #[derive(Clone, Default)]
pub struct BlockCacheLifecycle; pub struct BlockCacheLifecycle;
impl Lifecycle<(u32, u16), ArcBytes> for BlockCacheLifecycle { impl Lifecycle<(u32, u16), ArcBytes<'static>> for BlockCacheLifecycle {
type RequestState = (); type RequestState = ();
#[inline] #[inline]
fn is_pinned(&self, _key: &(u32, u16), val: &ArcBytes) -> bool { fn is_pinned(&self, _key: &(u32, u16), val: &ArcBytes<'static>) -> bool {
val.is_shared_arc() val.is_shared_arc()
} }
...@@ -178,12 +177,13 @@ impl Lifecycle<(u32, u16), ArcBytes> for BlockCacheLifecycle { ...@@ -178,12 +177,13 @@ impl Lifecycle<(u32, u16), ArcBytes> for BlockCacheLifecycle {
fn begin_request(&self) -> Self::RequestState {} fn begin_request(&self) -> Self::RequestState {}
#[inline] #[inline]
fn on_evict(&self, _state: &mut Self::RequestState, _key: (u32, u16), _val: ArcBytes) {} fn on_evict(&self, _state: &mut Self::RequestState, _key: (u32, u16), _val: ArcBytes<'static>) {
}
} }
pub type BlockCache = quick_cache::sync::Cache< pub type BlockCache = quick_cache::sync::Cache<
(u32, u16), (u32, u16),
ArcBytes, ArcBytes<'static>,
BlockWeighter, BlockWeighter,
BuildHasherDefault<FxHasher>, BuildHasherDefault<FxHasher>,
BlockCacheLifecycle, BlockCacheLifecycle,
...@@ -213,13 +213,13 @@ struct ArcBlockCacheReader<'a> { ...@@ -213,13 +213,13 @@ struct ArcBlockCacheReader<'a> {
verified_blocks: &'a [AtomicU64], verified_blocks: &'a [AtomicU64],
} }
impl ValueBlockCache<ArcBytes> for ArcBlockCacheReader<'_> { impl ValueBlockCache<ArcBytes<'static>> for ArcBlockCacheReader<'_> {
fn get_or_read( fn get_or_read(
self, self,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<ArcBytes> { ) -> Result<ArcBytes<'static>> {
get_or_cache_block( get_or_cache_block(
self.backing, self.backing,
meta, meta,
...@@ -235,7 +235,7 @@ impl ValueBlockCache<ArcBytes> for ArcBlockCacheReader<'_> { ...@@ -235,7 +235,7 @@ impl ValueBlockCache<ArcBytes> for ArcBlockCacheReader<'_> {
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<ArcBytes> { ) -> Result<ArcBytes<'static>> {
read_block_lookup(self.backing, meta, block_index, compression) read_block_lookup(self.backing, meta, block_index, compression)
} }
} }
...@@ -455,7 +455,7 @@ impl StaticSortedFile { ...@@ -455,7 +455,7 @@ impl StaticSortedFile {
/// If `FIND_ALL` is true, collects all entries with the same key. /// If `FIND_ALL` is true, collects all entries with the same key.
fn lookup_key_block<K: QueryKey, const FIND_ALL: bool>( fn lookup_key_block<K: QueryKey, const FIND_ALL: bool>(
&self, &self,
block: ArcBytes, block: ArcBytes<'static>,
key_hash: u64, key_hash: u64,
key: &K, key: &K,
layout: KeyBlockLayout, layout: KeyBlockLayout,
...@@ -489,7 +489,7 @@ impl StaticSortedFile { ...@@ -489,7 +489,7 @@ impl StaticSortedFile {
/// enabling direct indexing during binary search. /// enabling direct indexing during binary search.
fn lookup_fixed_key_block<K: QueryKey, const FIND_ALL: bool>( fn lookup_fixed_key_block<K: QueryKey, const FIND_ALL: bool>(
&self, &self,
block: ArcBytes, block: ArcBytes<'static>,
key_hash: u64, key_hash: u64,
key: &K, key: &K,
layout: KeyBlockLayout, layout: KeyBlockLayout,
...@@ -529,7 +529,7 @@ impl StaticSortedFile { ...@@ -529,7 +529,7 @@ impl StaticSortedFile {
/// key blocks (offset table lookup) and fixed-size key blocks (stride-based indexing). /// key blocks (offset table lookup) and fixed-size key blocks (stride-based indexing).
fn lookup_block_inner<'a, K: QueryKey, const FIND_ALL: bool>( fn lookup_block_inner<'a, K: QueryKey, const FIND_ALL: bool>(
&self, &self,
block: &ArcBytes, block: &ArcBytes<'static>,
entry_count: usize, entry_count: usize,
key_hash: u64, key_hash: u64,
key: &K, key: &K,
...@@ -610,7 +610,7 @@ impl StaticSortedFile { ...@@ -610,7 +610,7 @@ impl StaticSortedFile {
&self, &self,
ty: u8, ty: u8,
val: &[u8], val: &[u8],
key_block_arc: &ArcBytes, key_block_arc: &ArcBytes<'static>,
reader: ArcBlockCacheReader<'_>, reader: ArcBlockCacheReader<'_>,
) -> Result<LookupValue> { ) -> Result<LookupValue> {
handle_key_match_generic(&self.meta, ty, val, key_block_arc, self.compression, reader) handle_key_match_generic(&self.meta, ty, val, key_block_arc, self.compression, reader)
...@@ -632,10 +632,10 @@ fn get_or_cache_block( ...@@ -632,10 +632,10 @@ fn get_or_cache_block(
cache: &BlockCache, cache: &BlockCache,
verified_blocks: &[AtomicU64], verified_blocks: &[AtomicU64],
compression: Compression, compression: Compression,
) -> Result<ArcBytes> { ) -> Result<ArcBytes<'static>> {
let mmap_block = if let StaticSortedFileBacking::Mmap(mmap) = backing { let mmap_block = if matches!(backing, StaticSortedFileBacking::Mmap(_)) {
let (uncompressed_length, checksum, block_data) = let (uncompressed_length, checksum, block_data) = get_raw_block(backing, meta, block_index)
get_raw_block_slice(mmap, meta, block_index).with_context(|| { .with_context(|| {
format!( format!(
"Failed to read raw block {} from {:08}.sst", "Failed to read raw block {} from {:08}.sst",
block_index, meta.sequence_number block_index, meta.sequence_number
...@@ -644,9 +644,8 @@ fn get_or_cache_block( ...@@ -644,9 +644,8 @@ fn get_or_cache_block(
if uncompressed_length == 0 { if uncompressed_length == 0 {
// Uncompressed: serve directly from mmap. Verify CRC only once per file open. // Uncompressed: serve directly from mmap. Verify CRC only once per file open.
verify_checksum_once(meta, block_data, checksum, block_index, verified_blocks)?; verify_checksum_once(meta, &block_data, checksum, block_index, verified_blocks)?;
// SAFETY: block_data points into the mmap backing `mmap`. return Ok(block_data.into_static());
return Ok(unsafe { ArcBytes::from_mmap(mmap, block_data) });
} }
Some((uncompressed_length, checksum, block_data)) Some((uncompressed_length, checksum, block_data))
} else { } else {
...@@ -660,9 +659,7 @@ fn get_or_cache_block( ...@@ -660,9 +659,7 @@ fn get_or_cache_block(
GuardResult::Value(block) => block, GuardResult::Value(block) => block,
GuardResult::Guard(guard) => { GuardResult::Guard(guard) => {
let (uncompressed_length, checksum, block_data) = match mmap_block { let (uncompressed_length, checksum, block_data) = match mmap_block {
Some((uncompressed_length, checksum, block_data)) => { Some(block) => block,
(uncompressed_length, checksum, Cow::Borrowed(block_data))
}
None => get_raw_block(backing, meta, block_index)?, None => get_raw_block(backing, meta, block_index)?,
}; };
// A cached block may have been evicted, so re-reading still // A cached block may have been evicted, so re-reading still
...@@ -680,7 +677,7 @@ fn get_or_cache_block( ...@@ -680,7 +677,7 @@ fn get_or_cache_block(
} }
} }
let block = if uncompressed_length == 0 { let block = if uncompressed_length == 0 {
ArcBytes::from(block_data.into_owned().into_boxed_slice()) block_data.into_static()
} else { } else {
ArcBytes::from_decompressed(compression, uncompressed_length, &block_data) ArcBytes::from_decompressed(compression, uncompressed_length, &block_data)
.with_context(|| { .with_context(|| {
...@@ -764,12 +761,14 @@ fn get_raw_block<'a>( ...@@ -764,12 +761,14 @@ fn get_raw_block<'a>(
backing: &'a StaticSortedFileBacking, backing: &'a StaticSortedFileBacking,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
) -> Result<(u32, u32, Cow<'a, [u8]>)> { ) -> Result<(u32, u32, ArcBytes<'a>)> {
match backing { match backing {
StaticSortedFileBacking::Mmap(mmap) => { StaticSortedFileBacking::Mmap(mmap) => {
let (uncompressed_length, checksum, block) = let (uncompressed_length, checksum, block) =
get_raw_block_slice(mmap, meta, block_index)?; get_raw_block_slice(mmap, meta, block_index)?;
Ok((uncompressed_length, checksum, Cow::Borrowed(block))) Ok((uncompressed_length, checksum, unsafe {
ArcBytes::from_mmap_ref(mmap, block)
}))
} }
StaticSortedFileBacking::File { StaticSortedFileBacking::File {
file, file,
...@@ -799,7 +798,11 @@ fn get_raw_block<'a>( ...@@ -799,7 +798,11 @@ fn get_raw_block<'a>(
let uncompressed_length = be::read_u32(&bytes); let uncompressed_length = be::read_u32(&bytes);
let checksum = be::read_u32(&bytes[4..]); let checksum = be::read_u32(&bytes[4..]);
let block = bytes.split_off(BLOCK_HEADER_SIZE); let block = bytes.split_off(BLOCK_HEADER_SIZE);
Ok((uncompressed_length, checksum, Cow::Owned(block))) Ok((
uncompressed_length,
checksum,
ArcBytes::from(block.into_boxed_slice()),
))
} }
} }
} }
...@@ -883,17 +886,11 @@ fn read_block_lookup( ...@@ -883,17 +886,11 @@ fn read_block_lookup(
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
block_index: u16, block_index: u16,
compression: Compression, compression: Compression,
) -> Result<ArcBytes> { ) -> Result<ArcBytes<'static>> {
let (uncompressed_length, checksum, block) = get_raw_block(backing, meta, block_index)?; let (uncompressed_length, checksum, block) = get_raw_block(backing, meta, block_index)?;
verify_checksum(meta, &block, checksum, block_index)?; verify_checksum(meta, &block, checksum, block_index)?;
if uncompressed_length == 0 { if uncompressed_length == 0 {
return match (backing, block) { return Ok(block.into_static());
(StaticSortedFileBacking::Mmap(mmap), Cow::Borrowed(block)) => {
// SAFETY: block points into mmap.
Ok(unsafe { ArcBytes::from_mmap(mmap, block) })
}
(_, block) => Ok(ArcBytes::from(block.into_owned().into_boxed_slice())),
};
} }
ArcBytes::from_decompressed(compression, uncompressed_length, &block).with_context(|| { ArcBytes::from_decompressed(compression, uncompressed_length, &block).with_context(|| {
format!( format!(
......
...@@ -91,18 +91,26 @@ impl TurboKeyValueDatabase { ...@@ -91,18 +91,26 @@ impl TurboKeyValueDatabase {
self.db.is_empty() self.db.is_empty()
} }
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> { pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes<'static>>> {
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
pub fn batch_get(&self, key_space: KeySpace, keys: &[&[u8]]) -> Result<Vec<Option<ArcBytes>>> { pub fn batch_get(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<ArcBytes<'static>>>> {
self.db.batch_get(key_space as usize, keys) self.db.batch_get(key_space as usize, keys)
} }
/// Looks up a key and returns all matching values. /// Looks up a key and returns all matching values.
/// ///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache). /// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
pub fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> { pub fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
self.db.get_multiple(key_space as usize, &key) self.db.get_multiple(key_space as usize, &key)
} }
...@@ -213,7 +221,7 @@ pub struct TurboWriteBatch<'a> { ...@@ -213,7 +221,7 @@ pub struct TurboWriteBatch<'a> {
} }
impl<'a> TurboWriteBatch<'a> { impl<'a> TurboWriteBatch<'a> {
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> { pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes<'static>>> {
self.db.get(key_space as usize, &key) self.db.get(key_space as usize, &key)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment