Unverified Commit dcb0d012 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-persistence: add borrowed mmap byte slices

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 4d621240
......@@ -13,30 +13,34 @@ use crate::{
compression::decompress_into_arc,
shared_bytes::{SharedBytes, is_subslice_of},
};
/// The backing storage for an `ArcBytes`.
///
/// The inner values are never read directly — they exist solely to keep the
/// backing memory alive while the raw `data` pointer in `ArcBytes` references it.
#[derive(Clone)]
enum Backing {
Arc { _backing: Arc<[u8]> },
Mmap { _backing: Arc<Mmap> },
enum Backing<'l> {
Arc {
_backing: Arc<[u8]>,
},
Mmap {
_backing: Arc<Mmap>,
},
/// Borrows an mmap handle when the byte view cannot escape the caller's scope.
MmapRef {
_backing: &'l Arc<Mmap>,
},
}
/// An owned byte slice backed by either an `Arc<[u8]>` or a memory-mapped file.
/// A byte slice backed by owned bytes, an owned mmap handle, or a borrowed mmap handle.
#[derive(Clone)]
pub struct ArcBytes {
pub struct ArcBytes<'l> {
data: *const [u8],
// Safety: Backing should come last so that it is dropped after the data pointer so we don't
// create a dangling pointer. This isn't really a problem since it is technically ok to have
// dangling _pointers_.
backing: Backing,
// Keep the backing after the raw pointer so it is dropped last.
backing: Backing<'l>,
}
unsafe impl Send for ArcBytes {}
unsafe impl Sync for ArcBytes {}
unsafe impl Send for ArcBytes<'_> {}
unsafe impl Sync for ArcBytes<'_> {}
impl From<Arc<[u8]>> for ArcBytes {
impl From<Arc<[u8]>> for ArcBytes<'static> {
fn from(arc: Arc<[u8]>) -> Self {
Self {
data: &*arc as *const [u8],
......@@ -45,13 +49,13 @@ impl From<Arc<[u8]>> for ArcBytes {
}
}
impl From<Box<[u8]>> for ArcBytes {
fn from(b: Box<[u8]>) -> Self {
Self::from(Arc::from(b))
impl From<Box<[u8]>> for ArcBytes<'static> {
fn from(bytes: Box<[u8]>) -> Self {
Self::from(Arc::from(bytes))
}
}
impl Deref for ArcBytes {
impl Deref for ArcBytes<'_> {
type Target = [u8];
fn deref(&self) -> &Self::Target {
......@@ -59,90 +63,118 @@ impl Deref for ArcBytes {
}
}
impl Borrow<[u8]> for ArcBytes {
impl Borrow<[u8]> for ArcBytes<'_> {
fn borrow(&self) -> &[u8] {
self
}
}
impl Hash for ArcBytes {
impl Hash for ArcBytes<'_> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.deref().hash(state)
}
}
impl PartialEq for ArcBytes {
impl PartialEq for ArcBytes<'_> {
fn eq(&self, other: &Self) -> bool {
self.deref().eq(other.deref())
}
}
impl Debug for ArcBytes {
impl Debug for ArcBytes<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
Debug::fmt(&**self, f)
}
}
impl Eq for ArcBytes {}
impl Eq for ArcBytes<'_> {}
fn backing_as_slice<'a>(backing: &'a Backing<'a>) -> &'a [u8] {
match backing {
Backing::Arc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
Backing::MmapRef { _backing } => _backing,
}
}
impl<'l> ArcBytes<'l> {
pub fn slice(self, range: Range<usize>) -> Self {
let data = &self[range] as *const [u8];
Self {
data,
backing: self.backing,
}
}
/// # Safety
/// `subslice` must point into this value's backing storage.
pub unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
debug_assert!(is_subslice_of(subslice, backing_as_slice(&self.backing)));
Self {
data: subslice as *const [u8],
backing: self.backing.clone(),
}
}
/// Promotes a borrowed mmap view by cloning its mmap handle.
pub fn into_static(self) -> ArcBytes<'static> {
ArcBytes {
data: self.data,
backing: match self.backing {
Backing::Arc { _backing } => Backing::Arc { _backing },
Backing::Mmap { _backing } => Backing::Mmap { _backing },
Backing::MmapRef { _backing } => Backing::Mmap {
_backing: _backing.clone(),
},
},
}
}
impl ArcBytes {
/// Returns `true` if this `ArcBytes` is backed by a memory-mapped file.
pub fn is_mmap_backed(&self) -> bool {
matches!(self.backing, Backing::Mmap { .. })
matches!(self.backing, Backing::Mmap { .. } | Backing::MmapRef { .. })
}
/// Returns `true` if the backing `Arc` allocation is shared (i.e., there
/// are other `Arc` clones referencing the same data outside the cache).
/// Always returns `false` for mmap-backed bytes, since the mmap `Arc` is
/// shared across all slices from the same file and is not a useful signal.
pub fn is_shared_arc(&self) -> bool {
match &self.backing {
Backing::Arc { _backing } => Arc::strong_count(_backing) > 1,
Backing::Mmap { .. } => false,
Backing::Mmap { .. } | Backing::MmapRef { .. } => false,
}
}
/// # Safety
/// `subslice` must point into `mmap`.
pub unsafe fn from_mmap(mmap: Arc<Mmap>, subslice: &[u8]) -> ArcBytes<'static> {
debug_assert!(is_subslice_of(subslice, &mmap));
ArcBytes {
data: subslice as *const [u8],
backing: Backing::Mmap { _backing: mmap },
}
}
/// # Safety
/// `subslice` must point into `mmap`.
pub unsafe fn from_mmap_ref(mmap: &'l Arc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!(is_subslice_of(subslice, mmap));
Self {
data: subslice as *const [u8],
backing: Backing::MmapRef { _backing: mmap },
}
}
}
impl SharedBytes for ArcBytes {
impl SharedBytes for ArcBytes<'static> {
type MmapHandle = Arc<Mmap>;
fn slice(self, range: Range<usize>) -> Self {
let data = &*self;
let data = &data[range] as *const [u8];
Self {
data,
backing: self.backing,
}
ArcBytes::slice(self, range)
}
unsafe fn slice_from_subslice(&self, subslice: &[u8]) -> Self {
debug_assert!(
is_subslice_of(
subslice,
match &self.backing {
Backing::Arc { _backing } => _backing,
Backing::Mmap { _backing } => _backing,
}
),
"slice_from_subslice: subslice is not within the backing storage"
);
Self {
data: subslice as *const [u8],
backing: self.backing.clone(),
}
unsafe { ArcBytes::slice_from_subslice(self, subslice) }
}
unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!(
is_subslice_of(subslice, mmap),
"from_mmap: subslice is not within the mmap"
);
ArcBytes {
data: subslice as *const [u8],
backing: Backing::Mmap {
_backing: mmap.clone(),
},
}
unsafe { ArcBytes::from_mmap(mmap.clone(), subslice) }
}
fn from_decompressed(
......@@ -157,3 +189,22 @@ impl SharedBytes for ArcBytes {
)?))
}
}
#[cfg(test)]
mod tests {
use std::io::Write;
use super::*;
#[test]
fn borrowed_mmap_can_be_sliced_and_promoted() -> anyhow::Result<()> {
let mut file = tempfile::tempfile()?;
file.write_all(b"0123456789")?;
let mmap = Arc::new(unsafe { Mmap::map(&file)? });
let borrowed = unsafe { ArcBytes::from_mmap_ref(&mmap, &mmap[2..8]) };
let promoted = borrowed.slice(1..5).into_static();
drop(mmap);
assert_eq!(&*promoted, b"3456");
Ok(())
}
}
......@@ -659,7 +659,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Reads and decompresses a blob file. This is not backed by any cache.
#[tracing::instrument(level = "info", name = "reading database blob", skip_all)]
fn read_blob(&self, seq: u32, compression: Compression) -> Result<ArcBytes> {
fn read_blob(&self, seq: u32, compression: Compression) -> Result<ArcBytes<'static>> {
let path = self.path.join(format!("{seq:08}.blob"));
let file = File::open(&path)?;
let data: Either<Mmap, Vec<u8>> = match self.config.access_mode {
......@@ -1983,7 +1983,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Get a value from the database. Returns None if the key is not found. The returned value
/// might hold onto a block of the database and it should not be hold long-term.
pub fn get<K: QueryKey>(&self, family: usize, key: &K) -> Result<Option<ArcBytes>> {
pub fn get<K: QueryKey>(&self, family: usize, key: &K) -> Result<Option<ArcBytes<'static>>> {
debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic
......@@ -2015,7 +2015,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
&self,
family: usize,
key: &K,
) -> Result<SmallVec<[ArcBytes; 1]>> {
) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::MultiValue {
// This is an error in our caller so just panic
......@@ -2041,10 +2041,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
family: usize,
key: &K,
span: &EnteredSpan,
) -> Result<SmallVec<[ArcBytes; 1]>> {
) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
let hash = hash_key(key);
let inner = self.inner.read();
let mut output: SmallVec<[ArcBytes; 1]> = SmallVec::new();
let mut output: SmallVec<[ArcBytes<'static>; 1]> = SmallVec::new();
// Track whether we found the key in any SST (even if deleted).
// Used for miss_global stat: only fires if key was never found anywhere.
#[cfg(feature = "stats")]
......@@ -2053,7 +2053,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// Values deleted by key-value tombstones seen so far. Because we walk meta files newest
// first, and tombstones sort first within a key group, every tombstone that could apply to
// a value has already been seen by the time we reach that value.
let mut deleted_values: AutoSet<ArcBytes, BuildHasherDefault<FxHasher>, 1> =
let mut deleted_values: AutoSet<ArcBytes<'static>, BuildHasherDefault<FxHasher>, 1> =
AutoSet::default();
let mut size = 0;
......@@ -2172,7 +2172,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
&self,
family: usize,
keys: &[K],
) -> Result<Vec<Option<ArcBytes>>> {
) -> Result<Vec<Option<ArcBytes<'static>>>> {
debug_assert!(family < FAMILIES, "Family index out of bounds");
if self.config.family_configs[family].kind != FamilyKind::SingleValue {
// This is an error in our caller so just panic
......
......@@ -9,7 +9,7 @@ use crate::{
/// `ArcBytes` for the lookup path. The compaction/iteration path uses
/// `LookupValue<RcBytes>` which is convertible to `IterValue`.
#[derive(PartialEq)]
pub enum LookupValue<B = ArcBytes> {
pub enum LookupValue<B = ArcBytes<'static>> {
/// The value was deleted.
KeyDeleted,
/// A single value was deleted from this key's group (MultiValue families only). Other values
......
use std::{
borrow::Cow,
cmp::Ordering,
hash::BuildHasherDefault,
io,
......@@ -145,8 +144,8 @@ impl From<LookupValue> for SstLookupResult {
#[derive(Clone, Default)]
pub struct BlockWeighter;
impl quick_cache::Weighter<(u32, u16), ArcBytes> for BlockWeighter {
fn weight(&self, _key: &(u32, u16), val: &ArcBytes) -> u64 {
impl quick_cache::Weighter<(u32, u16), ArcBytes<'static>> for BlockWeighter {
fn weight(&self, _key: &(u32, u16), val: &ArcBytes<'static>) -> u64 {
if val.is_mmap_backed() {
// Mmap-backed blocks bypass the cache (served directly from mmap),
// so this branch should never be reached.
......@@ -166,11 +165,11 @@ impl quick_cache::Weighter<(u32, u16), ArcBytes> for BlockWeighter {
#[derive(Clone, Default)]
pub struct BlockCacheLifecycle;
impl Lifecycle<(u32, u16), ArcBytes> for BlockCacheLifecycle {
impl Lifecycle<(u32, u16), ArcBytes<'static>> for BlockCacheLifecycle {
type RequestState = ();
#[inline]
fn is_pinned(&self, _key: &(u32, u16), val: &ArcBytes) -> bool {
fn is_pinned(&self, _key: &(u32, u16), val: &ArcBytes<'static>) -> bool {
val.is_shared_arc()
}
......@@ -178,12 +177,13 @@ impl Lifecycle<(u32, u16), ArcBytes> for BlockCacheLifecycle {
fn begin_request(&self) -> Self::RequestState {}
#[inline]
fn on_evict(&self, _state: &mut Self::RequestState, _key: (u32, u16), _val: ArcBytes) {}
fn on_evict(&self, _state: &mut Self::RequestState, _key: (u32, u16), _val: ArcBytes<'static>) {
}
}
pub type BlockCache = quick_cache::sync::Cache<
(u32, u16),
ArcBytes,
ArcBytes<'static>,
BlockWeighter,
BuildHasherDefault<FxHasher>,
BlockCacheLifecycle,
......@@ -213,13 +213,13 @@ struct ArcBlockCacheReader<'a> {
verified_blocks: &'a [AtomicU64],
}
impl ValueBlockCache<ArcBytes> for ArcBlockCacheReader<'_> {
impl ValueBlockCache<ArcBytes<'static>> for ArcBlockCacheReader<'_> {
fn get_or_read(
self,
meta: &StaticSortedFileMetaData,
block_index: u16,
compression: Compression,
) -> Result<ArcBytes> {
) -> Result<ArcBytes<'static>> {
get_or_cache_block(
self.backing,
meta,
......@@ -235,7 +235,7 @@ impl ValueBlockCache<ArcBytes> for ArcBlockCacheReader<'_> {
meta: &StaticSortedFileMetaData,
block_index: u16,
compression: Compression,
) -> Result<ArcBytes> {
) -> Result<ArcBytes<'static>> {
read_block_lookup(self.backing, meta, block_index, compression)
}
}
......@@ -455,7 +455,7 @@ impl StaticSortedFile {
/// If `FIND_ALL` is true, collects all entries with the same key.
fn lookup_key_block<K: QueryKey, const FIND_ALL: bool>(
&self,
block: ArcBytes,
block: ArcBytes<'static>,
key_hash: u64,
key: &K,
layout: KeyBlockLayout,
......@@ -489,7 +489,7 @@ impl StaticSortedFile {
/// enabling direct indexing during binary search.
fn lookup_fixed_key_block<K: QueryKey, const FIND_ALL: bool>(
&self,
block: ArcBytes,
block: ArcBytes<'static>,
key_hash: u64,
key: &K,
layout: KeyBlockLayout,
......@@ -529,7 +529,7 @@ impl StaticSortedFile {
/// key blocks (offset table lookup) and fixed-size key blocks (stride-based indexing).
fn lookup_block_inner<'a, K: QueryKey, const FIND_ALL: bool>(
&self,
block: &ArcBytes,
block: &ArcBytes<'static>,
entry_count: usize,
key_hash: u64,
key: &K,
......@@ -610,7 +610,7 @@ impl StaticSortedFile {
&self,
ty: u8,
val: &[u8],
key_block_arc: &ArcBytes,
key_block_arc: &ArcBytes<'static>,
reader: ArcBlockCacheReader<'_>,
) -> Result<LookupValue> {
handle_key_match_generic(&self.meta, ty, val, key_block_arc, self.compression, reader)
......@@ -632,10 +632,10 @@ fn get_or_cache_block(
cache: &BlockCache,
verified_blocks: &[AtomicU64],
compression: Compression,
) -> Result<ArcBytes> {
let mmap_block = if let StaticSortedFileBacking::Mmap(mmap) = backing {
let (uncompressed_length, checksum, block_data) =
get_raw_block_slice(mmap, meta, block_index).with_context(|| {
) -> Result<ArcBytes<'static>> {
let mmap_block = if matches!(backing, StaticSortedFileBacking::Mmap(_)) {
let (uncompressed_length, checksum, block_data) = get_raw_block(backing, meta, block_index)
.with_context(|| {
format!(
"Failed to read raw block {} from {:08}.sst",
block_index, meta.sequence_number
......@@ -644,9 +644,8 @@ fn get_or_cache_block(
if uncompressed_length == 0 {
// Uncompressed: serve directly from mmap. Verify CRC only once per file open.
verify_checksum_once(meta, block_data, checksum, block_index, verified_blocks)?;
// SAFETY: block_data points into the mmap backing `mmap`.
return Ok(unsafe { ArcBytes::from_mmap(mmap, block_data) });
verify_checksum_once(meta, &block_data, checksum, block_index, verified_blocks)?;
return Ok(block_data.into_static());
}
Some((uncompressed_length, checksum, block_data))
} else {
......@@ -660,9 +659,7 @@ fn get_or_cache_block(
GuardResult::Value(block) => block,
GuardResult::Guard(guard) => {
let (uncompressed_length, checksum, block_data) = match mmap_block {
Some((uncompressed_length, checksum, block_data)) => {
(uncompressed_length, checksum, Cow::Borrowed(block_data))
}
Some(block) => block,
None => get_raw_block(backing, meta, block_index)?,
};
// A cached block may have been evicted, so re-reading still
......@@ -680,7 +677,7 @@ fn get_or_cache_block(
}
}
let block = if uncompressed_length == 0 {
ArcBytes::from(block_data.into_owned().into_boxed_slice())
block_data.into_static()
} else {
ArcBytes::from_decompressed(compression, uncompressed_length, &block_data)
.with_context(|| {
......@@ -764,12 +761,14 @@ fn get_raw_block<'a>(
backing: &'a StaticSortedFileBacking,
meta: &StaticSortedFileMetaData,
block_index: u16,
) -> Result<(u32, u32, Cow<'a, [u8]>)> {
) -> Result<(u32, u32, ArcBytes<'a>)> {
match backing {
StaticSortedFileBacking::Mmap(mmap) => {
let (uncompressed_length, checksum, block) =
get_raw_block_slice(mmap, meta, block_index)?;
Ok((uncompressed_length, checksum, Cow::Borrowed(block)))
Ok((uncompressed_length, checksum, unsafe {
ArcBytes::from_mmap_ref(mmap, block)
}))
}
StaticSortedFileBacking::File {
file,
......@@ -799,7 +798,11 @@ fn get_raw_block<'a>(
let uncompressed_length = be::read_u32(&bytes);
let checksum = be::read_u32(&bytes[4..]);
let block = bytes.split_off(BLOCK_HEADER_SIZE);
Ok((uncompressed_length, checksum, Cow::Owned(block)))
Ok((
uncompressed_length,
checksum,
ArcBytes::from(block.into_boxed_slice()),
))
}
}
}
......@@ -883,17 +886,11 @@ fn read_block_lookup(
meta: &StaticSortedFileMetaData,
block_index: u16,
compression: Compression,
) -> Result<ArcBytes> {
) -> Result<ArcBytes<'static>> {
let (uncompressed_length, checksum, block) = get_raw_block(backing, meta, block_index)?;
verify_checksum(meta, &block, checksum, block_index)?;
if uncompressed_length == 0 {
return match (backing, block) {
(StaticSortedFileBacking::Mmap(mmap), Cow::Borrowed(block)) => {
// SAFETY: block points into mmap.
Ok(unsafe { ArcBytes::from_mmap(mmap, block) })
}
(_, block) => Ok(ArcBytes::from(block.into_owned().into_boxed_slice())),
};
return Ok(block.into_static());
}
ArcBytes::from_decompressed(compression, uncompressed_length, &block).with_context(|| {
format!(
......
......@@ -91,18 +91,26 @@ impl TurboKeyValueDatabase {
self.db.is_empty()
}
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> {
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes<'static>>> {
self.db.get(key_space as usize, &key)
}
pub fn batch_get(&self, key_space: KeySpace, keys: &[&[u8]]) -> Result<Vec<Option<ArcBytes>>> {
pub fn batch_get(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<ArcBytes<'static>>>> {
self.db.batch_get(key_space as usize, keys)
}
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
pub fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
pub fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[ArcBytes<'static>; 1]>> {
self.db.get_multiple(key_space as usize, &key)
}
......@@ -213,7 +221,7 @@ pub struct TurboWriteBatch<'a> {
}
impl<'a> TurboWriteBatch<'a> {
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> {
pub fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes<'static>>> {
self.db.get(key_space as usize, &key)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment