Unverified Commit 1fa24a41 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: Lazy decompress medium value compressed blocks (#82257)

### What?

During compaction we can lazily recompress medium-sized values to avoid the large temporary memory usage.
parent a874eb51
use std::{mem::MaybeUninit, sync::Arc};
use anyhow::{Context, Result};
use lzzzz::lz4::{ACC_LEVEL_DEFAULT, decompress, decompress_with_dict};
#[tracing::instrument(level = "trace", skip_all)]
pub fn decompress_into_arc(
uncompressed_length: u32,
block: &[u8],
compression_dictionary: Option<&[u8]>,
_long_term: bool,
) -> Result<Arc<[u8]>> {
// We directly allocate the buffer in an Arc to avoid copying it into an Arc and avoiding
// double indirection. This is a dynamically sized arc.
let buffer: Arc<[MaybeUninit<u8>]> = Arc::new_zeroed_slice(uncompressed_length as usize);
// Assume that the buffer is initialized.
let buffer = Arc::into_raw(buffer);
// Safety: Assuming that the buffer is initialized is safe because we just created it as
// zeroed slice and u8 doesn't require initialization.
let mut buffer = unsafe { Arc::from_raw(buffer as *mut [u8]) };
// Safety: We know that the buffer is not shared yet.
let decompressed = unsafe { Arc::get_mut_unchecked(&mut buffer) };
let bytes_writes = if let Some(dict) = compression_dictionary {
// Safety: decompress_with_dict will only write to `decompressed` and not read from it.
decompress_with_dict(block, decompressed, dict)?
} else {
// Safety: decompress will only write to `decompressed` and not read from it.
decompress(block, decompressed)?
};
assert_eq!(
bytes_writes, uncompressed_length as usize,
"Decompressed length does not match expected length"
);
// Safety: The buffer is now fully initialized and can be used.
Ok(buffer)
}
#[tracing::instrument(level = "trace", skip_all)]
pub fn compress_into_buffer(
block: &[u8],
dict: Option<&[u8]>,
_long_term: bool,
buffer: &mut Vec<u8>,
) -> Result<()> {
let mut compressor = if let Some(dict) = dict {
lzzzz::lz4::Compressor::with_dict(dict)
} else {
lzzzz::lz4::Compressor::new()
}
.context("LZ4 compressor creation failed")?;
let acc_factor = ACC_LEVEL_DEFAULT;
compressor
.next_to_vec(block, buffer, acc_factor)
.context("Compression failed")?;
Ok(())
}
......@@ -3,19 +3,15 @@ use std::{
collections::HashSet,
fs::{self, File, OpenOptions, ReadDir},
io::{BufWriter, Write},
mem::{MaybeUninit, swap, transmute},
mem::swap,
ops::RangeInclusive,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, AtomicU32, Ordering},
},
sync::atomic::{AtomicBool, AtomicU32, Ordering},
};
use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt, WriteBytesExt};
use jiff::Timestamp;
use lzzzz::lz4::decompress;
use memmap2::Mmap;
use parking_lot::{Mutex, RwLock};
......@@ -24,6 +20,7 @@ use crate::{
QueryKey,
arc_slice::ArcSlice,
compaction::selector::{Compactable, compute_metrics, get_merge_segments},
compression::decompress_into_arc,
constants::{
AMQF_AVG_SIZE, AMQF_CACHE_SIZE, DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE,
KEY_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE,
......@@ -392,14 +389,9 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
#[cfg(target_os = "linux")]
mmap.advise(memmap2::Advice::Unmergeable)?;
let mut compressed = &mmap[..];
let uncompressed_length = compressed.read_u32::<BE>()? as usize;
let buffer = Arc::new_zeroed_slice(uncompressed_length);
// Safety: MaybeUninit<u8> can be safely transmuted to u8.
let mut buffer = unsafe { transmute::<Arc<[MaybeUninit<u8>]>, Arc<[u8]>>(buffer) };
// Safety: We know that the buffer is not shared yet.
let decompressed = unsafe { Arc::get_mut_unchecked(&mut buffer) };
decompress(compressed, decompressed)?;
let uncompressed_length = compressed.read_u32::<BE>()?;
let buffer = decompress_into_arc(uncompressed_length, compressed, None, true)?;
Ok(ArcSlice::from(buffer))
}
......@@ -918,9 +910,9 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
});
}
fn create_sst_file<S: ParallelScheduler>(
fn create_sst_file<'l, S: ParallelScheduler>(
parallel_scheduler: &S,
entries: &[LookupEntry],
entries: &[LookupEntry<'l>],
total_key_size: usize,
total_value_size: usize,
path: &Path,
......@@ -964,7 +956,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
let mut total_key_size = 0;
let mut total_value_size = 0;
let mut current: Option<LookupEntry> = None;
let mut current: Option<LookupEntry<'_>> = None;
let mut entries = Vec::new();
let mut last_entries = Vec::new();
let mut last_entries_total_sizes = (0, 0);
......@@ -975,7 +967,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
if let Some(current) = current.take() {
if current.key != entry.key {
let key_size = current.key.len();
let value_size = current.value.size_in_sst();
let value_size = current.value.uncompressed_size_in_sst();
total_key_size += key_size;
total_value_size += value_size;
......@@ -1023,7 +1015,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
}
if let Some(entry) = current {
total_key_size += entry.key.len();
total_value_size += entry.value.size_in_sst();
total_value_size += entry.value.uncompressed_size_in_sst();
entries.push(entry);
}
......
......@@ -8,6 +8,7 @@ mod arc_slice;
mod collector;
mod collector_entry;
mod compaction;
mod compression;
mod constants;
mod db;
mod key;
......
......@@ -14,28 +14,43 @@ pub enum LookupValue {
Blob { sequence_number: u32 },
}
impl LookupValue {
/// A value from a SST file lookup.
pub enum LazyLookupValue<'l> {
/// A LookupValue
Eager(LookupValue),
/// A medium sized value that is still compressed.
Medium {
uncompressed_size: u32,
block: &'l [u8],
dictionary: &'l [u8],
},
}
impl LazyLookupValue<'_> {
/// Returns the size of the value in the SST file.
pub fn size_in_sst(&self) -> usize {
pub fn uncompressed_size_in_sst(&self) -> usize {
match self {
LookupValue::Slice { value } => value.len(),
LookupValue::Deleted => 0,
LookupValue::Blob { .. } => 0,
LazyLookupValue::Eager(LookupValue::Slice { value }) => value.len(),
LazyLookupValue::Eager(LookupValue::Deleted) => 0,
LazyLookupValue::Eager(LookupValue::Blob { .. }) => 0,
LazyLookupValue::Medium {
uncompressed_size, ..
} => *uncompressed_size as usize,
}
}
}
/// An entry from a SST file lookup.
pub struct LookupEntry {
pub struct LookupEntry<'l> {
/// The hash of the key.
pub hash: u64,
/// The key.
pub key: ArcSlice<u8>,
/// The value.
pub value: LookupValue,
pub value: LazyLookupValue<'l>,
}
impl Entry for LookupEntry {
impl Entry for LookupEntry<'_> {
fn key_hash(&self) -> u64 {
self.hash
}
......@@ -50,17 +65,26 @@ impl Entry for LookupEntry {
fn value(&self) -> EntryValue<'_> {
match &self.value {
LookupValue::Deleted => EntryValue::Deleted,
LookupValue::Slice { value } => {
LazyLookupValue::Eager(LookupValue::Deleted) => EntryValue::Deleted,
LazyLookupValue::Eager(LookupValue::Slice { value }) => {
if value.len() > MAX_SMALL_VALUE_SIZE {
EntryValue::Medium { value }
} else {
EntryValue::Small { value }
}
}
LookupValue::Blob { sequence_number } => EntryValue::Large {
LazyLookupValue::Eager(LookupValue::Blob { sequence_number }) => EntryValue::Large {
blob: *sequence_number,
},
LazyLookupValue::Medium {
uncompressed_size,
block,
dictionary,
} => EntryValue::MediumCompressed {
uncompressed_size: *uncompressed_size,
block,
dictionary,
},
}
}
}
......@@ -6,27 +6,27 @@ use crate::lookup_entry::LookupEntry;
/// An active iterator that is being merged. It has peeked the next element and can be compared
/// according to that element. The `order` is used when multiple iterators have the same key.
struct ActiveIterator<T: Iterator<Item = Result<LookupEntry>>> {
struct ActiveIterator<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> {
iter: T,
order: usize,
entry: LookupEntry,
entry: LookupEntry<'l>,
}
impl<T: Iterator<Item = Result<LookupEntry>>> PartialEq for ActiveIterator<T> {
impl<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> PartialEq for ActiveIterator<'l, T> {
fn eq(&self, other: &Self) -> bool {
self.entry.hash == other.entry.hash && *self.entry.key == *other.entry.key
}
}
impl<T: Iterator<Item = Result<LookupEntry>>> Eq for ActiveIterator<T> {}
impl<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> Eq for ActiveIterator<'l, T> {}
impl<T: Iterator<Item = Result<LookupEntry>>> PartialOrd for ActiveIterator<T> {
impl<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> PartialOrd for ActiveIterator<'l, T> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl<T: Iterator<Item = Result<LookupEntry>>> Ord for ActiveIterator<T> {
impl<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> Ord for ActiveIterator<'l, T> {
fn cmp(&self, other: &Self) -> Ordering {
self.entry
.hash
......@@ -39,11 +39,11 @@ impl<T: Iterator<Item = Result<LookupEntry>>> Ord for ActiveIterator<T> {
/// An iterator that merges multiple sorted iterators into a single sorted iterator. Internal it
/// uses an heap of iterators to iterate them in order.
pub struct MergeIter<T: Iterator<Item = Result<LookupEntry>>> {
heap: BinaryHeap<ActiveIterator<T>>,
pub struct MergeIter<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> {
heap: BinaryHeap<ActiveIterator<'l, T>>,
}
impl<T: Iterator<Item = Result<LookupEntry>>> MergeIter<T> {
impl<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> MergeIter<'l, T> {
pub fn new(iters: impl Iterator<Item = T>) -> Result<Self> {
let mut heap = BinaryHeap::new();
for (order, mut iter) in iters.enumerate() {
......@@ -56,8 +56,8 @@ impl<T: Iterator<Item = Result<LookupEntry>>> MergeIter<T> {
}
}
impl<T: Iterator<Item = Result<LookupEntry>>> Iterator for MergeIter<T> {
type Item = Result<LookupEntry>;
impl<'l, T: Iterator<Item = Result<LookupEntry<'l>>>> Iterator for MergeIter<'l, T> {
type Item = Result<LookupEntry<'l>>;
fn next(&mut self) -> Option<Self::Item> {
let ActiveIterator {
......
......@@ -2,15 +2,12 @@ use std::{
cmp::Ordering,
fs::File,
hash::BuildHasherDefault,
mem::{MaybeUninit, transmute},
ops::Range,
path::{Path, PathBuf},
sync::Arc,
};
use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt};
use lzzzz::lz4::decompress_with_dict;
use memmap2::Mmap;
use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher;
......@@ -18,7 +15,8 @@ use rustc_hash::FxHasher;
use crate::{
QueryKey,
arc_slice::ArcSlice,
lookup_entry::{LookupEntry, LookupValue},
compression::decompress_into_arc,
lookup_entry::{LazyLookupValue, LookupEntry, LookupValue},
};
/// The block header for an index block.
......@@ -326,6 +324,7 @@ impl StaticSortedFile {
self.read_block(
block_index,
&self.mmap[self.meta.key_compression_dictionary_range()],
false,
)
}
......@@ -334,11 +333,30 @@ impl StaticSortedFile {
self.read_block(
block_index,
&self.mmap[self.meta.value_compression_dictionary_range()],
false,
)
}
/// Reads a block from the file.
fn read_block(&self, block_index: u16, compression_dictionary: &[u8]) -> Result<ArcSlice<u8>> {
fn read_block(
&self,
block_index: u16,
compression_dictionary: &[u8],
long_term: bool,
) -> Result<ArcSlice<u8>> {
let (uncompressed_length, block) = self.get_compressed_block(block_index)?;
let buffer = decompress_into_arc(
uncompressed_length,
block,
Some(compression_dictionary),
long_term,
)?;
Ok(ArcSlice::from(buffer))
}
/// Gets the slice of the compressed block from the memory mapped file.
fn get_compressed_block(&self, block_index: u16) -> Result<(u32, &[u8])> {
#[cfg(feature = "strict_checks")]
if block_index >= self.meta.block_count {
bail!(
......@@ -386,17 +404,9 @@ impl StaticSortedFile {
self.meta.blocks_start()
);
}
let uncompressed_length =
(&self.mmap[block_start..block_start + 4]).read_u32::<BE>()? as usize;
let block = self.mmap[block_start + 4..block_end].to_vec();
let buffer = Arc::new_zeroed_slice(uncompressed_length);
// Safety: MaybeUninit<u8> can be safely transmuted to u8.
let mut buffer = unsafe { transmute::<Arc<[MaybeUninit<u8>]>, Arc<[u8]>>(buffer) };
// Safety: We know that the buffer is not shared yet.
let decompressed = unsafe { Arc::get_mut_unchecked(&mut buffer) };
decompress_with_dict(&block, decompressed, compression_dictionary)?;
Ok(ArcSlice::from(buffer))
let uncompressed_length = (&self.mmap[block_start..block_start + 4]).read_u32::<BE>()?;
let block = &self.mmap[block_start + 4..block_end];
Ok((uncompressed_length, block))
}
}
......@@ -423,15 +433,15 @@ struct CurrentIndexBlock {
index: usize,
}
impl Iterator for StaticSortedFileIter<'_> {
type Item = Result<LookupEntry>;
impl<'l> Iterator for StaticSortedFileIter<'l> {
type Item = Result<LookupEntry<'l>>;
fn next(&mut self) -> Option<Self::Item> {
self.next_internal().transpose()
}
}
impl StaticSortedFileIter<'_> {
impl<'l> StaticSortedFileIter<'l> {
/// Enters a block at the given index.
fn enter_block(&mut self, block_index: u16) -> Result<()> {
let block_arc = self.this.get_key_block(block_index, self.key_block_cache)?;
......@@ -468,7 +478,7 @@ impl StaticSortedFileIter<'_> {
}
/// Gets the next entry in the file and moves the cursor.
fn next_internal(&mut self) -> Result<Option<LookupEntry>> {
fn next_internal(&mut self) -> Result<Option<LookupEntry<'l>>> {
loop {
if let Some(CurrentKeyBlock {
offsets,
......@@ -479,9 +489,22 @@ impl StaticSortedFileIter<'_> {
{
let GetKeyEntryResult { hash, key, ty, val } =
get_key_entry(&offsets, &entries, entry_count, index)?;
let value = self
.this
.handle_key_match(ty, val, self.value_block_cache)?;
let value = if ty == KEY_BLOCK_ENTRY_TYPE_MEDIUM {
let mut val = val;
let block = val.read_u16::<BE>()?;
let (uncompressed_size, block) = self.this.get_compressed_block(block)?;
LazyLookupValue::Medium {
uncompressed_size,
block,
dictionary: &self.this.mmap
[self.this.meta.value_compression_dictionary_range()],
}
} else {
let value = self
.this
.handle_key_match(ty, val, self.value_block_cache)?;
LazyLookupValue::Eager(value)
};
let entry = LookupEntry {
hash,
// Safety: The key is a valid slice of the entries.
......
......@@ -8,11 +8,13 @@ use std::{
use anyhow::{Context, Result};
use byteorder::{BE, ByteOrder, WriteBytesExt};
use lzzzz::lz4::{ACC_LEVEL_DEFAULT, max_compressed_size};
use crate::static_sorted_file::{
BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL,
use crate::{
compression::{compress_into_buffer, decompress_into_arc},
static_sorted_file::{
BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL,
},
};
/// The maximum number of entries that should go into a single key block
......@@ -68,6 +70,12 @@ pub enum EntryValue<'l> {
Small { value: &'l [u8] },
/// Medium-sized value. They are stored in their own value block.
Medium { value: &'l [u8] },
/// Medium-sized value. They are stored in their own value block. Precompressed.
MediumCompressed {
uncompressed_size: u32,
block: &'l [u8],
dictionary: &'l [u8],
},
/// Large-sized value. They are stored in a blob file.
Large { blob: u32 },
/// Tombstone. The value was removed.
......@@ -293,25 +301,25 @@ impl<'l> BlockWriter<'l> {
#[tracing::instrument(level = "trace", skip_all)]
fn write_key_block(&mut self, block: &[u8], dict: &[u8]) -> Result<()> {
self.write_block(block, dict)
self.write_block(block, dict, false)
.context("Failed to write key block")
}
#[tracing::instrument(level = "trace", skip_all)]
fn write_index_block(&mut self, block: &[u8], dict: &[u8]) -> Result<()> {
self.write_block(block, dict)
self.write_block(block, dict, false)
.context("Failed to write index block")
}
#[tracing::instrument(level = "trace", skip_all)]
fn write_value_block(&mut self, block: &[u8], dict: &[u8]) -> Result<()> {
self.write_block(block, dict)
self.write_block(block, dict, false)
.context("Failed to write value block")
}
fn write_block(&mut self, block: &[u8], dict: &[u8]) -> Result<()> {
fn write_block(&mut self, block: &[u8], dict: &[u8], long_term: bool) -> Result<()> {
let uncompressed_size = block.len().try_into().unwrap();
self.compress_block_into_buffer(block, dict);
self.compress_block_into_buffer(block, dict, long_term)?;
let len = (self.buffer.len() + 4).try_into().unwrap();
let offset = self
.block_offsets
......@@ -333,14 +341,13 @@ impl<'l> BlockWriter<'l> {
}
/// Compresses a block with a compression dictionary.
#[tracing::instrument(level = "trace", skip_all)]
fn compress_block_into_buffer(&mut self, block: &[u8], dict: &[u8]) {
let mut compressor =
lzzzz::lz4::Compressor::with_dict(dict).expect("LZ4 compressor creation failed");
self.buffer.reserve(max_compressed_size(block.len()));
compressor
.next_to_vec(block, self.buffer, ACC_LEVEL_DEFAULT)
.expect("Compression failed");
fn compress_block_into_buffer(
&mut self,
block: &[u8],
dict: &[u8],
long_term: bool,
) -> Result<()> {
compress_into_buffer(block, Some(dict), long_term, self.buffer)
}
}
......@@ -386,7 +393,19 @@ fn write_value_blocks(
value_locations.push((block_index, 0));
writer.write_value_block(value, value_compression_dictionary)?;
}
_ => {
EntryValue::MediumCompressed {
uncompressed_size,
block,
dictionary,
} => {
let block_index = writer.next_block_index();
value_locations.push((block_index, 0));
// Recompress block with a different dictionary
let decompressed =
decompress_into_arc(uncompressed_size, block, Some(dictionary), false)?;
writer.write_value_block(&decompressed, value_compression_dictionary)?;
}
EntryValue::Deleted | EntryValue::Large { .. } => {
value_locations.push((0, 0));
}
}
......@@ -438,7 +457,7 @@ fn write_key_blocks_and_compute_amqf(
value.len().try_into().unwrap(),
);
}
EntryValue::Medium { .. } => {
EntryValue::Medium { .. } | EntryValue::MediumCompressed { .. } => {
block.put_medium(entry, value_location.0);
}
EntryValue::Large { blob } => {
......
......@@ -10,7 +10,6 @@ use std::{
use anyhow::{Context, Result};
use byteorder::{BE, WriteBytesExt};
use either::Either;
use lzzzz::lz4::{self, ACC_LEVEL_DEFAULT};
use parking_lot::Mutex;
use smallvec::SmallVec;
use thread_local::ThreadLocal;
......@@ -19,6 +18,7 @@ use crate::{
ValueBuffer,
collector::Collector,
collector_entry::CollectorEntry,
compression::compress_into_buffer,
constants::{MAX_MEDIUM_VALUE_SIZE, THREAD_LOCAL_SIZE_SHIFT},
key::StoreKey,
meta_file_builder::MetaFileBuilder,
......@@ -390,7 +390,7 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let mut buffer = Vec::new();
buffer.write_u32::<BE>(value.len() as u32)?;
lz4::compress_to_vec(value, &mut buffer, ACC_LEVEL_DEFAULT)
compress_into_buffer(value, None, true, &mut buffer)
.context("Compression of value for blob file failed")?;
let file = self.db_path.join(format!("{seq:08}.blob"));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment