Unverified Commit 6631f105 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Optimize LZ4 buffer handling

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 998a9da6
use std::{cell::RefCell, mem::MaybeUninit, rc::Rc, sync::Arc}; use std::{cell::RefCell, ffi::c_char, mem::MaybeUninit, rc::Rc, sync::Arc};
use anyhow::{Context, Result, ensure}; use anyhow::{Context, Result, ensure};
use lz4::block as lz4; use lz4::liblz4::{LZ4_compress_default, LZ4_compressBound, LZ4_decompress_safe};
/// Compression algorithm used for a family's SST blocks and blob values. /// Compression algorithm used for a family's SST blocks and blob values.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
...@@ -35,16 +35,31 @@ fn decompress_block( ...@@ -35,16 +35,31 @@ fn decompress_block(
zero-copy mmap path" zero-copy mmap path"
); );
let bytes_written = match compression { let bytes_written = match compression {
Compression::Lz4 => lz4::decompress_to_buffer( Compression::Lz4 => {
block, let compressed_len = block
Some( .len()
expected_len .try_into()
.try_into() .context("LZ4 compressed length exceeds i32::MAX")?;
.context("LZ4 uncompressed length exceeds i32::MAX")?, let dest_len = dest
), .len()
dest, .try_into()
) .context("LZ4 uncompressed length exceeds i32::MAX")?;
.map_err(anyhow::Error::from), // Safety: both pointers are valid for the checked lengths passed to liblz4, the
// destination is writable, and liblz4 does not retain either pointer.
let bytes_written = unsafe {
LZ4_decompress_safe(
block.as_ptr().cast::<c_char>(),
dest.as_mut_ptr().cast::<c_char>(),
compressed_len,
dest_len,
)
};
ensure!(
bytes_written >= 0,
"LZ4 decompression failed with code {bytes_written}"
);
Ok(bytes_written as usize)
}
Compression::Zstd3 => ZSTD_DECOMPRESSOR.with_borrow_mut(|decompressor| { Compression::Zstd3 => ZSTD_DECOMPRESSOR.with_borrow_mut(|decompressor| {
decompressor decompressor
.decompress_to_buffer(block, dest) .decompress_to_buffer(block, dest)
...@@ -125,24 +140,44 @@ impl Compressor { ...@@ -125,24 +140,44 @@ impl Compressor {
Ok(Self { compression, zstd }) Ok(Self { compression, zstd })
} }
/// Compresses `block` into the start of `buffer` and returns the compressed length. /// Compresses `block` into `buffer`.
///
/// The buffer may be longer than the returned length. Keeping the initialized LZ4 scratch
/// space avoids zero-filling the destination again on every block.
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
pub(crate) fn compress_into_buffer( pub(crate) fn compress_into_buffer(
&mut self, &mut self,
block: &[u8], block: &[u8],
buffer: &mut Vec<u8>, buffer: &mut Vec<u8>,
) -> Result<usize> { ) -> Result<()> {
match self.compression { match self.compression {
Compression::Lz4 => { Compression::Lz4 => {
let bound = lz4::compress_bound(block.len()).context("LZ4 input is too large")?; let input_len = block
if buffer.len() < bound { .len()
buffer.resize(bound, 0); .try_into()
} .context("LZ4 input length exceeds i32::MAX")?;
lz4::compress_to_buffer(block, None, false, &mut buffer[..bound]) // Safety: LZ4_compressBound only reads its integer argument.
.context("LZ4 compression failed") let bound = unsafe { LZ4_compressBound(input_len) };
ensure!(bound > 0, "LZ4 input is too large");
buffer.clear();
buffer.reserve(bound as usize);
// Safety: the input pointer is valid for input_len bytes. reserve() established at
// least bound bytes of spare capacity, whose pointer is writable. liblz4 does not
// retain either pointer and reports how many destination bytes it initialized.
let bytes_written = unsafe {
LZ4_compress_default(
block.as_ptr().cast::<c_char>(),
buffer.spare_capacity_mut().as_mut_ptr().cast::<c_char>(),
input_len,
bound,
)
};
ensure!(
bytes_written > 0 && bytes_written <= bound,
"LZ4 compression failed with invalid output length {bytes_written}"
);
// Safety: liblz4 initialized exactly bytes_written bytes, and the check above
// verifies that they fit within the reserved bound.
unsafe { buffer.set_len(bytes_written as usize) };
Ok(())
} }
Compression::Zstd3 => { Compression::Zstd3 => {
buffer.clear(); buffer.clear();
...@@ -151,7 +186,8 @@ impl Compressor { ...@@ -151,7 +186,8 @@ impl Compressor {
.as_mut() .as_mut()
.expect("zstd compressor not initialized") .expect("zstd compressor not initialized")
.compress_to_buffer(block, buffer) .compress_to_buffer(block, buffer)
.context("zstd compression failed") .context("zstd compression failed")?;
Ok(())
} }
} }
} }
...@@ -167,16 +203,24 @@ mod tests { ...@@ -167,16 +203,24 @@ mod tests {
for compression in [Compression::Lz4, Compression::Zstd3] { for compression in [Compression::Lz4, Compression::Zstd3] {
let mut compressor = Compressor::new(compression).unwrap(); let mut compressor = Compressor::new(compression).unwrap();
let mut compressed = Vec::new(); let mut compressed = Vec::new();
let compressed_len = compressor compressor
.compress_into_buffer(&input, &mut compressed) .compress_into_buffer(&input, &mut compressed)
.unwrap(); .unwrap();
let output = decompress_into_arc( let output = decompress_into_arc(compression, input.len() as u32, &compressed).unwrap();
compression,
input.len() as u32,
&compressed[..compressed_len],
)
.unwrap();
assert_eq!(&*output, input); assert_eq!(&*output, input);
} }
} }
#[test]
fn truncated_lz4_block_returns_an_error() {
let input = b"turbo persistence compression ".repeat(1024);
let mut compressed = Vec::new();
Compressor::new(Compression::Lz4)
.unwrap()
.compress_into_buffer(&input, &mut compressed)
.unwrap();
compressed.truncate(compressed.len() / 2);
assert!(decompress_into_arc(Compression::Lz4, input.len() as u32, &compressed).is_err());
}
} }
...@@ -398,13 +398,10 @@ fn write_block_to_file( ...@@ -398,13 +398,10 @@ fn write_block_to_file(
compressor: &mut Compressor, compressor: &mut Compressor,
) -> Result<u16> { ) -> Result<u16> {
let (uncompressed_size, data_to_write): (u32, &[u8]) = if try_compress { let (uncompressed_size, data_to_write): (u32, &[u8]) = if try_compress {
let compressed_len = compressor.compress_into_buffer(block, compress_buffer)?; compressor.compress_into_buffer(block, compress_buffer)?;
// Same threshold as LevelDB/RocksDB: require at least 12.5% savings. // Same threshold as LevelDB/RocksDB: require at least 12.5% savings.
if compressed_len < block.len() - (block.len() / 8) { if compress_buffer.len() < block.len() - (block.len() / 8) {
( (block.len().try_into().unwrap(), compress_buffer.as_slice())
block.len().try_into().unwrap(),
&compress_buffer[..compressed_len],
)
} else { } else {
(0, block) (0, block)
} }
......
...@@ -488,10 +488,9 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize ...@@ -488,10 +488,9 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1; let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let mut compressed = Vec::new(); let mut compressed = Vec::new();
let compression = self.family_configs[usize_from_u32(family)].compression; let compression = self.family_configs[usize_from_u32(family)].compression;
let compressed_len = Compressor::new(compression)? Compressor::new(compression)?
.compress_into_buffer(value, &mut compressed) .compress_into_buffer(value, &mut compressed)
.context("Compression of value for blob file failed")?; .context("Compression of value for blob file failed")?;
compressed.truncate(compressed_len);
let mut buffer = Vec::with_capacity(8 + compressed.len()); let mut buffer = Vec::with_capacity(8 + compressed.len());
buffer.write_u32::<BE>(value.len() as u32)?; buffer.write_u32::<BE>(value.len() as u32)?;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment