Unverified Commit f2468331 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: remove value compression dictionary (#82338)

### What?

remove value compression dictionary

It doesn't have benefit as each block is already large as we merge small values together and medium size values are already large.

This also gives the benefit that we do not need to recompress medium value blocks when doing compaction
parent 1400faa4
......@@ -35,7 +35,6 @@ fn main() -> Result<()> {
amqf_entries,
sst_size,
key_compression_dictionary_size,
value_compression_dictionary_size,
block_count,
} in meta_file.entries
{
......@@ -45,15 +44,11 @@ fn main() -> Result<()> {
);
println!(" AMQF {amqf_entries} entries = {} KiB", amqf_size / 1024);
println!(
" {} KiB = {} kiB key compression dict + {} KiB value compression dict + \
{block_count} blocks (avg {} bytes/block)",
" {} KiB = {} kiB key compression dict + {block_count} blocks (avg {} \
bytes/block)",
sst_size / 1024,
key_compression_dictionary_size / 1024,
value_compression_dictionary_size / 1024,
(sst_size
- key_compression_dictionary_size as u64
- value_compression_dictionary_size as u64)
/ block_count as u64
(sst_size - key_compression_dictionary_size as u64) / block_count as u64
);
}
if !meta_file.obsolete_sst_files.is_empty() {
......
......@@ -45,7 +45,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store
- foreach described SST file
- 4 bytes sequence number of the SST file
- 2 bytes key Compression Dictionary length
- 2 bytes value Compression Dictionary length
- 2 bytes block count
- 8 bytes min hash
- 8 bytes max hash
......@@ -59,7 +58,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store
The SST file contains only data without any header.
- serialized key Compression Dictionary
- serialized value Compression Dictionary
- foreach block
- 4 bytes uncompressed block length
- compressed data
......
......@@ -92,11 +92,11 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
self.entries.push(entry);
}
/// Sorts the entries and returns them along with the total key and value sizes. This doesn't
/// Sorts the entries and returns them along with the total key size. This doesn't
/// clear the entries.
pub fn sorted(&mut self) -> (&[CollectorEntry<K>], usize, usize) {
pub fn sorted(&mut self) -> (&[CollectorEntry<K>], usize) {
self.entries.sort_unstable_by(|a, b| a.key.cmp(&b.key));
(&self.entries, self.total_key_size, self.total_value_size)
(&self.entries, self.total_key_size)
}
/// Clears the collector.
......
......@@ -898,8 +898,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
amqf,
key_compression_dictionary_length: entry
.key_compression_dictionary_length(),
value_compression_dictionary_length: entry
.value_compression_dictionary_length(),
block_count: entry.block_count(),
size: entry.size(),
entries: 0,
......@@ -914,7 +912,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
parallel_scheduler: &S,
entries: &[LookupEntry<'l>],
total_key_size: usize,
total_value_size: usize,
path: &Path,
seq: u32,
) -> Result<(u32, File, StaticSortedFileBuilderMeta<'static>)>
......@@ -924,7 +921,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
write_static_stored_file(
entries,
total_key_size,
total_value_size,
&path.join(format!("{seq:08}.sst")),
)
})?;
......@@ -959,7 +955,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
let mut current: Option<LookupEntry<'_>> = None;
let mut entries = Vec::new();
let mut last_entries = Vec::new();
let mut last_entries_total_sizes = (0, 0);
let mut last_entries_total_key_size = 0;
for entry in iter {
let entry = entry?;
......@@ -975,15 +971,10 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
> DATA_THRESHOLD_PER_COMPACTED_FILE
|| entries.len() >= MAX_ENTRIES_PER_COMPACTED_FILE
{
let (
selected_total_key_size,
selected_total_value_size,
) = last_entries_total_sizes;
let selected_total_key_size =
last_entries_total_key_size;
swap(&mut entries, &mut last_entries);
last_entries_total_sizes = (
total_key_size - key_size,
total_value_size - value_size,
);
last_entries_total_key_size = total_key_size - key_size;
total_key_size = key_size;
total_value_size = value_size;
......@@ -997,7 +988,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
&self.parallel_scheduler,
&entries,
selected_total_key_size,
selected_total_value_size,
path,
seq,
)?);
......@@ -1015,7 +1005,8 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
}
if let Some(entry) = current {
total_key_size += entry.key.len();
total_value_size += entry.value.uncompressed_size_in_sst();
// Obsolete as we no longer need total_value_size
// total_value_size += entry.value.uncompressed_size_in_sst();
entries.push(entry);
}
......@@ -1028,7 +1019,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
&self.parallel_scheduler,
&entries,
total_key_size,
total_value_size,
path,
seq,
)?);
......@@ -1039,8 +1029,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
if !last_entries.is_empty() {
last_entries.append(&mut entries);
last_entries_total_sizes.0 += total_key_size;
last_entries_total_sizes.1 += total_value_size;
last_entries_total_key_size += total_key_size;
let (part1, part2) = last_entries.split_at(last_entries.len() / 2);
......@@ -1052,8 +1041,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
&self.parallel_scheduler,
part1,
// We don't know the exact sizes so we estimate them
last_entries_total_sizes.0 / 2,
last_entries_total_sizes.1 / 2,
last_entries_total_key_size / 2,
path,
seq1,
)?);
......@@ -1062,8 +1050,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
new_sst_files.push(create_sst_file(
&self.parallel_scheduler,
part2,
last_entries_total_sizes.0 / 2,
last_entries_total_sizes.1 / 2,
last_entries_total_key_size / 2,
path,
seq2,
)?);
......@@ -1263,8 +1250,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
amqf_entries: amqf.len(),
key_compression_dictionary_size: entry
.key_compression_dictionary_length(),
value_compression_dictionary_size: entry
.value_compression_dictionary_length(),
block_count: entry.block_count(),
}
})
......@@ -1302,6 +1287,5 @@ pub struct MetaFileEntryInfo {
pub amqf_entries: usize,
pub sst_size: u64,
pub key_compression_dictionary_size: u16,
pub value_compression_dictionary_size: u16,
pub block_count: u16,
}
......@@ -22,7 +22,6 @@ pub enum LazyLookupValue<'l> {
Medium {
uncompressed_size: u32,
block: &'l [u8],
dictionary: &'l [u8],
},
}
......@@ -79,11 +78,9 @@ impl Entry for LookupEntry<'_> {
LazyLookupValue::Medium {
uncompressed_size,
block,
dictionary,
} => EntryValue::MediumCompressed {
uncompressed_size: *uncompressed_size,
block,
dictionary,
},
}
}
......
......@@ -144,10 +144,6 @@ impl MetaEntry {
self.sst_data.key_compression_dictionary_length
}
pub fn value_compression_dictionary_length(&self) -> u16 {
self.sst_data.value_compression_dictionary_length
}
pub fn block_count(&self) -> u16 {
self.sst_data.block_count
}
......@@ -222,7 +218,6 @@ impl MetaFile {
sst_data: StaticSortedFileMetaData {
sequence_number: file.read_u32::<BE>()?,
key_compression_dictionary_length: file.read_u16::<BE>()?,
value_compression_dictionary_length: file.read_u16::<BE>()?,
block_count: file.read_u16::<BE>()?,
},
family,
......
......@@ -58,7 +58,6 @@ impl<'a> MetaFileBuilder<'a> {
for (sequence_number, sst) in &self.entries {
file.write_u32::<BE>(*sequence_number)?;
file.write_u16::<BE>(sst.key_compression_dictionary_length)?;
file.write_u16::<BE>(sst.value_compression_dictionary_length)?;
file.write_u16::<BE>(sst.block_count)?;
file.write_u64::<BE>(sst.min_hash)?;
file.write_u64::<BE>(sst.max_hash)?;
......
......@@ -65,8 +65,6 @@ pub struct StaticSortedFileMetaData {
pub sequence_number: u32,
/// The length of the key compression dictionary.
pub key_compression_dictionary_length: u16,
/// The length of the value compression dictionary.
pub value_compression_dictionary_length: u16,
/// The number of blocks in the SST file.
pub block_count: u16,
}
......@@ -79,8 +77,7 @@ impl StaticSortedFileMetaData {
pub fn blocks_start(&self) -> usize {
let k: usize = self.key_compression_dictionary_length.into();
let v: usize = self.value_compression_dictionary_length.into();
k + v
k
}
pub fn key_compression_dictionary_range(&self) -> Range<usize> {
......@@ -88,12 +85,6 @@ impl StaticSortedFileMetaData {
let end: usize = self.key_compression_dictionary_length.into();
start..end
}
pub fn value_compression_dictionary_range(&self) -> Range<usize> {
let start = self.key_compression_dictionary_length as usize;
let end = start + self.value_compression_dictionary_length as usize;
start..end
}
}
/// A memory mapped SST file.
......@@ -310,7 +301,7 @@ impl StaticSortedFile {
match value_block_cache.get_value_or_guard(&(self.meta.sequence_number, block), None) {
GuardResult::Value(block) => block,
GuardResult::Guard(guard) => {
let block = self.read_value_block(block)?;
let block = self.read_small_value_block(block)?;
let _ = guard.insert(block.clone());
block
}
......@@ -323,25 +314,26 @@ impl StaticSortedFile {
fn read_key_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
self.read_block(
block_index,
&self.mmap[self.meta.key_compression_dictionary_range()],
Some(&self.mmap[self.meta.key_compression_dictionary_range()]),
false,
)
}
/// Reads a value block from the file.
fn read_small_value_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
self.read_block(block_index, None, false)
}
/// Reads a value block from the file.
fn read_value_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
self.read_block(
block_index,
&self.mmap[self.meta.value_compression_dictionary_range()],
false,
)
self.read_block(block_index, None, true)
}
/// Reads a block from the file.
fn read_block(
&self,
block_index: u16,
compression_dictionary: &[u8],
compression_dictionary: Option<&[u8]>,
long_term: bool,
) -> Result<ArcSlice<u8>> {
let (uncompressed_length, block) = self.get_compressed_block(block_index)?;
......@@ -349,7 +341,7 @@ impl StaticSortedFile {
let buffer = decompress_into_arc(
uncompressed_length,
block,
Some(compression_dictionary),
compression_dictionary,
long_term,
)?;
Ok(ArcSlice::from(buffer))
......@@ -496,8 +488,6 @@ impl<'l> StaticSortedFileIter<'l> {
LazyLookupValue::Medium {
uncompressed_size,
block,
dictionary: &self.this.mmap
[self.this.meta.value_compression_dictionary_range()],
}
} else {
let value = self
......
......@@ -407,17 +407,15 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
fn create_sst_file(
&self,
family: u32,
collector_data: (&[CollectorEntry<K>], usize, usize),
collector_data: (&[CollectorEntry<K>], usize),
) -> Result<(u32, File)> {
let (entries, total_key_size, total_value_size) = collector_data;
let (entries, total_key_size) = collector_data;
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let path = self.db_path.join(format!("{seq:08}.sst"));
let (meta, file) = self
.parallel_scheduler
.block_in_place(|| {
write_static_stored_file(entries, total_key_size, total_value_size, &path)
})
.block_in_place(|| write_static_stored_file(entries, total_key_size, &path))
.with_context(|| format!("Unable to write SST file {seq:08}.sst"))?;
#[cfg(feature = "verify_sst_content")]
......@@ -440,7 +438,6 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
StaticSortedFileMetaData {
sequence_number: seq,
key_compression_dictionary_length: meta.key_compression_dictionary_length,
value_compression_dictionary_length: meta.value_compression_dictionary_length,
block_count: meta.block_count,
},
)?;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment