Unverified Commit f2468331 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: remove value compression dictionary (#82338)

### What?

remove value compression dictionary

It doesn't have benefit as each block is already large as we merge small values together and medium size values are already large.

This also gives the benefit that we do not need to recompress medium value blocks when doing compaction
parent 1400faa4
...@@ -35,7 +35,6 @@ fn main() -> Result<()> { ...@@ -35,7 +35,6 @@ fn main() -> Result<()> {
amqf_entries, amqf_entries,
sst_size, sst_size,
key_compression_dictionary_size, key_compression_dictionary_size,
value_compression_dictionary_size,
block_count, block_count,
} in meta_file.entries } in meta_file.entries
{ {
...@@ -45,15 +44,11 @@ fn main() -> Result<()> { ...@@ -45,15 +44,11 @@ fn main() -> Result<()> {
); );
println!(" AMQF {amqf_entries} entries = {} KiB", amqf_size / 1024); println!(" AMQF {amqf_entries} entries = {} KiB", amqf_size / 1024);
println!( println!(
" {} KiB = {} kiB key compression dict + {} KiB value compression dict + \ " {} KiB = {} kiB key compression dict + {block_count} blocks (avg {} \
{block_count} blocks (avg {} bytes/block)", bytes/block)",
sst_size / 1024, sst_size / 1024,
key_compression_dictionary_size / 1024, key_compression_dictionary_size / 1024,
value_compression_dictionary_size / 1024, (sst_size - key_compression_dictionary_size as u64) / block_count as u64
(sst_size
- key_compression_dictionary_size as u64
- value_compression_dictionary_size as u64)
/ block_count as u64
); );
} }
if !meta_file.obsolete_sst_files.is_empty() { if !meta_file.obsolete_sst_files.is_empty() {
......
...@@ -45,7 +45,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store ...@@ -45,7 +45,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store
- foreach described SST file - foreach described SST file
- 4 bytes sequence number of the SST file - 4 bytes sequence number of the SST file
- 2 bytes key Compression Dictionary length - 2 bytes key Compression Dictionary length
- 2 bytes value Compression Dictionary length
- 2 bytes block count - 2 bytes block count
- 8 bytes min hash - 8 bytes min hash
- 8 bytes max hash - 8 bytes max hash
...@@ -59,7 +58,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store ...@@ -59,7 +58,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store
The SST file contains only data without any header. The SST file contains only data without any header.
- serialized key Compression Dictionary - serialized key Compression Dictionary
- serialized value Compression Dictionary
- foreach block - foreach block
- 4 bytes uncompressed block length - 4 bytes uncompressed block length
- compressed data - compressed data
......
...@@ -92,11 +92,11 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> { ...@@ -92,11 +92,11 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
self.entries.push(entry); self.entries.push(entry);
} }
/// Sorts the entries and returns them along with the total key and value sizes. This doesn't /// Sorts the entries and returns them along with the total key size. This doesn't
/// clear the entries. /// clear the entries.
pub fn sorted(&mut self) -> (&[CollectorEntry<K>], usize, usize) { pub fn sorted(&mut self) -> (&[CollectorEntry<K>], usize) {
self.entries.sort_unstable_by(|a, b| a.key.cmp(&b.key)); self.entries.sort_unstable_by(|a, b| a.key.cmp(&b.key));
(&self.entries, self.total_key_size, self.total_value_size) (&self.entries, self.total_key_size)
} }
/// Clears the collector. /// Clears the collector.
......
...@@ -898,8 +898,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -898,8 +898,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
amqf, amqf,
key_compression_dictionary_length: entry key_compression_dictionary_length: entry
.key_compression_dictionary_length(), .key_compression_dictionary_length(),
value_compression_dictionary_length: entry
.value_compression_dictionary_length(),
block_count: entry.block_count(), block_count: entry.block_count(),
size: entry.size(), size: entry.size(),
entries: 0, entries: 0,
...@@ -914,7 +912,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -914,7 +912,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
parallel_scheduler: &S, parallel_scheduler: &S,
entries: &[LookupEntry<'l>], entries: &[LookupEntry<'l>],
total_key_size: usize, total_key_size: usize,
total_value_size: usize,
path: &Path, path: &Path,
seq: u32, seq: u32,
) -> Result<(u32, File, StaticSortedFileBuilderMeta<'static>)> ) -> Result<(u32, File, StaticSortedFileBuilderMeta<'static>)>
...@@ -924,7 +921,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -924,7 +921,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
write_static_stored_file( write_static_stored_file(
entries, entries,
total_key_size, total_key_size,
total_value_size,
&path.join(format!("{seq:08}.sst")), &path.join(format!("{seq:08}.sst")),
) )
})?; })?;
...@@ -959,7 +955,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -959,7 +955,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
let mut current: Option<LookupEntry<'_>> = None; let mut current: Option<LookupEntry<'_>> = None;
let mut entries = Vec::new(); let mut entries = Vec::new();
let mut last_entries = Vec::new(); let mut last_entries = Vec::new();
let mut last_entries_total_sizes = (0, 0); let mut last_entries_total_key_size = 0;
for entry in iter { for entry in iter {
let entry = entry?; let entry = entry?;
...@@ -975,15 +971,10 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -975,15 +971,10 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
> DATA_THRESHOLD_PER_COMPACTED_FILE > DATA_THRESHOLD_PER_COMPACTED_FILE
|| entries.len() >= MAX_ENTRIES_PER_COMPACTED_FILE || entries.len() >= MAX_ENTRIES_PER_COMPACTED_FILE
{ {
let ( let selected_total_key_size =
selected_total_key_size, last_entries_total_key_size;
selected_total_value_size,
) = last_entries_total_sizes;
swap(&mut entries, &mut last_entries); swap(&mut entries, &mut last_entries);
last_entries_total_sizes = ( last_entries_total_key_size = total_key_size - key_size;
total_key_size - key_size,
total_value_size - value_size,
);
total_key_size = key_size; total_key_size = key_size;
total_value_size = value_size; total_value_size = value_size;
...@@ -997,7 +988,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -997,7 +988,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
&self.parallel_scheduler, &self.parallel_scheduler,
&entries, &entries,
selected_total_key_size, selected_total_key_size,
selected_total_value_size,
path, path,
seq, seq,
)?); )?);
...@@ -1015,7 +1005,8 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1015,7 +1005,8 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
} }
if let Some(entry) = current { if let Some(entry) = current {
total_key_size += entry.key.len(); total_key_size += entry.key.len();
total_value_size += entry.value.uncompressed_size_in_sst(); // Obsolete as we no longer need total_value_size
// total_value_size += entry.value.uncompressed_size_in_sst();
entries.push(entry); entries.push(entry);
} }
...@@ -1028,7 +1019,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1028,7 +1019,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
&self.parallel_scheduler, &self.parallel_scheduler,
&entries, &entries,
total_key_size, total_key_size,
total_value_size,
path, path,
seq, seq,
)?); )?);
...@@ -1039,8 +1029,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1039,8 +1029,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
if !last_entries.is_empty() { if !last_entries.is_empty() {
last_entries.append(&mut entries); last_entries.append(&mut entries);
last_entries_total_sizes.0 += total_key_size; last_entries_total_key_size += total_key_size;
last_entries_total_sizes.1 += total_value_size;
let (part1, part2) = last_entries.split_at(last_entries.len() / 2); let (part1, part2) = last_entries.split_at(last_entries.len() / 2);
...@@ -1052,8 +1041,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1052,8 +1041,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
&self.parallel_scheduler, &self.parallel_scheduler,
part1, part1,
// We don't know the exact sizes so we estimate them // We don't know the exact sizes so we estimate them
last_entries_total_sizes.0 / 2, last_entries_total_key_size / 2,
last_entries_total_sizes.1 / 2,
path, path,
seq1, seq1,
)?); )?);
...@@ -1062,8 +1050,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1062,8 +1050,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
new_sst_files.push(create_sst_file( new_sst_files.push(create_sst_file(
&self.parallel_scheduler, &self.parallel_scheduler,
part2, part2,
last_entries_total_sizes.0 / 2, last_entries_total_key_size / 2,
last_entries_total_sizes.1 / 2,
path, path,
seq2, seq2,
)?); )?);
...@@ -1263,8 +1250,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -1263,8 +1250,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
amqf_entries: amqf.len(), amqf_entries: amqf.len(),
key_compression_dictionary_size: entry key_compression_dictionary_size: entry
.key_compression_dictionary_length(), .key_compression_dictionary_length(),
value_compression_dictionary_size: entry
.value_compression_dictionary_length(),
block_count: entry.block_count(), block_count: entry.block_count(),
} }
}) })
...@@ -1302,6 +1287,5 @@ pub struct MetaFileEntryInfo { ...@@ -1302,6 +1287,5 @@ pub struct MetaFileEntryInfo {
pub amqf_entries: usize, pub amqf_entries: usize,
pub sst_size: u64, pub sst_size: u64,
pub key_compression_dictionary_size: u16, pub key_compression_dictionary_size: u16,
pub value_compression_dictionary_size: u16,
pub block_count: u16, pub block_count: u16,
} }
...@@ -22,7 +22,6 @@ pub enum LazyLookupValue<'l> { ...@@ -22,7 +22,6 @@ pub enum LazyLookupValue<'l> {
Medium { Medium {
uncompressed_size: u32, uncompressed_size: u32,
block: &'l [u8], block: &'l [u8],
dictionary: &'l [u8],
}, },
} }
...@@ -79,11 +78,9 @@ impl Entry for LookupEntry<'_> { ...@@ -79,11 +78,9 @@ impl Entry for LookupEntry<'_> {
LazyLookupValue::Medium { LazyLookupValue::Medium {
uncompressed_size, uncompressed_size,
block, block,
dictionary,
} => EntryValue::MediumCompressed { } => EntryValue::MediumCompressed {
uncompressed_size: *uncompressed_size, uncompressed_size: *uncompressed_size,
block, block,
dictionary,
}, },
} }
} }
......
...@@ -144,10 +144,6 @@ impl MetaEntry { ...@@ -144,10 +144,6 @@ impl MetaEntry {
self.sst_data.key_compression_dictionary_length self.sst_data.key_compression_dictionary_length
} }
pub fn value_compression_dictionary_length(&self) -> u16 {
self.sst_data.value_compression_dictionary_length
}
pub fn block_count(&self) -> u16 { pub fn block_count(&self) -> u16 {
self.sst_data.block_count self.sst_data.block_count
} }
...@@ -222,7 +218,6 @@ impl MetaFile { ...@@ -222,7 +218,6 @@ impl MetaFile {
sst_data: StaticSortedFileMetaData { sst_data: StaticSortedFileMetaData {
sequence_number: file.read_u32::<BE>()?, sequence_number: file.read_u32::<BE>()?,
key_compression_dictionary_length: file.read_u16::<BE>()?, key_compression_dictionary_length: file.read_u16::<BE>()?,
value_compression_dictionary_length: file.read_u16::<BE>()?,
block_count: file.read_u16::<BE>()?, block_count: file.read_u16::<BE>()?,
}, },
family, family,
......
...@@ -58,7 +58,6 @@ impl<'a> MetaFileBuilder<'a> { ...@@ -58,7 +58,6 @@ impl<'a> MetaFileBuilder<'a> {
for (sequence_number, sst) in &self.entries { for (sequence_number, sst) in &self.entries {
file.write_u32::<BE>(*sequence_number)?; file.write_u32::<BE>(*sequence_number)?;
file.write_u16::<BE>(sst.key_compression_dictionary_length)?; file.write_u16::<BE>(sst.key_compression_dictionary_length)?;
file.write_u16::<BE>(sst.value_compression_dictionary_length)?;
file.write_u16::<BE>(sst.block_count)?; file.write_u16::<BE>(sst.block_count)?;
file.write_u64::<BE>(sst.min_hash)?; file.write_u64::<BE>(sst.min_hash)?;
file.write_u64::<BE>(sst.max_hash)?; file.write_u64::<BE>(sst.max_hash)?;
......
...@@ -65,8 +65,6 @@ pub struct StaticSortedFileMetaData { ...@@ -65,8 +65,6 @@ pub struct StaticSortedFileMetaData {
pub sequence_number: u32, pub sequence_number: u32,
/// The length of the key compression dictionary. /// The length of the key compression dictionary.
pub key_compression_dictionary_length: u16, pub key_compression_dictionary_length: u16,
/// The length of the value compression dictionary.
pub value_compression_dictionary_length: u16,
/// The number of blocks in the SST file. /// The number of blocks in the SST file.
pub block_count: u16, pub block_count: u16,
} }
...@@ -79,8 +77,7 @@ impl StaticSortedFileMetaData { ...@@ -79,8 +77,7 @@ impl StaticSortedFileMetaData {
pub fn blocks_start(&self) -> usize { pub fn blocks_start(&self) -> usize {
let k: usize = self.key_compression_dictionary_length.into(); let k: usize = self.key_compression_dictionary_length.into();
let v: usize = self.value_compression_dictionary_length.into(); k
k + v
} }
pub fn key_compression_dictionary_range(&self) -> Range<usize> { pub fn key_compression_dictionary_range(&self) -> Range<usize> {
...@@ -88,12 +85,6 @@ impl StaticSortedFileMetaData { ...@@ -88,12 +85,6 @@ impl StaticSortedFileMetaData {
let end: usize = self.key_compression_dictionary_length.into(); let end: usize = self.key_compression_dictionary_length.into();
start..end start..end
} }
pub fn value_compression_dictionary_range(&self) -> Range<usize> {
let start = self.key_compression_dictionary_length as usize;
let end = start + self.value_compression_dictionary_length as usize;
start..end
}
} }
/// A memory mapped SST file. /// A memory mapped SST file.
...@@ -310,7 +301,7 @@ impl StaticSortedFile { ...@@ -310,7 +301,7 @@ impl StaticSortedFile {
match value_block_cache.get_value_or_guard(&(self.meta.sequence_number, block), None) { match value_block_cache.get_value_or_guard(&(self.meta.sequence_number, block), None) {
GuardResult::Value(block) => block, GuardResult::Value(block) => block,
GuardResult::Guard(guard) => { GuardResult::Guard(guard) => {
let block = self.read_value_block(block)?; let block = self.read_small_value_block(block)?;
let _ = guard.insert(block.clone()); let _ = guard.insert(block.clone());
block block
} }
...@@ -323,25 +314,26 @@ impl StaticSortedFile { ...@@ -323,25 +314,26 @@ impl StaticSortedFile {
fn read_key_block(&self, block_index: u16) -> Result<ArcSlice<u8>> { fn read_key_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
self.read_block( self.read_block(
block_index, block_index,
&self.mmap[self.meta.key_compression_dictionary_range()], Some(&self.mmap[self.meta.key_compression_dictionary_range()]),
false, false,
) )
} }
/// Reads a value block from the file.
fn read_small_value_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
self.read_block(block_index, None, false)
}
/// Reads a value block from the file. /// Reads a value block from the file.
fn read_value_block(&self, block_index: u16) -> Result<ArcSlice<u8>> { fn read_value_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
self.read_block( self.read_block(block_index, None, true)
block_index,
&self.mmap[self.meta.value_compression_dictionary_range()],
false,
)
} }
/// Reads a block from the file. /// Reads a block from the file.
fn read_block( fn read_block(
&self, &self,
block_index: u16, block_index: u16,
compression_dictionary: &[u8], compression_dictionary: Option<&[u8]>,
long_term: bool, long_term: bool,
) -> Result<ArcSlice<u8>> { ) -> Result<ArcSlice<u8>> {
let (uncompressed_length, block) = self.get_compressed_block(block_index)?; let (uncompressed_length, block) = self.get_compressed_block(block_index)?;
...@@ -349,7 +341,7 @@ impl StaticSortedFile { ...@@ -349,7 +341,7 @@ impl StaticSortedFile {
let buffer = decompress_into_arc( let buffer = decompress_into_arc(
uncompressed_length, uncompressed_length,
block, block,
Some(compression_dictionary), compression_dictionary,
long_term, long_term,
)?; )?;
Ok(ArcSlice::from(buffer)) Ok(ArcSlice::from(buffer))
...@@ -496,8 +488,6 @@ impl<'l> StaticSortedFileIter<'l> { ...@@ -496,8 +488,6 @@ impl<'l> StaticSortedFileIter<'l> {
LazyLookupValue::Medium { LazyLookupValue::Medium {
uncompressed_size, uncompressed_size,
block, block,
dictionary: &self.this.mmap
[self.this.meta.value_compression_dictionary_range()],
} }
} else { } else {
let value = self let value = self
......
...@@ -407,17 +407,15 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -407,17 +407,15 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
fn create_sst_file( fn create_sst_file(
&self, &self,
family: u32, family: u32,
collector_data: (&[CollectorEntry<K>], usize, usize), collector_data: (&[CollectorEntry<K>], usize),
) -> Result<(u32, File)> { ) -> Result<(u32, File)> {
let (entries, total_key_size, total_value_size) = collector_data; let (entries, total_key_size) = collector_data;
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1; let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let path = self.db_path.join(format!("{seq:08}.sst")); let path = self.db_path.join(format!("{seq:08}.sst"));
let (meta, file) = self let (meta, file) = self
.parallel_scheduler .parallel_scheduler
.block_in_place(|| { .block_in_place(|| write_static_stored_file(entries, total_key_size, &path))
write_static_stored_file(entries, total_key_size, total_value_size, &path)
})
.with_context(|| format!("Unable to write SST file {seq:08}.sst"))?; .with_context(|| format!("Unable to write SST file {seq:08}.sst"))?;
#[cfg(feature = "verify_sst_content")] #[cfg(feature = "verify_sst_content")]
...@@ -440,7 +438,6 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -440,7 +438,6 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
StaticSortedFileMetaData { StaticSortedFileMetaData {
sequence_number: seq, sequence_number: seq,
key_compression_dictionary_length: meta.key_compression_dictionary_length, key_compression_dictionary_length: meta.key_compression_dictionary_length,
value_compression_dictionary_length: meta.value_compression_dictionary_length,
block_count: meta.block_count, block_count: meta.block_count,
}, },
)?; )?;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment