Unverified Commit 9d8f9509 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-persistence: radix sort AMQF fingerprints

Building an SST file's AMQF sorted up to a million u32 fingerprints with
sort_unstable, ~10% of the CPU of a compaction of small entries. An LSD
radix sort is ~4x faster at that size, and produces identical filters.
parent 05482087
...@@ -25,6 +25,7 @@ mod meta_file_builder; ...@@ -25,6 +25,7 @@ mod meta_file_builder;
#[cfg(feature = "mmap")] #[cfg(feature = "mmap")]
pub mod mmap_helper; pub mod mmap_helper;
mod parallel_scheduler; mod parallel_scheduler;
mod radix_sort;
mod rc_bytes; mod rc_bytes;
pub mod shard; pub mod shard;
mod shared_bytes; mod shared_bytes;
......
//! A radix sort for `u32` values of a known bit width.
/// Sorts `values`, which must all be below `1 << bits`, ascending.
///
/// This is a least significant digit radix sort, which is much faster than a comparison sort for
/// the up to a million fingerprints of an SST file's AMQF. It takes as few passes as digits of at
/// most 11 bits allow, so `ceil(bits / 11)`, with the bits split evenly between them: digits of
/// equal width spread the values evenly over their histogram, while a narrow last digit would make
/// consecutive values often increment the same counter. One pass upfront counts all digits, and
/// passes over digits that are equal in all values are skipped. Short inputs use a comparison sort,
/// since clearing the histograms would dominate.
pub(crate) fn radix_sort_u32(values: &mut Vec<u32>, bits: u32) {
// Measured on Apple silicon, the radix sort wins from ~250 to ~550 values depending on `bits`,
// and from ~400 for the ~17 bit fingerprints of a 512 entry AMQF.
const MIN_LEN: usize = 512;
debug_assert!(bits <= u32::BITS);
debug_assert!(values.iter().all(|&v| bits == u32::BITS || v >> bits == 0));
if values.len() < MIN_LEN {
values.sort_unstable();
return;
}
match bits.div_ceil(MAX_DIGIT_BITS) {
0 | 1 => radix_sort_u32_passes::<1>(values, bits),
2 => radix_sort_u32_passes::<2>(values, bits),
_ => radix_sort_u32_passes::<3>(values, bits),
}
}
/// The widest digit [`radix_sort_u32`] uses, so that a histogram (8 KiB) fits in L1.
const MAX_DIGIT_BITS: u32 = 11;
/// [`radix_sort_u32`] with `PASSES` digits.
fn radix_sort_u32_passes<const PASSES: usize>(values: &mut Vec<u32>, bits: u32) {
const BUCKETS: usize = 1 << MAX_DIGIT_BITS;
let digit_bits = bits.div_ceil(PASSES as u32).max(1);
// Masking with the histogram size too keeps every digit provably within the histogram.
let mask = ((1usize << digit_bits) - 1) & (BUCKETS - 1);
let digit = |value: u32, pass: usize| (value as usize >> (pass as u32 * digit_bits)) & mask;
// Sized for the widest digit, the digits of short `bits` only use the start. A file holds far
// fewer than `u32::MAX` entries, and `u32` counts halve the cache footprint.
let mut counts = [[0u32; BUCKETS]; PASSES];
for &value in values.iter() {
for (pass, counts) in counts.iter_mut().enumerate() {
counts[digit(value, pass)] += 1;
}
}
let mut scratch = vec![0u32; values.len()];
for (pass, counts) in counts.iter_mut().enumerate() {
let counts = &mut counts[..=mask];
// All values have the same digit, so this pass wouldn't move anything.
if counts[digit(values[0], pass)] as usize == values.len() {
continue;
}
// Turn the counts into the start offset of each bucket.
let mut offset = 0;
for count in counts.iter_mut() {
let start = offset;
offset += *count;
*count = start;
}
for &value in values.iter() {
let bucket = &mut counts[digit(value, pass)];
scratch[*bucket as usize] = value;
*bucket += 1;
}
std::mem::swap(values, &mut scratch);
}
}
#[cfg(test)]
mod tests {
use rand::{RngExt, SeedableRng, rngs::SmallRng};
use super::radix_sort_u32;
fn check(mut values: Vec<u32>, bits: u32) {
let mut expected = values.clone();
expected.sort_unstable();
radix_sort_u32(&mut values, bits);
assert_eq!(values, expected, "{bits} bits, {} values", values.len());
}
#[test]
fn test_matches_sorting() {
let mut rng = SmallRng::seed_from_u64(1);
// Lengths below and above the comparison sort cutoff, and bit counts with one, two and
// three passes, including ones that don't split evenly between the digits.
for len in [0, 1, 7, 511, 512, 513, 1024, 5000, 100_000] {
for bits in [1, 8, 11, 12, 21, 22, 27, 31, 32] {
let max = if bits == 32 {
u32::MAX
} else {
(1 << bits) - 1
};
check((0..len).map(|_| rng.random_range(0..=max)).collect(), bits);
}
}
}
#[test]
fn test_skips_constant_digits() {
// 27 bits are three 9 bit digits. The low and high digits are the same in all values, only
// the middle one varies.
let values = (0..5000u32).map(|i| (i % 512) << 9 | 0x5).collect();
check(values, 27);
check(vec![42; 3000], 27);
}
}
...@@ -15,6 +15,7 @@ use crate::{ ...@@ -15,6 +15,7 @@ use crate::{
compression::{Compressor, checksum_block}, compression::{Compressor, checksum_block},
constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE}, constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE},
meta_file::MetaEntryFlags, meta_file::MetaEntryFlags,
radix_sort::radix_sort_u32,
static_sorted_file::{ static_sorted_file::{
BLOB_VALUE_REF_SIZE, BLOCK_TYPE_INDEX, FIXED_KEY_BLOCK_MIXED_VALUE_TYPE, FixedRegions, BLOB_VALUE_REF_SIZE, BLOCK_TYPE_INDEX, FIXED_KEY_BLOCK_MIXED_VALUE_TYPE, FixedRegions,
KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN,
...@@ -1131,9 +1132,11 @@ impl<E: Entry> StreamingSstWriter<E> { ...@@ -1131,9 +1132,11 @@ impl<E: Entry> StreamingSstWriter<E> {
let fp_size = builder.fingerprint_size(); let fp_size = builder.fingerprint_size();
assert!(fp_size < 32, "fp_size {fp_size} exceeds u32"); assert!(fp_size < 32, "fp_size {fp_size} exceeds u32");
let fp_mask = (1u32 << fp_size) - 1; let fp_mask = (1u32 << fp_size) - 1;
// Mask in-place to fingerprint size and sort. // Mask in-place to fingerprint size and sort. The builder only uses these low bits.
self.collected_fingerprints for h in &mut self.collected_fingerprints {
.sort_unstable_by_key(|&h| h & fp_mask); *h &= fp_mask;
}
radix_sort_u32(&mut self.collected_fingerprints, u32::from(fp_size));
for &h in &self.collected_fingerprints { for &h in &self.collected_fingerprints {
builder builder
.insert_fingerprint(false, h as u64) .insert_fingerprint(false, h as u64)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment