Unverified Commit 3d3405a1 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] use zero copy qfilter deserialization (#90574)

## What

Switch turbo-persistence AMQF filters from owned `qfilter::Filter` (heap-allocated) to zero-copy `qfilter::FilterRef` that borrows directly from the memory-mapped meta file.

## Why

- **Lower memory usage** — no heap copy of filter data; `FilterRef` is just a pointer into the mmap
- **Faster open time** — no deserialization/allocation, just pointer math over the mmap
- **OS-managed memory** — mmap pages can be cheaply evicted under pressure (free LRU behavior), unlike heap-allocated `Filter` data which needs to be swapped

## How

- Update `qfilter` dependency to the latest alpha release with `FilterRef` support 
- Switch per-entry AMQF serialization from `turbo_bincode` to `pot` format, which supports zero-copy deserialization 
- Store `qfilter::FilterRef<'static>` directly in `MetaEntry` (lifetime transmuted from the mmap borrow)
- Rely on Rust's struct field drop order guarantee: `MetaFile::entries` is declared before `MetaFile::mmap`, so all `FilterRef`s are dropped before the mmap is unmapped
- Update compaction code to work with `FilterRef` avoiding many allocations when merging

### Safety invariants

The `FilterRef<'static>` lifetime is transmuted 🙀 — the actual borrow is from `MetaFile::mmap`. This is safe because:

1. `MetaEntry` is never moved out of `MetaFile` (only accessed by `&` reference via `entries()` / `entry()`)
2. Rust drops struct fields in declaration order, and `entries` is declared before `mmap`
3. This is the same pattern used by `ArcBytes`/`RcBytes` in this crate (raw pointer into backing storage)

## Benchmark Results

I ran a number of the Read benchmarks and compaction benchmarks and it is all in the noise, which makes sense.  We might get a slight benefit from avoiding the `OnceLock` and lazy initialization, we should also have slightly lower maxrss and offer the OS more flexibility during memory pressure

From measuring vercel-site, a warm build saves ~40m of MaxRSS
parent 3c301acc
......@@ -1309,7 +1309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
dependencies = [
"lazy_static",
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]
......@@ -3053,7 +3053,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2 0.6.2",
"tokio",
"tower-service",
"tracing",
......@@ -5406,17 +5406,6 @@ dependencies = [
"serde",
]
[[package]]
name = "pot"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf741fa415952eb20f27fbc210dc85f31cc7cdc80aa3ce81d5e27d28a6f45dc2"
dependencies = [
"byteorder",
"half",
"serde",
]
[[package]]
name = "potential_utf"
version = "0.1.4"
......@@ -5674,9 +5663,9 @@ dependencies = [
[[package]]
name = "qfilter"
version = "0.3.0-alpha.2"
version = "0.3.0-alpha.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bd0ea20afc2eab082dde00f58d9e83ed70dcb693fa88c23890e122e19355635"
checksum = "9fb8c6e59c09e93a6dbc7ff5feff747fc4efc3f5a90c00851e1472ce1bdf2eff"
dependencies = [
"serde",
"serde_bytes",
......@@ -5723,7 +5712,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.1",
"rustls",
"socket2 0.5.10",
"socket2 0.6.2",
"thiserror 2.0.12",
"tokio",
"tracing",
......@@ -5761,7 +5750,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.10",
"socket2 0.6.2",
"tracing",
"windows-sys 0.60.2",
]
......@@ -9441,7 +9430,6 @@ name = "turbo-persistence"
version = "0.1.0"
dependencies = [
"anyhow",
"bincode 2.0.1",
"bitfield",
"byteorder",
"codspeed-criterion-compat",
......@@ -9453,7 +9441,7 @@ dependencies = [
"memmap2 0.9.5",
"nohash-hasher",
"parking_lot",
"pot",
"postcard",
"qfilter",
"quick_cache",
"rand 0.10.0",
......@@ -9463,9 +9451,9 @@ dependencies = [
"tempfile",
"thread_local",
"tracing",
"turbo-bincode",
"turbo-tasks-malloc",
"xxhash-rust",
"zerocopy",
]
[[package]]
......
......@@ -14,7 +14,6 @@ verbose_log = []
[dependencies]
anyhow = { workspace = true }
bincode = { workspace = true }
bitfield = { workspace = true }
byteorder = { workspace = true }
crc32fast = { workspace = true }
......@@ -25,15 +24,15 @@ lzzzz = { workspace = true }
memmap2 = "0.9.5"
nohash-hasher = { workspace = true }
parking_lot = { workspace = true }
pot = "3.0.0"
# See https://github.com/vercel/next.js/issues/91708 for why we need the legacy_x86_64_support feature
qfilter = { version = "0.3.0-alpha.2", features = ["serde", "legacy_x86_64_support"] }
qfilter = { version = "0.3.0-alpha.3", features = ["serde", "legacy_x86_64_support"] }
postcard = { workspace = true, features = ["alloc", "use-std"] }
zerocopy = { version = "0.8", features = ["derive"] }
quick_cache = { workspace = true }
rustc-hash = { workspace = true }
smallvec = { workspace = true }
thread_local = { workspace = true }
tracing = { workspace = true }
turbo-bincode= { workspace = true }
xxhash-rust = { workspace = true }
[dev-dependencies]
......
......@@ -26,6 +26,9 @@ enum Backing {
#[derive(Clone)]
pub struct ArcBytes {
data: *const [u8],
// Safety: Backing should come last so that it is dropped after the data pointer so we don't
// create a dangling pointer. This isn't really a problem since it is technically ok to have
// dangling _pointers_.
backing: Backing,
}
......
......@@ -528,6 +528,11 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// len is only a snapshot at that time and it can change while we create the filter.
// So we give it 5% more space to make resizes less likely.
let initial_capacity = set.len() * 20 / 19;
// TODO: Using u64::BITS as fingerprint size is wasteful for a
// probabilistic membership filter. A smaller fingerprint (e.g. via
// Filter::new with a target fp_rate) would significantly reduce size,
// but would make merging slower since mismatched fingerprint sizes
// fall back to one-by-one insertion instead of sorted merge.
let mut amqf =
qfilter::Filter::with_fingerprint_size(initial_capacity as u64, u64::BITS as u8)
.unwrap();
......@@ -987,7 +992,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
// during the merge loop. Empty filters (from commits with no
// reads) are discarded.
let used_key_hashes: Option<qfilter::Filter> = {
let filters: Vec<qfilter::Filter> = meta_files
let filters: Vec<qfilter::FilterRef<'_>> = meta_files
.iter()
.filter(|m| m.family() == family)
.filter_map(|meta_file| {
......@@ -1001,9 +1006,11 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
None
} else if filters.len() == 1 {
// Just directly use the single item
filters.into_iter().next()
Some(filters[0].to_owned())
} else {
let total_len: u64 = filters.iter().map(|f| f.len()).sum();
// Fingerprint size must match the source filters to
// enable the efficient sorted merge path in qfilter.
let mut merged =
qfilter::Filter::with_fingerprint_size(total_len, u64::BITS as u8)
.expect("Failed to create merged AMQF filter");
......
......@@ -7,8 +7,9 @@ use std::{
use anyhow::{Context, Result};
use byteorder::{BE, WriteBytesExt};
use qfilter::Filter;
use zerocopy::IntoBytes;
use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta;
use crate::{meta_file::EntryHeader, static_sorted_file_builder::StaticSortedFileBuilderMeta};
pub struct MetaFileBuilder<'a> {
family: u32,
......@@ -64,19 +65,22 @@ impl<'a> MetaFileBuilder<'a> {
let mut amqf_offset = 0;
for (sequence_number, sst) in &self.entries {
file.write_u32::<BE>(*sequence_number)?;
file.write_u16::<BE>(sst.block_count)?;
file.write_u64::<BE>(sst.min_hash)?;
file.write_u64::<BE>(sst.max_hash)?;
file.write_u64::<BE>(sst.size)?;
file.write_u32::<BE>(sst.flags.0)?;
amqf_offset += sst.amqf.len();
file.write_u32::<BE>(amqf_offset as u32)?;
let header = EntryHeader::new(
*sequence_number,
sst.block_count,
sst.min_hash,
sst.max_hash,
sst.size,
sst.flags,
amqf_offset as u32,
);
file.write_all(header.as_bytes())?;
}
let serialized_used_key_hashes = self
.used_key_hashes_amqf
.as_ref()
.map(|f| pot::to_vec(f).expect("AMQF serialization failed"));
.map(|f| postcard::to_allocvec(f).expect("AMQF serialization failed"));
amqf_offset += serialized_used_key_hashes
.as_ref()
.map(|bytes| bytes.len())
......
......@@ -8,12 +8,11 @@ use std::{
use anyhow::{Context, Result};
use byteorder::{BE, ByteOrder, WriteBytesExt};
use turbo_bincode::turbo_bincode_encode;
use crate::{
compression::{checksum_block, compress_into_buffer},
constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE},
meta_file::{AmqfBincodeWrapper, MetaEntryFlags},
meta_file::MetaEntryFlags,
static_sorted_file::{
BLOB_VALUE_REF_SIZE, BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH,
BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY_NO_HASH, BLOCK_TYPE_KEY_WITH_HASH, DELETED_VALUE_REF_SIZE,
......@@ -956,9 +955,8 @@ impl<E: Entry> StreamingSstWriter<E> {
let mut filter = self.filter.take().unwrap();
filter.shrink_to_fit();
// Serialize AMQF
let amqf =
turbo_bincode_encode(&AmqfBincodeWrapper(filter)).expect("AMQF serialization failed");
// Serialize AMQF using postcard for zero-copy deserialization via FilterRef
let amqf = postcard::to_allocvec(&filter).expect("AMQF serialization failed");
// Compute file size from block offsets rather than calling stream_position()
// (which requires a flush + seek).
......@@ -969,7 +967,7 @@ impl<E: Entry> StreamingSstWriter<E> {
let meta = StaticSortedFileBuilderMeta {
min_hash: self.min_hash,
max_hash: self.max_hash,
amqf: Cow::Owned(amqf.into_vec()),
amqf: Cow::Owned(amqf),
block_count,
size: file_size,
flags: self.flags,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment