Unverified Commit 3829047f authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Optimize persistence CPU (#100737)

A few small optimizations from profiling

* Replace SmallVec with Vec for our encoding buffer
- During persistence these are pooled and preallocated so we never
benefit from the inline storage
- SmallVec places branches around core operations which complicate
codegen
* Avoid temporary allocations during encoding via a new 'filtered
collection encode primitive'. This allows us to guess a size and fix it
later by overwriting the varint.
* Expand the SST Writer output buffer to 256K to reduce the number of
`write` syscalls by about 90%
* Update to the new bincode-fork location in vercel-labs, which also
picks up a small varint encoding win

Altogether these saved about 10% cpu time and 25% wall time for v0
persistence
parent 1c9408a1
......@@ -581,7 +581,7 @@ dependencies = [
[[package]]
name = "bincode"
version = "2.0.1"
source = "git+https://github.com/bgw/bincode.git?branch=bgw%2Fpatches#19f09c5f6895d769883c10b3d374f761ab7fe83d"
source = "git+https://github.com/vercel-labs/bincode-fork.git?branch=main#64e3c5e11da1715cd06d319552e780f5e9349281"
dependencies = [
"bincode_derive",
"serde",
......@@ -591,7 +591,7 @@ dependencies = [
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "git+https://github.com/bgw/bincode.git?branch=bgw%2Fpatches#19f09c5f6895d769883c10b3d374f761ab7fe83d"
source = "git+https://github.com/vercel-labs/bincode-fork.git?branch=main#64e3c5e11da1715cd06d319552e780f5e9349281"
dependencies = [
"virtue",
]
......
......@@ -374,7 +374,7 @@ vergen-gitcl = { version = "1.0.8", features = ["cargo"] }
webbrowser = "1.0.6"
[patch.crates-io]
bincode = { git = "https://github.com/bgw/bincode.git", branch = "bgw/patches" }
bincode = { git = "https://github.com/vercel-labs/bincode-fork.git", branch = "main" }
virtue = { git = "https://github.com/bgw/virtue.git", branch = "bgw/fix-generic-default-parsing" }
# We have to very frequently bump the swc major version of mdxjs-rs
mdxjs = { git = "https://github.com/vercel-labs/mdxjs-rs-turbopack.git", branch = "turbopack" }
......
use bincode::{Encode, enc::Encoder, error::EncodeError};
use crate::TurboBincodeEncoder;
/// Encodes a length-prefixed sequence whose length is only known after its elements are written,
/// so a filtered collection can be encoded in a single pass without collecting the surviving
/// elements first.
///
/// `max_len` is written as the length prefix up front but if fewer elements are actually written we
/// overwrite in place with a smaller value. Because the length is written as a varint and the size
/// can only go down we never have to move anything but this can lead to writing 'overlong' varints.
/// Bincode tolerates this but it isn't well specified.
///
/// A consequence of this is that the encoding is 'non-canonical'. e.g. roundtripping can produce a
/// different byte sequence. So if deterministic encoding is required this utility cannot be used.
#[inline]
pub fn encode_len_prefixed<'a>(
encoder: &mut TurboBincodeEncoder<'a>,
max_len: usize,
encode_elements: impl FnOnce(&mut TurboBincodeEncoder<'a>) -> Result<usize, EncodeError>,
) -> Result<usize, EncodeError> {
let start = encoder.writer().buffer.len();
max_len.encode(encoder)?;
let end = encoder.writer().buffer.len();
let len = encode_elements(encoder)?;
assert!(
len <= max_len,
"encoded {len} elements but only reserved a length of {max_len}"
);
if len != max_len {
patch_len_prefix(&mut encoder.writer().buffer[start..end], len);
}
Ok(len)
}
fn patch_len_prefix(prefix: &mut [u8], len: usize) {
// This mirrors bincode's little-endian varint encoding (`TURBO_BINCODE_CONFIG`): values up to
// 250 are a single byte, larger ones are a discriminant byte followed by a u16, u32 or u64.
// Keep the discriminant and write `len` at that width.
match prefix.len() {
// `len < max_len <= 250`, so it is a single byte too
1 => prefix[0] = len as u8,
3 => prefix[1..].copy_from_slice(&(len as u16).to_le_bytes()),
5 => prefix[1..].copy_from_slice(&(len as u32).to_le_bytes()),
9 => prefix[1..].copy_from_slice(&(len as u64).to_le_bytes()),
width => unreachable!("unexpected varint width {width}"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
TurboBincodeBuffer, new_turbo_bincode_encoder, turbo_bincode_decode, turbo_bincode_encode,
};
type Framed = (u8, Vec<u32>, u8);
/// Encodes the first `len` of `max_len` elements as a length-prefixed sequence, framed by a
/// leading and trailing byte to catch writes outside the prefix and elements.
fn encode_reserved(max_len: usize, len: usize) -> TurboBincodeBuffer {
let mut buffer = TurboBincodeBuffer::new();
let mut encoder = new_turbo_bincode_encoder(&mut buffer);
0xAAu8.encode(&mut encoder).unwrap();
let written = encode_len_prefixed(&mut encoder, max_len, |encoder| {
for i in 0..len {
(i as u32).encode(encoder)?;
}
Ok(len)
})
.unwrap();
assert_eq!(written, len);
0xBBu8.encode(&mut encoder).unwrap();
buffer
}
fn expected(len: usize) -> Framed {
(0xAA, (0..len as u32).collect(), 0xBB)
}
/// This also guards that bincode accepts overlong varints: the cases that cross a width
/// boundary would fail to decode (or decode wrongly) if bincode started rejecting them.
#[test]
fn decodes_to_actual_length() {
// Varint widths change above 250 (1 -> 3 bytes) and above u16::MAX (3 -> 5 bytes).
let cases = [
(0, 0),
(5, 5),
(5, 3),
(5, 0),
(250, 0),
(251, 251),
(251, 250),
(251, 0),
(300, 251),
(65535, 251),
(65535, 250),
(65536, 65536),
(65536, 65535),
(65536, 250),
(70000, 66000),
];
for (max_len, len) in cases {
let encoded = encode_reserved(max_len, len);
let decoded: Framed = turbo_bincode_decode(&encoded)
.unwrap_or_else(|e| panic!("max_len={max_len} len={len}: {e}"));
assert_eq!(decoded, expected(len), "max_len={max_len} len={len}");
}
}
#[test]
fn keeps_reserved_width() {
// 250 fits in one byte, but the 3-byte reservation for 251 is kept: [251, 250, 0]
let encoded = encode_reserved(251, 250);
assert_eq!(&encoded[1..4], &[251, 250, 0]);
// Without a width change the encoding is canonical
assert_eq!(
encode_reserved(5, 3),
turbo_bincode_encode(&expected(3)).unwrap()
);
}
}
mod len_prefixed;
#[doc(hidden)]
pub mod macro_helpers;
pub mod serde_self_describing;
......@@ -13,6 +14,8 @@ use bincode::{
};
use turbo_tasks_hash::DeterministicHasher;
pub use crate::len_prefixed::encode_len_prefixed;
pub const TURBO_BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
/// Same as standard config, but since we aren't encoding we don't benefit from varint
/// optimizations.
......@@ -21,7 +24,7 @@ pub const TURBO_BINCODE_HASH_CONFIG: bincode::config::Configuration<
bincode::config::Fixint,
bincode::config::NoLimit,
> = TURBO_BINCODE_CONFIG.with_fixed_int_encoding();
pub type TurboBincodeBuffer = SmallVec<[u8; 16]>;
pub type TurboBincodeBuffer = Vec<u8>;
pub type TurboBincodeEncoder<'a> =
EncoderImpl<TurboBincodeWriter<'a>, bincode::config::Configuration>;
pub type TurboBincodeDecoder<'a> =
......@@ -37,10 +40,10 @@ pub fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> {
DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ())
}
/// Encode the value into a new [`SmallVec`] using a [`TurboBincodeEncoder`].
/// Encode the value into a new [`TurboBincodeBuffer`] using a [`TurboBincodeEncoder`].
///
/// Note: If you can re-use a buffer, you should. That will always be cheaper than creating a new
/// [`SmallVec`].
/// [`TurboBincodeBuffer`].
pub fn turbo_bincode_encode<T: Encode>(value: &T) -> Result<TurboBincodeBuffer, EncodeError> {
let mut buffer = TurboBincodeBuffer::new();
turbo_bincode_encode_into(value, &mut buffer)?;
......@@ -82,6 +85,7 @@ impl<'a> TurboBincodeWriter<'a> {
}
impl Writer for TurboBincodeWriter<'_> {
#[inline(always)]
fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> {
self.buffer.extend_from_slice(bytes);
Ok(())
......
......@@ -28,6 +28,11 @@ use crate::{
/// Size of the per-block header on disk: 4 bytes uncompressed_size + 4 bytes CRC32 checksum.
pub const BLOCK_HEADER_SIZE: usize = 8;
/// Capacity of the buffered writer for SST files. Must be well above the typical block size, so
/// that each block (8–12 KB small-value blocks, up to 16 KB key blocks) doesn't force its own
/// `write` syscall.
const FILE_WRITE_BUFFER_SIZE: usize = 256 * 1024;
/// The maximum number of entries that should go into a single key block
const MAX_KEY_BLOCK_ENTRIES: usize = MAX_KEY_BLOCK_SIZE / KEY_BLOCK_ENTRY_META_OVERHEAD;
/// The maximum bytes that should go into a single key block
......@@ -641,7 +646,7 @@ impl<E: Entry> StreamingSstWriter<E> {
compression: Compression,
) -> Result<Self> {
let file_path = file.to_owned();
let file = BufWriter::new(File::create(file)?);
let file = BufWriter::with_capacity(FILE_WRITE_BUFFER_SIZE, File::create(file)?);
let compressor = Compressor::new(compression)?;
// Estimate number of key blocks based on max entry count.
......
use std::{borrow::Cow, ops::Deref};
use smallvec::SmallVec;
pub enum ValueBuffer<'l> {
Borrowed(&'l [u8]),
Vec(Vec<u8>),
SmallVec(SmallVec<[u8; 16]>),
Owned(Box<[u8]>),
}
impl ValueBuffer<'_> {
pub fn into_boxed_slice(self) -> Box<[u8]> {
match self {
ValueBuffer::Borrowed(b) => b.into(),
ValueBuffer::Vec(v) => v.into_boxed_slice(),
ValueBuffer::SmallVec(sv) => sv.into_boxed_slice(),
ValueBuffer::Owned(b) => b,
}
}
}
......@@ -26,19 +22,13 @@ impl<'l> From<&'l [u8]> for ValueBuffer<'l> {
impl From<Vec<u8>> for ValueBuffer<'_> {
fn from(v: Vec<u8>) -> Self {
ValueBuffer::Vec(v)
ValueBuffer::Owned(v.into_boxed_slice())
}
}
impl From<Box<[u8]>> for ValueBuffer<'_> {
fn from(v: Box<[u8]>) -> Self {
ValueBuffer::Vec(v.into_vec())
}
}
impl From<SmallVec<[u8; 16]>> for ValueBuffer<'_> {
fn from(sv: SmallVec<[u8; 16]>) -> Self {
ValueBuffer::SmallVec(sv)
fn from(b: Box<[u8]>) -> Self {
ValueBuffer::Owned(b)
}
}
......@@ -46,7 +36,7 @@ impl<'l> From<Cow<'l, [u8]>> for ValueBuffer<'l> {
fn from(c: Cow<'l, [u8]>) -> Self {
match c {
Cow::Borrowed(b) => ValueBuffer::Borrowed(b),
Cow::Owned(o) => ValueBuffer::Vec(o),
Cow::Owned(o) => ValueBuffer::Owned(o.into_boxed_slice()),
}
}
}
......@@ -57,8 +47,7 @@ impl Deref for ValueBuffer<'_> {
fn deref(&self) -> &Self::Target {
match self {
ValueBuffer::Borrowed(b) => b,
ValueBuffer::Vec(v) => v,
ValueBuffer::SmallVec(sv) => sv,
ValueBuffer::Owned(b) => b,
}
}
}
......@@ -2037,8 +2037,8 @@ mod must_exist_tests {
.chain(data_only_ids.iter().map(|&task_id| (task_id, false)))
.map(|(task_id, with_meta)| SnapshotItem::Put {
task_id,
data: Some(encoded_data.clone()),
meta: with_meta.then(|| encoded_meta.clone()),
data: Some(Box::from(&encoded_data[..])),
meta: with_meta.then(|| Box::from(&encoded_meta[..])),
task_type_hash: None,
#[cfg(feature = "print_cache_item_size")]
stats: Default::default(),
......
......@@ -11,7 +11,6 @@ use std::{
use anyhow::{Context, Result};
use hashbrown::hash_table;
use smallvec::SmallVec;
use thread_local::ThreadLocal;
use tracing::span::Id;
use turbo_bincode::{TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder};
......@@ -195,7 +194,7 @@ impl TaskDataCategory {
/// resulting buffer sizes.
///
/// TODO: The `Result` return type is an artifact of the bincode `Encode` trait requiring
/// fallible encoding. In practice, encoding to a `SmallVec` is infallible (no I/O), and the only
/// fallible encoding. In practice, encoding to a `Vec` is infallible (no I/O), and the only
/// real failure mode — a `TypedSharedReference` whose value type has no bincode impl — is a
/// programmer error caught by the panic in the caller. Consider making the bincode encoding trait
/// infallible (i.e. returning `()` instead of `Result<(), EncodeError>`) to eliminate the
......@@ -205,7 +204,7 @@ pub(crate) fn encode_task_contents(
data: &TaskStorage,
category: SpecificTaskDataCategory,
scratch_buffer: &mut TurboBincodeBuffer,
) -> Result<TurboBincodeBuffer> {
) -> Result<Box<[u8]>> {
scratch_buffer.clear();
let mut encoder = new_turbo_bincode_encoder(scratch_buffer);
data.encode(category, &mut encoder)?;
......@@ -223,7 +222,7 @@ pub(crate) fn encode_task_contents(
)
})?;
}
Ok(SmallVec::from_slice(scratch_buffer))
Ok(Box::from(&scratch_buffer[..]))
}
/// Converts a task's current state into the [`SnapshotItem`] that persistence writes for it.
......@@ -1349,7 +1348,7 @@ mod tests {
) -> SnapshotItem {
SnapshotItem::Put {
task_id,
meta: Some(TurboBincodeBuffer::default()),
meta: Some(Box::default()),
data: None,
task_type_hash: None,
#[cfg(feature = "print_cache_item_size")]
......
......@@ -743,10 +743,10 @@ impl TaskStorage {
}
/// Encode fields for the specified category
pub fn encode<E: bincode::enc::Encoder>(
pub fn encode(
&self,
category: SpecificTaskDataCategory,
encoder: &mut E,
encoder: &mut turbo_bincode::TurboBincodeEncoder<'_>,
) -> Result<(), bincode::error::EncodeError> {
match category {
SpecificTaskDataCategory::Meta => self.encode_meta(encoder),
......
use std::cmp::max;
use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{
DynTaskInputs, RawVc, TaskId, backend::CachedTaskType, macro_helpers::NativeFunction,
};
......@@ -16,9 +15,9 @@ pub enum SnapshotItem {
Put {
task_id: TaskId,
/// Serialized task meta data, if modified
meta: Option<TurboBincodeBuffer>,
meta: Option<Box<[u8]>>,
/// Serialized task data, if modified
data: Option<TurboBincodeBuffer>,
data: Option<Box<[u8]>>,
/// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>,
/// Cache size statistics captured when the item was encoded.
......
......@@ -300,8 +300,7 @@ impl<'l> From<WriteBuffer<'l>> for ValueBuffer<'l> {
fn from(val: WriteBuffer<'l>) -> Self {
match val {
WriteBuffer::Borrowed(b) => ValueBuffer::Borrowed(b),
WriteBuffer::Vec(v) => ValueBuffer::Vec(v),
WriteBuffer::SmallVec(sv) => ValueBuffer::SmallVec(sv),
WriteBuffer::Owned(b) => ValueBuffer::Owned(b),
}
}
}
use std::{borrow::Cow, ops::Deref};
use smallvec::SmallVec;
pub enum WriteBuffer<'a> {
Borrowed(&'a [u8]),
Vec(Vec<u8>),
SmallVec(smallvec::SmallVec<[u8; 16]>),
Owned(Box<[u8]>),
}
impl WriteBuffer<'_> {
pub fn into_static(self) -> WriteBuffer<'static> {
match self {
WriteBuffer::Borrowed(b) => WriteBuffer::SmallVec(SmallVec::from_slice(b)),
WriteBuffer::Vec(v) => WriteBuffer::Vec(v),
WriteBuffer::SmallVec(sv) => WriteBuffer::Vec(sv.into_vec()),
WriteBuffer::Borrowed(b) => WriteBuffer::Owned(b.into()),
WriteBuffer::Owned(b) => WriteBuffer::Owned(b),
}
}
}
......@@ -24,8 +20,7 @@ impl Deref for WriteBuffer<'_> {
fn deref(&self) -> &Self::Target {
match self {
WriteBuffer::Borrowed(b) => b,
WriteBuffer::Vec(v) => v,
WriteBuffer::SmallVec(sv) => sv,
WriteBuffer::Owned(b) => b,
}
}
}
......@@ -34,7 +29,7 @@ impl<'l> From<Cow<'l, [u8]>> for WriteBuffer<'l> {
fn from(c: Cow<'l, [u8]>) -> Self {
match c {
Cow::Borrowed(b) => WriteBuffer::Borrowed(b),
Cow::Owned(o) => WriteBuffer::Vec(o),
Cow::Owned(o) => WriteBuffer::Owned(o.into_boxed_slice()),
}
}
}
......@@ -284,7 +284,7 @@ impl TurboBackingStorage {
batch.put(
KeySpace::TaskMeta,
WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(meta),
WriteBuffer::Owned(meta),
)?;
meta_items += 1;
}
......@@ -292,7 +292,7 @@ impl TurboBackingStorage {
batch.put(
KeySpace::TaskData,
WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(data),
WriteBuffer::Owned(data),
)?;
data_items += 1;
}
......@@ -533,7 +533,7 @@ fn save_infra(
.put(
KeySpace::Infra,
WriteBuffer::Borrowed(InfraKey::GcRoots.key().as_ref()),
WriteBuffer::SmallVec(roots),
WriteBuffer::Owned(roots.into_boxed_slice()),
)
.context("Unable to write GC roots")?;
}
......
......@@ -3472,8 +3472,8 @@ fn gen_decode_body(grouped_fields: &GroupedFields, category: Category) -> TokenS
/// Generate encode/decode methods for TaskStorage serialization.
///
/// Generates four methods:
/// - `encode_meta<E>(&self, encoder: &mut E)` - Encode meta category fields
/// - `encode_data<E>(&self, encoder: &mut E)` - Encode data category fields
/// - `encode_meta(&self, encoder: &mut TurboBincodeEncoder)` - Encode meta category fields
/// - `encode_data(&self, encoder: &mut TurboBincodeEncoder)` - Encode data category fields
/// - `decode_meta<D>(&mut self, decoder: &mut D)` - Decode meta category fields
/// - `decode_data<D>(&mut self, decoder: &mut D)` - Decode data category fields
///
......@@ -3531,9 +3531,9 @@ fn generate_encode_decode_methods(grouped_fields: &GroupedFields) -> TokenStream
impl TaskStorage {
/// Encode meta category fields directly to bincode.
/// Only persistent (non-transient) fields are encoded.
pub fn encode_meta<E: bincode::enc::Encoder>(
pub fn encode_meta(
&self,
encoder: &mut E,
encoder: &mut turbo_bincode::TurboBincodeEncoder<'_>,
) -> Result<(), bincode::error::EncodeError> {
#encode_meta_body
#encode_meta_flags
......@@ -3542,9 +3542,9 @@ fn generate_encode_decode_methods(grouped_fields: &GroupedFields) -> TokenStream
/// Encode data category fields directly to bincode.
/// Only persistent (non-transient) fields are encoded.
pub fn encode_data<E: bincode::enc::Encoder>(
pub fn encode_data(
&self,
encoder: &mut E,
encoder: &mut turbo_bincode::TurboBincodeEncoder<'_>,
) -> Result<(), bincode::error::EncodeError> {
#encode_data_body
#encode_data_flags
......@@ -3636,8 +3636,9 @@ fn generate_filter_predicate(field: &FieldInfo) -> Option<(TokenStream, FilterPr
/// is already a reference from the match arm).
///
/// For non-filtered fields, encodes the value directly.
/// For filtered fields, uses a single-pass collect to a Vec, then encodes.
/// This avoids multiple iterations (check non-empty + count + encode).
/// For filtered collections, reserves the unfiltered length as the length prefix, encodes the
/// surviving entries in a single pass, then patches the prefix with the actual count. This avoids
/// both iterating twice (count + encode) and collecting the entries into a temporary `Vec`.
fn generate_encode_value(field: &FieldInfo, value_ref: TokenStream) -> TokenStream {
let Some((predicate, pred_type)) = generate_filter_predicate(field) else {
// No filtering needed, just encode normally
......@@ -3657,40 +3658,35 @@ fn generate_encode_value(field: &FieldInfo, value_ref: TokenStream) -> TokenStre
}
}
FilterPredicateType::Set => {
// For AutoSet<K>, filter out transient keys - collect once then encode
// For AutoSet<K>, filter out transient keys
quote! {
{
let filtered: Vec<_> = (#value_ref).iter().filter(#predicate).collect();
bincode::Encode::encode(&filtered.len(), encoder)?;
for key in filtered {
bincode::Encode::encode(key, encoder)?;
}
}
}
}
FilterPredicateType::CounterMap => {
// For counter maps, filter out entries with transient keys - collect once
quote! {
{
let filtered: Vec<_> = (#value_ref).iter().filter(#predicate).collect();
bincode::Encode::encode(&filtered.len(), encoder)?;
for (key, value) in filtered {
bincode::Encode::encode(key, encoder)?;
bincode::Encode::encode(value, encoder)?;
}
let collection = #value_ref;
turbo_bincode::encode_len_prefixed(encoder, collection.len(), |encoder| {
let mut count = 0usize;
for key in collection.iter().filter(#predicate) {
bincode::Encode::encode(key, encoder)?;
count += 1;
}
Ok(count)
})?;
}
}
}
FilterPredicateType::Map => {
// For maps, filter out entries with transient keys or values - collect once
FilterPredicateType::CounterMap | FilterPredicateType::Map => {
// For maps, filter out entries with transient keys (or values, for AutoMap)
quote! {
{
let filtered: Vec<_> = (#value_ref).iter().filter(#predicate).collect();
bincode::Encode::encode(&filtered.len(), encoder)?;
for (key, value) in filtered {
bincode::Encode::encode(key, encoder)?;
bincode::Encode::encode(value, encoder)?;
}
let collection = #value_ref;
turbo_bincode::encode_len_prefixed(encoder, collection.len(), |encoder| {
let mut count = 0usize;
for (key, value) in collection.iter().filter(#predicate) {
bincode::Encode::encode(key, encoder)?;
bincode::Encode::encode(value, encoder)?;
count += 1;
}
Ok(count)
})?;
}
}
}
......@@ -3707,8 +3703,8 @@ fn generate_encode_inline_field(field: &FieldInfo) -> TokenStream {
/// Generate code to encode a lazy field value with index.
///
/// For filtered fields, collects to a Vec first, then checks if non-empty before
/// writing index. This avoids multiple iterations over the data.
/// For filtered fields, writes the index and entries in a single pass and truncates them away
/// again if every entry was filtered out. This avoids multiple iterations over the data.
fn generate_encode_lazy_field_with_index(field: &FieldInfo, index: u8) -> TokenStream {
let Some((predicate, pred_type)) = generate_filter_predicate(field) else {
// No filtering needed - encode directly
......@@ -3732,32 +3728,46 @@ fn generate_encode_lazy_field_with_index(field: &FieldInfo, index: u8) -> TokenS
}
}
FilterPredicateType::Set => {
// Collect once, check if non-empty, then encode
// Encode optimistically in a single pass, then roll back the index too if every entry
// was filtered out
quote! {
{
let filtered: Vec<_> = data.iter().filter(#predicate).collect();
if !filtered.is_empty() {
bincode::Encode::encode(&#index, encoder)?;
bincode::Encode::encode(&filtered.len(), encoder)?;
for key in filtered {
bincode::Encode::encode(key, encoder)?;
}
let start = bincode::enc::Encoder::writer(encoder).buffer.len();
bincode::Encode::encode(&#index, encoder)?;
let count =
turbo_bincode::encode_len_prefixed(encoder, data.len(), |encoder| {
let mut count = 0usize;
for key in data.iter().filter(#predicate) {
bincode::Encode::encode(key, encoder)?;
count += 1;
}
Ok(count)
})?;
if count == 0 {
bincode::enc::Encoder::writer(encoder).buffer.truncate(start);
}
}
}
}
FilterPredicateType::CounterMap | FilterPredicateType::Map => {
// Collect once, check if non-empty, then encode
// Encode optimistically in a single pass, then roll back the index too if every entry
// was filtered out
quote! {
{
let filtered: Vec<_> = data.iter().filter(#predicate).collect();
if !filtered.is_empty() {
bincode::Encode::encode(&#index, encoder)?;
bincode::Encode::encode(&filtered.len(), encoder)?;
for (key, value) in filtered {
bincode::Encode::encode(key, encoder)?;
bincode::Encode::encode(value, encoder)?;
}
let start = bincode::enc::Encoder::writer(encoder).buffer.len();
bincode::Encode::encode(&#index, encoder)?;
let count =
turbo_bincode::encode_len_prefixed(encoder, data.len(), |encoder| {
let mut count = 0usize;
for (key, value) in data.iter().filter(#predicate) {
bincode::Encode::encode(key, encoder)?;
bincode::Encode::encode(value, encoder)?;
count += 1;
}
Ok(count)
})?;
if count == 0 {
bincode::enc::Encoder::writer(encoder).buffer.truncate(start);
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment