Unverified Commit 3829047f authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Optimize persistence CPU (#100737)

A few small optimizations from profiling

* Replace SmallVec with Vec for our encoding buffer
- During persistence these are pooled and preallocated so we never
benefit from the inline storage
- SmallVec places branches around core operations which complicate
codegen
* Avoid temporary allocations during encoding via a new 'filtered
collection encode primitive'. This allows us to guess a size and fix it
later by overwriting the varint.
* Expand the SST Writer output buffer to 256K to reduce the number of
`write` syscalls by about 90%
* Update to the new bincode-fork location in vercel-labs, which also
picks up a small varint encoding win

Altogether these saved about 10% cpu time and 25% wall time for v0
persistence
parent 1c9408a1
...@@ -581,7 +581,7 @@ dependencies = [ ...@@ -581,7 +581,7 @@ dependencies = [
[[package]] [[package]]
name = "bincode" name = "bincode"
version = "2.0.1" version = "2.0.1"
source = "git+https://github.com/bgw/bincode.git?branch=bgw%2Fpatches#19f09c5f6895d769883c10b3d374f761ab7fe83d" source = "git+https://github.com/vercel-labs/bincode-fork.git?branch=main#64e3c5e11da1715cd06d319552e780f5e9349281"
dependencies = [ dependencies = [
"bincode_derive", "bincode_derive",
"serde", "serde",
...@@ -591,7 +591,7 @@ dependencies = [ ...@@ -591,7 +591,7 @@ dependencies = [
[[package]] [[package]]
name = "bincode_derive" name = "bincode_derive"
version = "2.0.1" version = "2.0.1"
source = "git+https://github.com/bgw/bincode.git?branch=bgw%2Fpatches#19f09c5f6895d769883c10b3d374f761ab7fe83d" source = "git+https://github.com/vercel-labs/bincode-fork.git?branch=main#64e3c5e11da1715cd06d319552e780f5e9349281"
dependencies = [ dependencies = [
"virtue", "virtue",
] ]
......
...@@ -374,7 +374,7 @@ vergen-gitcl = { version = "1.0.8", features = ["cargo"] } ...@@ -374,7 +374,7 @@ vergen-gitcl = { version = "1.0.8", features = ["cargo"] }
webbrowser = "1.0.6" webbrowser = "1.0.6"
[patch.crates-io] [patch.crates-io]
bincode = { git = "https://github.com/bgw/bincode.git", branch = "bgw/patches" } bincode = { git = "https://github.com/vercel-labs/bincode-fork.git", branch = "main" }
virtue = { git = "https://github.com/bgw/virtue.git", branch = "bgw/fix-generic-default-parsing" } virtue = { git = "https://github.com/bgw/virtue.git", branch = "bgw/fix-generic-default-parsing" }
# We have to very frequently bump the swc major version of mdxjs-rs # We have to very frequently bump the swc major version of mdxjs-rs
mdxjs = { git = "https://github.com/vercel-labs/mdxjs-rs-turbopack.git", branch = "turbopack" } mdxjs = { git = "https://github.com/vercel-labs/mdxjs-rs-turbopack.git", branch = "turbopack" }
......
use bincode::{Encode, enc::Encoder, error::EncodeError};
use crate::TurboBincodeEncoder;
/// Encodes a length-prefixed sequence whose length is only known after its elements are written,
/// so a filtered collection can be encoded in a single pass without collecting the surviving
/// elements first.
///
/// `max_len` is written as the length prefix up front but if fewer elements are actually written we
/// overwrite in place with a smaller value. Because the length is written as a varint and the size
/// can only go down we never have to move anything but this can lead to writing 'overlong' varints.
/// Bincode tolerates this but it isn't well specified.
///
/// A consequence of this is that the encoding is 'non-canonical'. e.g. roundtripping can produce a
/// different byte sequence. So if deterministic encoding is required this utility cannot be used.
#[inline]
pub fn encode_len_prefixed<'a>(
encoder: &mut TurboBincodeEncoder<'a>,
max_len: usize,
encode_elements: impl FnOnce(&mut TurboBincodeEncoder<'a>) -> Result<usize, EncodeError>,
) -> Result<usize, EncodeError> {
let start = encoder.writer().buffer.len();
max_len.encode(encoder)?;
let end = encoder.writer().buffer.len();
let len = encode_elements(encoder)?;
assert!(
len <= max_len,
"encoded {len} elements but only reserved a length of {max_len}"
);
if len != max_len {
patch_len_prefix(&mut encoder.writer().buffer[start..end], len);
}
Ok(len)
}
fn patch_len_prefix(prefix: &mut [u8], len: usize) {
// This mirrors bincode's little-endian varint encoding (`TURBO_BINCODE_CONFIG`): values up to
// 250 are a single byte, larger ones are a discriminant byte followed by a u16, u32 or u64.
// Keep the discriminant and write `len` at that width.
match prefix.len() {
// `len < max_len <= 250`, so it is a single byte too
1 => prefix[0] = len as u8,
3 => prefix[1..].copy_from_slice(&(len as u16).to_le_bytes()),
5 => prefix[1..].copy_from_slice(&(len as u32).to_le_bytes()),
9 => prefix[1..].copy_from_slice(&(len as u64).to_le_bytes()),
width => unreachable!("unexpected varint width {width}"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
TurboBincodeBuffer, new_turbo_bincode_encoder, turbo_bincode_decode, turbo_bincode_encode,
};
type Framed = (u8, Vec<u32>, u8);
/// Encodes the first `len` of `max_len` elements as a length-prefixed sequence, framed by a
/// leading and trailing byte to catch writes outside the prefix and elements.
fn encode_reserved(max_len: usize, len: usize) -> TurboBincodeBuffer {
let mut buffer = TurboBincodeBuffer::new();
let mut encoder = new_turbo_bincode_encoder(&mut buffer);
0xAAu8.encode(&mut encoder).unwrap();
let written = encode_len_prefixed(&mut encoder, max_len, |encoder| {
for i in 0..len {
(i as u32).encode(encoder)?;
}
Ok(len)
})
.unwrap();
assert_eq!(written, len);
0xBBu8.encode(&mut encoder).unwrap();
buffer
}
fn expected(len: usize) -> Framed {
(0xAA, (0..len as u32).collect(), 0xBB)
}
/// This also guards that bincode accepts overlong varints: the cases that cross a width
/// boundary would fail to decode (or decode wrongly) if bincode started rejecting them.
#[test]
fn decodes_to_actual_length() {
// Varint widths change above 250 (1 -> 3 bytes) and above u16::MAX (3 -> 5 bytes).
let cases = [
(0, 0),
(5, 5),
(5, 3),
(5, 0),
(250, 0),
(251, 251),
(251, 250),
(251, 0),
(300, 251),
(65535, 251),
(65535, 250),
(65536, 65536),
(65536, 65535),
(65536, 250),
(70000, 66000),
];
for (max_len, len) in cases {
let encoded = encode_reserved(max_len, len);
let decoded: Framed = turbo_bincode_decode(&encoded)
.unwrap_or_else(|e| panic!("max_len={max_len} len={len}: {e}"));
assert_eq!(decoded, expected(len), "max_len={max_len} len={len}");
}
}
#[test]
fn keeps_reserved_width() {
// 250 fits in one byte, but the 3-byte reservation for 251 is kept: [251, 250, 0]
let encoded = encode_reserved(251, 250);
assert_eq!(&encoded[1..4], &[251, 250, 0]);
// Without a width change the encoding is canonical
assert_eq!(
encode_reserved(5, 3),
turbo_bincode_encode(&expected(3)).unwrap()
);
}
}
mod len_prefixed;
#[doc(hidden)] #[doc(hidden)]
pub mod macro_helpers; pub mod macro_helpers;
pub mod serde_self_describing; pub mod serde_self_describing;
...@@ -13,6 +14,8 @@ use bincode::{ ...@@ -13,6 +14,8 @@ use bincode::{
}; };
use turbo_tasks_hash::DeterministicHasher; use turbo_tasks_hash::DeterministicHasher;
pub use crate::len_prefixed::encode_len_prefixed;
pub const TURBO_BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard(); pub const TURBO_BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
/// Same as standard config, but since we aren't encoding we don't benefit from varint /// Same as standard config, but since we aren't encoding we don't benefit from varint
/// optimizations. /// optimizations.
...@@ -21,7 +24,7 @@ pub const TURBO_BINCODE_HASH_CONFIG: bincode::config::Configuration< ...@@ -21,7 +24,7 @@ pub const TURBO_BINCODE_HASH_CONFIG: bincode::config::Configuration<
bincode::config::Fixint, bincode::config::Fixint,
bincode::config::NoLimit, bincode::config::NoLimit,
> = TURBO_BINCODE_CONFIG.with_fixed_int_encoding(); > = TURBO_BINCODE_CONFIG.with_fixed_int_encoding();
pub type TurboBincodeBuffer = SmallVec<[u8; 16]>; pub type TurboBincodeBuffer = Vec<u8>;
pub type TurboBincodeEncoder<'a> = pub type TurboBincodeEncoder<'a> =
EncoderImpl<TurboBincodeWriter<'a>, bincode::config::Configuration>; EncoderImpl<TurboBincodeWriter<'a>, bincode::config::Configuration>;
pub type TurboBincodeDecoder<'a> = pub type TurboBincodeDecoder<'a> =
...@@ -37,10 +40,10 @@ pub fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> { ...@@ -37,10 +40,10 @@ pub fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> {
DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ()) DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ())
} }
/// Encode the value into a new [`SmallVec`] using a [`TurboBincodeEncoder`]. /// Encode the value into a new [`TurboBincodeBuffer`] using a [`TurboBincodeEncoder`].
/// ///
/// Note: If you can re-use a buffer, you should. That will always be cheaper than creating a new /// Note: If you can re-use a buffer, you should. That will always be cheaper than creating a new
/// [`SmallVec`]. /// [`TurboBincodeBuffer`].
pub fn turbo_bincode_encode<T: Encode>(value: &T) -> Result<TurboBincodeBuffer, EncodeError> { pub fn turbo_bincode_encode<T: Encode>(value: &T) -> Result<TurboBincodeBuffer, EncodeError> {
let mut buffer = TurboBincodeBuffer::new(); let mut buffer = TurboBincodeBuffer::new();
turbo_bincode_encode_into(value, &mut buffer)?; turbo_bincode_encode_into(value, &mut buffer)?;
...@@ -82,6 +85,7 @@ impl<'a> TurboBincodeWriter<'a> { ...@@ -82,6 +85,7 @@ impl<'a> TurboBincodeWriter<'a> {
} }
impl Writer for TurboBincodeWriter<'_> { impl Writer for TurboBincodeWriter<'_> {
#[inline(always)]
fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> { fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> {
self.buffer.extend_from_slice(bytes); self.buffer.extend_from_slice(bytes);
Ok(()) Ok(())
......
...@@ -28,6 +28,11 @@ use crate::{ ...@@ -28,6 +28,11 @@ use crate::{
/// Size of the per-block header on disk: 4 bytes uncompressed_size + 4 bytes CRC32 checksum. /// Size of the per-block header on disk: 4 bytes uncompressed_size + 4 bytes CRC32 checksum.
pub const BLOCK_HEADER_SIZE: usize = 8; pub const BLOCK_HEADER_SIZE: usize = 8;
/// Capacity of the buffered writer for SST files. Must be well above the typical block size, so
/// that each block (8–12 KB small-value blocks, up to 16 KB key blocks) doesn't force its own
/// `write` syscall.
const FILE_WRITE_BUFFER_SIZE: usize = 256 * 1024;
/// The maximum number of entries that should go into a single key block /// The maximum number of entries that should go into a single key block
const MAX_KEY_BLOCK_ENTRIES: usize = MAX_KEY_BLOCK_SIZE / KEY_BLOCK_ENTRY_META_OVERHEAD; const MAX_KEY_BLOCK_ENTRIES: usize = MAX_KEY_BLOCK_SIZE / KEY_BLOCK_ENTRY_META_OVERHEAD;
/// The maximum bytes that should go into a single key block /// The maximum bytes that should go into a single key block
...@@ -641,7 +646,7 @@ impl<E: Entry> StreamingSstWriter<E> { ...@@ -641,7 +646,7 @@ impl<E: Entry> StreamingSstWriter<E> {
compression: Compression, compression: Compression,
) -> Result<Self> { ) -> Result<Self> {
let file_path = file.to_owned(); let file_path = file.to_owned();
let file = BufWriter::new(File::create(file)?); let file = BufWriter::with_capacity(FILE_WRITE_BUFFER_SIZE, File::create(file)?);
let compressor = Compressor::new(compression)?; let compressor = Compressor::new(compression)?;
// Estimate number of key blocks based on max entry count. // Estimate number of key blocks based on max entry count.
......
use std::{borrow::Cow, ops::Deref}; use std::{borrow::Cow, ops::Deref};
use smallvec::SmallVec;
pub enum ValueBuffer<'l> { pub enum ValueBuffer<'l> {
Borrowed(&'l [u8]), Borrowed(&'l [u8]),
Vec(Vec<u8>), Owned(Box<[u8]>),
SmallVec(SmallVec<[u8; 16]>),
} }
impl ValueBuffer<'_> { impl ValueBuffer<'_> {
pub fn into_boxed_slice(self) -> Box<[u8]> { pub fn into_boxed_slice(self) -> Box<[u8]> {
match self { match self {
ValueBuffer::Borrowed(b) => b.into(), ValueBuffer::Borrowed(b) => b.into(),
ValueBuffer::Vec(v) => v.into_boxed_slice(), ValueBuffer::Owned(b) => b,
ValueBuffer::SmallVec(sv) => sv.into_boxed_slice(),
} }
} }
} }
...@@ -26,19 +22,13 @@ impl<'l> From<&'l [u8]> for ValueBuffer<'l> { ...@@ -26,19 +22,13 @@ impl<'l> From<&'l [u8]> for ValueBuffer<'l> {
impl From<Vec<u8>> for ValueBuffer<'_> { impl From<Vec<u8>> for ValueBuffer<'_> {
fn from(v: Vec<u8>) -> Self { fn from(v: Vec<u8>) -> Self {
ValueBuffer::Vec(v) ValueBuffer::Owned(v.into_boxed_slice())
} }
} }
impl From<Box<[u8]>> for ValueBuffer<'_> { impl From<Box<[u8]>> for ValueBuffer<'_> {
fn from(v: Box<[u8]>) -> Self { fn from(b: Box<[u8]>) -> Self {
ValueBuffer::Vec(v.into_vec()) ValueBuffer::Owned(b)
}
}
impl From<SmallVec<[u8; 16]>> for ValueBuffer<'_> {
fn from(sv: SmallVec<[u8; 16]>) -> Self {
ValueBuffer::SmallVec(sv)
} }
} }
...@@ -46,7 +36,7 @@ impl<'l> From<Cow<'l, [u8]>> for ValueBuffer<'l> { ...@@ -46,7 +36,7 @@ impl<'l> From<Cow<'l, [u8]>> for ValueBuffer<'l> {
fn from(c: Cow<'l, [u8]>) -> Self { fn from(c: Cow<'l, [u8]>) -> Self {
match c { match c {
Cow::Borrowed(b) => ValueBuffer::Borrowed(b), Cow::Borrowed(b) => ValueBuffer::Borrowed(b),
Cow::Owned(o) => ValueBuffer::Vec(o), Cow::Owned(o) => ValueBuffer::Owned(o.into_boxed_slice()),
} }
} }
} }
...@@ -57,8 +47,7 @@ impl Deref for ValueBuffer<'_> { ...@@ -57,8 +47,7 @@ impl Deref for ValueBuffer<'_> {
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
match self { match self {
ValueBuffer::Borrowed(b) => b, ValueBuffer::Borrowed(b) => b,
ValueBuffer::Vec(v) => v, ValueBuffer::Owned(b) => b,
ValueBuffer::SmallVec(sv) => sv,
} }
} }
} }
...@@ -2037,8 +2037,8 @@ mod must_exist_tests { ...@@ -2037,8 +2037,8 @@ mod must_exist_tests {
.chain(data_only_ids.iter().map(|&task_id| (task_id, false))) .chain(data_only_ids.iter().map(|&task_id| (task_id, false)))
.map(|(task_id, with_meta)| SnapshotItem::Put { .map(|(task_id, with_meta)| SnapshotItem::Put {
task_id, task_id,
data: Some(encoded_data.clone()), data: Some(Box::from(&encoded_data[..])),
meta: with_meta.then(|| encoded_meta.clone()), meta: with_meta.then(|| Box::from(&encoded_meta[..])),
task_type_hash: None, task_type_hash: None,
#[cfg(feature = "print_cache_item_size")] #[cfg(feature = "print_cache_item_size")]
stats: Default::default(), stats: Default::default(),
......
...@@ -11,7 +11,6 @@ use std::{ ...@@ -11,7 +11,6 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use hashbrown::hash_table; use hashbrown::hash_table;
use smallvec::SmallVec;
use thread_local::ThreadLocal; use thread_local::ThreadLocal;
use tracing::span::Id; use tracing::span::Id;
use turbo_bincode::{TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder}; use turbo_bincode::{TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder};
...@@ -195,7 +194,7 @@ impl TaskDataCategory { ...@@ -195,7 +194,7 @@ impl TaskDataCategory {
/// resulting buffer sizes. /// resulting buffer sizes.
/// ///
/// TODO: The `Result` return type is an artifact of the bincode `Encode` trait requiring /// TODO: The `Result` return type is an artifact of the bincode `Encode` trait requiring
/// fallible encoding. In practice, encoding to a `SmallVec` is infallible (no I/O), and the only /// fallible encoding. In practice, encoding to a `Vec` is infallible (no I/O), and the only
/// real failure mode — a `TypedSharedReference` whose value type has no bincode impl — is a /// real failure mode — a `TypedSharedReference` whose value type has no bincode impl — is a
/// programmer error caught by the panic in the caller. Consider making the bincode encoding trait /// programmer error caught by the panic in the caller. Consider making the bincode encoding trait
/// infallible (i.e. returning `()` instead of `Result<(), EncodeError>`) to eliminate the /// infallible (i.e. returning `()` instead of `Result<(), EncodeError>`) to eliminate the
...@@ -205,7 +204,7 @@ pub(crate) fn encode_task_contents( ...@@ -205,7 +204,7 @@ pub(crate) fn encode_task_contents(
data: &TaskStorage, data: &TaskStorage,
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
scratch_buffer: &mut TurboBincodeBuffer, scratch_buffer: &mut TurboBincodeBuffer,
) -> Result<TurboBincodeBuffer> { ) -> Result<Box<[u8]>> {
scratch_buffer.clear(); scratch_buffer.clear();
let mut encoder = new_turbo_bincode_encoder(scratch_buffer); let mut encoder = new_turbo_bincode_encoder(scratch_buffer);
data.encode(category, &mut encoder)?; data.encode(category, &mut encoder)?;
...@@ -223,7 +222,7 @@ pub(crate) fn encode_task_contents( ...@@ -223,7 +222,7 @@ pub(crate) fn encode_task_contents(
) )
})?; })?;
} }
Ok(SmallVec::from_slice(scratch_buffer)) Ok(Box::from(&scratch_buffer[..]))
} }
/// Converts a task's current state into the [`SnapshotItem`] that persistence writes for it. /// Converts a task's current state into the [`SnapshotItem`] that persistence writes for it.
...@@ -1349,7 +1348,7 @@ mod tests { ...@@ -1349,7 +1348,7 @@ mod tests {
) -> SnapshotItem { ) -> SnapshotItem {
SnapshotItem::Put { SnapshotItem::Put {
task_id, task_id,
meta: Some(TurboBincodeBuffer::default()), meta: Some(Box::default()),
data: None, data: None,
task_type_hash: None, task_type_hash: None,
#[cfg(feature = "print_cache_item_size")] #[cfg(feature = "print_cache_item_size")]
......
...@@ -743,10 +743,10 @@ impl TaskStorage { ...@@ -743,10 +743,10 @@ impl TaskStorage {
} }
/// Encode fields for the specified category /// Encode fields for the specified category
pub fn encode<E: bincode::enc::Encoder>( pub fn encode(
&self, &self,
category: SpecificTaskDataCategory, category: SpecificTaskDataCategory,
encoder: &mut E, encoder: &mut turbo_bincode::TurboBincodeEncoder<'_>,
) -> Result<(), bincode::error::EncodeError> { ) -> Result<(), bincode::error::EncodeError> {
match category { match category {
SpecificTaskDataCategory::Meta => self.encode_meta(encoder), SpecificTaskDataCategory::Meta => self.encode_meta(encoder),
......
use std::cmp::max; use std::cmp::max;
use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{ use turbo_tasks::{
DynTaskInputs, RawVc, TaskId, backend::CachedTaskType, macro_helpers::NativeFunction, DynTaskInputs, RawVc, TaskId, backend::CachedTaskType, macro_helpers::NativeFunction,
}; };
...@@ -16,9 +15,9 @@ pub enum SnapshotItem { ...@@ -16,9 +15,9 @@ pub enum SnapshotItem {
Put { Put {
task_id: TaskId, task_id: TaskId,
/// Serialized task meta data, if modified /// Serialized task meta data, if modified
meta: Option<TurboBincodeBuffer>, meta: Option<Box<[u8]>>,
/// Serialized task data, if modified /// Serialized task data, if modified
data: Option<TurboBincodeBuffer>, data: Option<Box<[u8]>>,
/// Task type for new tasks that need to be added to the task cache /// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>, task_type_hash: Option<TaskTypeHash>,
/// Cache size statistics captured when the item was encoded. /// Cache size statistics captured when the item was encoded.
......
...@@ -300,8 +300,7 @@ impl<'l> From<WriteBuffer<'l>> for ValueBuffer<'l> { ...@@ -300,8 +300,7 @@ impl<'l> From<WriteBuffer<'l>> for ValueBuffer<'l> {
fn from(val: WriteBuffer<'l>) -> Self { fn from(val: WriteBuffer<'l>) -> Self {
match val { match val {
WriteBuffer::Borrowed(b) => ValueBuffer::Borrowed(b), WriteBuffer::Borrowed(b) => ValueBuffer::Borrowed(b),
WriteBuffer::Vec(v) => ValueBuffer::Vec(v), WriteBuffer::Owned(b) => ValueBuffer::Owned(b),
WriteBuffer::SmallVec(sv) => ValueBuffer::SmallVec(sv),
} }
} }
} }
use std::{borrow::Cow, ops::Deref}; use std::{borrow::Cow, ops::Deref};
use smallvec::SmallVec;
pub enum WriteBuffer<'a> { pub enum WriteBuffer<'a> {
Borrowed(&'a [u8]), Borrowed(&'a [u8]),
Vec(Vec<u8>), Owned(Box<[u8]>),
SmallVec(smallvec::SmallVec<[u8; 16]>),
} }
impl WriteBuffer<'_> { impl WriteBuffer<'_> {
pub fn into_static(self) -> WriteBuffer<'static> { pub fn into_static(self) -> WriteBuffer<'static> {
match self { match self {
WriteBuffer::Borrowed(b) => WriteBuffer::SmallVec(SmallVec::from_slice(b)), WriteBuffer::Borrowed(b) => WriteBuffer::Owned(b.into()),
WriteBuffer::Vec(v) => WriteBuffer::Vec(v), WriteBuffer::Owned(b) => WriteBuffer::Owned(b),
WriteBuffer::SmallVec(sv) => WriteBuffer::Vec(sv.into_vec()),
} }
} }
} }
...@@ -24,8 +20,7 @@ impl Deref for WriteBuffer<'_> { ...@@ -24,8 +20,7 @@ impl Deref for WriteBuffer<'_> {
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
match self { match self {
WriteBuffer::Borrowed(b) => b, WriteBuffer::Borrowed(b) => b,
WriteBuffer::Vec(v) => v, WriteBuffer::Owned(b) => b,
WriteBuffer::SmallVec(sv) => sv,
} }
} }
} }
...@@ -34,7 +29,7 @@ impl<'l> From<Cow<'l, [u8]>> for WriteBuffer<'l> { ...@@ -34,7 +29,7 @@ impl<'l> From<Cow<'l, [u8]>> for WriteBuffer<'l> {
fn from(c: Cow<'l, [u8]>) -> Self { fn from(c: Cow<'l, [u8]>) -> Self {
match c { match c {
Cow::Borrowed(b) => WriteBuffer::Borrowed(b), Cow::Borrowed(b) => WriteBuffer::Borrowed(b),
Cow::Owned(o) => WriteBuffer::Vec(o), Cow::Owned(o) => WriteBuffer::Owned(o.into_boxed_slice()),
} }
} }
} }
...@@ -284,7 +284,7 @@ impl TurboBackingStorage { ...@@ -284,7 +284,7 @@ impl TurboBackingStorage {
batch.put( batch.put(
KeySpace::TaskMeta, KeySpace::TaskMeta,
WriteBuffer::Borrowed(key), WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(meta), WriteBuffer::Owned(meta),
)?; )?;
meta_items += 1; meta_items += 1;
} }
...@@ -292,7 +292,7 @@ impl TurboBackingStorage { ...@@ -292,7 +292,7 @@ impl TurboBackingStorage {
batch.put( batch.put(
KeySpace::TaskData, KeySpace::TaskData,
WriteBuffer::Borrowed(key), WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(data), WriteBuffer::Owned(data),
)?; )?;
data_items += 1; data_items += 1;
} }
...@@ -533,7 +533,7 @@ fn save_infra( ...@@ -533,7 +533,7 @@ fn save_infra(
.put( .put(
KeySpace::Infra, KeySpace::Infra,
WriteBuffer::Borrowed(InfraKey::GcRoots.key().as_ref()), WriteBuffer::Borrowed(InfraKey::GcRoots.key().as_ref()),
WriteBuffer::SmallVec(roots), WriteBuffer::Owned(roots.into_boxed_slice()),
) )
.context("Unable to write GC roots")?; .context("Unable to write GC roots")?;
} }
......
...@@ -3472,8 +3472,8 @@ fn gen_decode_body(grouped_fields: &GroupedFields, category: Category) -> TokenS ...@@ -3472,8 +3472,8 @@ fn gen_decode_body(grouped_fields: &GroupedFields, category: Category) -> TokenS
/// Generate encode/decode methods for TaskStorage serialization. /// Generate encode/decode methods for TaskStorage serialization.
/// ///
/// Generates four methods: /// Generates four methods:
/// - `encode_meta<E>(&self, encoder: &mut E)` - Encode meta category fields /// - `encode_meta(&self, encoder: &mut TurboBincodeEncoder)` - Encode meta category fields
/// - `encode_data<E>(&self, encoder: &mut E)` - Encode data category fields /// - `encode_data(&self, encoder: &mut TurboBincodeEncoder)` - Encode data category fields
/// - `decode_meta<D>(&mut self, decoder: &mut D)` - Decode meta category fields /// - `decode_meta<D>(&mut self, decoder: &mut D)` - Decode meta category fields
/// - `decode_data<D>(&mut self, decoder: &mut D)` - Decode data category fields /// - `decode_data<D>(&mut self, decoder: &mut D)` - Decode data category fields
/// ///
...@@ -3531,9 +3531,9 @@ fn generate_encode_decode_methods(grouped_fields: &GroupedFields) -> TokenStream ...@@ -3531,9 +3531,9 @@ fn generate_encode_decode_methods(grouped_fields: &GroupedFields) -> TokenStream
impl TaskStorage { impl TaskStorage {
/// Encode meta category fields directly to bincode. /// Encode meta category fields directly to bincode.
/// Only persistent (non-transient) fields are encoded. /// Only persistent (non-transient) fields are encoded.
pub fn encode_meta<E: bincode::enc::Encoder>( pub fn encode_meta(
&self, &self,
encoder: &mut E, encoder: &mut turbo_bincode::TurboBincodeEncoder<'_>,
) -> Result<(), bincode::error::EncodeError> { ) -> Result<(), bincode::error::EncodeError> {
#encode_meta_body #encode_meta_body
#encode_meta_flags #encode_meta_flags
...@@ -3542,9 +3542,9 @@ fn generate_encode_decode_methods(grouped_fields: &GroupedFields) -> TokenStream ...@@ -3542,9 +3542,9 @@ fn generate_encode_decode_methods(grouped_fields: &GroupedFields) -> TokenStream
/// Encode data category fields directly to bincode. /// Encode data category fields directly to bincode.
/// Only persistent (non-transient) fields are encoded. /// Only persistent (non-transient) fields are encoded.
pub fn encode_data<E: bincode::enc::Encoder>( pub fn encode_data(
&self, &self,
encoder: &mut E, encoder: &mut turbo_bincode::TurboBincodeEncoder<'_>,
) -> Result<(), bincode::error::EncodeError> { ) -> Result<(), bincode::error::EncodeError> {
#encode_data_body #encode_data_body
#encode_data_flags #encode_data_flags
...@@ -3636,8 +3636,9 @@ fn generate_filter_predicate(field: &FieldInfo) -> Option<(TokenStream, FilterPr ...@@ -3636,8 +3636,9 @@ fn generate_filter_predicate(field: &FieldInfo) -> Option<(TokenStream, FilterPr
/// is already a reference from the match arm). /// is already a reference from the match arm).
/// ///
/// For non-filtered fields, encodes the value directly. /// For non-filtered fields, encodes the value directly.
/// For filtered fields, uses a single-pass collect to a Vec, then encodes. /// For filtered collections, reserves the unfiltered length as the length prefix, encodes the
/// This avoids multiple iterations (check non-empty + count + encode). /// surviving entries in a single pass, then patches the prefix with the actual count. This avoids
/// both iterating twice (count + encode) and collecting the entries into a temporary `Vec`.
fn generate_encode_value(field: &FieldInfo, value_ref: TokenStream) -> TokenStream { fn generate_encode_value(field: &FieldInfo, value_ref: TokenStream) -> TokenStream {
let Some((predicate, pred_type)) = generate_filter_predicate(field) else { let Some((predicate, pred_type)) = generate_filter_predicate(field) else {
// No filtering needed, just encode normally // No filtering needed, just encode normally
...@@ -3657,40 +3658,35 @@ fn generate_encode_value(field: &FieldInfo, value_ref: TokenStream) -> TokenStre ...@@ -3657,40 +3658,35 @@ fn generate_encode_value(field: &FieldInfo, value_ref: TokenStream) -> TokenStre
} }
} }
FilterPredicateType::Set => { FilterPredicateType::Set => {
// For AutoSet<K>, filter out transient keys - collect once then encode // For AutoSet<K>, filter out transient keys
quote! { quote! {
{ {
let filtered: Vec<_> = (#value_ref).iter().filter(#predicate).collect(); let collection = #value_ref;
bincode::Encode::encode(&filtered.len(), encoder)?; turbo_bincode::encode_len_prefixed(encoder, collection.len(), |encoder| {
for key in filtered { let mut count = 0usize;
bincode::Encode::encode(key, encoder)?; for key in collection.iter().filter(#predicate) {
} bincode::Encode::encode(key, encoder)?;
} count += 1;
} }
} Ok(count)
FilterPredicateType::CounterMap => { })?;
// For counter maps, filter out entries with transient keys - collect once
quote! {
{
let filtered: Vec<_> = (#value_ref).iter().filter(#predicate).collect();
bincode::Encode::encode(&filtered.len(), encoder)?;
for (key, value) in filtered {
bincode::Encode::encode(key, encoder)?;
bincode::Encode::encode(value, encoder)?;
}
} }
} }
} }
FilterPredicateType::Map => { FilterPredicateType::CounterMap | FilterPredicateType::Map => {
// For maps, filter out entries with transient keys or values - collect once // For maps, filter out entries with transient keys (or values, for AutoMap)
quote! { quote! {
{ {
let filtered: Vec<_> = (#value_ref).iter().filter(#predicate).collect(); let collection = #value_ref;
bincode::Encode::encode(&filtered.len(), encoder)?; turbo_bincode::encode_len_prefixed(encoder, collection.len(), |encoder| {
for (key, value) in filtered { let mut count = 0usize;
bincode::Encode::encode(key, encoder)?; for (key, value) in collection.iter().filter(#predicate) {
bincode::Encode::encode(value, encoder)?; bincode::Encode::encode(key, encoder)?;
} bincode::Encode::encode(value, encoder)?;
count += 1;
}
Ok(count)
})?;
} }
} }
} }
...@@ -3707,8 +3703,8 @@ fn generate_encode_inline_field(field: &FieldInfo) -> TokenStream { ...@@ -3707,8 +3703,8 @@ fn generate_encode_inline_field(field: &FieldInfo) -> TokenStream {
/// Generate code to encode a lazy field value with index. /// Generate code to encode a lazy field value with index.
/// ///
/// For filtered fields, collects to a Vec first, then checks if non-empty before /// For filtered fields, writes the index and entries in a single pass and truncates them away
/// writing index. This avoids multiple iterations over the data. /// again if every entry was filtered out. This avoids multiple iterations over the data.
fn generate_encode_lazy_field_with_index(field: &FieldInfo, index: u8) -> TokenStream { fn generate_encode_lazy_field_with_index(field: &FieldInfo, index: u8) -> TokenStream {
let Some((predicate, pred_type)) = generate_filter_predicate(field) else { let Some((predicate, pred_type)) = generate_filter_predicate(field) else {
// No filtering needed - encode directly // No filtering needed - encode directly
...@@ -3732,32 +3728,46 @@ fn generate_encode_lazy_field_with_index(field: &FieldInfo, index: u8) -> TokenS ...@@ -3732,32 +3728,46 @@ fn generate_encode_lazy_field_with_index(field: &FieldInfo, index: u8) -> TokenS
} }
} }
FilterPredicateType::Set => { FilterPredicateType::Set => {
// Collect once, check if non-empty, then encode // Encode optimistically in a single pass, then roll back the index too if every entry
// was filtered out
quote! { quote! {
{ {
let filtered: Vec<_> = data.iter().filter(#predicate).collect(); let start = bincode::enc::Encoder::writer(encoder).buffer.len();
if !filtered.is_empty() { bincode::Encode::encode(&#index, encoder)?;
bincode::Encode::encode(&#index, encoder)?; let count =
bincode::Encode::encode(&filtered.len(), encoder)?; turbo_bincode::encode_len_prefixed(encoder, data.len(), |encoder| {
for key in filtered { let mut count = 0usize;
bincode::Encode::encode(key, encoder)?; for key in data.iter().filter(#predicate) {
} bincode::Encode::encode(key, encoder)?;
count += 1;
}
Ok(count)
})?;
if count == 0 {
bincode::enc::Encoder::writer(encoder).buffer.truncate(start);
} }
} }
} }
} }
FilterPredicateType::CounterMap | FilterPredicateType::Map => { FilterPredicateType::CounterMap | FilterPredicateType::Map => {
// Collect once, check if non-empty, then encode // Encode optimistically in a single pass, then roll back the index too if every entry
// was filtered out
quote! { quote! {
{ {
let filtered: Vec<_> = data.iter().filter(#predicate).collect(); let start = bincode::enc::Encoder::writer(encoder).buffer.len();
if !filtered.is_empty() { bincode::Encode::encode(&#index, encoder)?;
bincode::Encode::encode(&#index, encoder)?; let count =
bincode::Encode::encode(&filtered.len(), encoder)?; turbo_bincode::encode_len_prefixed(encoder, data.len(), |encoder| {
for (key, value) in filtered { let mut count = 0usize;
bincode::Encode::encode(key, encoder)?; for (key, value) in data.iter().filter(#predicate) {
bincode::Encode::encode(value, encoder)?; bincode::Encode::encode(key, encoder)?;
} bincode::Encode::encode(value, encoder)?;
count += 1;
}
Ok(count)
})?;
if count == 0 {
bincode::enc::Encoder::writer(encoder).buffer.truncate(start);
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment