Unverified Commit 18b94630 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] add a new hasher implementation to eliminate allocations (#89059)

We only store hashes for TaskTypes in the TaskCache keyspace, so this introduces an allocation free path to compute it. 

For writing out the TaskCache this wont make much of a difference since we were reusing a scratch buffer, but this should eliminate a source of allocations from the read path.

This does of course cause a small binary size regression `157M (160,952K) -> 158M (161,800K)  = +1M (+848K)`
parent 32d8a9bf
......@@ -9412,6 +9412,7 @@ dependencies = [
"ringmap",
"serde",
"smallvec",
"turbo-tasks-hash",
"unty",
]
......@@ -9599,6 +9600,7 @@ dependencies = [
"turbo-persistence",
"turbo-rcstr",
"turbo-tasks",
"turbo-tasks-hash",
"turbo-tasks-malloc",
"turbo-tasks-testing",
]
......
......@@ -18,4 +18,5 @@ mime = { workspace = true }
ringmap = { workspace = true }
serde = { workspace = true }
smallvec = { workspace = true }
turbo-tasks-hash = { workspace = true }
unty = { workspace = true }
......@@ -11,8 +11,16 @@ use bincode::{
enc::{Encoder, EncoderImpl, write::Writer},
error::{DecodeError, EncodeError},
};
use turbo_tasks_hash::DeterministicHasher;
pub const TURBO_BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
/// Same as standard config, but since we aren't encoding we don't benefit from varint
/// optimizations.
pub const TURBO_BINCODE_HASH_CONFIG: bincode::config::Configuration<
bincode::config::LittleEndian,
bincode::config::Fixint,
bincode::config::NoLimit,
> = TURBO_BINCODE_CONFIG.with_fixed_int_encoding();
pub type TurboBincodeBuffer = SmallVec<[u8; 16]>;
pub type TurboBincodeEncoder<'a> =
EncoderImpl<TurboBincodeWriter<'a>, bincode::config::Configuration>;
......@@ -121,6 +129,44 @@ impl Reader for TurboBincodeReader<'_> {
}
}
/// A [`Writer`] that sinks bytes directly into a [`DeterministicHasher`] instead of a buffer.
///
/// This allows encoding values directly to a hash without intermediate buffer allocation.
pub struct HashWriter<'a, H: DeterministicHasher + ?Sized> {
hasher: &'a mut H,
}
impl<'a, H: DeterministicHasher + ?Sized> HashWriter<'a, H> {
/// Creates a new `HashWriter` that writes to the given hasher.
pub fn new(hasher: &'a mut H) -> Self {
Self { hasher }
}
}
impl<H: DeterministicHasher + ?Sized> Writer for HashWriter<'_, H> {
fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> {
self.hasher.write_bytes(bytes);
Ok(())
}
}
/// An encoder that writes directly to a [`DeterministicHasher`].
pub type HashEncoder<'a, H> = EncoderImpl<
HashWriter<'a, H>,
bincode::config::Configuration<
bincode::config::LittleEndian,
bincode::config::Fixint,
bincode::config::NoLimit,
>,
>;
/// Creates a new [`HashEncoder`] that encodes directly to the given hasher.
///
/// This is useful for computing hashes of encoded values without allocating a buffer.
pub fn new_hash_encoder<H: DeterministicHasher + ?Sized>(hasher: &mut H) -> HashEncoder<'_, H> {
EncoderImpl::new(HashWriter::new(hasher), TURBO_BINCODE_HASH_CONFIG)
}
/// Represents a type that can only be encoded with a [`TurboBincodeEncoder`].
///
/// All traits implementing this must also implement the more generic [`Encode`] trait, but they
......
......@@ -61,6 +61,7 @@ turbo-bincode = { workspace = true }
turbo-persistence = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] }
......
......@@ -9,7 +9,6 @@ use anyhow::{Context, Result};
use smallvec::SmallVec;
use turbo_bincode::{
TurboBincodeBuffer, new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode,
turbo_bincode_encode_into,
};
use turbo_tasks::{
TaskId,
......@@ -17,6 +16,7 @@ use turbo_tasks::{
panic_hooks::{PanicHookGuard, register_panic_hook},
parallel,
};
use turbo_tasks_hash::Xxh3Hash64Hasher;
use crate::{
GitVersionInfo,
......@@ -269,11 +269,6 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
{
let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered();
let mut batch = self.inner.database.write_batch()?;
// these buffers should be large, because they're temporary and re-used.
// From measuring a large application the largest TaskType was ~365b, so this should be big
// enough to trigger no resizes in the loop.
const INITIAL_ENCODE_BUFFER_CAPACITY: usize = 512;
// Start organizing the updates in parallel
match &mut batch {
&mut WriteBatch::Concurrent(ref batch, _) => {
......@@ -308,14 +303,8 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
|updates| {
let _span = _span.clone().entered();
let mut max_task_id = 0;
// Re-use the same buffer across every `compute_task_type_hash` call in
// this chunk. `ConcurrentWriteBatch::put` will copy the data out of
// this buffer into smaller exact-sized vecs.
let mut task_type_bytes =
TurboBincodeBuffer::with_capacity(INITIAL_ENCODE_BUFFER_CAPACITY);
for (task_type, task_id) in updates {
let hash = compute_task_type_hash(&task_type, &mut task_type_bytes);
let hash = compute_task_type_hash(&task_type);
let task_id: u32 = *task_id;
batch
......@@ -385,13 +374,8 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>()
)
.entered();
// Re-use the same buffer across every `serialize_task_type` call.
// `ConcurrentWriteBatch::put` will copy the data out of this buffer into
// smaller exact-sized vecs.
let mut task_type_bytes =
TurboBincodeBuffer::with_capacity(INITIAL_ENCODE_BUFFER_CAPACITY);
for (task_type, task_id) in task_cache_updates.into_iter().flatten() {
let hash = compute_task_type_hash(&task_type, &mut task_type_bytes);
let hash = compute_task_type_hash(&task_type);
let task_id = *task_id;
batch
......@@ -437,7 +421,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
tx: &D::ReadTransaction<'_>,
task_type: &CachedTaskType,
) -> Result<SmallVec<[TaskId; 1]>> {
let hash = compute_task_type_hash(task_type, &mut TurboBincodeBuffer::new());
let hash = compute_task_type_hash(task_type);
let buffers = database.get_multiple(tx, KeySpace::TaskCache, &hash.to_le_bytes())?;
let mut task_ids = SmallVec::with_capacity(buffers.len());
......@@ -590,28 +574,18 @@ where
/// Computes a deterministic 64-bit hash of a CachedTaskType for use as a TaskCache key.
///
/// This uses the existing TurboBincodeEncode implementation which is deterministic
/// (function IDs from registry, bincode argument encoding), then hashes the result
/// with XxHash64.
fn compute_task_type_hash(
task_type: &CachedTaskType,
scratch_buffer: &mut TurboBincodeBuffer,
) -> u64 {
// TODO: use a custom encoder that can directly hash without filling a buffer
// This should not fail for valid task types - the encoding is deterministic
turbo_bincode_encode_into(task_type, scratch_buffer)
.expect("CachedTaskType encoding should not fail");
let hash = turbo_persistence::hash_key(&scratch_buffer.as_slice());
scratch_buffer.clear();
/// This encodes the task type directly to a hasher, avoiding intermediate buffer allocation.
/// The encoding is deterministic (function IDs from registry, bincode argument encoding).
fn compute_task_type_hash(task_type: &CachedTaskType) -> u64 {
let mut hasher = Xxh3Hash64Hasher::new();
task_type.hash_encode(&mut hasher);
let hash = hasher.finish();
if cfg!(feature = "verify_serialization") {
turbo_bincode_encode_into(task_type, scratch_buffer)
.expect("CachedTaskType encoding should not fail");
let hash2 = turbo_persistence::hash_key(&scratch_buffer.as_slice());
scratch_buffer.clear();
task_type.hash_encode(&mut hasher);
let hash2 = hasher.finish();
assert_eq!(
hash, hash2,
"Encoding TaskType twice was non-deterministic: \n{:?}\ngot hashes {} != {}",
"Hashing TaskType twice was non-deterministic: \n{:?}\ngot hashes {} != {}",
task_type, hash, hash2
);
}
......
......@@ -22,9 +22,10 @@ use smallvec::SmallVec;
use tracing::Span;
use turbo_bincode::{
TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder,
impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode,
impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode, new_hash_encoder,
};
use turbo_rcstr::RcStr;
use turbo_tasks_hash::DeterministicHasher;
use crate::{
RawVc, ReadCellOptions, ReadOutputOptions, ReadRef, SharedReference, TaskId, TaskIdSet,
......@@ -81,6 +82,20 @@ impl CachedTaskType {
pub fn get_name(&self) -> &'static str {
self.native_fn.name
}
/// Encodes this task type directly to a hasher, avoiding buffer allocation.
///
/// This uses the same encoding logic as [`TurboBincodeEncode`] but writes
/// directly to a [`DeterministicHasher`] instead of a buffer.
pub fn hash_encode<H: DeterministicHasher>(&self, hasher: &mut H) {
let fn_id = registry::get_function_id(self.native_fn);
{
let mut encoder = new_hash_encoder(hasher);
Encode::encode(&fn_id, &mut encoder).expect("fn_id encoding should not fail");
Encode::encode(&self.this, &mut encoder).expect("this encoding should not fail");
}
(self.native_fn.arg_meta.hash_encode)(&*self.arg, hasher);
}
}
impl TurboBincodeEncode for CachedTaskType {
......
......@@ -5,7 +5,8 @@ use bincode::{Decode, Encode};
use futures::Future;
use once_cell::sync::Lazy;
use tracing::Span;
use turbo_bincode::{AnyDecodeFn, AnyEncodeFn};
use turbo_bincode::{AnyDecodeFn, AnyEncodeFn, new_hash_encoder};
use turbo_tasks_hash::DeterministicHasher;
use crate::{
RawVc, TaskExecutionReason, TaskInput, TaskPersistence, TaskPriority,
......@@ -24,9 +25,17 @@ type IsResolvedFunctor = fn(&dyn MagicAny) -> bool;
type FilterOwnedArgsFunctor = for<'a> fn(Box<dyn MagicAny>) -> Box<dyn MagicAny>;
type FilterAndResolveFunctor = ResolveFunctor;
/// Function pointer that encodes a task argument directly to a hasher.
///
/// This allows computing hashes of task arguments without intermediate buffer allocation.
pub type AnyHashEncodeFn = fn(&dyn Any, &mut dyn DeterministicHasher);
pub struct ArgMeta {
// TODO: This should be an `Option` with `None` for transient tasks. We can skip some codegen.
pub bincode: (AnyEncodeFn, AnyDecodeFn<Box<dyn MagicAny>>),
/// Encodes the argument directly to a hasher, avoiding buffer allocation.
/// Uses the same encoding logic as bincode but writes to a [`DeterministicHasher`].
pub hash_encode: AnyHashEncodeFn,
is_resolved: IsResolvedFunctor,
resolve: ResolveFunctor,
/// Used for trait methods, filters out unused arguments.
......@@ -70,6 +79,11 @@ impl ArgMeta {
Ok(Box::new(val))
},
),
hash_encode: |this, hasher| {
let mut encoder = new_hash_encoder(hasher);
T::encode(any_as_encode::<T>(this), &mut encoder)
.expect("encoding to hasher should not fail");
},
is_resolved: |value| downcast_args_ref::<T>(value).is_resolved(),
resolve: resolve_functor_impl::<T>,
filter_owned,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment