Commit 444a4263 authored by Luke Sandberg's avatar Luke Sandberg

add a new hasher implemntation to eliminate allocations

parent 42ccc8d6
...@@ -9257,6 +9257,7 @@ dependencies = [ ...@@ -9257,6 +9257,7 @@ dependencies = [
"ringmap", "ringmap",
"serde", "serde",
"smallvec", "smallvec",
"turbo-tasks-hash",
"unty", "unty",
] ]
...@@ -9445,6 +9446,7 @@ dependencies = [ ...@@ -9445,6 +9446,7 @@ dependencies = [
"turbo-persistence", "turbo-persistence",
"turbo-rcstr", "turbo-rcstr",
"turbo-tasks", "turbo-tasks",
"turbo-tasks-hash",
"turbo-tasks-malloc", "turbo-tasks-malloc",
"turbo-tasks-testing", "turbo-tasks-testing",
] ]
......
...@@ -18,4 +18,5 @@ mime = { workspace = true } ...@@ -18,4 +18,5 @@ mime = { workspace = true }
ringmap = { workspace = true } ringmap = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true }
turbo-tasks-hash = { workspace = true }
unty = { workspace = true } unty = { workspace = true }
...@@ -11,6 +11,7 @@ use bincode::{ ...@@ -11,6 +11,7 @@ use bincode::{
enc::{Encoder, EncoderImpl, write::Writer}, enc::{Encoder, EncoderImpl, write::Writer},
error::{DecodeError, EncodeError}, error::{DecodeError, EncodeError},
}; };
use turbo_tasks_hash::DeterministicHasher;
pub const TURBO_BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard(); pub const TURBO_BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
pub type TurboBincodeBuffer = SmallVec<[u8; 16]>; pub type TurboBincodeBuffer = SmallVec<[u8; 16]>;
...@@ -121,6 +122,37 @@ impl Reader for TurboBincodeReader<'_> { ...@@ -121,6 +122,37 @@ impl Reader for TurboBincodeReader<'_> {
} }
} }
/// A [`Writer`] that sinks bytes directly into a [`DeterministicHasher`] instead of a buffer.
///
/// This allows encoding values directly to a hash without intermediate buffer allocation.
pub struct HashWriter<'a, H: DeterministicHasher + ?Sized> {
hasher: &'a mut H,
}
impl<'a, H: DeterministicHasher + ?Sized> HashWriter<'a, H> {
/// Creates a new `HashWriter` that writes to the given hasher.
pub fn new(hasher: &'a mut H) -> Self {
Self { hasher }
}
}
impl<H: DeterministicHasher + ?Sized> Writer for HashWriter<'_, H> {
fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> {
self.hasher.write_bytes(bytes);
Ok(())
}
}
/// An encoder that writes directly to a [`DeterministicHasher`].
pub type HashEncoder<'a, H> = EncoderImpl<HashWriter<'a, H>, bincode::config::Configuration>;
/// Creates a new [`HashEncoder`] that encodes directly to the given hasher.
///
/// This is useful for computing hashes of encoded values without allocating a buffer.
pub fn new_hash_encoder<H: DeterministicHasher + ?Sized>(hasher: &mut H) -> HashEncoder<'_, H> {
EncoderImpl::new(HashWriter::new(hasher), TURBO_BINCODE_CONFIG)
}
/// Represents a type that can only be encoded with a [`TurboBincodeEncoder`]. /// Represents a type that can only be encoded with a [`TurboBincodeEncoder`].
/// ///
/// All traits implementing this must also implement the more generic [`Encode`] trait, but they /// All traits implementing this must also implement the more generic [`Encode`] trait, but they
......
...@@ -60,6 +60,7 @@ turbo-bincode = { workspace = true } ...@@ -60,6 +60,7 @@ turbo-bincode = { workspace = true }
turbo-persistence = { workspace = true } turbo-persistence = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true }
[dev-dependencies] [dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] }
......
...@@ -9,7 +9,6 @@ use anyhow::{Context, Result}; ...@@ -9,7 +9,6 @@ use anyhow::{Context, Result};
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::{ use turbo_bincode::{
TurboBincodeBuffer, new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode, TurboBincodeBuffer, new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode,
turbo_bincode_encode_into,
}; };
use turbo_tasks::{ use turbo_tasks::{
TaskId, TaskId,
...@@ -17,6 +16,7 @@ use turbo_tasks::{ ...@@ -17,6 +16,7 @@ use turbo_tasks::{
panic_hooks::{PanicHookGuard, register_panic_hook}, panic_hooks::{PanicHookGuard, register_panic_hook},
parallel, parallel,
}; };
use turbo_tasks_hash::Xxh3Hash64Hasher;
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
...@@ -269,11 +269,6 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -269,11 +269,6 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
{ {
let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered(); let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered();
let mut batch = self.inner.database.write_batch()?; let mut batch = self.inner.database.write_batch()?;
// these buffers should be large, because they're temporary and re-used.
// From measuring a large application the largest TaskType was ~365b, so this should be big
// enough to trigger no resizes in the loop.
const INITIAL_ENCODE_BUFFER_CAPACITY: usize = 512;
// Start organizing the updates in parallel // Start organizing the updates in parallel
match &mut batch { match &mut batch {
&mut WriteBatch::Concurrent(ref batch, _) => { &mut WriteBatch::Concurrent(ref batch, _) => {
...@@ -308,13 +303,8 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -308,13 +303,8 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
|updates| { |updates| {
let _span = _span.clone().entered(); let _span = _span.clone().entered();
let mut max_task_id = 0; let mut max_task_id = 0;
// Re-use the same buffer across every `compute_task_type_hash` call in
// this chunk. `ConcurrentWriteBatch::put` will copy the data out of
// this buffer into smaller exact-sized vecs.
let mut task_type_bytes =
TurboBincodeBuffer::with_capacity(INITIAL_ENCODE_BUFFER_CAPACITY);
for (task_type, task_id) in updates { for (task_type, task_id) in updates {
let hash = compute_task_type_hash(&task_type, &mut task_type_bytes); let hash = compute_task_type_hash(&task_type);
let task_id: u32 = *task_id; let task_id: u32 = *task_id;
batch batch
...@@ -384,13 +374,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -384,13 +374,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>() items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>()
) )
.entered(); .entered();
// Re-use the same buffer across every `serialize_task_type` call.
// `ConcurrentWriteBatch::put` will copy the data out of this buffer into
// smaller exact-sized vecs.
let mut task_type_bytes =
TurboBincodeBuffer::with_capacity(INITIAL_ENCODE_BUFFER_CAPACITY);
for (task_type, task_id) in task_cache_updates.into_iter().flatten() { for (task_type, task_id) in task_cache_updates.into_iter().flatten() {
let hash = compute_task_type_hash(&task_type, &mut task_type_bytes); let hash = compute_task_type_hash(&task_type);
let task_id = *task_id; let task_id = *task_id;
batch batch
...@@ -436,16 +422,16 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -436,16 +422,16 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
tx: &D::ReadTransaction<'_>, tx: &D::ReadTransaction<'_>,
task_type: &CachedTaskType, task_type: &CachedTaskType,
) -> Result<SmallVec<[TaskId; 1]>> { ) -> Result<SmallVec<[TaskId; 1]>> {
let hash = compute_task_type_hash(task_type, &mut TurboBincodeBuffer::new()); let hash = compute_task_type_hash(task_type);
let buffers = database.get_multiple(tx, KeySpace::TaskCache, &hash.to_le_bytes())?; let results = database.get_multiple(tx, KeySpace::TaskCache, &hash.to_le_bytes())?;
let mut task_ids = SmallVec::with_capacity(buffers.len()); Ok(results
for bytes in buffers { .iter()
let bytes = bytes.borrow().try_into()?; .map(|bytes| {
let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap(); let as_array = bytes.borrow().try_into().unwrap();
task_ids.push(id); TaskId::try_from(u32::from_le_bytes(as_array)).unwrap()
} })
Ok(task_ids) .collect())
} }
if inner.database.is_empty() { if inner.database.is_empty() {
// Checking if the database is empty is a performance optimization // Checking if the database is empty is a performance optimization
...@@ -589,20 +575,12 @@ where ...@@ -589,20 +575,12 @@ where
/// Computes a deterministic 64-bit hash of a CachedTaskType for use as a TaskCache key. /// Computes a deterministic 64-bit hash of a CachedTaskType for use as a TaskCache key.
/// ///
/// This uses the existing TurboBincodeEncode implementation which is deterministic /// This encodes the task type directly to a hasher, avoiding intermediate buffer allocation.
/// (function IDs from registry, bincode argument encoding), then hashes the result /// The encoding is deterministic (function IDs from registry, bincode argument encoding).
/// with XxHash64. fn compute_task_type_hash(task_type: &CachedTaskType) -> u64 {
fn compute_task_type_hash( let mut hasher = Xxh3Hash64Hasher::new();
task_type: &CachedTaskType, task_type.hash_encode(&mut hasher);
scratch_buffer: &mut TurboBincodeBuffer, hasher.finish()
) -> u64 {
// TODO: use a custom encoder that can directly
// This should not fail for valid task types - the encoding is deterministic
turbo_bincode_encode_into(task_type, scratch_buffer)
.expect("CachedTaskType encoding should not fail");
let r = turbo_persistence::hash_key(&scratch_buffer.as_slice());
scratch_buffer.clear();
r
} }
type SerializedTasks = Vec< type SerializedTasks = Vec<
......
...@@ -20,9 +20,10 @@ use smallvec::SmallVec; ...@@ -20,9 +20,10 @@ use smallvec::SmallVec;
use tracing::Span; use tracing::Span;
use turbo_bincode::{ use turbo_bincode::{
TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder, TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder,
impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode, impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode, new_hash_encoder,
}; };
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks_hash::DeterministicHasher;
use crate::{ use crate::{
RawVc, ReadCellOptions, ReadOutputOptions, ReadRef, SharedReference, TaskId, TaskIdSet, RawVc, ReadCellOptions, ReadOutputOptions, ReadRef, SharedReference, TaskId, TaskIdSet,
...@@ -79,6 +80,20 @@ impl CachedTaskType { ...@@ -79,6 +80,20 @@ impl CachedTaskType {
pub fn get_name(&self) -> &'static str { pub fn get_name(&self) -> &'static str {
self.native_fn.name self.native_fn.name
} }
/// Encodes this task type directly to a hasher, avoiding buffer allocation.
///
/// This uses the same encoding logic as [`TurboBincodeEncode`] but writes
/// directly to a [`DeterministicHasher`] instead of a buffer.
pub fn hash_encode<H: DeterministicHasher>(&self, hasher: &mut H) {
let fn_id = registry::get_function_id(self.native_fn);
{
let mut encoder = new_hash_encoder(hasher);
Encode::encode(&fn_id, &mut encoder).expect("fn_id encoding should not fail");
Encode::encode(&self.this, &mut encoder).expect("this encoding should not fail");
}
(self.native_fn.arg_meta.hash_encode)(&*self.arg, hasher);
}
} }
impl TurboBincodeEncode for CachedTaskType { impl TurboBincodeEncode for CachedTaskType {
......
...@@ -5,7 +5,8 @@ use bincode::{Decode, Encode}; ...@@ -5,7 +5,8 @@ use bincode::{Decode, Encode};
use futures::Future; use futures::Future;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use tracing::Span; use tracing::Span;
use turbo_bincode::{AnyDecodeFn, AnyEncodeFn}; use turbo_bincode::{AnyDecodeFn, AnyEncodeFn, new_hash_encoder};
use turbo_tasks_hash::DeterministicHasher;
use crate::{ use crate::{
RawVc, TaskExecutionReason, TaskInput, TaskPersistence, TaskPriority, RawVc, TaskExecutionReason, TaskInput, TaskPersistence, TaskPriority,
...@@ -24,9 +25,17 @@ type IsResolvedFunctor = fn(&dyn MagicAny) -> bool; ...@@ -24,9 +25,17 @@ type IsResolvedFunctor = fn(&dyn MagicAny) -> bool;
type FilterOwnedArgsFunctor = for<'a> fn(Box<dyn MagicAny>) -> Box<dyn MagicAny>; type FilterOwnedArgsFunctor = for<'a> fn(Box<dyn MagicAny>) -> Box<dyn MagicAny>;
type FilterAndResolveFunctor = ResolveFunctor; type FilterAndResolveFunctor = ResolveFunctor;
/// Function pointer that encodes a task argument directly to a hasher.
///
/// This allows computing hashes of task arguments without intermediate buffer allocation.
pub type AnyHashEncodeFn = fn(&dyn Any, &mut dyn DeterministicHasher);
pub struct ArgMeta { pub struct ArgMeta {
// TODO: This should be an `Option` with `None` for transient tasks. We can skip some codegen. // TODO: This should be an `Option` with `None` for transient tasks. We can skip some codegen.
pub bincode: (AnyEncodeFn, AnyDecodeFn<Box<dyn MagicAny>>), pub bincode: (AnyEncodeFn, AnyDecodeFn<Box<dyn MagicAny>>),
/// Encodes the argument directly to a hasher, avoiding buffer allocation.
/// Uses the same encoding logic as bincode but writes to a [`DeterministicHasher`].
pub hash_encode: AnyHashEncodeFn,
is_resolved: IsResolvedFunctor, is_resolved: IsResolvedFunctor,
resolve: ResolveFunctor, resolve: ResolveFunctor,
/// Used for trait methods, filters out unused arguments. /// Used for trait methods, filters out unused arguments.
...@@ -70,6 +79,11 @@ impl ArgMeta { ...@@ -70,6 +79,11 @@ impl ArgMeta {
Ok(Box::new(val)) Ok(Box::new(val))
}, },
), ),
hash_encode: |this, hasher| {
let mut encoder = new_hash_encoder(hasher);
T::encode(any_as_encode::<T>(this), &mut encoder)
.expect("encoding to hasher should not fail");
},
is_resolved: |value| downcast_args_ref::<T>(value).is_resolved(), is_resolved: |value| downcast_args_ref::<T>(value).is_resolved(),
resolve: resolve_functor_impl::<T>, resolve: resolve_functor_impl::<T>,
filter_owned, filter_owned,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment