Unverified Commit 4de22017 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Share scratch buffer across shards using thread local (#90167)

Instead of allocating a scratch buffer for every shard, buffer them in a thread local.
parent 66898148
...@@ -9564,6 +9564,7 @@ dependencies = [ ...@@ -9564,6 +9564,7 @@ dependencies = [
"serde_path_to_error", "serde_path_to_error",
"smallvec", "smallvec",
"tempfile", "tempfile",
"thread_local",
"tokio", "tokio",
"tracing", "tracing",
"turbo-bincode", "turbo-bincode",
......
...@@ -57,6 +57,7 @@ turbo-persistence = { workspace = true } ...@@ -57,6 +57,7 @@ turbo-persistence = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true } turbo-tasks-hash = { workspace = true }
thread_local = { workspace = true }
[dev-dependencies] [dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] }
......
...@@ -1209,7 +1209,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1209,7 +1209,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
) )
}; };
// take_snapshot already filters empty items and empty shards in parallel // take_snapshot filters empty shards (no modified/snapshot entries) in parallel.
// Individual empty SnapshotItems are filtered by the iterator.
let task_snapshots = self.storage.take_snapshot(&process, &process_snapshot); let task_snapshots = self.storage.take_snapshot(&process, &process_snapshot);
swap_retain(&mut persisted_task_cache_log, |shard| !shard.is_empty()); swap_retain(&mut persisted_task_cache_log, |shard| !shard.is_empty());
......
use std::{ use std::{
cell::Cell,
hash::Hash, hash::Hash,
iter::Peekable,
ops::{Deref, DerefMut}, ops::{Deref, DerefMut},
sync::{Arc, atomic::AtomicBool}, sync::{Arc, atomic::AtomicBool},
}; };
use smallvec::SmallVec; use smallvec::SmallVec;
use thread_local::ThreadLocal;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{FxDashMap, TaskId, parallel}; use turbo_tasks::{FxDashMap, TaskId, parallel};
...@@ -113,7 +114,9 @@ impl Storage { ...@@ -113,7 +114,9 @@ impl Storage {
/// receives an owned Box<TaskStorage> snapshot. /// receives an owned Box<TaskStorage> snapshot.
/// Both callbacks receive a mutable scratch buffer that can be reused across iterations /// Both callbacks receive a mutable scratch buffer that can be reused across iterations
/// to avoid repeated allocations. /// to avoid repeated allocations.
/// The returned iterators are guaranteed to be non-empty and only yield non-empty items. /// The returned shards implement `IntoIterator`. Empty shards (no modified or snapshot
/// entries) are filtered out, but shards may still yield no items if all entries produce
/// empty `SnapshotItem`s (this is rare and only happens under error conditions).
pub fn take_snapshot< pub fn take_snapshot<
'l, 'l,
P: for<'a> Fn(TaskId, &'a TaskStorage, &mut TurboBincodeBuffer) -> SnapshotItem + Sync, P: for<'a> Fn(TaskId, &'a TaskStorage, &mut TurboBincodeBuffer) -> SnapshotItem + Sync,
...@@ -122,12 +125,15 @@ impl Storage { ...@@ -122,12 +125,15 @@ impl Storage {
&'l self, &'l self,
process: &'l P, process: &'l P,
process_snapshot: &'l PS, process_snapshot: &'l PS,
) -> Vec<Peekable<SnapshotShard<'l, P, PS>>> { ) -> Vec<SnapshotShard<'l, P, PS>> {
if !self.snapshot_mode() { if !self.snapshot_mode() {
self.start_snapshot(); self.start_snapshot();
} }
let guard = Arc::new(SnapshotGuard { storage: self }); let guard = Arc::new(SnapshotGuard {
storage: self,
scratch_buffers: ThreadLocal::new(),
});
// The number of shards is much larger than the number of threads, so the effect of the // The number of shards is much larger than the number of threads, so the effect of the
// locks held is negligible. // locks held is negligible.
...@@ -163,22 +169,14 @@ impl Storage { ...@@ -163,22 +169,14 @@ impl Storage {
return None; return None;
} }
/// How big of a buffer to allocate initially. Based on metrics from a large Some(SnapshotShard {
/// application this should cover about 98% of values with no resizes
const SCRATCH_BUFFER_SIZE: usize = 4096;
let shard = SnapshotShard {
direct_snapshots, direct_snapshots,
modified, modified,
storage: self, storage: self,
guard: Some(guard.clone()),
process, process,
process_snapshot, process_snapshot,
scratch_buffer: TurboBincodeBuffer::with_capacity(SCRATCH_BUFFER_SIZE), _guard: guard.clone(),
}; })
// Peek to filter out shards that only produce empty items
let mut iter = shard.peekable();
iter.peek().is_some().then_some(iter)
}) })
.into_iter() .into_iter()
.flatten() .flatten()
...@@ -382,8 +380,69 @@ impl DerefMut for StorageWriteGuard<'_> { ...@@ -382,8 +380,69 @@ impl DerefMut for StorageWriteGuard<'_> {
} }
} }
/// How big of a buffer to allocate initially. Based on metrics from a large
/// application this should cover about 98% of values with no resizes.
const SCRATCH_BUFFER_INITIAL_SIZE: usize = 4096;
/// State machine for a per-thread scratch buffer slot.
///
/// Transitions:
/// - `Uninit` → `Taken` (first take)
/// - `Available` → `Taken` (subsequent takes)
/// - `Taken` → `Available` (return)
///
/// Any other transition is a bug (e.g. double-take or double-return).
#[derive(Default)]
enum ScratchBufferSlot {
/// No buffer has been allocated on this thread yet.
#[default]
Uninit,
/// The buffer is currently checked out.
Taken,
/// The buffer is available for reuse.
Available(TurboBincodeBuffer),
}
pub struct SnapshotGuard<'l> { pub struct SnapshotGuard<'l> {
storage: &'l Storage, storage: &'l Storage,
/// Per-thread scratch buffers for encoding task data. Buffers are taken
/// by `SnapshotShardIter` on creation and returned on drop, allowing reuse
/// across multiple shards processed by the same thread. When the guard is
/// dropped (after all iterators are done), the `ThreadLocal` drops too,
/// freeing all buffers.
scratch_buffers: ThreadLocal<Cell<ScratchBufferSlot>>,
}
impl SnapshotGuard<'_> {
fn take_scratch_buffer(&self) -> TurboBincodeBuffer {
let cell = self.scratch_buffers.get_or_default();
match cell.take() {
ScratchBufferSlot::Available(buf) => {
cell.set(ScratchBufferSlot::Taken);
buf
}
ScratchBufferSlot::Uninit => {
cell.set(ScratchBufferSlot::Taken);
TurboBincodeBuffer::with_capacity(SCRATCH_BUFFER_INITIAL_SIZE)
}
ScratchBufferSlot::Taken => {
panic!("scratch buffer taken twice without being returned");
}
}
}
fn return_scratch_buffer(&self, buffer: TurboBincodeBuffer) {
let cell = self.scratch_buffers.get_or_default();
match cell.take() {
ScratchBufferSlot::Taken => cell.set(ScratchBufferSlot::Available(buffer)),
ScratchBufferSlot::Available(_) => {
panic!("scratch buffer returned without being taken (already available)");
}
ScratchBufferSlot::Uninit => {
panic!("scratch buffer returned without being taken (uninit)");
}
}
}
} }
impl Drop for SnapshotGuard<'_> { impl Drop for SnapshotGuard<'_> {
...@@ -396,45 +455,71 @@ pub struct SnapshotShard<'l, P, PS> { ...@@ -396,45 +455,71 @@ pub struct SnapshotShard<'l, P, PS> {
direct_snapshots: Vec<(TaskId, Box<TaskStorage>)>, direct_snapshots: Vec<(TaskId, Box<TaskStorage>)>,
modified: SmallVec<[TaskId; 4]>, modified: SmallVec<[TaskId; 4]>,
storage: &'l Storage, storage: &'l Storage,
guard: Option<Arc<SnapshotGuard<'l>>>,
process: &'l P, process: &'l P,
process_snapshot: &'l PS, process_snapshot: &'l PS,
/// Scratch buffer for encoding task data, reused across iterations to avoid allocations /// Held for its `Drop` impl — ensures snapshot mode ends when all shards are done.
scratch_buffer: TurboBincodeBuffer, _guard: Arc<SnapshotGuard<'l>>,
} }
impl<'l, P, PS> SnapshotShard<'l, P, PS> impl<'l, P, PS> IntoIterator for SnapshotShard<'l, P, PS>
where where
P: Fn(TaskId, &TaskStorage, &mut TurboBincodeBuffer) -> SnapshotItem + Sync, P: Fn(TaskId, &TaskStorage, &mut TurboBincodeBuffer) -> SnapshotItem + Sync,
PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> SnapshotItem + Sync, PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> SnapshotItem + Sync,
{ {
fn next_item(&mut self) -> Option<SnapshotItem> { type Item = SnapshotItem;
if let Some((task_id, snapshot)) = self.direct_snapshots.pop() { type IntoIter = SnapshotShardIter<'l, P, PS>;
return Some((self.process_snapshot)(
task_id, fn into_iter(self) -> Self::IntoIter {
snapshot, let buffer = self._guard.take_scratch_buffer();
&mut self.scratch_buffer, SnapshotShardIter {
)); shard: self,
} buffer,
while let Some(task_id) = self.modified.pop() { }
let inner = self.storage.map.get(&task_id).unwrap(); }
}
/// Iterator over a single shard's snapshot items. Holds a thread-local scratch
/// buffer for the duration of iteration and returns it on drop.
pub struct SnapshotShardIter<'l, P, PS> {
shard: SnapshotShard<'l, P, PS>,
buffer: TurboBincodeBuffer,
}
impl<'l, P, PS> Iterator for SnapshotShardIter<'l, P, PS>
where
P: Fn(TaskId, &TaskStorage, &mut TurboBincodeBuffer) -> SnapshotItem + Sync,
PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> SnapshotItem + Sync,
{
type Item = SnapshotItem;
fn next(&mut self) -> Option<Self::Item> {
while let Some((task_id, snapshot)) = self.shard.direct_snapshots.pop() {
let item = (self.shard.process_snapshot)(task_id, snapshot, &mut self.buffer);
if !item.is_empty() {
return Some(item);
}
}
while let Some(task_id) = self.shard.modified.pop() {
let inner = self.shard.storage.map.get(&task_id).unwrap();
if !inner.flags.any_snapshot() { if !inner.flags.any_snapshot() {
return Some((self.process)(task_id, &inner, &mut self.scratch_buffer)); let item = (self.shard.process)(task_id, &inner, &mut self.buffer);
if !item.is_empty() {
return Some(item);
}
} else { } else {
drop(inner); drop(inner);
let maybe_snapshot = { let maybe_snapshot = {
let mut modified_state = self.storage.modified.get_mut(&task_id).unwrap(); let mut modified_state = self.shard.storage.modified.get_mut(&task_id).unwrap();
let ModifiedState::Snapshot(snapshot) = &mut *modified_state else { let ModifiedState::Snapshot(snapshot) = &mut *modified_state else {
unreachable!("The snapshot bit was set, so it must be in Snapshot state"); unreachable!("The snapshot bit was set, so it must be in Snapshot state");
}; };
snapshot.take() snapshot.take()
}; };
if let Some(snapshot) = maybe_snapshot { if let Some(snapshot) = maybe_snapshot {
return Some((self.process_snapshot)( let item = (self.shard.process_snapshot)(task_id, snapshot, &mut self.buffer);
task_id, if !item.is_empty() {
snapshot, return Some(item);
&mut self.scratch_buffer, }
));
} }
} }
} }
...@@ -442,20 +527,10 @@ where ...@@ -442,20 +527,10 @@ where
} }
} }
impl<'l, P, PS> Iterator for SnapshotShard<'l, P, PS> impl<P, PS> Drop for SnapshotShardIter<'_, P, PS> {
where fn drop(&mut self) {
P: Fn(TaskId, &TaskStorage, &mut TurboBincodeBuffer) -> SnapshotItem + Sync, self.shard
PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> SnapshotItem + Sync, ._guard
{ .return_scratch_buffer(std::mem::take(&mut self.buffer));
type Item = SnapshotItem;
fn next(&mut self) -> Option<Self::Item> {
while let Some(item) = self.next_item() {
if !item.is_empty() {
return Some(item);
}
}
self.guard = None;
None
} }
} }
...@@ -62,7 +62,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -62,7 +62,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<()> ) -> Result<()>
where where
I: Iterator<Item = SnapshotItem> + Send + Sync; I: IntoIterator<Item = SnapshotItem> + Send + Sync;
/// Returns all task IDs that match the given task type (hash collision candidates). /// Returns all task IDs that match the given task type (hash collision candidates).
/// ///
/// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key. /// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key.
...@@ -125,7 +125,7 @@ where ...@@ -125,7 +125,7 @@ where
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<()> ) -> Result<()>
where where
I: Iterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{ {
either::for_both!(self, this => this.save_snapshot( either::for_both!(self, this => this.save_snapshot(
operations, operations,
......
...@@ -231,7 +231,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -231,7 +231,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<()> ) -> Result<()>
where where
I: Iterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{ {
let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered(); let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered();
let batch = self.inner.database.write_batch()?; let batch = self.inner.database.write_batch()?;
...@@ -451,7 +451,7 @@ fn process_task_data<'a, B: ConcurrentWriteBatch<'a> + Send + Sync, I>( ...@@ -451,7 +451,7 @@ fn process_task_data<'a, B: ConcurrentWriteBatch<'a> + Send + Sync, I>(
batch: &B, batch: &B,
) -> Result<()> ) -> Result<()>
where where
I: Iterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{ {
parallel::try_for_each_owned(tasks, |tasks| { parallel::try_for_each_owned(tasks, |tasks| {
for SnapshotItem { for SnapshotItem {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment