Unverified Commit 9cee46cd authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Encode task storage directly during persistence (skip clone) (#89412)

# Don't clone TaskStorage objects when snapshotting the database

## What

Instead of cloning `TaskStorage` objects when preparing to snapshot, we directly encode from a reference to the `TaskStorage` in the backend storage map.

## Why

From a performance profile of `save_snapshot`, I observed that ~22% of the time was spent in `clone_meta_snapshot` or `clone_data_snapshot`, while ~28% of the time was spent in `encode_task_data`. The whole reason we are cloning is so we can drop the lock on the dashmap shard while encoding. In principle this makes sense, however:

* We are only holding *read* locks on the dashmap, so we don't block other encoding threads
* Holding the read locks for slightly longer will just slow down racing writing threads, but those are rare and already incur a high cost due to the extra cloning in `track_modification`. So slowing them down isn't too risky.

## Perf

On a vercel-site cold build i don't actually see a real change in persistence time


Before: 6.6s 7.8s 7.1s
After: 6.9s 6.6s 7.2s

So we are saving cpu time and allocations from this change but the overall task is bound on IO, so we should really only expect a win on more CPU constrained systems.
parent f1f83236
...@@ -63,7 +63,7 @@ use crate::{ ...@@ -63,7 +63,7 @@ use crate::{
storage::Storage, storage::Storage,
storage_schema::{TaskStorage, TaskStorageAccessors}, storage_schema::{TaskStorage, TaskStorageAccessors},
}, },
backing_storage::BackingStorage, backing_storage::{BackingStorage, SnapshotItem},
data::{ data::{
ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState, ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressState, InProgressStateInner, OutputValue, TransientTask, InProgressState, InProgressStateInner, OutputValue, TransientTask,
...@@ -1110,121 +1110,111 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1110,121 +1110,111 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let task_cache_stats: Mutex<FxHashMap<_, TaskCacheStats>> = let task_cache_stats: Mutex<FxHashMap<_, TaskCacheStats>> =
Mutex::new(FxHashMap::default()); Mutex::new(FxHashMap::default());
let preprocess = |task_id: TaskId, inner: &TaskStorage| { // Helper to encode a TaskStorage into a SnapshotItem
if task_id.is_transient() { // encode_meta/encode_data control whether to encode each category
return (None, None); let encode_snapshot_item =
} |task_id: TaskId,
inner: &TaskStorage,
let meta_restored = inner.flags.meta_restored(); encode_meta: bool,
let data_restored = inner.flags.data_restored(); encode_data: bool,
buffer: &mut TurboBincodeBuffer| {
// Encode meta/data directly from TaskStorage let encode_category = |task_id: TaskId,
let meta = meta_restored.then(|| inner.clone_meta_snapshot()); data: &TaskStorage,
let data = data_restored.then(|| inner.clone_data_snapshot()); category: SpecificTaskDataCategory,
buffer: &mut TurboBincodeBuffer|
(meta, data) -> Option<TurboBincodeBuffer> {
}; match encode_task_data(task_id, data, category, buffer) {
let process = |task_id: TaskId, Ok(encoded) => {
(meta, data): (Option<TaskStorage>, Option<TaskStorage>), #[cfg(feature = "print_cache_item_size")]
buffer: &mut TurboBincodeBuffer| { {
#[cfg(feature = "print_cache_item_size")] let mut stats = task_cache_stats.lock();
if let Some(ref m) = meta { let entry = stats
task_cache_stats .entry(self.get_task_name(task_id, turbo_tasks))
.lock() .or_default();
.entry(self.get_task_name(task_id, turbo_tasks)) match category {
.or_default() SpecificTaskDataCategory::Meta => entry.add_meta(&encoded),
.add_counts(m); SpecificTaskDataCategory::Data => entry.add_data(&encoded),
} }
( }
task_id, Some(encoded)
meta.map(|d| encode_task_data(task_id, &d, SpecificTaskDataCategory::Meta, buffer)), }
data.map(|d| encode_task_data(task_id, &d, SpecificTaskDataCategory::Data, buffer)), Err(err) => {
) eprintln!(
}; "Serializing task {} failed ({:?}): {:?}",
let process_snapshot = self.debug_get_task_description(task_id),
|task_id: TaskId, inner: Box<TaskStorage>, buffer: &mut TurboBincodeBuffer| { category,
err
);
None
}
}
};
if task_id.is_transient() { if task_id.is_transient() {
return (task_id, None, None); return SnapshotItem {
task_id,
data: None,
meta: None,
};
} }
#[cfg(feature = "print_cache_item_size")] #[cfg(feature = "print_cache_item_size")]
if inner.flags.meta_modified() { if encode_meta {
task_cache_stats task_cache_stats
.lock() .lock()
.entry(self.get_task_name(task_id, turbo_tasks)) .entry(self.get_task_name(task_id, turbo_tasks))
.or_default() .or_default()
.add_counts(&inner); .add_counts(inner);
} }
// Encode meta/data directly from TaskStorage snapshot let meta = if encode_meta {
( encode_category(task_id, inner, SpecificTaskDataCategory::Meta, buffer)
} else {
None
};
let data = if encode_data {
encode_category(task_id, inner, SpecificTaskDataCategory::Data, buffer)
} else {
None
};
SnapshotItem {
task_id,
meta,
data,
}
};
// Process tasks from the main storage map (uses restored flags)
let process = |task_id: TaskId, inner: &TaskStorage, buffer: &mut TurboBincodeBuffer| {
encode_snapshot_item(
task_id,
inner,
inner.flags.meta_restored(),
inner.flags.data_restored(),
buffer,
)
};
// Process tasks that were accessed during snapshot mode (uses modified flags)
let process_snapshot =
|task_id: TaskId, inner: Box<TaskStorage>, buffer: &mut TurboBincodeBuffer| {
encode_snapshot_item(
task_id, task_id,
inner.flags.meta_modified().then(|| { &inner,
encode_task_data(task_id, &inner, SpecificTaskDataCategory::Meta, buffer) inner.flags.meta_modified(),
}), inner.flags.data_modified(),
inner.flags.data_modified().then(|| { buffer,
encode_task_data(task_id, &inner, SpecificTaskDataCategory::Data, buffer)
}),
) )
}; };
let snapshot = self let snapshot = self.storage.take_snapshot(&process, &process_snapshot);
.storage
.take_snapshot(&preprocess, &process, &process_snapshot);
let task_snapshots = snapshot let task_snapshots = snapshot
.into_iter() .into_iter()
.filter_map(|iter| { .filter_map(|iter| {
let mut iter = iter let mut iter = iter
.filter_map( .filter_map(|item: SnapshotItem| (!item.is_empty()).then_some(item))
|(task_id, meta, data): (
_,
Option<Result<SmallVec<_>>>,
Option<Result<SmallVec<_>>>,
)| {
let meta = match meta {
Some(Ok(meta)) => {
#[cfg(feature = "print_cache_item_size")]
task_cache_stats
.lock()
.entry(self.get_task_name(task_id, turbo_tasks))
.or_default()
.add_meta(&meta);
Some(meta)
}
None => None,
Some(Err(err)) => {
println!(
"Serializing task {} failed (meta): {:?}",
self.debug_get_task_description(task_id),
err
);
None
}
};
let data = match data {
Some(Ok(data)) => {
#[cfg(feature = "print_cache_item_size")]
task_cache_stats
.lock()
.entry(self.get_task_name(task_id, turbo_tasks))
.or_default()
.add_data(&data);
Some(data)
}
None => None,
Some(Err(err)) => {
println!(
"Serializing task {} failed (data): {:?}",
self.debug_get_task_description(task_id),
err
);
None
}
};
(meta.is_some() || data.is_some()).then_some((task_id, meta, data))
},
)
.peekable(); .peekable();
iter.peek().is_some().then_some(iter) iter.peek().is_some().then_some(iter)
}) })
......
...@@ -105,23 +105,22 @@ impl Storage { ...@@ -105,23 +105,22 @@ impl Storage {
/// Processes every modified item (resp. a snapshot of it) with the given functions and returns /// Processes every modified item (resp. a snapshot of it) with the given functions and returns
/// the results. Ends snapshot mode afterwards. /// the results. Ends snapshot mode afterwards.
/// preprocess is potentially called within a lock, so it should be fast. /// process is called while holding a read lock on the task storage, so it can access
/// process is called outside of locks, so it could do more expensive operations. /// the TaskStorage directly without cloning.
/// Both process and process_snapshot receive a mutable scratch buffer that can be reused /// process_snapshot is called for tasks that were accessed during snapshot mode and
/// across iterations to avoid repeated allocations. /// receives an owned Box<TaskStorage> snapshot.
/// Both callbacks receive a mutable scratch buffer that can be reused across iterations
/// to avoid repeated allocations.
pub fn take_snapshot< pub fn take_snapshot<
'l, 'l,
T,
R, R,
PP: for<'a> Fn(TaskId, &'a TaskStorage) -> T + Sync, P: for<'a> Fn(TaskId, &'a TaskStorage, &mut TurboBincodeBuffer) -> R + Sync,
P: Fn(TaskId, T, &mut TurboBincodeBuffer) -> R + Sync,
PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> R + Sync, PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> R + Sync,
>( >(
&'l self, &'l self,
preprocess: &'l PP,
process: &'l P, process: &'l P,
process_snapshot: &'l PS, process_snapshot: &'l PS,
) -> Vec<SnapshotShard<'l, PP, P, PS>> { ) -> Vec<SnapshotShard<'l, P, PS>> {
if !self.snapshot_mode() { if !self.snapshot_mode() {
self.start_snapshot(); self.start_snapshot();
} }
...@@ -165,7 +164,6 @@ impl Storage { ...@@ -165,7 +164,6 @@ impl Storage {
storage: self, storage: self,
guard: Some(guard.clone()), guard: Some(guard.clone()),
process, process,
preprocess,
process_snapshot, process_snapshot,
scratch_buffer: TurboBincodeBuffer::with_capacity(SCRATCH_BUFFER_SIZE), scratch_buffer: TurboBincodeBuffer::with_capacity(SCRATCH_BUFFER_SIZE),
} }
...@@ -379,22 +377,20 @@ impl Drop for SnapshotGuard<'_> { ...@@ -379,22 +377,20 @@ impl Drop for SnapshotGuard<'_> {
} }
} }
pub struct SnapshotShard<'l, PP, P, PS> { pub struct SnapshotShard<'l, P, PS> {
direct_snapshots: Vec<(TaskId, Box<TaskStorage>)>, direct_snapshots: Vec<(TaskId, Box<TaskStorage>)>,
modified: SmallVec<[TaskId; 4]>, modified: SmallVec<[TaskId; 4]>,
storage: &'l Storage, storage: &'l Storage,
guard: Option<Arc<SnapshotGuard<'l>>>, guard: Option<Arc<SnapshotGuard<'l>>>,
process: &'l P, process: &'l P,
preprocess: &'l PP,
process_snapshot: &'l PS, process_snapshot: &'l PS,
/// Scratch buffer for encoding task data, reused across iterations to avoid allocations /// Scratch buffer for encoding task data, reused across iterations to avoid allocations
scratch_buffer: TurboBincodeBuffer, scratch_buffer: TurboBincodeBuffer,
} }
impl<'l, T, R, PP, P, PS> Iterator for SnapshotShard<'l, PP, P, PS> impl<'l, R, P, PS> Iterator for SnapshotShard<'l, P, PS>
where where
PP: for<'a> Fn(TaskId, &'a TaskStorage) -> T + Sync, P: for<'a> Fn(TaskId, &'a TaskStorage, &mut TurboBincodeBuffer) -> R + Sync,
P: Fn(TaskId, T, &mut TurboBincodeBuffer) -> R + Sync,
PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> R + Sync, PS: Fn(TaskId, Box<TaskStorage>, &mut TurboBincodeBuffer) -> R + Sync,
{ {
type Item = R; type Item = R;
...@@ -410,13 +406,7 @@ where ...@@ -410,13 +406,7 @@ where
while let Some(task_id) = self.modified.pop() { while let Some(task_id) = self.modified.pop() {
let inner = self.storage.map.get(&task_id).unwrap(); let inner = self.storage.map.get(&task_id).unwrap();
if !inner.flags.any_snapshot() { if !inner.flags.any_snapshot() {
let preprocessed = (self.preprocess)(task_id, &inner); return Some((self.process)(task_id, &inner, &mut self.scratch_buffer));
drop(inner);
return Some((self.process)(
task_id,
preprocessed,
&mut self.scratch_buffer,
));
} else { } else {
drop(inner); drop(inner);
let maybe_snapshot = { let maybe_snapshot = {
......
...@@ -524,14 +524,6 @@ impl TaskStorage { ...@@ -524,14 +524,6 @@ impl TaskStorage {
} }
} }
/// Clone only the fields for the specified category
pub fn clone_category_snapshot(&self, category: SpecificTaskDataCategory) -> TaskStorage {
match category {
SpecificTaskDataCategory::Meta => self.clone_meta_snapshot(),
SpecificTaskDataCategory::Data => self.clone_data_snapshot(),
}
}
/// Initialize a transient task with the given root type and activeness tracking. /// Initialize a transient task with the given root type and activeness tracking.
/// ///
/// This sets up the activeness state for root/once tasks. /// This sets up the activeness state for root/once tasks.
......
...@@ -3,6 +3,7 @@ use std::{any::type_name, sync::Arc}; ...@@ -3,6 +3,7 @@ use std::{any::type_name, sync::Arc};
use anyhow::Result; use anyhow::Result;
use either::Either; use either::Either;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{TaskId, backend::CachedTaskType}; use turbo_tasks::{TaskId, backend::CachedTaskType};
use crate::{ use crate::{
...@@ -10,6 +11,18 @@ use crate::{ ...@@ -10,6 +11,18 @@ use crate::{
utils::chunked_vec::ChunkedVec, utils::chunked_vec::ChunkedVec,
}; };
pub struct SnapshotItem {
pub task_id: TaskId,
pub data: Option<TurboBincodeBuffer>,
pub meta: Option<TurboBincodeBuffer>,
}
impl SnapshotItem {
pub fn is_empty(&self) -> bool {
self.meta.is_none() && self.data.is_none()
}
}
/// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by /// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by
/// [`default_backing_storage`] or [`noop_backing_storage`]. /// [`default_backing_storage`] or [`noop_backing_storage`].
/// ///
...@@ -51,14 +64,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -51,14 +64,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<()> ) -> Result<()>
where where
I: Iterator< I: Iterator<Item = SnapshotItem> + Send + Sync;
Item = (
TaskId,
Option<SmallVec<[u8; 16]>>,
Option<SmallVec<[u8; 16]>>,
),
> + Send
+ Sync;
fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>; fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>;
/// Returns all task IDs that match the given task type (hash collision candidates). /// Returns all task IDs that match the given task type (hash collision candidates).
/// ///
...@@ -134,14 +140,7 @@ where ...@@ -134,14 +140,7 @@ where
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<()> ) -> Result<()>
where where
I: Iterator< I: Iterator<Item = SnapshotItem> + Send + Sync,
Item = (
TaskId,
Option<SmallVec<[u8; 16]>>,
Option<SmallVec<[u8; 16]>>,
),
> + Send
+ Sync,
{ {
either::for_both!(self, this => this.save_snapshot( either::for_both!(self, this => this.save_snapshot(
operations, operations,
......
...@@ -7,9 +7,7 @@ use std::{ ...@@ -7,9 +7,7 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::{ use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode};
TurboBincodeBuffer, new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode,
};
use turbo_tasks::{ use turbo_tasks::{
TaskId, TaskId,
backend::CachedTaskType, backend::CachedTaskType,
...@@ -21,7 +19,7 @@ use turbo_tasks_hash::Xxh3Hash64Hasher; ...@@ -21,7 +19,7 @@ use turbo_tasks_hash::Xxh3Hash64Hasher;
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage}, backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backing_storage::{BackingStorage, BackingStorageSealed}, backing_storage::{BackingStorage, BackingStorageSealed, SnapshotItem},
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
db_versioning::handle_db_versioning, db_versioning::handle_db_versioning,
...@@ -258,14 +256,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -258,14 +256,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<()> ) -> Result<()>
where where
I: Iterator< I: Iterator<Item = SnapshotItem> + Send + Sync,
Item = (
TaskId,
Option<TurboBincodeBuffer>,
Option<TurboBincodeBuffer>,
),
> + Send
+ Sync,
{ {
let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered(); let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered();
let mut batch = self.inner.database.write_batch()?; let mut batch = self.inner.database.write_batch()?;
...@@ -607,18 +598,16 @@ fn process_task_data<'a, B: ConcurrentWriteBatch<'a> + Send + Sync, I>( ...@@ -607,18 +598,16 @@ fn process_task_data<'a, B: ConcurrentWriteBatch<'a> + Send + Sync, I>(
batch: Option<&B>, batch: Option<&B>,
) -> Result<SerializedTasks> ) -> Result<SerializedTasks>
where where
I: Iterator< I: Iterator<Item = SnapshotItem> + Send + Sync,
Item = (
TaskId,
Option<TurboBincodeBuffer>,
Option<TurboBincodeBuffer>,
),
> + Send
+ Sync,
{ {
parallel::map_collect_owned::<_, _, Result<Vec<_>>>(tasks, |tasks| { parallel::map_collect_owned::<_, _, Result<Vec<_>>>(tasks, |tasks| {
let mut result = Vec::new(); let mut result = Vec::new();
for (task_id, meta, data) in tasks { for SnapshotItem {
task_id,
meta,
data,
} in tasks
{
if let Some(batch) = batch { if let Some(batch) = batch {
let key = IntKey::new(*task_id); let key = IntKey::new(*task_id);
let key = key.as_ref(); let key = key.as_ref();
......
...@@ -3111,25 +3111,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3111,25 +3111,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
let restore_meta_inline = gen_restore_inline_for_category(grouped_fields, Category::Meta); let restore_meta_inline = gen_restore_inline_for_category(grouped_fields, Category::Meta);
let restore_data_inline = gen_restore_inline_for_category(grouped_fields, Category::Data); let restore_data_inline = gen_restore_inline_for_category(grouped_fields, Category::Data);
// Generate flags handling for clone - per category
let clone_meta_flags = if has_meta_flags {
quote! {
// Clone persisted meta flags
snapshot.flags.set_persisted_meta_bits(self.flags.persisted_meta_bits());
}
} else {
quote! {}
};
let clone_data_flags = if has_data_flags {
quote! {
// Clone persisted data flags
snapshot.flags.set_persisted_data_bits(self.flags.persisted_data_bits());
}
} else {
quote! {}
};
let clone_all_flags = if has_any_flags { let clone_all_flags = if has_any_flags {
quote! { quote! {
// Clone all persisted flags // Clone all persisted flags
...@@ -3170,11 +3151,7 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3170,11 +3151,7 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
quote! { quote! {
#[automatically_derived] #[automatically_derived]
impl TaskStorage { impl TaskStorage {
/// Create a snapshot containing all persistent fields (both meta and data). /// Create a snapshot containing all persistent fields
///
/// This clones all persistent fields into a new TaskStorage, skipping
/// transient fields that may not be cloneable. Use this for the `Both`
/// snapshot case where both meta and data are dirty.
pub fn clone_snapshot(&self) -> TaskStorage { pub fn clone_snapshot(&self) -> TaskStorage {
let mut snapshot = TaskStorage::new(); let mut snapshot = TaskStorage::new();
...@@ -3186,6 +3163,9 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3186,6 +3163,9 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
#clone_all_flags #clone_all_flags
// Pre-allocate lazy vec (upper bound - some may be transient and skipped)
snapshot.lazy.reserve(self.lazy.len());
// Clone all persistent lazy fields (both meta and data) // Clone all persistent lazy fields (both meta and data)
for field in &self.lazy { for field in &self.lazy {
match field { match field {
...@@ -3199,54 +3179,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3199,54 +3179,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
snapshot snapshot
} }
/// Create a snapshot containing only meta category fields for serialization.
///
/// This clones only the persistent meta fields into a new TaskStorage,
/// which can then be serialized outside the lock.
pub fn clone_meta_snapshot(&self) -> TaskStorage {
let mut snapshot = TaskStorage::new();
// Clone inline meta fields
#(#clone_meta_inline)*
#clone_meta_flags
// Clone lazy meta fields (only persistent ones)
for field in &self.lazy {
match field {
#(#clone_meta_lazy_arms)*
// Skip transient and data fields
_ => {}
}
}
snapshot
}
/// Create a snapshot containing only data category fields for serialization.
///
/// This clones only the persistent data fields into a new TaskStorage,
/// which can then be serialized outside the lock.
pub fn clone_data_snapshot(&self) -> TaskStorage {
let mut snapshot = TaskStorage::new();
// Clone inline data fields
#(#clone_data_inline)*
#clone_data_flags
// Clone lazy data fields (only persistent ones)
for field in &self.lazy {
match field {
#(#clone_data_lazy_arms)*
// Skip transient and meta fields
_ => {}
}
}
snapshot
}
/// Restore persisted data from a decoded TaskStorage. /// Restore persisted data from a decoded TaskStorage.
/// ///
/// This is used during restore operations to copy decoded persisted data /// This is used during restore operations to copy decoded persisted data
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment