Unverified Commit a572b53c authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks: bincode-encode copy-on-write snapshots instead of cloning (#99644)

### What?

When a task that is already part of an in-progress persistence snapshot
is modified again, `track_modification_internal` now bincode-encodes the
task's pre-mutation state into the `SnapshotItem` that persistence will
write, and stores that in the `snapshots` map. Before, it stored a
`clone_snapshot()` copy of the `TaskStorage`.

### Why?

The copy-on-write path exists so persistence sees a consistent snapshot
even if a task is mutated while the snapshot is being written. Cloning
`TaskStorage` doesn't guarantee that: cell contents are held behind
`SharedReference` (`Arc`), so the clone shares them with the live task.
If a cell value has interior mutability, a later mutation shows up in
the "frozen" copy and the persisted state is no longer consistent.

Encoding the state right away captures plain bytes that later mutations
can't reach. The snapshot iterator hands the stored item straight to
persistence, so nothing is encoded twice. The only cost is holding the
encoded buffers in memory until the racing persistence picks them up,
and this race is rare.

<!-- NEXT_JS_LLM -->


<!-- fleet 98a896e8-00fa-4f81-97f0-48bd49e2a9a2 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent d43f2e6f
...@@ -1911,6 +1911,8 @@ mod must_exist_tests { ...@@ -1911,6 +1911,8 @@ mod must_exist_tests {
data: Some(encoded_data.clone()), data: Some(encoded_data.clone()),
meta: Some(encoded_meta.clone()), meta: Some(encoded_meta.clone()),
task_type_hash: None, task_type_hash: None,
#[cfg(feature = "print_cache_item_size")]
stats: Default::default(),
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
backing.save_snapshot(None, vec![items]).unwrap(); backing.save_snapshot(None, vec![items]).unwrap();
...@@ -2104,7 +2106,10 @@ mod filter_transient_tracking_tests { ...@@ -2104,7 +2106,10 @@ mod filter_transient_tracking_tests {
use super::*; use super::*;
use crate::{ use crate::{
backend::{TaskDataCategory, storage::Storage}, backend::{
TaskDataCategory,
storage::{Storage, StorageOptions},
},
data::{CellRef, OutputValue}, data::{CellRef, OutputValue},
}; };
...@@ -2140,7 +2145,7 @@ mod filter_transient_tracking_tests { ...@@ -2140,7 +2145,7 @@ mod filter_transient_tracking_tests {
#[test] #[test]
fn autoset_add_remove_only_tracks_persistent_keys() { fn autoset_add_remove_only_tracks_persistent_keys() {
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
let task_id = persistent_task(1); let task_id = persistent_task(1);
// `children` is an AutoSet<TaskId> meta field with filter_transient. // `children` is an AutoSet<TaskId> meta field with filter_transient.
...@@ -2190,7 +2195,7 @@ mod filter_transient_tracking_tests { ...@@ -2190,7 +2195,7 @@ mod filter_transient_tracking_tests {
#[test] #[test]
fn autoset_extend_tracks_iff_any_persistent() { fn autoset_extend_tracks_iff_any_persistent() {
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
// Extend with only transient children: no meta modification. // Extend with only transient children: no meta modification.
let task_id = persistent_task(1); let task_id = persistent_task(1);
...@@ -2218,7 +2223,7 @@ mod filter_transient_tracking_tests { ...@@ -2218,7 +2223,7 @@ mod filter_transient_tracking_tests {
#[test] #[test]
fn countermap_update_count_only_tracks_persistent_keys() { fn countermap_update_count_only_tracks_persistent_keys() {
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
let task_id = persistent_task(1); let task_id = persistent_task(1);
// `upper` is a CounterMap<TaskId> meta field with filter_transient. // `upper` is a CounterMap<TaskId> meta field with filter_transient.
...@@ -2241,7 +2246,7 @@ mod filter_transient_tracking_tests { ...@@ -2241,7 +2246,7 @@ mod filter_transient_tracking_tests {
#[test] #[test]
fn countermap_update_counts_batch_tracks_iff_any_persistent() { fn countermap_update_counts_batch_tracks_iff_any_persistent() {
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
// followers: CounterMap<TaskId>, filter_transient, has update_counts. // followers: CounterMap<TaskId>, filter_transient, has update_counts.
let task_id = persistent_task(1); let task_id = persistent_task(1);
...@@ -2267,7 +2272,7 @@ mod filter_transient_tracking_tests { ...@@ -2267,7 +2272,7 @@ mod filter_transient_tracking_tests {
#[test] #[test]
fn direct_option_set_take_tracks_by_value_transience() { fn direct_option_set_take_tracks_by_value_transience() {
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
// `output` is a direct Option<OutputValue> meta field with filter_transient. // `output` is a direct Option<OutputValue> meta field with filter_transient.
// Setting a transient output: no meta modification. // Setting a transient output: no meta modification.
...@@ -2352,19 +2357,40 @@ mod filter_transient_tracking_tests { ...@@ -2352,19 +2357,40 @@ mod filter_transient_tracking_tests {
mod cell_data_tracking_tests { mod cell_data_tracking_tests {
//! `cell_data` writes track a Data modification only when the cell's value //! `cell_data` writes track a Data modification only when the cell's value
//! type persists something. //! type persists something.
use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{ use turbo_tasks::{
self as turbo_tasks, CellId, TRANSIENT_TASK_BIT, TaskId, ValueTypePersistence, VcValueType, self as turbo_tasks, CellId, TRANSIENT_TASK_BIT, TaskId, ValueTypePersistence, VcValueType,
registry, registry,
}; };
use super::*; use super::*;
use crate::backend::{ use crate::{
TaskDataCategory, storage::Storage, storage_schema::TaskStorageAccessors, backend::{
TaskDataCategory,
storage::{Storage, StorageOptions, encode_task_contents},
storage_schema::TaskStorageAccessors,
},
backing_storage::SnapshotItem,
}; };
#[turbo_tasks::value] #[turbo_tasks::value]
struct PersistableV(#[allow(dead_code)] u32); struct PersistableV(#[allow(dead_code)] u32);
#[turbo_tasks::value(eq = "manual")]
struct InteriorMutableV {
#[bincode(with = "turbo_tasks::parking_lot_mutex_bincode")]
#[turbo_tasks(debug_ignore)]
value: parking_lot::Mutex<u32>,
}
impl PartialEq for InteriorMutableV {
fn eq(&self, _other: &Self) -> bool {
false
}
}
impl Eq for InteriorMutableV {}
#[turbo_tasks::value(serialization = "skip")] #[turbo_tasks::value(serialization = "skip")]
struct SkipCheapV(#[allow(dead_code)] u32); struct SkipCheapV(#[allow(dead_code)] u32);
...@@ -2409,7 +2435,7 @@ mod cell_data_tracking_tests { ...@@ -2409,7 +2435,7 @@ mod cell_data_tracking_tests {
// (HashOnly persists via the separate `cell_data_hash` field) — but both // (HashOnly persists via the separate `cell_data_hash` field) — but both
// are still stored in memory. Tracking is monotonic: a later persistable // are still stored in memory. Tracking is monotonic: a later persistable
// write flips the flag even after skipped writes left it clean. // write flips the flag even after skipped writes left it clean.
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
let mut g = guard_for(&storage, persistent_task(1)); let mut g = guard_for(&storage, persistent_task(1));
let skip = cell_of::<SkipCheapV>(0); let skip = cell_of::<SkipCheapV>(0);
...@@ -2432,7 +2458,7 @@ mod cell_data_tracking_tests { ...@@ -2432,7 +2458,7 @@ mod cell_data_tracking_tests {
#[test] #[test]
fn remove_tracks_only_for_persistable() { fn remove_tracks_only_for_persistable() {
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
let skip = cell_of::<SkipCheapV>(0); let skip = cell_of::<SkipCheapV>(0);
let persistable = cell_of::<PersistableV>(0); let persistable = cell_of::<PersistableV>(0);
...@@ -2471,7 +2497,7 @@ mod cell_data_tracking_tests { ...@@ -2471,7 +2497,7 @@ mod cell_data_tracking_tests {
// `drop_partial` (which keys on Evictability, not the modified flag), so // `drop_partial` (which keys on Evictability, not the modified flag), so
// a task that only wrote such a cell is both evictable-clean AND keeps the // a task that only wrote such a cell is both evictable-clean AND keeps the
// value. This is the core safety property of not tracking Skip writes. // value. This is the core safety property of not tracking Skip writes.
let storage = Storage::new(2, true); let storage = Storage::new(StorageOptions::for_tests());
let task_id = persistent_task(1); let task_id = persistent_task(1);
let cell = cell_of::<SkipNeverV>(0); let cell = cell_of::<SkipNeverV>(0);
...@@ -2495,4 +2521,105 @@ mod cell_data_tracking_tests { ...@@ -2495,4 +2521,105 @@ mod cell_data_tracking_tests {
"Skip + evict=never cell must survive eviction even though the task was never modified" "Skip + evict=never cell must survive eviction even though the task was never modified"
); );
} }
/// Regression test for copy-on-write snapshots with interior-mutable cell contents.
///
/// Cloning `TaskStorage` only clones the cell's `SharedReference` pointer, so a mutation
/// through the shared value would also change the supposedly frozen snapshot copy. The
/// pre-mutation state must instead be encoded when the task is modified during a snapshot.
#[tokio::test(flavor = "multi_thread")]
async fn modify_during_snapshot_encodes_interior_mutable_cell_contents() {
let storage = Storage::new(StorageOptions::for_tests());
let task_id = persistent_task(1);
let cell = cell_of::<InteriorMutableV>(0);
let value = SharedReference::new(triomphe::Arc::new(InteriorMutableV {
value: parking_lot::Mutex::new(1),
}));
{
let mut g = guard_for(&storage, task_id);
g.insert_cell_data(cell, value.clone(), persistence_of(&cell));
assert!(g.data_modified());
}
let expected = {
let task = storage.access_mut(task_id);
encode_task_contents(
task_id,
&task,
SpecificTaskDataCategory::Data,
&mut TurboBincodeBuffer::new(),
)
.unwrap()
};
let (snapshot_guard, has_modifications) = storage.start_snapshot();
assert!(has_modifications);
let process = |_: TaskId, _: &TaskStorage, _: &mut TurboBincodeBuffer| -> SnapshotItem {
panic!("the pre-encoded snapshot item must be used")
};
let shards = storage.take_snapshot(snapshot_guard, &process, &|_| {}, false);
{
let mut g = guard_for(&storage, task_id);
let _ = g.track_modification(SpecificTaskDataCategory::Data, "test");
*value
.downcast_ref::<InteriorMutableV>()
.unwrap()
.value
.lock() = 2;
}
let items: Vec<_> = shards.into_iter().flatten().collect();
assert_eq!(items.len(), 1);
let SnapshotItem::Put { data, .. } = &items[0] else {
panic!("expected a Put item");
};
assert_eq!(data.as_deref(), Some(&expected[..]));
let live = {
let task = storage.access_mut(task_id);
encode_task_contents(
task_id,
&task,
SpecificTaskDataCategory::Data,
&mut TurboBincodeBuffer::new(),
)
.unwrap()
};
assert_ne!(live, expected, "the shared cell value really changed");
}
/// A pre-encoded (copy-on-write) item carries the cache size stats of the state it encoded,
/// not the live state after the mutation that triggered the copy.
#[cfg(feature = "print_cache_item_size")]
#[tokio::test(flavor = "multi_thread")]
async fn pre_encoded_item_carries_stats_captured_at_copy_time() {
let storage = Storage::new(StorageOptions::for_tests());
let task_id = persistent_task(1);
{
let mut g = guard_for(&storage, task_id);
let _ = g.track_modification(SpecificTaskDataCategory::Meta, "test");
}
let (snapshot_guard, _) = storage.start_snapshot();
let process = |_: TaskId, _: &TaskStorage, _: &mut TurboBincodeBuffer| -> SnapshotItem {
panic!("the pre-encoded snapshot item must be used")
};
let shards = storage.take_snapshot(snapshot_guard, &process, &|_| {}, false);
// The tracked add copies the task (no children yet) before inserting the child.
{
let mut g = guard_for(&storage, task_id);
assert!(g.add_children(persistent_task(2)));
}
let items: Vec<_> = shards.into_iter().flatten().collect();
assert_eq!(items.len(), 1);
let SnapshotItem::Put { stats, .. } = &items[0] else {
panic!("expected a Put item");
};
assert_eq!(stats.counts.children, 0);
assert_eq!(storage.access_mut(task_id).meta_counts().children, 1);
}
} }
...@@ -21,6 +21,9 @@ pub enum SnapshotItem { ...@@ -21,6 +21,9 @@ pub enum SnapshotItem {
data: Option<TurboBincodeBuffer>, data: Option<TurboBincodeBuffer>,
/// Task type for new tasks that need to be added to the task cache /// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>, task_type_hash: Option<TaskTypeHash>,
/// Cache size statistics captured when the item was encoded.
#[cfg(feature = "print_cache_item_size")]
stats: Box<SnapshotItemStats>,
}, },
Delete { Delete {
task_id: TaskId, task_id: TaskId,
...@@ -30,6 +33,18 @@ pub enum SnapshotItem { ...@@ -30,6 +33,18 @@ pub enum SnapshotItem {
}, },
} }
/// Per-task cache size statistics, captured by `encode_snapshot_item` from the same task state
/// that was encoded, so pre-encoded (copy-on-write) items report what was actually persisted.
#[cfg(feature = "print_cache_item_size")]
#[derive(Default)]
pub struct SnapshotItemStats {
/// The task name, used as the stats grouping key.
pub task_name: String,
pub counts: crate::backend::storage_schema::MetaCounts,
/// Size of the encoded task output, if any.
pub output_size: usize,
}
impl SnapshotItem { impl SnapshotItem {
/// The task this item persists or tombstones. (Currently only used by tests, which assert on /// The task this item persists or tombstones. (Currently only used by tests, which assert on
/// the id of items yielded by the snapshot iterator.) /// the id of items yielded by the snapshot iterator.)
......
...@@ -276,6 +276,7 @@ impl TurboBackingStorage { ...@@ -276,6 +276,7 @@ impl TurboBackingStorage {
meta, meta,
data, data,
task_type_hash, task_type_hash,
..
} => { } => {
let key = IntKey::new(*task_id); let key = IntKey::new(*task_id);
let key = key.as_ref(); let key = key.as_ref();
......
...@@ -898,18 +898,6 @@ impl GroupedFields { ...@@ -898,18 +898,6 @@ impl GroupedFields {
// Code Generation Helpers // Code Generation Helpers
// ============================================================================= // =============================================================================
/// Generate inline field clone assignments: `snapshot.field = self.field.clone();`
fn gen_clone_inline_fields<'a>(fields: impl Iterator<Item = &'a FieldInfo>) -> Vec<TokenStream> {
fields
.map(|field| {
let field_name = &field.field_name;
quote! {
snapshot.#field_name = self.#field_name.clone();
}
})
.collect()
}
fn gen_restore_inline_field(field: &FieldInfo) -> TokenStream { fn gen_restore_inline_field(field: &FieldInfo) -> TokenStream {
let field_name = &field.field_name; let field_name = &field.field_name;
if !field.filter_transient { if !field.filter_transient {
...@@ -3844,25 +3832,6 @@ fn generate_decode_lazy_fields(fields: &[&FieldInfo]) -> TokenStream { ...@@ -3844,25 +3832,6 @@ fn generate_decode_lazy_fields(fields: &[&FieldInfo]) -> TokenStream {
} }
} }
/// Generate clone inline statements for a category.
fn gen_clone_inline_for_category(
grouped_fields: &GroupedFields,
category: Category,
) -> Vec<TokenStream> {
gen_clone_inline_fields(grouped_fields.persistent_inline(category))
}
/// Generate clone lazy match arms for a category.
fn gen_clone_lazy_arms_for_category(
grouped_fields: &GroupedFields,
category: Category,
) -> Vec<TokenStream> {
gen_lazy_match_arms(grouped_fields.persistent_lazy(category), |_, field| {
let variant_name = &field.variant_name;
quote! { snapshot.lazy.push(LazyField::#variant_name(data.clone())); }
})
}
/// Generate restore inline statements for a category. /// Generate restore inline statements for a category.
fn gen_restore_inline_for_category( fn gen_restore_inline_for_category(
grouped_fields: &GroupedFields, grouped_fields: &GroupedFields,
...@@ -3874,11 +3843,9 @@ fn gen_restore_inline_for_category( ...@@ -3874,11 +3843,9 @@ fn gen_restore_inline_for_category(
.collect() .collect()
} }
/// Generate snapshot clone and restore methods for TaskStorage. /// Generate snapshot restore methods for TaskStorage.
/// ///
/// Generates: /// Generates:
/// - `clone_meta_snapshot(&self) -> TaskStorage` - Clone only persistent meta fields
/// - `clone_data_snapshot(&self) -> TaskStorage` - Clone only persistent data fields
/// - `restore_from(&mut self, source, category)` - Restore data by category from decoded storage /// - `restore_from(&mut self, source, category)` - Restore data by category from decoded storage
/// - `restore_meta_from(&mut self, source)` - Restore meta fields from source /// - `restore_meta_from(&mut self, source)` - Restore meta fields from source
/// - `restore_data_from(&mut self, source)` - Restore data fields from source /// - `restore_data_from(&mut self, source)` - Restore data fields from source
...@@ -3886,14 +3853,8 @@ fn gen_restore_inline_for_category( ...@@ -3886,14 +3853,8 @@ fn gen_restore_inline_for_category(
fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStream { fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStream {
let has_meta_flags = grouped_fields.persisted_meta_flags().next().is_some(); let has_meta_flags = grouped_fields.persisted_meta_flags().next().is_some();
let has_data_flags = grouped_fields.persisted_data_flags().next().is_some(); let has_data_flags = grouped_fields.persisted_data_flags().next().is_some();
let has_any_flags = has_meta_flags || has_data_flags;
// Generate field operations by category // Generate field operations by category
let clone_meta_inline = gen_clone_inline_for_category(grouped_fields, Category::Meta);
let clone_data_inline = gen_clone_inline_for_category(grouped_fields, Category::Data);
let clone_meta_lazy_arms = gen_clone_lazy_arms_for_category(grouped_fields, Category::Meta);
let clone_data_lazy_arms = gen_clone_lazy_arms_for_category(grouped_fields, Category::Data);
let restore_meta_inline = gen_restore_inline_for_category(grouped_fields, Category::Meta); let restore_meta_inline = gen_restore_inline_for_category(grouped_fields, Category::Meta);
let restore_data_inline = gen_restore_inline_for_category(grouped_fields, Category::Data); let restore_data_inline = gen_restore_inline_for_category(grouped_fields, Category::Data);
...@@ -3912,15 +3873,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3912,15 +3873,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
.map(|(idx, f)| gen_restore_lazy_merge_arm(f, idx as u8)) .map(|(idx, f)| gen_restore_lazy_merge_arm(f, idx as u8))
.collect(); .collect();
let clone_all_flags = if has_any_flags {
quote! {
// Clone all persisted flags
snapshot.flags.set_persisted_bits(self.flags.persisted_bits());
}
} else {
quote! {}
};
// Generate flags handling for restore - per category // Generate flags handling for restore - per category
let restore_meta_flags = if has_meta_flags { let restore_meta_flags = if has_meta_flags {
quote! { quote! {
...@@ -3943,34 +3895,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -3943,34 +3895,6 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
quote! { quote! {
#[automatically_derived] #[automatically_derived]
impl TaskStorage { impl TaskStorage {
/// Create a snapshot containing all persistent fields
pub fn clone_snapshot(&self) -> TaskStorage {
let mut snapshot = TaskStorage::new();
// Clone inline meta fields
#(#clone_meta_inline)*
// Clone inline data fields
#(#clone_data_inline)*
#clone_all_flags
// Clone all persistent lazy fields (both meta and data).
// (No pre-`reserve`: the schema has ≤24 lazy fields, so at most 3 grows
// (0→4→8→16→24) total — cheaper than complicating the public API surface
// of `TinyVec`.)
for field in &self.lazy {
match field {
#(#clone_data_lazy_arms)*
#(#clone_meta_lazy_arms)*
// Skip transient fields
_ => {}
}
}
snapshot
}
/// Restore persisted data from a decoded TaskStorage. /// Restore persisted data from a decoded TaskStorage.
/// ///
/// This is used during restore operations to copy decoded persisted data /// This is used during restore operations to copy decoded persisted data
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment