Unverified Commit 44da33c4 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-tasks-backend: keep collectibles strongly consistent without waiting

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent d39a2e66
...@@ -70,7 +70,8 @@ use crate::{ ...@@ -70,7 +70,8 @@ use crate::{
AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext, LeafDistanceUpdateQueue, AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext, LeafDistanceUpdateQueue,
OutdatedEdge, TaskGuard, TaskType, TaskTypeRef, capture_all_edges, cleanup_old_edges, OutdatedEdge, TaskGuard, TaskType, TaskTypeRef, capture_all_edges, cleanup_old_edges,
compute_leaf_distance_update, connect_child, connect_children, get_aggregation_number, compute_leaf_distance_update, connect_child, connect_children, get_aggregation_number,
get_uppers, invalidate, make_task_dirty_internal, prepare_new_children, update_cell, get_uppers, invalidate, make_task_dirty_internal, prepare_new_children,
release_held_dirty_uppers, update_cell,
}, },
snapshot_coordinator::{OperationGuard, SlowSettle, SnapshotCoordinator}, snapshot_coordinator::{OperationGuard, SlowSettle, SnapshotCoordinator},
storage::{Storage, StorageOptions, encode_snapshot_item}, storage::{Storage, StorageOptions, encode_snapshot_item},
...@@ -2010,13 +2011,20 @@ impl TurboTasksBackend { ...@@ -2010,13 +2011,20 @@ impl TurboTasksBackend {
) { ) {
let mut ctx = self.execute_context(turbo_tasks); let mut ctx = self.execute_context(turbo_tasks);
let mut task = ctx.task(task_id, TaskDataCategory::All); let mut task = ctx.task(task_id, TaskDataCategory::All);
let mut held_dirty_uppers = Vec::new();
if let Some(in_progress) = task.take_in_progress() { if let Some(in_progress) = task.take_in_progress() {
match in_progress { match in_progress {
InProgressState::Scheduled { InProgressState::Scheduled {
done_event, done_event,
reason: _, reason: _,
} => done_event.notify(usize::MAX), } => done_event.notify(usize::MAX),
InProgressState::InProgress(InProgressStateInner { done_event, .. }) => { InProgressState::InProgress(inner) => {
let InProgressStateInner {
done_event,
held_dirty_uppers: held,
..
} = *inner;
held_dirty_uppers = held;
done_event.notify(usize::MAX) done_event.notify(usize::MAX)
} }
InProgressState::Canceled => {} InProgressState::Canceled => {}
...@@ -2062,6 +2070,7 @@ impl TurboTasksBackend { ...@@ -2062,6 +2070,7 @@ impl TurboTasksBackend {
if let Some(data_update) = data_update { if let Some(data_update) = data_update {
AggregationUpdateQueue::run(data_update, &mut ctx); AggregationUpdateQueue::run(data_update, &mut ctx);
} }
release_held_dirty_uppers(&mut ctx, task_id, held_dirty_uppers);
drop(in_progress_cells); drop(in_progress_cells);
} }
...@@ -2107,6 +2116,8 @@ impl TurboTasksBackend { ...@@ -2107,6 +2116,8 @@ impl TurboTasksBackend {
once_task, once_task,
done_event, done_event,
marked_as_completed: false, marked_as_completed: false,
completing: false,
held_dirty_uppers: Vec::new(),
new_children: Default::default(), new_children: Default::default(),
}, },
))); )));
...@@ -2370,6 +2381,7 @@ impl TurboTasksBackend { ...@@ -2370,6 +2381,7 @@ impl TurboTasksBackend {
let &mut InProgressState::InProgress(InProgressStateInner { let &mut InProgressState::InProgress(InProgressStateInner {
stale, stale,
ref mut new_children, ref mut new_children,
ref mut completing,
once_task: is_once_task, once_task: is_once_task,
.. ..
}) = in_progress }) = in_progress
...@@ -2412,6 +2424,11 @@ impl TurboTasksBackend { ...@@ -2412,6 +2424,11 @@ impl TurboTasksBackend {
return Err(stale_priority); return Err(stale_priority);
} }
// From here on the completion changes the task's edges and propagates these changes to the
// uppers it sees now. Uppers that drop the task until the completion ends keep its dirty
// state (see `hold_dirty_state_while_completing` in `aggregation_update.rs`).
*completing = true;
// take the children from the task to process them // take the children from the task to process them
let mut new_children = take(new_children); let mut new_children = take(new_children);
...@@ -2793,8 +2810,11 @@ impl TurboTasksBackend { ...@@ -2793,8 +2810,11 @@ impl TurboTasksBackend {
#[cfg(not(feature = "no_fast_stale"))] #[cfg(not(feature = "no_fast_stale"))]
if *stale && !is_once_task { if *stale && !is_once_task {
let stale_priority = compute_stale_priority(&task); let stale_priority = compute_stale_priority(&task);
let Some(InProgressState::InProgress(InProgressStateInner { done_event, .. })) = let Some(InProgressState::InProgress(InProgressStateInner {
task.take_in_progress() done_event,
held_dirty_uppers,
..
})) = task.take_in_progress()
else { else {
unreachable!(); unreachable!();
}; };
...@@ -2805,6 +2825,8 @@ impl TurboTasksBackend { ...@@ -2805,6 +2825,8 @@ impl TurboTasksBackend {
debug_assert!(old.is_none(), "InProgress already exists"); debug_assert!(old.is_none(), "InProgress already exists");
drop(task); drop(task);
release_held_dirty_uppers(ctx, task_id, held_dirty_uppers);
// All `new_children` are currently hold active with an active count and we need to undo // All `new_children` are currently hold active with an active count and we need to undo
// that. (We already filtered out the old children from that list) // that. (We already filtered out the old children from that list)
AggregationUpdateQueue::run( AggregationUpdateQueue::run(
...@@ -2870,6 +2892,8 @@ impl TurboTasksBackend { ...@@ -2870,6 +2892,8 @@ impl TurboTasksBackend {
once_task: is_once_task, once_task: is_once_task,
stale, stale,
marked_as_completed: _, marked_as_completed: _,
completing: _,
held_dirty_uppers,
new_children, new_children,
}) = in_progress }) = in_progress
else { else {
...@@ -2885,6 +2909,8 @@ impl TurboTasksBackend { ...@@ -2885,6 +2909,8 @@ impl TurboTasksBackend {
reason: TaskExecutionReason::Stale, reason: TaskExecutionReason::Stale,
}); });
debug_assert!(old.is_none(), "InProgress already exists"); debug_assert!(old.is_none(), "InProgress already exists");
drop(task);
release_held_dirty_uppers(ctx, task_id, held_dirty_uppers);
return (Some(stale_priority), None, Vec::new()); return (Some(stale_priority), None, Vec::new());
} }
...@@ -2956,6 +2982,7 @@ impl TurboTasksBackend { ...@@ -2956,6 +2982,7 @@ impl TurboTasksBackend {
if let Some(data_update) = data_update { if let Some(data_update) = data_update {
AggregationUpdateQueue::run(data_update, ctx); AggregationUpdateQueue::run(data_update, ctx);
} }
release_held_dirty_uppers(ctx, task_id, held_dirty_uppers);
// We return so the data can be dropped outside of critical sections // We return so the data can be dropped outside of critical sections
(stale_priority, in_progress_cells, removed_data) (stale_priority, in_progress_cells, removed_data)
...@@ -3343,54 +3370,63 @@ impl TurboTasksBackend { ...@@ -3343,54 +3370,63 @@ impl TurboTasksBackend {
) -> AutoMap<RawVc, i32, BuildHasherDefault<FxHasher>, 1> { ) -> AutoMap<RawVc, i32, BuildHasherDefault<FxHasher>, 1> {
let mut ctx = self.execute_context(turbo_tasks); let mut ctx = self.execute_context(turbo_tasks);
let mut collectibles = AutoMap::default(); let mut collectibles = AutoMap::default();
{ // Lock the reader together with the task, so that reading the collectibles and making the
let mut task = ctx.task(task_id, TaskDataCategory::All); // reader's dependency on them current happen atomically. Otherwise a collectibles change in
task.assert_not_deleted("read_task_collectibles"); // between would still see the dependency as outdated (not read yet) and make the reader
if task // dirty, but not stale. The reader's execution would then complete with the collectibles
.get_persistent_task_type() // it read before the change, and its dirty flag would be cleared.
.is_some_and(|t| !t.native_fn.is_root) let (mut task, mut reader) = match reader_id {
{ Some(reader_id) if reader_id != task_id => {
drop(task); let (task, reader) = ctx.task_pair(task_id, reader_id, TaskDataCategory::All);
panic!( (task, Some(reader))
"Reading collectibles of non-root task {} (reader: {}). The `root` attribute \
is missing on the task.",
self.debug_get_task_description(task_id),
reader_id.map_or_else(
|| "unknown".to_string(),
|r| self.debug_get_task_description(r)
)
);
} }
for (collectible, count) in task.iter_aggregated_collectibles() { _ => (ctx.task(task_id, TaskDataCategory::All), None),
if *count > 0 && collectible.collectible_type == collectible_type { };
*collectibles task.assert_not_deleted("read_task_collectibles");
.entry(RawVc::task_cell( if task
collectible.cell.task, .get_persistent_task_type()
collectible.cell.cell, .is_some_and(|t| !t.native_fn.is_root)
)) {
.or_insert(0) += 1; drop(task);
} drop(reader);
} panic!(
for (&collectible, &count) in task.iter_collectibles() { "Reading collectibles of non-root task {} (reader: {}). The `root` attribute is \
if collectible.collectible_type == collectible_type { missing on the task.",
*collectibles self.debug_get_task_description(task_id),
.entry(RawVc::task_cell( reader_id.map_or_else(
collectible.cell.task, || "unknown".to_string(),
collectible.cell.cell, |r| self.debug_get_task_description(r)
)) )
.or_insert(0) += count; );
} }
for (collectible, count) in task.iter_aggregated_collectibles() {
if *count > 0 && collectible.collectible_type == collectible_type {
*collectibles
.entry(RawVc::task_cell(
collectible.cell.task,
collectible.cell.cell,
))
.or_insert(0) += 1;
} }
if let Some(reader_id) = reader_id { }
let _ = task.add_collectibles_dependents((collectible_type, reader_id)); for (&collectible, &count) in task.iter_collectibles() {
if collectible.collectible_type == collectible_type {
*collectibles
.entry(RawVc::task_cell(
collectible.cell.task,
collectible.cell.cell,
))
.or_insert(0) += count;
} }
} }
if let Some(reader_id) = reader_id { if let Some(reader_id) = reader_id {
let mut reader = ctx.task(reader_id, TaskDataCategory::Data); let _ = task.add_collectibles_dependents((collectible_type, reader_id));
let target = CollectiblesRef { let target = CollectiblesRef {
task: task_id, task: task_id,
collectible_type, collectible_type,
}; };
// A task reading its own collectibles uses the same guard for both roles.
let reader = reader.as_mut().unwrap_or(&mut task);
if !reader.remove_outdated_collectibles_dependencies(&target) { if !reader.remove_outdated_collectibles_dependencies(&target) {
let _ = reader.add_collectibles_dependencies(target); let _ = reader.add_collectibles_dependencies(target);
} }
......
...@@ -1862,7 +1862,7 @@ impl TaskStorageAccessors for TaskGuard<'_> { ...@@ -1862,7 +1862,7 @@ impl TaskStorageAccessors for TaskGuard<'_> {
pub use self::{ pub use self::{
aggregation_update::{ aggregation_update::{
AggregatedDataUpdate, AggregationUpdateJob, CollectiblesUpdateVec, get_aggregation_number, AggregatedDataUpdate, AggregationUpdateJob, CollectiblesUpdateVec, get_aggregation_number,
get_uppers, is_aggregating_node, is_root_node, get_uppers, is_aggregating_node, is_root_node, release_held_dirty_uppers,
}, },
cleanup_old_edges::{ cleanup_old_edges::{
OutdatedEdge, capture_all_edges, cleanup_old_edges, cleanup_old_edges_deletions_only, OutdatedEdge, capture_all_edges, cleanup_old_edges, cleanup_old_edges_deletions_only,
......
...@@ -400,6 +400,8 @@ mod invalidate_cell_dependents_tests { ...@@ -400,6 +400,8 @@ mod invalidate_cell_dependents_tests {
once_task: false, once_task: false,
done_event: Event::new(|| || "test done_event".to_string()), done_event: Event::new(|| || "test done_event".to_string()),
marked_as_completed: false, marked_as_completed: false,
completing: false,
held_dirty_uppers: Vec::new(),
new_children: Default::default(), new_children: Default::default(),
}, },
))); )));
......
...@@ -247,6 +247,15 @@ pub struct InProgressStateInner { ...@@ -247,6 +247,15 @@ pub struct InProgressStateInner {
/// Early marking as completed. This is set before the output is available and will ignore full /// Early marking as completed. This is set before the output is available and will ignore full
/// task completion of the task for strongly consistent reads. /// task completion of the task for strongly consistent reads.
pub marked_as_completed: bool, pub marked_as_completed: bool,
/// Set once the execution has finished and its completion has started to change the task's
/// edges (removing old children, connecting new ones). These changes are propagated to the
/// uppers the task had at that time, while the task is still dirty.
pub completing: bool,
/// Uppers that lost their last inner edge to this task while it was `completing`. They still
/// count the task's dirty state, so they can't become clean before the completion's pending
/// changes have reached them. Released when the completion ends (finished, rescheduled or
/// canceled). Usually empty.
pub held_dirty_uppers: Vec<HeldDirtyUpper>,
/// Event that is triggered when the task output is available (completed flag set). /// Event that is triggered when the task output is available (completed flag set).
/// This is used to wait for completion when reading the task output before it's available. /// This is used to wait for completion when reading the task output before it's available.
pub done_event: Event, pub done_event: Event,
...@@ -255,6 +264,15 @@ pub struct InProgressStateInner { ...@@ -255,6 +264,15 @@ pub struct InProgressStateInner {
pub new_children: FxHashSet<TaskId>, pub new_children: FxHashSet<TaskId>,
} }
/// A dirty container contribution of a task that an upper keeps after it lost its last inner edge
/// to the task, see [`InProgressStateInner::held_dirty_uppers`].
#[derive(Debug, Clone, Copy)]
pub struct HeldDirtyUpper {
pub upper_id: TaskId,
pub dirty_count: i32,
pub current_session_clean_count: i32,
}
#[derive(Debug)] #[derive(Debug)]
pub enum InProgressState { pub enum InProgressState {
Scheduled { Scheduled {
......
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
//! After a strongly consistent read, the collectibles of an operation must reflect the current
//! graph, including the children that were connected or disconnected in the current change round.
//!
//! Several root operations share parents, and the parents share chains of children. Each change
//! re-selects which parents each root connects and which chains each parent connects. A root that
//! drops a parent can do so while the parent's own completion is still propagating its changed
//! children to the root. The root must not become clean before these changes have reached it.
//!
//! The race depends on timing and shows up reliably only when the machine is under CPU load.
use std::collections::BTreeSet;
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{CollectiblesSource, ResolvedVc, State, ValueToString, Vc, emit};
use turbo_tasks_testing::{Registration, register, run_once};
static REGISTRATION: Registration = register!();
const ROOTS: u32 = 4;
const PARENTS: u32 = 16;
const CHILDREN: u32 = 24;
const DEPTH: u32 = 4;
const ROUNDS: u32 = 10;
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn collectibles_include_children_connected_in_the_current_round() {
run_once(&REGISTRATION, async || {
let generation = Generation {
state: State::new(0),
}
.resolved_cell();
for round in 0..ROUNDS {
generation.await?.state.set(round);
for root in 0..ROOTS {
let collected = collect_operation(generation, root)
.read_strongly_consistent()
.await?;
assert_eq!(
*collected,
expected_collectibles(round, root),
"collectibles of root {root} in round {round}"
);
}
}
anyhow::Ok(())
})
.await
.unwrap()
}
/// Deterministically selects about one in `one_in` of the candidates.
fn selected(generation: u32, owner: u32, candidate: u32, one_in: u64) -> bool {
let mut x = (u64::from(generation) << 42) ^ (u64::from(owner) << 21) ^ u64::from(candidate);
// splitmix64 finalizer
x = x.wrapping_add(0x9e37_79b9_7f4a_7c15);
x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
x ^= x >> 31;
x % one_in == 0
}
fn selected_parents(generation: u32, root: u32) -> impl Iterator<Item = u32> {
(0..PARENTS).filter(move |&parent| selected(generation, root, parent, 2))
}
fn selected_children(generation: u32, parent: u32) -> impl Iterator<Item = u32> {
// Offset the owner so parent selections don't mirror root selections.
(0..CHILDREN).filter(move |&child| selected(generation, ROOTS + parent, child, 8))
}
fn expected_collectibles(generation: u32, root: u32) -> Vec<RcStr> {
selected_parents(generation, root)
.flat_map(|parent| selected_children(generation, parent))
.map(thing_name)
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
fn thing_name(index: u32) -> RcStr {
format!("thing {index}").into()
}
#[turbo_tasks::value]
struct Generation {
state: State<u32>,
}
#[turbo_tasks::value]
struct Thing(u32);
#[turbo_tasks::value_impl]
impl ValueToString for Thing {
#[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> {
Vc::cell(thing_name(self.0))
}
}
/// A chain of `depth` tasks; the end of the chain emits `Thing(index)`.
#[turbo_tasks::function]
async fn child(index: u32, depth: u32) -> Result<Vc<()>> {
if depth == 0 {
emit(ResolvedVc::upcast::<Box<dyn ValueToString>>(
Thing(index).resolved_cell(),
));
} else {
child(index, depth - 1).await?;
}
Ok(Vc::cell(()))
}
/// Connects the chains selected for this parent in the current generation.
#[turbo_tasks::function]
async fn parent(generation: ResolvedVc<Generation>, index: u32) -> Result<Vc<()>> {
let generation = *generation.await?.state.get();
for child_index in selected_children(generation, index) {
child(child_index, DEPTH).await?;
}
Ok(Vc::cell(()))
}
/// Connects the parents selected for this root in the current generation.
#[turbo_tasks::function(operation, root)]
async fn root_operation(generation: ResolvedVc<Generation>, root: u32) -> Result<Vc<()>> {
let generation_value = *generation.await?.state.get();
for parent_index in selected_parents(generation_value, root) {
parent(*generation, parent_index).await?;
}
Ok(Vc::cell(()))
}
#[turbo_tasks::value(transparent)]
struct Collected(Vec<RcStr>);
#[turbo_tasks::function(operation, root)]
async fn collect_operation(generation: ResolvedVc<Generation>, root: u32) -> Result<Vc<Collected>> {
let root_op = root_operation(generation, root);
root_op.connect().await?;
let mut names = Vec::new();
for collectible in root_op.peek_collectibles::<Box<dyn ValueToString>>() {
names.push((*collectible.to_string().await?).clone());
}
names.sort();
names.dedup();
Ok(Vc::cell(names))
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment