Unverified Commit f27683f2 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks: update reader leaf distance inline when adding output dependency (#99764)

### What?

Updates the reader's leaf distance inline when `try_read_task_output`
adds a new output-dependency edge. The `LeafDistanceUpdateQueue` is only
used to propagate to the reader's existing output dependents.

### Why?

The reader is already locked (with the `All` category) while the edge is
added. Before this change, the code allocated a fresh
`LeafDistanceUpdateQueue` and pushed the reader into it. After dropping
the guards it locked the reader again, just to update one field.

In the common case the reader has no output dependents yet, so all of
that work was redundant. This path runs for each new edge whose reader's
distance is not already greater than the dependency's, which is common
in cold builds with dependency tracking enabled.

### How?

- The leaf-distance arithmetic now lives in a single
`compute_leaf_distance_update`, used by both the queue and the inline
path. The arithmetic itself is unchanged.
- At the call site:
- The new distance is written on the reader guard that is already held.
  - Output-dependent IDs are snapshotted into a `SmallVec`.
  - The paired-lock dependency registration is unchanged.
- The queue is only created, seeded and executed after both guards are
dropped.
- The `trace_leaf_distance_update` tracing stays supported via an inline
span.

### Verification

- New behavioral unit tests for how leaf distance propagates:
  - a reader that is already ahead of its dependency is not updated;
  - a reader at or behind its dependency moves past it;
  - updates inside the buffer zone keep the buffer ceiling;
  - dependents are seeded with the reader's new distance.
- `cargo nextest run -p turbo-tasks-backend`: 291/291 pass.
- The existing ungated
`collector_is_not_re_executed_for_collectibles_it_has_not_read_yet` test
is load-sensitive on unmodified `canary` too: it failed 91 of 120 runs
with 4 copies in parallel. It flaked once locally during the rebase; it
did not fail in later runs.
- `cargo test -p turbo-tasks --lib`: 127/127 pass.
- `cargo check --tests` passes both with and without `--features
trace_leaf_distance_update`.
- `cargo clippy -p turbo-tasks-backend --tests -- -D warnings` passes.

Not run: a real build benchmark. This removes a redundant lock and
allocations on the hot path; no build-time speedup is claimed or
measured.

<!-- NEXT_JS_LLM -->


<!-- fleet b792e520-57be-40bf-8385-fad7b8fc11aa -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent e2be0379
...@@ -69,8 +69,8 @@ use crate::{ ...@@ -69,8 +69,8 @@ use crate::{
operation::{ operation::{
AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext, LeafDistanceUpdateQueue, AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext, LeafDistanceUpdateQueue,
OutdatedEdge, TaskGuard, TaskType, TaskTypeRef, capture_all_edges, cleanup_old_edges, OutdatedEdge, TaskGuard, TaskType, TaskTypeRef, capture_all_edges, cleanup_old_edges,
connect_child, connect_children, get_aggregation_number, get_uppers, invalidate, compute_leaf_distance_update, connect_child, connect_children, get_aggregation_number,
make_task_dirty_internal, prepare_new_children, update_cell, get_uppers, invalidate, make_task_dirty_internal, prepare_new_children, update_cell,
}, },
snapshot_coordinator::{OperationGuard, SlowSettle, SnapshotCoordinator}, snapshot_coordinator::{OperationGuard, SlowSettle, SnapshotCoordinator},
storage::{Storage, StorageOptions, encode_snapshot_item}, storage::{Storage, StorageOptions, encode_snapshot_item},
...@@ -79,7 +79,7 @@ use crate::{ ...@@ -79,7 +79,7 @@ use crate::{
backing_storage::SnapshotItem, backing_storage::SnapshotItem,
data::{ data::{
ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState, ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressState, InProgressStateInner, OutputValue, TransientTask, InProgressState, InProgressStateInner, LeafDistance, OutputValue, TransientTask,
}, },
error::{TaskError, TaskErrorItem}, error::{TaskError, TaskErrorItem},
kv_backing_storage::TurboBackingStorage, kv_backing_storage::TurboBackingStorage,
...@@ -945,19 +945,37 @@ impl TurboTasksBackend { ...@@ -945,19 +945,37 @@ impl TurboTasksBackend {
dependent_task = ?reader dependent_task = ?reader
) )
.entered(); .entered();
let mut queue = LeafDistanceUpdateQueue::new(); // The reader's updated leaf distance together with its output dependents, which
// need to be updated after all locks are released.
let mut leaf_distance_propagation: Option<(LeafDistance, SmallVec<[TaskId; 4]>)> =
None;
#[cfg(feature = "trace_leaf_distance_update")]
let mut leaf_distance_span = None;
let reader = reader.unwrap(); let reader = reader.unwrap();
if task.add_output_dependent(reader) { if task.add_output_dependent(reader) {
// Ensure that dependent leaf distance is strictly monotonic increasing // Ensure that dependent leaf distance is strictly monotonic increasing.
let leaf_distance = task.get_leaf_distance().copied().unwrap_or_default(); let leaf_distance = task.get_leaf_distance().copied().unwrap_or_default();
let reader_leaf_distance = if let Some(new_reader_leaf_distance) = compute_leaf_distance_update(
reader_task.get_leaf_distance().copied().unwrap_or_default(); reader_task.get_leaf_distance().copied().unwrap_or_default(),
if reader_leaf_distance.distance <= leaf_distance.distance { leaf_distance.distance,
queue.push( leaf_distance.max_distance_in_buffer,
reader, ) {
leaf_distance.distance, #[cfg(feature = "trace_leaf_distance_update")]
leaf_distance.max_distance_in_buffer, {
); leaf_distance_span = Some(tracing::trace_span!(
"update leaf distance (inline)",
dependencies_distance = leaf_distance.distance,
dependencies_max_distance_in_buffer =
leaf_distance.max_distance_in_buffer
));
}
reader_task.set_leaf_distance(new_reader_leaf_distance);
let dependents: SmallVec<[TaskId; 4]> =
reader_task.iter_output_dependent().collect();
if !dependents.is_empty() {
leaf_distance_propagation =
Some((new_reader_leaf_distance, dependents));
}
} }
} }
...@@ -973,7 +991,13 @@ impl TurboTasksBackend { ...@@ -973,7 +991,13 @@ impl TurboTasksBackend {
} }
drop(reader_task); drop(reader_task);
queue.execute(&mut ctx); if let Some((reader_leaf_distance, dependents)) = leaf_distance_propagation {
#[cfg(feature = "trace_leaf_distance_update")]
let _guard = leaf_distance_span.map(|span| span.entered());
let mut queue = LeafDistanceUpdateQueue::new();
queue.push_dependents(dependents, reader_leaf_distance);
queue.execute(&mut ctx);
}
} else { } else {
drop(task); drop(task);
} }
......
...@@ -8,8 +8,9 @@ use rustc_hash::FxHashMap; ...@@ -8,8 +8,9 @@ use rustc_hash::FxHashMap;
use tracing::{span::Span, trace_span}; use tracing::{span::Span, trace_span};
use turbo_tasks::TaskId; use turbo_tasks::TaskId;
use crate::backend::{ use crate::{
TaskDataCategory, operation::ExecuteContext, storage_schema::TaskStorageAccessors, backend::{TaskDataCategory, operation::ExecuteContext, storage_schema::TaskStorageAccessors},
data::LeafDistance,
}; };
/// We avoid incrementing the leaf distance by 1 each time to avoid frequent updates. /// We avoid incrementing the leaf distance by 1 each time to avoid frequent updates.
...@@ -17,6 +18,37 @@ use crate::backend::{ ...@@ -17,6 +18,37 @@ use crate::backend::{
/// This constant defines the size of that buffer zone at leaf distance 0. /// This constant defines the size of that buffer zone at leaf distance 0.
const BASE_LEAF_DISTANCE_BUFFER: u32 = 128; const BASE_LEAF_DISTANCE_BUFFER: u32 = 128;
/// Computes the new leaf distance of a task that depends (via an output dependency) on a task with
/// the given `dependency_distance` and `dependency_max_distance_in_buffer`.
///
/// Returns `None` when `current` is already strictly greater than the dependency distance, i.e.
/// the leaf distance is already strictly monotonic along this edge and no update is needed.
pub fn compute_leaf_distance_update(
current: LeafDistance,
dependency_distance: u32,
dependency_max_distance_in_buffer: u32,
) -> Option<LeafDistance> {
debug_assert!(dependency_max_distance_in_buffer < u32::MAX / 2);
if current.distance > dependency_distance {
// It is strictly monotonic. No need to update.
return None;
}
// It's not strictly monotonic, we need to update
let mut leaf_distance = current;
if leaf_distance.max_distance_in_buffer <= dependency_distance {
// We overshoot the buffer zone.
let old_value = leaf_distance.distance;
leaf_distance.distance = dependency_max_distance_in_buffer + 1;
let buffer_size = BASE_LEAF_DISTANCE_BUFFER
- BASE_LEAF_DISTANCE_BUFFER.saturating_mul(old_value) / leaf_distance.distance;
leaf_distance.max_distance_in_buffer = leaf_distance.distance + buffer_size;
} else {
// We are within the buffer zone, keep the max as is
leaf_distance.distance = dependency_distance + 1;
}
Some(leaf_distance)
}
/// An leaf distance update job that is enqueued. /// An leaf distance update job that is enqueued.
struct LeafDistanceUpdate { struct LeafDistanceUpdate {
dependencies_distance: u32, dependencies_distance: u32,
...@@ -77,6 +109,22 @@ impl LeafDistanceUpdateQueue { ...@@ -77,6 +109,22 @@ impl LeafDistanceUpdateQueue {
}; };
} }
/// Enqueues leaf distance updates for all `dependents` of a task whose leaf distance has just
/// been updated to `leaf_distance`.
pub fn push_dependents(
&mut self,
dependents: impl IntoIterator<Item = TaskId>,
leaf_distance: LeafDistance,
) {
for dependent_id in dependents {
self.push(
dependent_id,
leaf_distance.distance,
leaf_distance.max_distance_in_buffer,
);
}
}
/// Executes all jobs in the queue. /// Executes all jobs in the queue.
pub fn execute(&mut self, ctx: &mut ExecuteContext<'_>) { pub fn execute(&mut self, ctx: &mut ExecuteContext<'_>) {
while let Some((Reverse(queue_dependencies_distance), task_id)) = self.queue.pop() { while let Some((Reverse(queue_dependencies_distance), task_id)) = self.queue.pop() {
...@@ -124,34 +172,95 @@ impl LeafDistanceUpdateQueue { ...@@ -124,34 +172,95 @@ impl LeafDistanceUpdateQueue {
// For performance reasons this should stay `Data` and not `All` // For performance reasons this should stay `Data` and not `All`
TaskDataCategory::Data, TaskDataCategory::Data,
); );
debug_assert!(dependencies_max_distance_in_buffer < u32::MAX / 2); let Some(leaf_distance) = compute_leaf_distance_update(
let mut leaf_distance = task.get_leaf_distance().copied().unwrap_or_default(); task.get_leaf_distance().copied().unwrap_or_default(),
if leaf_distance.distance > dependencies_distance { dependencies_distance,
// It is strictly monotonic. No need to update. dependencies_max_distance_in_buffer,
) else {
return; return;
} };
// It's not strictly monotonic, we need to update
if leaf_distance.max_distance_in_buffer <= dependencies_distance {
// We overshoot the buffer zone.
let old_value = leaf_distance.distance;
leaf_distance.distance = dependencies_max_distance_in_buffer + 1;
let buffer_size = BASE_LEAF_DISTANCE_BUFFER
- BASE_LEAF_DISTANCE_BUFFER.saturating_mul(old_value) / leaf_distance.distance;
leaf_distance.max_distance_in_buffer = leaf_distance.distance + buffer_size;
} else {
// We are within the buffer zone, keep the max as is
leaf_distance.distance = dependencies_distance + 1;
}
// TODO Technically CellDependent is also needed, but there are cycles in the CellDependent // TODO Technically CellDependent is also needed, but there are cycles in the CellDependent
// graph. So we need to handle that properly first. When enabling this, make sure to also // graph. So we need to handle that properly first. When enabling this, make sure to also
// call the leaf update queue when adding CellDependents. // call the leaf update queue when adding CellDependents.
for dependent_id in task.iter_output_dependent() { self.push_dependents(task.iter_output_dependent(), leaf_distance);
self.push( task.set_leaf_distance(leaf_distance);
dependent_id, }
leaf_distance.distance, }
leaf_distance.max_distance_in_buffer,
#[cfg(test)]
mod tests {
use turbo_tasks::TaskId;
use super::{LeafDistanceUpdateQueue, compute_leaf_distance_update};
use crate::data::LeafDistance;
fn ld(distance: u32, max_distance_in_buffer: u32) -> LeafDistance {
LeafDistance {
distance,
max_distance_in_buffer,
}
}
fn task_id(id: u32) -> TaskId {
TaskId::new(id).unwrap()
}
#[test]
fn reader_ahead_of_dependency_is_not_updated() {
assert_eq!(compute_leaf_distance_update(ld(5, 100), 4, 50), None);
}
#[test]
fn reader_at_or_behind_dependency_moves_past_it() {
for (reader, dependency) in [
(ld(0, 0), ld(0, 0)),
(ld(3, 100), ld(3, 50)),
(ld(10, 20), ld(20, 150)),
] {
let updated = compute_leaf_distance_update(
reader,
dependency.distance,
dependency.max_distance_in_buffer,
)
.unwrap();
assert!(
updated.distance > dependency.distance,
"{reader:?} {dependency:?}"
);
assert!(updated.max_distance_in_buffer >= updated.distance);
}
}
#[test]
fn updates_within_buffer_keep_buffer_ceiling() {
let first = compute_leaf_distance_update(ld(2, 100), 10, 60).unwrap();
assert_eq!(first, ld(11, 100));
let second = compute_leaf_distance_update(first, 12, 70).unwrap();
assert_eq!(second, ld(13, 100));
}
#[test]
fn push_dependents_without_dependents_enqueues_nothing() {
let mut queue = LeafDistanceUpdateQueue::new();
queue.push_dependents([], ld(3, 130));
assert!(queue.queue.is_empty());
assert!(queue.leaf_distance_updates.is_empty());
}
#[test]
fn push_dependents_propagates_new_distance() {
let reader = compute_leaf_distance_update(ld(0, 0), 6, 134).unwrap();
let mut queue = LeafDistanceUpdateQueue::new();
queue.push_dependents([task_id(1), task_id(2)], reader);
assert_eq!(queue.queue.len(), 2);
for id in [task_id(1), task_id(2)] {
let update = &queue.leaf_distance_updates[&id];
assert_eq!(update.dependencies_distance, reader.distance);
assert_eq!(
update.dependencies_max_distance_in_buffer,
reader.max_distance_in_buffer
); );
} }
task.set_leaf_distance(leaf_distance);
} }
} }
...@@ -1844,7 +1844,7 @@ pub use self::{ ...@@ -1844,7 +1844,7 @@ pub use self::{
connect_child::connect_child, connect_child::connect_child,
connect_children::connect_children, connect_children::connect_children,
invalidate::{invalidate, make_task_dirty_internal}, invalidate::{invalidate, make_task_dirty_internal},
leaf_distance_update::LeafDistanceUpdateQueue, leaf_distance_update::{LeafDistanceUpdateQueue, compute_leaf_distance_update},
prepare_new_children::prepare_new_children, prepare_new_children::prepare_new_children,
update_cell::update_cell, update_cell::update_cell,
update_collectible::UpdateCollectibleOperation, update_collectible::UpdateCollectibleOperation,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment