Unverified Commit 1999173b authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks-gc: Defer most aggregation graph management until after GC has settled (#98591)

Fix AggregationGraph updates during GC.

During GC we aremoving a lot of edges so this can trigger a lot of
aggregation graph churn. Because of the highly parallel nature of GC
this can end up with multiple queues fighting over updates which can
exceed various retry limits and lead to things like trying to turn nodes
into `followers` that are actively being deleted.

Instead we just don't rebalance the aggregation graph during GC, instead
we buffer every `rebalance` request and set the `optimize` budget to
zero. Then after GC settles we process all the rebalances at once,
taking care to filter out any tasks that have been since deleted.

Because GC is only removing edges it can lead to inefficient graphs (too
many interior nodes) but not violate graph invariants.

Alternatives considered:

The initial idea was to ensure GC had finished with all 'children' of
the node before processing rebalance requests, but this was only a
partial solution (and complex!), since it didn't handle the case of an
aggregating node having multiple children become collectible in the
initial scan, this could still trigger 'optimization contention' and
turning dead nodes into followers.

Testing:

The test/benchmark are enhanced to produce non-trivial aggregation
graphs which improves test coverage (though the issue of racing deletes
was trivially found by CI also)

---------
Co-authored-by: 's avatarClaude Opus 5 (1M context) <noreply@anthropic.com>
parent 30c08f56
...@@ -68,9 +68,12 @@ fn create_generation() -> Vc<Generation> { ...@@ -68,9 +68,12 @@ fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell() Generation(State::new(0)).cell()
} }
// WIDE shape: root -> `width` intermediates, each -> a leaf (2 tasks per index). Bumping the // TREE shape: a `BRANCHING`-ary tree of `subtree` nodes over `width` leaves. Bumping the
// generation disconnects the whole previous generation, so a collect tears down ~2*width tasks and // generation disconnects the whole previous tree. The tree must be deep enough to push `effective`
// exercises the per-task fan-out plus a one-level cascade (intermediate -> its leaf). // past `LEAF_NUMBER` and produce follower edges, otherwise a collect's rebalance half goes
// unmeasured.
const BRANCHING: u32 = 4;
#[turbo_tasks::function] #[turbo_tasks::function]
fn wide_leaf(generation: u32, index: u32) -> Vc<u32> { fn wide_leaf(generation: u32, index: u32) -> Vc<u32> {
...@@ -78,20 +81,70 @@ fn wide_leaf(generation: u32, index: u32) -> Vc<u32> { ...@@ -78,20 +81,70 @@ fn wide_leaf(generation: u32, index: u32) -> Vc<u32> {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn wide_intermediate(generation: u32, index: u32) -> Result<Vc<u32>> { async fn subtree(generation: u32, index: u32, span: u32) -> Result<Vc<u32>> {
Ok(Vc::cell(1 + *wide_leaf(generation, index).await?)) if span <= 1 {
return Ok(Vc::cell(1 + *wide_leaf(generation, index).await?));
}
let child_span = span.div_ceil(BRANCHING);
let mut sum = 0u32;
let mut start = index;
while start < index + span {
let this_span = child_span.min(index + span - start);
sum = sum.wrapping_add(*subtree(generation, start, this_span).await?);
start += this_span;
}
Ok(Vc::cell(sum))
}
/// A *retained* interior node: keyed on `index`/`span` only, so the same task survives a generation
/// bump and re-executes with new children -- a live parent whose children are collected out from
/// under it.
#[turbo_tasks::function]
async fn live_parent(generation: ResolvedVc<Generation>, index: u32, span: u32) -> Result<Vc<u32>> {
let generation_value = *generation.await?.get();
Ok(Vc::cell(*subtree(generation_value, index, span).await?))
} }
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
async fn wide_root(generation: ResolvedVc<Generation>, width: u32) -> Result<Vc<u32>> { async fn wide_root(generation: ResolvedVc<Generation>, width: u32) -> Result<Vc<u32>> {
let generation = *generation.await?.get(); let child_span = width.div_ceil(BRANCHING);
let mut sum = 0u32; let mut sum = 0u32;
for index in 0..width { let mut start = 0;
sum = sum.wrapping_add(*wide_intermediate(generation, index).await?); while start < width {
let this_span = child_span.min(width - start);
sum = sum.wrapping_add(*live_parent(*generation, start, this_span).await?);
start += this_span;
} }
Ok(Vc::cell(sum)) Ok(Vc::cell(sum))
} }
/// Tasks in one generation: every `subtree` node plus every leaf. Mirrors `subtree`'s recursion.
fn generation_task_count(width: u32) -> u64 {
fn count(span: u32) -> u64 {
if span <= 1 {
return 2;
}
let child_span = span.div_ceil(BRANCHING);
let mut total = 1;
let mut start = 0;
while start < span {
let this_span = child_span.min(span - start);
total += count(this_span);
start += this_span;
}
total
}
let child_span = width.max(1).div_ceil(BRANCHING);
let mut total = 0;
let mut start = 0;
while start < width.max(1) {
let this_span = child_span.min(width.max(1) - start);
total += count(this_span);
start += this_span;
}
total
}
/// Build generation 0 of the WIDE graph then bump to generation 1, leaving generation 0 fully /// Build generation 0 of the WIDE graph then bump to generation 1, leaving generation 0 fully
/// disconnected (garbage) and resident. Returns the backend ready for a timed collect. /// disconnected (garbage) and resident. Returns the backend ready for a timed collect.
fn setup_wide_garbage( fn setup_wide_garbage(
...@@ -131,8 +184,8 @@ pub fn gc(c: &mut Criterion) { ...@@ -131,8 +184,8 @@ pub fn gc(c: &mut Criterion) {
group.sample_size(10); group.sample_size(10);
for width in [5_000u32, 25_000, 100_000] { for width in [5_000u32, 25_000, 100_000] {
// ~2 tasks per index become garbage (intermediate + leaf). // Every `subtree` node plus every leaf becomes garbage.
let garbage = (2 * width) as u64; let garbage = generation_task_count(width);
group.throughput(Throughput::Elements(garbage)); group.throughput(Throughput::Elements(garbage));
group.bench_with_input(BenchmarkId::new("wide", garbage), &width, |b, &width| { group.bench_with_input(BenchmarkId::new("wide", garbage), &width, |b, &width| {
// Must match `create_tt`'s `num_workers`. // Must match `create_tt`'s `num_workers`.
......
...@@ -25,7 +25,7 @@ use std::{ ...@@ -25,7 +25,7 @@ use std::{
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use auto_hash_map::{AutoMap, AutoSet}; use auto_hash_map::{AutoMap, AutoSet};
use gc::DEFAULT_GC_ROOT_TTL; use gc::DEFAULT_GC_ROOT_TTL;
pub use gc::{GcStats, TtlCounter}; pub use gc::{GcPassResult, GcStats, TtlCounter};
use hashbrown::hash_table::Entry; use hashbrown::hash_table::Entry;
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
...@@ -267,30 +267,32 @@ pub struct TurboTasksBackend { ...@@ -267,30 +267,32 @@ pub struct TurboTasksBackend {
/// What [`TurboTasksBackend::snapshot_and_evict_for_testing`] observed. /// What [`TurboTasksBackend::snapshot_and_evict_for_testing`] observed.
#[doc(hidden)] #[doc(hidden)]
#[derive(Debug, Default)] #[derive(Default)]
pub struct TestSnapshotOutcome { pub struct TestSnapshotOutcome {
/// Whether the snapshot found modifications to persist. /// Whether the snapshot found modifications to persist.
pub had_new_data: bool, pub had_new_data: bool,
/// Tasks evicted from memory at each level. /// Tasks evicted from memory at each level.
pub eviction_counts: EvictionCounts, pub eviction_counts: EvictionCounts,
/// The [`GcStats`] of the GC pass this snapshot ran, or `None` when GC is disabled for the /// The [`GcStats`] and [`GcPassResult`] of the GC pass this snapshot ran, or `None` when GC
/// backend. /// is disabled for the backend.
pub gc: Option<GcStats>, pub gc: Option<(GcStats, GcPassResult)>,
} }
impl TestSnapshotOutcome { impl TestSnapshotOutcome {
/// The [`GcStats`] for the GC pass, panicking if GC is disabled. For tests whose whole point /// The [`GcStats`] for the GC pass
/// is the pass, so a misconfigured backend fails loudly rather than silently asserting
/// nothing.
pub fn gc_stats(&self) -> &GcStats { pub fn gc_stats(&self) -> &GcStats {
self.gc &self
.gc
.as_ref() .as_ref()
.expect("no GC pass ran: the backend needs `BackendOptions::gc = Some(true)`") .expect("no GC pass ran: the backend needs `BackendOptions::gc = Some(true)`")
.0
} }
/// Whether the GC pass wound down early. `false` when GC is disabled. /// Whether the GC pass wound down early. `false` when GC is disabled.
pub fn gc_interrupted(&self) -> bool { pub fn gc_interrupted(&self) -> bool {
self.gc.as_ref().is_some_and(|stats| stats.interrupted) self.gc
.as_ref()
.is_some_and(|(_, result)| result.interrupted)
} }
} }
...@@ -404,12 +406,8 @@ impl TurboTasksBackend { ...@@ -404,12 +406,8 @@ impl TurboTasksBackend {
/// Perform a snapshot and then evict all evictable tasks from memory. /// Perform a snapshot and then evict all evictable tasks from memory.
/// ///
/// This is exposed for integration tests that need to verify the /// This is exposed for integration tests that need to verify the snapshot → evict → restore
/// snapshot → evict → restore cycle works correctly. /// cycle works correctly.
///
/// Returns [`TestSnapshotOutcome`], which carries the GC pass's own [`GcStats`] alongside the
/// eviction counts. A test needs the pass's numbers because the resident task count can't
/// distinguish "GC collected it" from "GC skipped it and eviction dropped it to disk".
#[doc(hidden)] #[doc(hidden)]
pub fn snapshot_and_evict_for_testing( pub fn snapshot_and_evict_for_testing(
&self, &self,
...@@ -420,8 +418,8 @@ impl TurboTasksBackend { ...@@ -420,8 +418,8 @@ impl TurboTasksBackend {
"snapshot_and_evict requires persistence" "snapshot_and_evict requires persistence"
); );
let snapshot_result = self.snapshot_and_persist(None, SnapshotReason::Test, turbo_tasks); let snapshot_result = self.snapshot_and_persist(None, SnapshotReason::Test, turbo_tasks);
let (had_new_data, gc_stats) = match snapshot_result { let (had_new_data, gc_outcome) = match snapshot_result {
Ok((_, new_data, gc_stats)) => (new_data, gc_stats), Ok((_, new_data, gc_outcome)) => (new_data, gc_outcome),
Err(_) => { Err(_) => {
// Snapshot/persist failed — skip eviction since the data may not // Snapshot/persist failed — skip eviction since the data may not
// be on disk yet. Evicting now could lose in-memory state that // be on disk yet. Evicting now could lose in-memory state that
...@@ -433,7 +431,7 @@ impl TurboTasksBackend { ...@@ -433,7 +431,7 @@ impl TurboTasksBackend {
TestSnapshotOutcome { TestSnapshotOutcome {
had_new_data, had_new_data,
eviction_counts, eviction_counts,
gc: gc_stats, gc: gc_outcome,
} }
} }
...@@ -1132,16 +1130,17 @@ impl TurboTasksBackend { ...@@ -1132,16 +1130,17 @@ impl TurboTasksBackend {
/// Runs a persistence cycle /// Runs a persistence cycle
/// ///
/// Returns `(snapshot_start, had_new_data, gc_stats)`. `gc_stats` is `None` when GC is /// Returns `(snapshot_start, had_new_data, gc_outcome)`. `gc_outcome` is `None` when GC is
/// disabled; it is returned rather than stashed on `self` so a test can inspect the pass it /// disabled; it is returned rather than stashed on `self` so a test can inspect the pass it
/// just triggered without the backend carrying test-only state. Production reads the same /// just triggered without the backend carrying test-only state. Production reads the same
/// numbers off the `gc` span. /// numbers off the `gc` span.
#[allow(clippy::type_complexity, reason = "only used for tests")]
fn snapshot_and_persist( fn snapshot_and_persist(
&self, &self,
parent_span: Option<tracing::Id>, parent_span: Option<tracing::Id>,
reason: SnapshotReason, reason: SnapshotReason,
turbo_tasks: &TurboTasks<TurboTasksBackend>, turbo_tasks: &TurboTasks<TurboTasksBackend>,
) -> Result<(Instant, bool, Option<GcStats>), anyhow::Error> { ) -> Result<(Instant, bool, Option<(GcStats, GcPassResult)>), anyhow::Error> {
let snapshot_span = let snapshot_span =
tracing::trace_span!(parent: parent_span.clone(), "snapshot", reason = reason.as_str()) tracing::trace_span!(parent: parent_span.clone(), "snapshot", reason = reason.as_str())
.entered(); .entered();
...@@ -1159,24 +1158,26 @@ impl TurboTasksBackend { ...@@ -1159,24 +1158,26 @@ impl TurboTasksBackend {
// can't be used for cross-process trace correlation. // can't be used for cross-process trace correlation.
let wall_start = SystemTime::now(); let wall_start = SystemTime::now();
let mut snapshot_phase = self.snapshot_coord.begin_snapshot(); let mut snapshot_phase = self.snapshot_coord.begin_snapshot();
let (gc_elapsed, gc_roots_to_persist, gc_stats) = if self.gc_enabled { let (gc_elapsed, gc_roots_to_persist, gc_outcome) = if self.gc_enabled {
let gc_span = tracing::info_span!( let gc_span = tracing::info_span!(
parent: parent_span.clone(), parent: parent_span.clone(),
"gc", "gc",
stats = tracing::field::Empty, stats = tracing::field::Empty,
interrupted = tracing::field::Empty
) )
.entered(); .entered();
let (stats, roots) = let (stats, result, roots) =
self.gc_collect(turbo_tasks, &snapshot_phase, reason.gc_is_interruptible()); self.gc_collect(turbo_tasks, &snapshot_phase, reason.gc_is_interruptible());
gc_span.record("stats", display(&stats)); gc_span.record("stats", display(&stats));
if stats.interrupted { gc_span.record("interrupted", result.interrupted);
if result.interrupted {
// If we were interrupted also abandon the persistence loop. // If we were interrupted also abandon the persistence loop.
// This ensures that we don't persist roots that were not completely validated. // This ensures that we don't persist roots that were not completely validated.
drop(snapshot_phase); drop(snapshot_phase);
drop(gc_span); drop(gc_span);
return Ok((start, false, Some(stats))); return Ok((start, false, Some((stats, result))));
} }
(Some(start.elapsed()), roots, Some(stats)) (Some(start.elapsed()), roots, Some((stats, result)))
} else { } else {
(None, None, None) (None, None, None)
}; };
...@@ -1195,7 +1196,7 @@ impl TurboTasksBackend { ...@@ -1195,7 +1196,7 @@ impl TurboTasksBackend {
// No tasks modified since the last snapshot — drop the guard (which // No tasks modified since the last snapshot — drop the guard (which
// calls end_snapshot) and skip the expensive O(N) scan. // calls end_snapshot) and skip the expensive O(N) scan.
drop(snapshot_guard); drop(snapshot_guard);
return Ok((start, false, gc_stats)); return Ok((start, false, gc_outcome));
} }
#[cfg(feature = "print_cache_item_size")] #[cfg(feature = "print_cache_item_size")]
...@@ -1486,7 +1487,7 @@ impl TurboTasksBackend { ...@@ -1486,7 +1487,7 @@ impl TurboTasksBackend {
// was present, and every modification that increments the count also failed // was present, and every modification that increments the count also failed
// during encoding. // during encoding.
std::hint::cold_path(); std::hint::cold_path();
return Ok((snapshot_time, false, gc_stats)); return Ok((snapshot_time, false, gc_outcome));
} }
let persist_start = Instant::now(); let persist_start = Instant::now();
...@@ -1622,7 +1623,7 @@ impl TurboTasksBackend { ...@@ -1622,7 +1623,7 @@ impl TurboTasksBackend {
]), ]),
))); )));
Ok((snapshot_time, true, gc_stats)) Ok((snapshot_time, true, gc_outcome))
} }
fn startup(&self, turbo_tasks: &TurboTasks<TurboTasksBackend>) { fn startup(&self, turbo_tasks: &TurboTasks<TurboTasksBackend>) {
...@@ -3104,7 +3105,7 @@ impl TurboTasksBackend { ...@@ -3104,7 +3105,7 @@ impl TurboTasksBackend {
Self::log_unrecoverable_persist_error(); Self::log_unrecoverable_persist_error();
return; return;
} }
Ok((snapshot_start, new_data, _gc_stats)) => { Ok((snapshot_start, new_data, _gc_outcome)) => {
// if we see 'new_data' then the next idle transition is 'fresh' // if we see 'new_data' then the next idle transition is 'fresh'
fresh_idle = new_data; fresh_idle = new_data;
is_first = false; is_first = false;
......
...@@ -983,6 +983,16 @@ impl AggregationUpdateQueue { ...@@ -983,6 +983,16 @@ impl AggregationUpdateQueue {
} }
} }
/// A queue that defers every optimization: each request is recorded on its task's
/// `optimization_pending` flag instead of being enqueued.
pub fn new_without_optimizations() -> Self {
Self {
// Start the budget already spent, so every optimization is refused.
optimizations_executed: MAX_OPTIMIZATIONS_PER_QUEUE,
..Self::new()
}
}
/// Returns true, when the queue is empty. /// Returns true, when the queue is empty.
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
let Self { let Self {
...@@ -1169,6 +1179,74 @@ impl AggregationUpdateQueue { ...@@ -1169,6 +1179,74 @@ impl AggregationUpdateQueue {
queue.execute(ctx); queue.execute(ctx);
} }
/// Whether only rebalance work (`balance_edge` / `optimize`) is left.
///
/// GC uses this to stop once edge removal is done and defer the rebalance until the parallel
/// collect is quiescent.
pub fn only_rebalance_remains(&self) -> bool {
let Self {
jobs,
aggregation_number_updates,
find_and_schedule,
balance_queue: _,
optimize_queue: _,
optimizations_executed: _,
done_aggregation_number_updates: _,
scheduled_tasks,
#[cfg(feature = "trace_aggregation_update_stats")]
stats: _,
} = self;
assert!(
find_and_schedule.is_empty() && scheduled_tasks.is_empty(),
"GC deletion must not schedule task executions, but {} find_and_schedule and {} \
scheduled_tasks jobs are pending",
find_and_schedule.len(),
scheduled_tasks.len(),
);
jobs.is_empty() && aggregation_number_updates.is_empty()
}
/// Takes the deferred balance edges, leaving the queue empty of them.
///
/// Only meaningful for a queue stopped at the rebalance boundary; see
/// [`CleanupOldEdgesOperation::run_edge_deletions_only`], which is the only caller.
pub(super) fn take_deferred_balance_edges(
&mut self,
) -> impl Iterator<Item = (TaskId, TaskId)> + use<> {
assert!(
self.only_rebalance_remains(),
"take_deferred_balance_edges expects a queue stopped at the rebalance boundary"
);
assert!(
self.optimize_queue.is_empty(),
"a queue built with `new_without_optimizations` must never hold optimize jobs"
);
take(&mut self.balance_queue)
.into_iter()
.map(|job| (job.upper_id, job.task_id))
}
/// Queues balance jobs for `edges`, skipping any whose endpoints have since been collected:
/// the edge went with them, and balancing a deleted task is what the deferral exists to avoid.
pub fn extend_balance_edges(
&mut self,
edges: impl IntoIterator<Item = (TaskId, TaskId)>,
ctx: &mut impl ExecuteContext<'_>,
) {
for (upper_id, task_id) in edges {
// structure as 2 if statements to ensure the first guard is dropped before the second
// one is fetched.
if ctx.task(upper_id, TaskDataCategory::Meta).deleted() {
continue;
}
if ctx.task(task_id, TaskDataCategory::Meta).deleted() {
continue;
}
self.push(AggregationUpdateJob::BalanceEdge { upper_id, task_id });
}
}
/// Executes a single step of the queue. Returns true, when the queue is empty. /// Executes a single step of the queue. Returns true, when the queue is empty.
pub fn process(&mut self, ctx: &mut impl ExecuteContext<'_>) -> bool { pub fn process(&mut self, ctx: &mut impl ExecuteContext<'_>) -> bool {
if let Some(job) = self.jobs.pop_front() { if let Some(job) = self.jobs.pop_front() {
......
...@@ -107,7 +107,37 @@ impl CleanupOldEdgesOperation { ...@@ -107,7 +107,37 @@ impl CleanupOldEdgesOperation {
.execute_with_stats(ctx) .execute_with_stats(ctx)
} }
fn execute_with_stats(mut self, ctx: &mut impl ExecuteContext<'_>) -> Stats { /// GC variant: tears down `outdated`, running only the edge deletions.
///
/// Returns the balance jobs that the deletions produced, for the caller to replay once the
/// parallel collect is quiescent. Deletion is safe to run concurrently, but `balance_edge`
/// *adds* edges, which is not while other workers are still collecting.
pub fn run_edge_deletions_only<'a, C: ExecuteContext<'a>>(
task_id: TaskId,
outdated: Vec<OutdatedEdge>,
ctx: &mut C,
) -> impl Iterator<Item = (TaskId, TaskId)> + use<C> {
let op = CleanupOldEdgesOperation::RemoveEdges {
task_id,
outdated,
queue: AggregationUpdateQueue::new_without_optimizations(),
};
op.execute_inner(ctx, true)
.1
.map(|mut queue| queue.take_deferred_balance_edges())
.into_iter()
.flatten()
}
fn execute_with_stats(self, ctx: &mut impl ExecuteContext<'_>) -> Stats {
self.execute_inner(ctx, false).0
}
fn execute_inner(
mut self,
ctx: &mut impl ExecuteContext<'_>,
stop_when_only_rebalance_remains: bool,
) -> (Stats, Option<AggregationUpdateQueue>) {
loop { loop {
ctx.operation_suspend_point(&self); ctx.operation_suspend_point(&self);
match self { match self {
...@@ -320,6 +350,12 @@ impl CleanupOldEdgesOperation { ...@@ -320,6 +350,12 @@ impl CleanupOldEdgesOperation {
} }
} }
CleanupOldEdgesOperation::AggregationUpdate { ref mut queue } => { CleanupOldEdgesOperation::AggregationUpdate { ref mut queue } => {
if stop_when_only_rebalance_remains && queue.only_rebalance_remains() {
// Edge removal is done; hand the rebalance back to the caller. Any
// other pending work would be dropped here, so `only_rebalance_remains`
// asserts that nothing else is left.
return (Default::default(), Some(take(queue)));
}
if queue.process(ctx) { if queue.process(ctx) {
self = CleanupOldEdgesOperation::Done { self = CleanupOldEdgesOperation::Done {
#[cfg(feature = "trace_aggregation_update_stats")] #[cfg(feature = "trace_aggregation_update_stats")]
...@@ -330,7 +366,7 @@ impl CleanupOldEdgesOperation { ...@@ -330,7 +366,7 @@ impl CleanupOldEdgesOperation {
} }
} }
CleanupOldEdgesOperation::Done { stats } => { CleanupOldEdgesOperation::Done { stats } => {
return stats; return (stats, None);
} }
} }
} }
......
...@@ -16,6 +16,8 @@ use tracing::span::Id; ...@@ -16,6 +16,8 @@ use tracing::span::Id;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, parallel}; use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, parallel};
#[cfg(debug_assertions)]
use crate::backend::storage_schema::GcRootHolders;
use crate::{ use crate::{
backend::storage_schema::{ backend::storage_schema::{
DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability, DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability,
...@@ -580,18 +582,68 @@ impl Storage { ...@@ -580,18 +582,68 @@ impl Storage {
/// Return the set of all known live roots. /// Return the set of all known live roots.
pub fn gc_scan_roots(&self) -> impl Iterator<Item = TaskId> { pub fn gc_scan_roots(&self) -> impl Iterator<Item = TaskId> {
// Roots that failed the "held by a transient pin" expectation, with the referencing tasks
// that kept them un-collectible. Collected during the scan and reported *after* it: naming
// a dependent means reading its storage, and the shard locks are held inside the closure.
#[cfg(debug_assertions)]
let unexpected = std::sync::Mutex::new(Vec::<(TaskId, GcRootHolders)>::new());
let per_shard: Vec<Vec<TaskId>> = let per_shard: Vec<Vec<TaskId>> =
parallel::map_collect(&(0..self.shard_count()).collect::<Vec<_>>(), |&index| { parallel::map_collect(&(0..self.shard_count()).collect::<Vec<_>>(), |&index| {
let mut roots = Vec::new(); let mut roots = Vec::new();
self.for_each_resident_persistent_in_shard(index, |task_id, storage| { self.for_each_resident_persistent_in_shard(index, |task_id, storage| {
if storage.gc_is_root() { if storage.gc_is_root() {
// The `is_root` criteria is conservative, in debug assert that we aren't m // The `is_root` criteria is conservative, in debug assert that we aren't
storage.gc_debug_assert_root_held_by_transient_pin(); // marking things as roots for surprising reasons
#[cfg(debug_assertions)]
if !storage.gc_is_held_by_transient_pin() {
unexpected
.lock()
.unwrap()
.push((task_id, storage.gc_root_holders()));
}
roots.push(task_id); roots.push(task_id);
} }
}); });
roots roots
}); });
#[cfg(debug_assertions)]
{
use std::fmt::Write as _;
let unexpected = unexpected.into_inner().unwrap();
if !unexpected.is_empty() {
let mut report = String::new();
fn describe_task(storage: &Storage, task_id: TaskId) -> String {
storage
.access_mut(task_id)
.get_persistent_task_type()
// `NativeFunction`'s `Debug` is the public view of its name fields.
.map(|t| format!("{:?}", t.native_fn))
.unwrap_or_else(|| "<unknown>".to_string())
}
for (task_id, holders) in &unexpected {
let _ = writeln!(
report,
" {} ({task_id:?}) is held by:",
describe_task(self, *task_id)
);
for (kind, holder) in holders.iter() {
let _ = writeln!(
report,
" via {kind}: {} ({holder:?})",
describe_task(self, *holder)
);
}
}
panic!(
"{} GC root(s) held by a non-transient pin.\nBeing held by another kind of \
reference implies a bug in GC or the aggregation graph.\n{report}",
unexpected.len()
);
}
}
per_shard.into_iter().flatten() per_shard.into_iter().flatten()
} }
......
...@@ -887,9 +887,9 @@ impl TaskStorage { ...@@ -887,9 +887,9 @@ impl TaskStorage {
&& self.gc_transient_ref_count() == 0 && self.gc_transient_ref_count() == 0
&& self.get_activeness().is_none() && self.get_activeness().is_none()
&& self.get_in_progress().is_none() && self.get_in_progress().is_none()
// It is rare for upper/followers to be present when the ref counts are 0 but it can happen transiently during a concurrent GC pass as uppers are moved around during the cascade. // It is rare for an upper to be present when the ref counts are 0, but it
// happens transiently during a concurrent GC pass as uppers move around in the cascade.
&& self.upper().is_empty() && self.upper().is_empty()
&& self.followers().is_none_or(|f| f.is_empty())
// `collectibles_dependents` is Meta, so it is always checkable here. // `collectibles_dependents` is Meta, so it is always checkable here.
&& self && self
.collectibles_dependents() .collectibles_dependents()
...@@ -908,7 +908,7 @@ impl TaskStorage { ...@@ -908,7 +908,7 @@ impl TaskStorage {
/// ///
/// NOTE: this is a conservative classification. The typical reason is that there is a /// NOTE: this is a conservative classification. The typical reason is that there is a
/// [`TaskStorage::transient_ref`] live, but this will return true if there is merely an /// [`TaskStorage::transient_ref`] live, but this will return true if there is merely an
/// `upper/follower`. /// `upper`.
pub fn gc_is_root(&self) -> bool { pub fn gc_is_root(&self) -> bool {
self.flags.is_restored(TaskDataCategory::Meta) self.flags.is_restored(TaskDataCategory::Meta)
&& !self.flags.deleted() && !self.flags.deleted()
...@@ -916,17 +916,60 @@ impl TaskStorage { ...@@ -916,17 +916,60 @@ impl TaskStorage {
&& !self.gc_maybe_collectible() && !self.gc_maybe_collectible()
} }
/// Assert that a task classified by [`TaskStorage::gc_is_root`] is held by a pin that eviction /// Whether this task is held by a pin that eviction cannot drop, which is what a task
/// cannot drop. /// classified by [`TaskStorage::gc_is_root`] is expected to be held by.
pub fn gc_debug_assert_root_held_by_transient_pin(&self) { #[cfg(debug_assertions)]
debug_assert!(self.gc_is_root()); // sanity for our caller pub fn gc_is_held_by_transient_pin(&self) -> bool {
debug_assert!( self.gc_transient_ref_count() > 0
self.gc_transient_ref_count() > 0 || self.get_in_progress().is_some()
|| self.get_in_progress().is_some() || self.get_activeness().is_some()
|| self.get_activeness().is_some(), }
"GC root is held by a non-transient pin: {self:?}.\nBeing held by another kind of \
reference implies a bug in GC or the aggregation graph." /// The concrete references keeping this task un-collectible, as `(edge kind, holder task)`
); /// pairs.
#[cfg(debug_assertions)]
pub fn gc_root_holders(&self) -> GcRootHolders {
let mut holders = GcRootHolders::default();
for (&upper, _) in self.upper().iter() {
holders.push("upper", upper);
}
if let Some(deps) = self.collectibles_dependents() {
for &(_, task) in deps.iter() {
holders.push("collectibles_dependent", task);
}
}
for &task in self.output_dependent().iter() {
holders.push("output_dependent", task);
}
if let Some(deps) = self.cell_dependents() {
// In a `cell_dependents` entry `CellRef.task` is the DEPENDENT's id, not this task's.
for cell_ref in deps.iter() {
holders.push("cell_dependent", cell_ref.task);
}
}
if let Some(deps) = self.cell_dependents_hashed() {
for (cell_ref, _) in deps.iter() {
holders.push("cell_dependent_hashed", cell_ref.task);
}
}
holders
}
}
/// `(edge kind, holder task)` pairs explaining why a task is not collectible.
/// See [`TaskStorage::gc_root_holders`].
#[cfg(debug_assertions)]
#[derive(Default, Debug)]
pub struct GcRootHolders(Vec<(&'static str, TaskId)>);
#[cfg(debug_assertions)]
impl GcRootHolders {
fn push(&mut self, kind: &'static str, task: TaskId) {
self.0.push((kind, task));
}
pub fn iter(&self) -> impl Iterator<Item = &(&'static str, TaskId)> {
self.0.iter()
} }
} }
......
...@@ -17,8 +17,8 @@ use turbo_persistence::{CompactConfig, TurboPersistence}; ...@@ -17,8 +17,8 @@ use turbo_persistence::{CompactConfig, TurboPersistence};
use crate::database::turbo::{self, TurboKeyValueDatabase}; use crate::database::turbo::{self, TurboKeyValueDatabase};
pub use crate::{ pub use crate::{
backend::{ backend::{
BackendOptions, EvictionMode, GcStats, StorageMode, TestSnapshotOutcome, TtlCounter, BackendOptions, EvictionMode, GcPassResult, GcStats, StorageMode, TestSnapshotOutcome,
TurboTasksBackend, TtlCounter, TurboTasksBackend,
}, },
database::{ database::{
db_invalidation, db_invalidation,
......
//! Shared graph shapes for the GC tests. //! Shared graph shapes for the GC tests.
//!
//! This module is compiled into each test binary separately, so any helper a given binary doesn't
//! call reads as dead code there.
#![allow(dead_code)]
use anyhow::Result; use anyhow::Result;
use turbo_tasks::{ResolvedVc, State, Vc}; use turbo_tasks::{ResolvedVc, State, Vc};
...@@ -72,29 +76,115 @@ pub fn create_generation() -> Vc<Generation> { ...@@ -72,29 +76,115 @@ pub fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell() Generation(State::new(0)).cell()
} }
/// A leaf keyed by (generation, index). Bumping the generation makes `wide_root` read an entirely /// Children per interior node in [`wide_root`]'s tree.
/// fresh set of these, disconnecting the whole previous generation. ///
/// The tree must be deep enough for `effective` to cross `LEAF_NUMBER`, the transition that
/// converts a node's children into `followers`. A flat tree never gets there however wide it is,
/// and so exercises no follower edges.
pub const BRANCHING: u32 = 4;
/// A leaf keyed by (generation, index).
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn leaf(generation: u32, index: u32) -> Vc<u32> { pub fn leaf(generation: u32, index: u32) -> Vc<u32> {
Vc::cell(generation.wrapping_mul(1000).wrapping_add(index)) Vc::cell(generation.wrapping_mul(1000).wrapping_add(index))
} }
/// A shared-dependency layer, so the disconnected garbage is a subtree (intermediate + leaf) and /// One interior node of the tree, covering `[index, index + span)`.
/// the tests exercise the cascade rather than single-node collection. ///
/// Recurses until the span fits in a single [`leaf`], making one generation's garbage a
/// multi-level subtree rather than a two-layer fan-out.
#[turbo_tasks::function]
pub async fn subtree(generation: u32, index: u32, span: u32) -> Result<Vc<u32>> {
if span <= 1 {
return Ok(Vc::cell(1 + *leaf(generation, index).await?));
}
let child_span = span.div_ceil(BRANCHING);
let mut sum = 0u32;
let mut start = index;
while start < index + span {
let this_span = child_span.min(index + span - start);
sum = sum.wrapping_add(*subtree(generation, start, this_span).await?);
start += this_span;
}
Ok(Vc::cell(sum))
}
/// A **live** interior node: keyed only on `index`/`span`, not on the generation, so the same task
/// survives every generation bump and simply re-executes with new children.
///
/// This is what makes the fixture reproduce the shape a real build has, and the one the flat
/// fixture never did: a *retained* parent whose children are collected out from under it. Two
/// sibling children of the same live parent go garbage in the same pass and are discovered
/// independently by the shard scan, so they are collected concurrently and both mutate this
/// parent's aggregation state while it stays live.
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn intermediate(generation: u32, index: u32) -> Result<Vc<u32>> { pub async fn live_parent(
Ok(Vc::cell(1 + *leaf(generation, index).await?)) generation: ResolvedVc<Generation>,
index: u32,
span: u32,
) -> Result<Vc<u32>> {
let generation_value = *generation.await?.get();
Ok(Vc::cell(*subtree(generation_value, index, span).await?))
} }
/// Reads `width` intermediates (each reading a leaf). Bumping the generation re-executes this and /// Reads a `width`-leaf tree of [`subtree`] nodes through a layer of retained [`live_parent`]
/// connects a fresh generation's worth of tasks, disconnecting the entire previous generation /// nodes. Bumping the generation re-executes the tree and connects a fresh generation's worth of
/// (`2 * width` tasks) as garbage. /// tasks, disconnecting the entire previous generation as garbage — while the `live_parent` layer
/// itself stays resident across passes.
///
/// The disconnected set is `interior + leaf` nodes; use [`generation_task_count`] to size
/// assertions rather than assuming a fixed multiple of `width`.
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
pub async fn wide_root(generation: ResolvedVc<Generation>, width: u32) -> Result<Vc<u32>> { pub async fn wide_root(generation: ResolvedVc<Generation>, width: u32) -> Result<Vc<u32>> {
let generation = *generation.await?.get(); let child_span = width.div_ceil(BRANCHING);
let mut sum = 0u32; let mut sum = 0u32;
for index in 0..width { let mut start = 0;
sum = sum.wrapping_add(*intermediate(generation, index).await?); while start < width {
let this_span = child_span.min(width - start);
sum = sum.wrapping_add(*live_parent(*generation, start, this_span).await?);
start += this_span;
} }
Ok(Vc::cell(sum)) Ok(Vc::cell(sum))
} }
/// Tasks that one generation makes **collectable**: every [`subtree`] node plus every [`leaf`]
/// under the retained [`live_parent`] layer. Mirrors the fixture's own recursion so the two
/// cannot drift.
///
/// The `live_parent`s themselves are *not* counted: they are keyed only on `index`/`span`, so the
/// same tasks are reused across generations and stay resident.
pub fn generation_task_count(width: u32) -> usize {
fn count(span: u32) -> usize {
if span <= 1 {
// A `subtree` node plus the `leaf` it reads.
return 2;
}
let child_span = span.div_ceil(BRANCHING);
let mut total = 1; // this interior node
let mut start = 0;
while start < span {
let this_span = child_span.min(span - start);
total += count(this_span);
start += this_span;
}
total
}
// `wide_root` fans out into `live_parent`s, each covering `child_span` leaves; only the
// `subtree` beneath each one is collectable.
let child_span = width.div_ceil(BRANCHING);
let mut total = 0;
let mut start = 0;
while start < width {
let this_span = child_span.min(width - start);
total += count(this_span);
start += this_span;
}
total
}
/// The value [`wide_root`] computes for `generation`, for correctness assertions.
pub fn expected_value(generation: u32, width: u32) -> u32 {
(0..width).fold(0u32, |acc, index| {
acc.wrapping_add(1u32.wrapping_add(generation.wrapping_mul(1000).wrapping_add(index)))
})
}
...@@ -13,7 +13,7 @@ use turbo_tasks::TurboTasks; ...@@ -13,7 +13,7 @@ use turbo_tasks::TurboTasks;
use turbo_tasks_backend::TurboTasksBackend; use turbo_tasks_backend::TurboTasksBackend;
use crate::{ use crate::{
gc_fixture::{create_generation, wide_root}, gc_fixture::{create_generation, generation_task_count, wide_root},
util::create_tt_with_gc_min_progress, util::create_tt_with_gc_min_progress,
}; };
...@@ -21,7 +21,7 @@ use crate::{ ...@@ -21,7 +21,7 @@ use crate::{
/// difference. /// difference.
const WIDTH: u32 = 400; const WIDTH: u32 = 400;
/// Builds generation `gen_value`, disconnecting the previous generation's `2 * WIDTH` tasks. /// Builds generation `gen_value`, disconnecting the previous generation's whole tree.
async fn build_generation(tt: &Arc<TurboTasks<TurboTasksBackend>>, gen_value: u32) { async fn build_generation(tt: &Arc<TurboTasks<TurboTasksBackend>>, gen_value: u32) {
turbo_tasks::run_once(tt.clone(), async move { turbo_tasks::run_once(tt.clone(), async move {
let generation_op = create_generation(); let generation_op = create_generation();
...@@ -76,16 +76,16 @@ async fn gc_min_progress_floor_beats_a_waiting_operation() { ...@@ -76,16 +76,16 @@ async fn gc_min_progress_floor_beats_a_waiting_operation() {
let stats = outcome.gc_stats(); let stats = outcome.gc_stats();
assert!( assert!(
!stats.interrupted, !outcome.gc_interrupted(),
"the min-progress floor must suppress the interrupt (collected={})", "the min-progress floor must suppress the interrupt (collected={})",
stats.collected stats.collected
); );
// Generation 0 is fully disconnected by generation 1, and an uninterrupted pass must take all // Generation 0 is fully disconnected by generation 1, and an uninterrupted pass must take all
// of it: `2 * WIDTH` tasks (an `intermediate` and a `leaf` per index). The live generation and // of it: every `subtree` node plus every `leaf`. The live generation and the transient
// the transient `run_once` roots are not collectible, so this is an exact count, not a floor. // `run_once` roots are not collectible, so this is an exact count, not a floor.
assert_eq!( assert_eq!(
stats.collected, stats.collected,
2 * WIDTH as usize, generation_task_count(WIDTH),
"a completing pass must collect the whole disconnected generation: {stats}" "a completing pass must collect the whole disconnected generation: {stats}"
); );
...@@ -116,13 +116,13 @@ async fn gc_interrupt_is_self_healing() { ...@@ -116,13 +116,13 @@ async fn gc_interrupt_is_self_healing() {
// first thing worth seeing. // first thing worth seeing.
println!("round {gen_value}: {stats}"); println!("round {gen_value}: {stats}");
collected_in_phase_1 += stats.collected; collected_in_phase_1 += stats.collected;
interrupted_rounds += usize::from(stats.interrupted); interrupted_rounds += usize::from(outcome.gc_interrupted());
} }
// Phase 2: a completing pass must recover exactly what phase 1 left behind. // Phase 2: a completing pass must recover exactly what phase 1 left behind.
let healed = tt.backend().gc_for_testing(&tt); let healed = tt.backend().gc_for_testing(&tt);
let produced = (2 * WIDTH as usize) * (ROUNDS as usize); let produced = generation_task_count(WIDTH) * (ROUNDS as usize);
let total_collected = collected_in_phase_1 + healed; let total_collected = collected_in_phase_1 + healed;
println!( println!(
"self-healing: collected_in_phase_1={collected_in_phase_1} healed={healed} \ "self-healing: collected_in_phase_1={collected_in_phase_1} healed={healed} \
......
...@@ -11,11 +11,11 @@ use turbo_tasks::TurboTasks; ...@@ -11,11 +11,11 @@ use turbo_tasks::TurboTasks;
use turbo_tasks_backend::TurboTasksBackend; use turbo_tasks_backend::TurboTasksBackend;
use crate::{ use crate::{
gc_fixture::{create_generation, wide_root}, gc_fixture::{create_generation, expected_value, generation_task_count, wide_root},
util::create_tt_with_gc_min_progress, util::create_tt_with_gc_min_progress,
}; };
const WIDTH: u32 = 24; const WIDTH: u32 = 256;
/// Repeatedly swapping a wide set of common dependencies (as happens when a project is re-rooted or /// Repeatedly swapping a wide set of common dependencies (as happens when a project is re-rooted or
/// its dependencies churn) must NOT grow the resident task set monotonically. Each round bumps the /// its dependencies churn) must NOT grow the resident task set monotonically. Each round bumps the
...@@ -92,23 +92,23 @@ async fn gc_re_rooting_stays_flat() { ...@@ -92,23 +92,23 @@ async fn gc_re_rooting_stays_flat() {
interrupted_rounds += usize::from(interrupted); interrupted_rounds += usize::from(interrupted);
} }
let no_gc_growth = baseline + (2 * WIDTH as usize) * (ROUNDS as usize); let no_gc_growth = baseline + generation_task_count(WIDTH) * (ROUNDS as usize);
println!( println!(
"baseline={baseline} max_resident={max_resident} total_collected={total_collected} \ "baseline={baseline} max_resident={max_resident} total_collected={total_collected} \
interrupted_rounds={interrupted_rounds} no_gc_growth_would_be={no_gc_growth}" interrupted_rounds={interrupted_rounds} no_gc_growth_would_be={no_gc_growth}"
); );
// Without GC the persistent resident set would climb by ~2*WIDTH per round (each generation's // Without GC the persistent resident set would climb by a whole generation per round (each
// intermediates + leaves persisted forever), so a bound within a small constant of the baseline // generation's subtree nodes + leaves persisted forever), so a bound within a small constant
// is only satisfiable if each generation's garbage subtree is actually collected. // of the baseline is only satisfiable if each generation's garbage is actually collected.
assert!( assert!(
max_resident <= baseline + 2 * WIDTH as usize, max_resident <= baseline + generation_task_count(WIDTH),
"persistent resident set grew too much across re-rooting (max={max_resident}, \ "persistent resident set grew too much across re-rooting (max={max_resident}, \
baseline={baseline}); GC is not returning to a flat baseline" baseline={baseline}); GC is not returning to a flat baseline"
); );
// Assert GC collected at least half of a full generation per round (slack for tasks that settle // Assert GC collected at least half of a full generation per round (slack for tasks that settle
// across passes), proving the flat baseline comes from GC collecting garbage — not merely from // across passes), proving the flat baseline comes from GC collecting garbage — not merely from
// eviction hiding it on disk. // eviction hiding it on disk.
let expected_min_collected = (2 * WIDTH as usize) * (ROUNDS as usize) / 2; let expected_min_collected = generation_task_count(WIDTH) * (ROUNDS as usize) / 2;
assert!( assert!(
total_collected >= expected_min_collected, total_collected >= expected_min_collected,
"GC collected too little ({total_collected}); expected >= {expected_min_collected} across \ "GC collected too little ({total_collected}); expected >= {expected_min_collected} across \
...@@ -124,10 +124,7 @@ async fn gc_re_rooting_stays_flat() { ...@@ -124,10 +124,7 @@ async fn gc_re_rooting_stays_flat() {
let generation_op = create_generation(); let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?; let generation_vc = generation_op.resolve().strongly_consistent().await?;
let output = wide_root(generation_vc, WIDTH); let output = wide_root(generation_vc, WIDTH);
// generation is ROUNDS; sum = WIDTH intermediates each = 1 + leaf(ROUNDS, i). let expected: u32 = expected_value(ROUNDS, WIDTH);
let expected: u32 = (0..WIDTH)
.map(|i| 1 + (ROUNDS.wrapping_mul(1000).wrapping_add(i)))
.fold(0u32, |a, b| a.wrapping_add(b));
assert_eq!(*output.read_strongly_consistent().await?, expected); assert_eq!(*output.read_strongly_consistent().await?, expected);
let _ = &tt3; let _ = &tt3;
anyhow::Ok(()) anyhow::Ok(())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment