Unverified Commit 2994f6be authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks-backend: let a GC pass wind down when it is blocking an operation (#96676)

A GC pass holds total operation exclusion, so every invalidation, cell
update and child connect
blocks for its whole duration. In dev that is an HMR edit stalled behind
the cascade, and the pass
had no way to notice or yield.

This makes a gc pass interruptible so we are guaranteed to not stall too
much work. To ensure progress we have a minimum guaranteed execution
time `100ms`
parent 552dd353
...@@ -19,7 +19,8 @@ ...@@ -19,7 +19,8 @@
use std::{ use std::{
fmt::Display, fmt::Display,
ops::ControlFlow, ops::ControlFlow,
time::{Duration, SystemTime, UNIX_EPOCH}, sync::atomic::{AtomicBool, Ordering},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
}; };
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
...@@ -70,9 +71,44 @@ enum GcJob { ...@@ -70,9 +71,44 @@ enum GcJob {
Collect(TaskId), Collect(TaskId),
} }
/// Decides when a GC pass should stop early because it is delaying real work.
struct GcBudget<'a> {
phase: &'a SnapshotPhase<'a, AnyOperation>,
started: Instant,
/// The minimum quantum of work this pass does before any interrupt is honoured.
min_progress: Duration,
/// Latched on the first trip.
stopped: AtomicBool,
}
impl GcBudget<'_> {
fn should_stop(&self) -> bool {
// We only stop if someone is waiting and we have already run for at least our
// `min_progress` To make querying the clock and phase cheaper we record it as a
// durable decision.
if self.stopped.load(Ordering::Relaxed) {
return true;
}
if !self.phase.operations_waiting() {
return false;
}
if self.started.elapsed() < self.min_progress {
return false;
}
// If we get here then there is an operation waiting _and_ we have already executed for at
// least our min_progress duration
self.stopped.store(true, Ordering::Relaxed);
true
}
fn was_interrupted(&self) -> bool {
self.stopped.load(Ordering::Relaxed)
}
}
/// Observability counters for one [`TurboTasksBackend::gc_collect`] pass. /// Observability counters for one [`TurboTasksBackend::gc_collect`] pass.
#[derive(Default, Debug)] #[derive(Default, Debug)]
pub(crate) struct GcStats { pub struct GcStats {
/// Number of roots detected by the pass /// Number of roots detected by the pass
pub gc_roots: usize, pub gc_roots: usize,
/// Tasks collected (marked soft-deleted). /// Tasks collected (marked soft-deleted).
...@@ -83,6 +119,8 @@ pub(crate) struct GcStats { ...@@ -83,6 +119,8 @@ pub(crate) struct GcStats {
pub aged_out_roots: usize, pub aged_out_roots: usize,
/// Persisted roots that this pass collected, to be dropped from the roots map. /// Persisted roots that this pass collected, to be dropped from the roots map.
pub deleted_roots: Vec<TaskId>, pub deleted_roots: Vec<TaskId>,
/// The gc loop was interrupted by competing work.
pub interrupted: bool,
} }
impl Display for GcStats { impl Display for GcStats {
...@@ -90,11 +128,12 @@ impl Display for GcStats { ...@@ -90,11 +128,12 @@ impl Display for GcStats {
write!( write!(
f, f,
"gc_roots = {gc_roots}, collected: {collected}, edges_deleted: {edges_deleted}, \ "gc_roots = {gc_roots}, collected: {collected}, edges_deleted: {edges_deleted}, \
aged_out_roots = {aged_out_roots}", aged_out_roots = {aged_out_roots}, interrupted = {interrupted}",
gc_roots = self.gc_roots, gc_roots = self.gc_roots,
collected = self.collected, collected = self.collected,
edges_deleted = self.edges_deleted, edges_deleted = self.edges_deleted,
aged_out_roots = self.aged_out_roots aged_out_roots = self.aged_out_roots,
interrupted = self.interrupted
) )
} }
} }
...@@ -123,11 +162,16 @@ impl TurboTasksBackend { ...@@ -123,11 +162,16 @@ impl TurboTasksBackend {
/// `phase` is the held exclusion; it is the caller's proof that no operation is running, which /// `phase` is the held exclusion; it is the caller's proof that no operation is running, which
/// is what makes it safe to mutate the graph here. /// is what makes it safe to mutate the graph here.
/// ///
/// Returns [`GcStats`] for the pass. /// `interruptible` controls whether we should abandon GC if other tasks are waiting to run.
/// Abandonment is controlled by [`GcBudget`] which ensures we can make a minimum amount of
/// progress even under load.
///
/// Returns [`GcStats`] for the pass and the new roots to persist if any
pub(crate) fn gc_collect( pub(crate) fn gc_collect(
&self, &self,
turbo_tasks: &TurboTasks<TurboTasksBackend>, turbo_tasks: &TurboTasks<TurboTasksBackend>,
phase: &SnapshotPhase<'_, AnyOperation>, phase: &SnapshotPhase<'_, AnyOperation>,
interruptible: bool,
) -> (GcStats, Option<Vec<(TaskId, TtlCounter)>>) { ) -> (GcStats, Option<Vec<(TaskId, TtlCounter)>>) {
// Record the time at the beginning of the loop to have a consistent timestamp for the roots // Record the time at the beginning of the loop to have a consistent timestamp for the roots
let now = SystemTime::now() let now = SystemTime::now()
...@@ -147,6 +191,17 @@ impl TurboTasksBackend { ...@@ -147,6 +191,17 @@ impl TurboTasksBackend {
let aged_out_count = aged_out.len(); let aged_out_count = aged_out.len();
// TODO(perf): recycle the task ids of collected tasks. // TODO(perf): recycle the task ids of collected tasks.
let budget = if interruptible {
Some(GcBudget {
phase,
started: Instant::now(),
min_progress: self.gc_min_progress,
stopped: AtomicBool::new(false),
})
} else {
None
};
let mut stats: GcStats = scope_unbounded_with( let mut stats: GcStats = scope_unbounded_with(
// Start by scanning all shards and collecting the aged out roots from prior sessions // Start by scanning all shards and collecting the aged out roots from prior sessions
(0..self.storage.shard_count()) (0..self.storage.shard_count())
...@@ -154,6 +209,12 @@ impl TurboTasksBackend { ...@@ -154,6 +209,12 @@ impl TurboTasksBackend {
.chain(aged_out.into_iter().map(GcJob::Collect)), .chain(aged_out.into_iter().map(GcJob::Collect)),
GcStats::default, GcStats::default,
|spawner, job, stats| { |spawner, job, stats| {
// Abort the gc loop if we are interrupted
if let Some(budget) = &budget
&& budget.should_stop()
{
return ControlFlow::Break(());
}
let collector = |task_id| spawner.spawn(GcJob::Collect(task_id)); let collector = |task_id| spawner.spawn(GcJob::Collect(task_id));
let task_id = match job { let task_id = match job {
GcJob::ScanShard(index) => { GcJob::ScanShard(index) => {
...@@ -224,6 +285,9 @@ impl TurboTasksBackend { ...@@ -224,6 +285,9 @@ impl TurboTasksBackend {
stats.gc_roots = roots.len(); stats.gc_roots = roots.len();
stats.aged_out_roots = aged_out_count; stats.aged_out_roots = aged_out_count;
stats.interrupted = budget
.as_ref()
.is_some_and(|budget| budget.was_interrupted());
// Only persist the roots map if it actually changed // Only persist the roots map if it actually changed
let roots_to_persist: Option<Vec<_>> = let roots_to_persist: Option<Vec<_>> =
...@@ -321,7 +385,7 @@ impl TurboTasksBackend { ...@@ -321,7 +385,7 @@ impl TurboTasksBackend {
); );
let _serialize = self.snapshot_in_progress.lock(); let _serialize = self.snapshot_in_progress.lock();
let phase = self.snapshot_coord.begin_snapshot(); let phase = self.snapshot_coord.begin_snapshot();
let (stats, roots) = self.gc_collect(turbo_tasks, &phase); let (stats, roots) = self.gc_collect(turbo_tasks, &phase, /* interruptible= */ false);
// Persist the roots map this pass produced. Some tests query the roots set and GC itself // Persist the roots map this pass produced. Some tests query the roots set and GC itself
// does as well, this ensures it is available to the next cycle. // does as well, this ensures it is available to the next cycle.
......
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
use std::sync::{ use std::sync::{
Arc, Arc,
atomic::{AtomicUsize, Ordering}, atomic::{AtomicBool, AtomicUsize, Ordering},
}; };
use parking_lot::{Condvar, Mutex}; use parking_lot::{Condvar, Mutex};
...@@ -51,6 +51,7 @@ struct State<O> { ...@@ -51,6 +51,7 @@ struct State<O> {
pub struct SnapshotCoordinator<O = AnyOperation> { pub struct SnapshotCoordinator<O = AnyOperation> {
/// Combined count + bit. See [`SNAPSHOT_REQUESTED_BIT`]. /// Combined count + bit. See [`SNAPSHOT_REQUESTED_BIT`].
in_progress_operations: AtomicUsize, in_progress_operations: AtomicUsize,
operations_waiting: AtomicBool,
state: Mutex<State<O>>, state: Mutex<State<O>>,
/// Notified by the last operation to drain (count drops to `BIT` while /// Notified by the last operation to drain (count drops to `BIT` while
/// `SNAPSHOT_REQUESTED_BIT` is set). Awaited by [`begin_snapshot`]. /// `SNAPSHOT_REQUESTED_BIT` is set). Awaited by [`begin_snapshot`].
...@@ -70,6 +71,7 @@ impl<O> SnapshotCoordinator<O> { ...@@ -70,6 +71,7 @@ impl<O> SnapshotCoordinator<O> {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
in_progress_operations: AtomicUsize::new(0), in_progress_operations: AtomicUsize::new(0),
operations_waiting: AtomicBool::new(false),
state: Mutex::new(State { state: Mutex::new(State {
snapshot_requested: false, snapshot_requested: false,
suspended_operations: FxHashSet::default(), suspended_operations: FxHashSet::default(),
...@@ -113,6 +115,8 @@ impl<O> SnapshotCoordinator<O> { ...@@ -113,6 +115,8 @@ impl<O> SnapshotCoordinator<O> {
if prev - 1 == SNAPSHOT_REQUESTED_BIT { if prev - 1 == SNAPSHOT_REQUESTED_BIT {
this.operations_drained.notify_all(); this.operations_drained.notify_all();
} }
this.operations_waiting.store(true, Ordering::Relaxed);
tokio::task::block_in_place(|| { tokio::task::block_in_place(|| {
this.snapshot_completed this.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested); .wait_while(&mut state, |s| s.snapshot_requested);
...@@ -161,6 +165,7 @@ impl<O> SnapshotCoordinator<O> { ...@@ -161,6 +165,7 @@ impl<O> SnapshotCoordinator<O> {
if prev - 1 == SNAPSHOT_REQUESTED_BIT { if prev - 1 == SNAPSHOT_REQUESTED_BIT {
this.operations_drained.notify_all(); this.operations_drained.notify_all();
} }
this.operations_waiting.store(true, Ordering::Relaxed);
// Wait for the snapshot to finish. // Wait for the snapshot to finish.
tokio::task::block_in_place(|| { tokio::task::block_in_place(|| {
this.snapshot_completed this.snapshot_completed
...@@ -304,12 +309,22 @@ impl<O> SnapshotPhase<'_, O> { ...@@ -304,12 +309,22 @@ impl<O> SnapshotPhase<'_, O> {
pub fn take_suspended_operations(&mut self) -> Vec<Arc<O>> { pub fn take_suspended_operations(&mut self) -> Vec<Arc<O>> {
std::mem::take(&mut self.suspended_operations) std::mem::take(&mut self.suspended_operations)
} }
/// Whether any operation is currently blocked waiting for this exclusion to end
pub fn operations_waiting(&self) -> bool {
self.coord.operations_waiting.load(Ordering::Relaxed)
}
} }
impl<O> Drop for SnapshotPhase<'_, O> { impl<O> Drop for SnapshotPhase<'_, O> {
fn drop(&mut self) { fn drop(&mut self) {
let mut state = self.coord.state.lock(); let mut state = self.coord.state.lock();
state.snapshot_requested = false; state.snapshot_requested = false;
// Clear the sticky waiter flag for the next exclusion. Everyone is about to be unblocked
// and because snapshot_requested is false no new waiters can arrive
self.coord
.operations_waiting
.store(false, Ordering::Relaxed);
let prev = self let prev = self
.coord .coord
.in_progress_operations .in_progress_operations
...@@ -643,6 +658,43 @@ mod tests { ...@@ -643,6 +658,43 @@ mod tests {
); );
} }
#[test]
fn operations_waiting_tracks_blocked_operations() {
let coord = Arc::new(SnapshotCoordinator::<Op>::new());
// Fast path: no exclusion in flight, so these never blocked and are not waiters.
let unblocked = coord.begin_operation();
assert!(
!coord.operations_waiting.load(Ordering::Relaxed),
"operations that never blocked are not waiters"
);
drop(unblocked);
let phase = coord.begin_snapshot();
assert!(
!phase.operations_waiting(),
"holding the exclusion alone is not a waiter"
);
let coord2 = coord.clone();
let op_thread = thread::spawn(move || {
let _guard = coord2.begin_operation();
});
// Wait until the operation is actually parked, not merely spawned: the flag is set under
// the state lock right before the park.
while !phase.operations_waiting() {
thread::yield_now();
}
drop(phase); // releases the waiter
assert!(
!coord.operations_waiting.load(Ordering::Relaxed),
"the sticky waiter flag must be cleared when the phase is dropped"
);
op_thread.join().unwrap();
}
#[test] #[test]
#[should_panic(expected = "already in flight")] #[should_panic(expected = "already in flight")]
fn overlapping_exclusions_panic() { fn overlapping_exclusions_panic() {
......
...@@ -16,7 +16,10 @@ use turbo_persistence::{CompactConfig, TurboPersistence}; ...@@ -16,7 +16,10 @@ use turbo_persistence::{CompactConfig, TurboPersistence};
use crate::database::turbo::{self, TurboKeyValueDatabase}; use crate::database::turbo::{self, TurboKeyValueDatabase};
pub use crate::{ pub use crate::{
backend::{BackendOptions, EvictionMode, StorageMode, TtlCounter, TurboTasksBackend}, backend::{
BackendOptions, EvictionMode, GcStats, StorageMode, TestSnapshotOutcome, TtlCounter,
TurboTasksBackend,
},
database::{ database::{
db_invalidation, db_invalidation,
db_invalidation::StartupCacheState, db_invalidation::StartupCacheState,
......
...@@ -11,6 +11,7 @@ use std::sync::{ ...@@ -11,6 +11,7 @@ use std::sync::{
use anyhow::Result; use anyhow::Result;
use turbo_tasks::{ResolvedVc, State, Vc}; use turbo_tasks::{ResolvedVc, State, Vc};
use turbo_tasks_backend::TestSnapshotOutcome;
use crate::util::{create_tt, create_tt_with_workers}; use crate::util::{create_tt, create_tt_with_workers};
...@@ -34,7 +35,11 @@ async fn eviction_recompute() { ...@@ -34,7 +35,11 @@ async fn eviction_recompute() {
let initial_random = read.random; let initial_random = read.random;
// Trigger snapshot + eviction // Trigger snapshot + eviction
let (had_data, counts) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data,
eviction_counts: counts,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("snapshot had_data={had_data}, evicted: {counts:?}"); println!("snapshot had_data={had_data}, evicted: {counts:?}");
assert!(had_data, "snapshot should have persisted data"); assert!(had_data, "snapshot should have persisted data");
...@@ -73,7 +78,11 @@ async fn eviction_deep_chain() { ...@@ -73,7 +78,11 @@ async fn eviction_deep_chain() {
let initial_random = read.random; let initial_random = read.random;
// Snapshot + evict — expect multiple intermediate tasks evicted // Snapshot + evict — expect multiple intermediate tasks evicted
let (had_data, counts) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data,
eviction_counts: counts,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("deep_chain: snapshot had_data={had_data}, evicted: {counts:?}"); println!("deep_chain: snapshot had_data={had_data}, evicted: {counts:?}");
assert!(had_data, "snapshot should have persisted data"); assert!(had_data, "snapshot should have persisted data");
assert!( assert!(
...@@ -91,7 +100,11 @@ async fn eviction_deep_chain() { ...@@ -91,7 +100,11 @@ async fn eviction_deep_chain() {
let random_after_first = read.random; let random_after_first = read.random;
// Evict again and change again // Evict again and change again
let (had_data2, counts2) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data2,
eviction_counts: counts2,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("deep_chain (2nd): snapshot had_data={had_data2}, evicted: {counts2:?}"); println!("deep_chain (2nd): snapshot had_data={had_data2}, evicted: {counts2:?}");
state.set(0); state.set(0);
...@@ -127,7 +140,11 @@ async fn eviction_dependency_chain() { ...@@ -127,7 +140,11 @@ async fn eviction_dependency_chain() {
let initial_random = read.random; let initial_random = read.random;
// Snapshot + evict // Snapshot + evict
let (had_data, counts) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data,
eviction_counts: counts,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("snapshot had_data={had_data}, evicted: {counts:?}"); println!("snapshot had_data={had_data}, evicted: {counts:?}");
assert!(had_data, "snapshot should have persisted data"); assert!(had_data, "snapshot should have persisted data");
assert!( assert!(
...@@ -144,7 +161,11 @@ async fn eviction_dependency_chain() { ...@@ -144,7 +161,11 @@ async fn eviction_dependency_chain() {
let random_after_first = read.random; let random_after_first = read.random;
// Evict again // Evict again
let (had_data2, counts2) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data2,
eviction_counts: counts2,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("snapshot (2nd) had_data={had_data2}, evicted: {counts2:?}"); println!("snapshot (2nd) had_data={had_data2}, evicted: {counts2:?}");
// Change again // Change again
...@@ -310,7 +331,11 @@ async fn eviction_session_stateful_survives() { ...@@ -310,7 +331,11 @@ async fn eviction_session_stateful_survives() {
assert_eq!(normal_read.value, 43); assert_eq!(normal_read.value, 43);
// Snapshot + evict // Snapshot + evict
let (had_data, counts) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data,
eviction_counts: counts,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("session_stateful: snapshot had_data={had_data}, evicted: {counts:?}"); println!("session_stateful: snapshot had_data={had_data}, evicted: {counts:?}");
assert!(had_data, "snapshot should have persisted data"); assert!(had_data, "snapshot should have persisted data");
// The normal intermediate tasks (add_one, times_three, plus_ten) should be // The normal intermediate tasks (add_one, times_three, plus_ten) should be
...@@ -358,7 +383,11 @@ async fn eviction_transient_reader_invalidated() { ...@@ -358,7 +383,11 @@ async fn eviction_transient_reader_invalidated() {
// (this run_once closure), so it may be blocked from full eviction. But we // (this run_once closure), so it may be blocked from full eviction. But we
// still exercise the evict path — some tasks (like create_state) may be // still exercise the evict path — some tasks (like create_state) may be
// data-only evicted. // data-only evicted.
let (had_data, counts) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data,
eviction_counts: counts,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("transient_reader: snapshot had_data={had_data}, evicted: {counts:?}"); println!("transient_reader: snapshot had_data={had_data}, evicted: {counts:?}");
assert!(had_data, "snapshot should have persisted data"); assert!(had_data, "snapshot should have persisted data");
...@@ -374,7 +403,10 @@ async fn eviction_transient_reader_invalidated() { ...@@ -374,7 +403,10 @@ async fn eviction_transient_reader_invalidated() {
); );
// Second eviction cycle // Second eviction cycle
let (_, counts2) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let counts2 = tt2
.backend()
.snapshot_and_evict_for_testing(&tt2)
.eviction_counts;
println!("transient_reader (2nd): evicted: {counts2:?}"); println!("transient_reader (2nd): evicted: {counts2:?}");
state.set(0); state.set(0);
...@@ -611,7 +643,11 @@ async fn eviction_persistable_never_preserves_live_cell() { ...@@ -611,7 +643,11 @@ async fn eviction_persistable_never_preserves_live_cell() {
// Snapshot + evict. `create_session_alive`'s `cell_data` should retain // Snapshot + evict. `create_session_alive`'s `cell_data` should retain
// the SessionAlive cell as residue (Evictability::Never), while // the SessionAlive cell as residue (Evictability::Never), while
// clearing `data_restored` and persisted data flag bits. // clearing `data_restored` and persisted data flag bits.
let (had_data, counts) = tt2.backend().snapshot_and_evict_for_testing(&tt2); let TestSnapshotOutcome {
had_new_data: had_data,
eviction_counts: counts,
..
} = tt2.backend().snapshot_and_evict_for_testing(&tt2);
println!("persistable_never: snapshot had_data={had_data}, evicted: {counts:?}"); println!("persistable_never: snapshot had_data={had_data}, evicted: {counts:?}");
assert!(had_data, "snapshot should have persisted data"); assert!(had_data, "snapshot should have persisted data");
......
...@@ -60,3 +60,41 @@ pub async fn diamond_root(constant: ResolvedVc<Constant>, fanout: u32) -> Result ...@@ -60,3 +60,41 @@ pub async fn diamond_root(constant: ResolvedVc<Constant>, fanout: u32) -> Result
pub async fn diamond_root_op(constant: ResolvedVc<Constant>, fanout: u32) -> Result<Vc<u32>> { pub async fn diamond_root_op(constant: ResolvedVc<Constant>, fanout: u32) -> Result<Vc<u32>> {
Ok(diamond_root(*constant, fanout)) Ok(diamond_root(*constant, fanout))
} }
// --- Generation fixture ---
/// The generation counter. A `State` so a test can bump it without rebuilding the graph.
#[turbo_tasks::value(transparent)]
pub struct Generation(State<u32>);
#[turbo_tasks::function(operation, root)]
pub fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell()
}
/// A leaf keyed by (generation, index). Bumping the generation makes `wide_root` read an entirely
/// fresh set of these, disconnecting the whole previous generation.
#[turbo_tasks::function]
pub fn leaf(generation: u32, index: u32) -> Vc<u32> {
Vc::cell(generation.wrapping_mul(1000).wrapping_add(index))
}
/// A shared-dependency layer, so the disconnected garbage is a subtree (intermediate + leaf) and
/// the tests exercise the cascade rather than single-node collection.
#[turbo_tasks::function]
pub async fn intermediate(generation: u32, index: u32) -> Result<Vc<u32>> {
Ok(Vc::cell(1 + *leaf(generation, index).await?))
}
/// Reads `width` intermediates (each reading a leaf). Bumping the generation re-executes this and
/// connects a fresh generation's worth of tasks, disconnecting the entire previous generation
/// (`2 * width` tasks) as garbage.
#[turbo_tasks::function(operation, root)]
pub async fn wide_root(generation: ResolvedVc<Generation>, width: u32) -> Result<Vc<u32>> {
let generation = *generation.await?.get();
let mut sum = 0u32;
for index in 0..width {
sum = sum.wrapping_add(*intermediate(generation, index).await?);
}
Ok(Vc::cell(sum))
}
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
//! Test the interruption mechanisms for GC
mod gc_fixture;
mod util;
use std::{sync::Arc, time::Duration};
use turbo_tasks::TurboTasks;
use turbo_tasks_backend::TurboTasksBackend;
use crate::{
gc_fixture::{create_generation, wide_root},
util::create_tt_with_gc_min_progress,
};
/// Wide enough that an interrupted pass leaves an observable shortfall rather than a rounding
/// difference.
const WIDTH: u32 = 400;
/// Builds generation `gen_value`, disconnecting the previous generation's `2 * WIDTH` tasks.
async fn build_generation(tt: &Arc<TurboTasks<TurboTasksBackend>>, gen_value: u32) {
turbo_tasks::run_once(tt.clone(), async move {
let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?;
if gen_value > 0 {
let generation = generation_op.read_strongly_consistent().await?;
generation.set(gen_value);
}
wide_root(generation_vc, WIDTH)
.read_strongly_consistent()
.await?;
anyhow::Ok(())
})
.await
.unwrap();
}
/// A waiter blocked for the whole pass must not interrupt it while the floor is unmet.
///
/// This *provokes* the interleaving rather than forcing it: the spawned operation may park on the
/// exclusion while the pass is running, or it may not be scheduled until the pass is already over,
/// in which case `operations_waiting()` is never true and there was no interrupt to suppress.
/// Forcing it would mean reaching into the coordinator from the test, which is a bigger intrusion
/// than the coverage is worth. Both interleavings must produce a completing pass, so the
/// assertions below hold either way — the floor is what makes the outcome independent of the
/// scheduling.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn gc_min_progress_floor_beats_a_waiting_operation() {
// A floor far longer than the pass: the interrupt must never be honoured.
let (tt, _persistence_dir) =
create_tt_with_gc_min_progress("gc_min_progress_floor", Duration::from_secs(60));
build_generation(&tt, 0).await;
build_generation(&tt, 1).await;
let tt_waiter = tt.clone();
let waiter = tokio::spawn(async move {
turbo_tasks::run_once(tt_waiter.clone(), async move {
let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?;
wide_root(generation_vc, WIDTH)
.read_strongly_consistent()
.await?;
anyhow::Ok(())
})
.await
.unwrap();
});
let outcome = tt.backend().snapshot_and_evict_for_testing(&tt);
waiter.await.unwrap();
let stats = outcome.gc_stats();
assert!(
!stats.interrupted,
"the min-progress floor must suppress the interrupt (collected={})",
stats.collected
);
// Generation 0 is fully disconnected by generation 1, and an uninterrupted pass must take all
// of it: `2 * WIDTH` tasks (an `intermediate` and a `leaf` per index). The live generation and
// the transient `run_once` roots are not collectible, so this is an exact count, not a floor.
assert_eq!(
stats.collected,
2 * WIDTH as usize,
"a completing pass must collect the whole disconnected generation: {stats}"
);
tt.stop_and_wait().await;
}
/// What an interrupted pass abandons must still be collectible by a later pass.
///
/// The two phases are sequential, not concurrent: nothing here forces a pass to overlap with an
/// operation. Phase 1 relies only on the zero `min_progress` floor, which makes *any* waiter
/// observed by `GcBudget::should_stop` interrupt the pass immediately, so interrupts happen
/// readily across the rounds instead of needing a mid-range floor that might interrupt none of
/// them and assert nothing. Because that is still scheduling-dependent, the loop asserts only
/// that at least one round interrupted — which is the premise phase 2 needs. Phase 2 then runs a
/// deliberately *uninterruptible* pass (`gc_for_testing`) after phase 1 is fully over, and that
/// pass is what must recover everything the interrupted rounds abandoned.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn gc_interrupt_is_self_healing() {
let (tt, _persistence_dir) =
create_tt_with_gc_min_progress("gc_interrupt_is_self_healing", Duration::ZERO);
build_generation(&tt, 0).await;
// Phase 1: accumulate garbage under passes that keep interrupting.
const ROUNDS: u32 = 10;
let mut collected_while_interrupting = 0usize;
let mut interrupted_rounds = 0usize;
for gen_value in 1..=ROUNDS {
build_generation(&tt, gen_value).await;
let outcome = tt.backend().snapshot_and_evict_for_testing(&tt);
let stats = outcome.gc_stats();
println!("round {gen_value}: {stats}");
collected_while_interrupting += stats.collected;
interrupted_rounds += usize::from(stats.interrupted);
}
// The premise of phase 2: without an interrupt, nothing was abandoned to heal from.
assert!(
interrupted_rounds > 0,
"no pass interrupted across {ROUNDS} rounds at a zero floor: nothing was abandoned, so \
this test proves nothing"
);
// Phase 2: a completing pass must recover what was left behind.
let healed = tt.backend().gc_for_testing(&tt);
let produced = (2 * WIDTH as usize) * (ROUNDS as usize);
let total_collected = collected_while_interrupting + healed;
println!(
"self-healing: collected_while_interrupting={collected_while_interrupting} \
healed={healed} total_collected={total_collected} produced={produced} \
interrupted_rounds={interrupted_rounds}/{ROUNDS}"
);
// Phase 2's pass is uninterruptible and runs once phase 1 is over, so nothing is left for a
// later pass to pick up: every generation but the live one is garbage, and all of it must be
// accounted for exactly. An interrupted pass that *lost* garbage rather than leaving it would
// show up here as a shortfall.
assert_eq!(
total_collected, produced,
"collected {total_collected} of {produced} garbage tasks ({collected_while_interrupting} \
across {interrupted_rounds} interrupted rounds, {healed} in the completing pass): \
interrupted passes are losing garbage rather than leaving it"
);
tt.stop_and_wait().await;
}
...@@ -2,66 +2,39 @@ ...@@ -2,66 +2,39 @@
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this #![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
mod gc_fixture;
mod util; mod util;
use std::sync::Arc; use std::{sync::Arc, time::Duration};
use anyhow::Result; use turbo_tasks::TurboTasks;
use turbo_tasks::{ResolvedVc, State, TurboTasks, Vc};
use turbo_tasks_backend::TurboTasksBackend; use turbo_tasks_backend::TurboTasksBackend;
use crate::util::create_tt; use crate::{
gc_fixture::{create_generation, wide_root},
#[turbo_tasks::value(transparent)] util::create_tt_with_gc_min_progress,
struct Generation(State<u32>); };
#[turbo_tasks::function(operation, root)]
fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell()
}
/// A leaf keyed by (generation, index). Bumping the generation makes `wide_root` read an entirely
/// fresh set of these, disconnecting the whole previous generation.
#[turbo_tasks::function]
fn leaf(generation: u32, index: u32) -> Vc<u32> {
Vc::cell(generation.wrapping_mul(1000).wrapping_add(index))
}
/// A shared-dependency layer, so the disconnected garbage is a subtree (intermediate + leaf) and
/// the test exercises the cascade rather than single-node collection.
#[turbo_tasks::function]
async fn intermediate(generation: u32, index: u32) -> Result<Vc<u32>> {
Ok(Vc::cell(1 + *leaf(generation, index).await?))
}
const WIDTH: u32 = 24; const WIDTH: u32 = 24;
/// Bumping the generation re-executes this and connects a fresh generation's worth of tasks,
/// disconnecting the entire previous generation (2*WIDTH tasks) as garbage.
#[turbo_tasks::function(operation, root)]
async fn wide_root(generation: ResolvedVc<Generation>) -> Result<Vc<u32>> {
let generation = *generation.await?.get();
let mut sum = 0u32;
for index in 0..WIDTH {
sum = sum.wrapping_add(*intermediate(generation, index).await?);
}
Ok(Vc::cell(sum))
}
/// Repeatedly swapping a wide set of common dependencies (as happens when a project is re-rooted or /// Repeatedly swapping a wide set of common dependencies (as happens when a project is re-rooted or
/// its dependencies churn) must NOT grow the resident task set monotonically. Each round bumps the /// its dependencies churn) must NOT grow the resident task set monotonically. Each round bumps the
/// generation (disconnecting the previous generation's 2*WIDTH tasks), then snapshots + GCs + /// generation (disconnecting the previous generation's 2*WIDTH tasks), then snapshots + GCs +
/// evicts, and the resident count must return to a flat baseline. /// evicts, and the resident count must return to a flat baseline.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn gc_re_rooting_stays_flat() { async fn gc_re_rooting_stays_flat() {
let (tt, _persistence_dir) = create_tt("gc_re_rooting_stays_flat"); let (tt, _persistence_dir) =
create_tt_with_gc_min_progress("gc_re_rooting_stays_flat", Duration::from_secs(60 * 60));
let tt2 = tt.clone(); let tt2 = tt.clone();
const ROUNDS: u32 = 20; const ROUNDS: u32 = 20;
// Each round runs in its own `run_once` so the root's activeness is released before GC (a // Each round runs in its own `run_once` so the root's activeness is released before GC (a
// `run_once` root keeps everything it touched active until it returns). // `run_once` root keeps everything it touched active until it returns).
async fn round(tt: &Arc<TurboTasks<TurboTasksBackend>>, gen_value: u32) -> (usize, usize) { async fn round(
tt: &Arc<TurboTasks<TurboTasksBackend>>,
gen_value: u32,
) -> (usize, usize, bool) {
let tt_inner = tt.clone(); let tt_inner = tt.clone();
turbo_tasks::run_once(tt.clone(), async move { turbo_tasks::run_once(tt.clone(), async move {
// `create_generation` is a cached root operation, so this returns the same task each // `create_generation` is a cached root operation, so this returns the same task each
...@@ -72,7 +45,7 @@ async fn gc_re_rooting_stays_flat() { ...@@ -72,7 +45,7 @@ async fn gc_re_rooting_stays_flat() {
let generation = generation_op.read_strongly_consistent().await?; let generation = generation_op.read_strongly_consistent().await?;
generation.set(gen_value); generation.set(gen_value);
} }
let output = wide_root(generation_vc); let output = wide_root(generation_vc, WIDTH);
output.read_strongly_consistent().await?; output.read_strongly_consistent().await?;
let _ = &tt_inner; let _ = &tt_inner;
anyhow::Ok(()) anyhow::Ok(())
...@@ -85,34 +58,44 @@ async fn gc_re_rooting_stays_flat() { ...@@ -85,34 +58,44 @@ async fn gc_re_rooting_stays_flat() {
// first would drop the garbage to disk-only where the in-memory GC can't collect or // first would drop the garbage to disk-only where the in-memory GC can't collect or
// tombstone it. // tombstone it.
let collected = tt.backend().gc_for_testing(tt); let collected = tt.backend().gc_for_testing(tt);
tt.backend().snapshot_and_evict_for_testing(tt);
let interrupted = tt
.backend()
.snapshot_and_evict_for_testing(tt)
.gc_interrupted();
// Measure the *persistent* resident count: GC only collects persistent tasks. Transient // Measure the *persistent* resident count: GC only collects persistent tasks. Transient
// roots (each round's `run_once`/Once task) are never collected and accumulate // roots (each round's `run_once`/Once task) are never collected and accumulate
// independently of GC — including them would mask the real signal. // independently of GC — including them would mask the real signal.
( (
collected, collected,
tt.backend().resident_persistent_task_count_for_testing(), tt.backend().resident_persistent_task_count_for_testing(),
interrupted,
) )
} }
// Warm up: build generation 0, then measure the steady-state baseline after one full cycle. // Warm up: build generation 0, then measure the steady-state baseline after one full cycle.
let (_, baseline) = round(&tt2, 0).await; let (_, baseline, _) = round(&tt2, 0).await;
println!("baseline persistent resident after gen 0: {baseline}"); println!("baseline persistent resident after gen 0: {baseline}");
// Churn: swap the whole dependency set many times. // Churn: swap the whole dependency set many times.
let mut max_resident = baseline; let mut max_resident = baseline;
let mut total_collected = 0usize; let mut total_collected = 0usize;
let mut interrupted_rounds = 0usize;
for gen_value in 1..=ROUNDS { for gen_value in 1..=ROUNDS {
let (collected, resident) = round(&tt2, gen_value).await; let (collected, resident, interrupted) = round(&tt2, gen_value).await;
println!("gen {gen_value}: collected={collected} persistent_resident={resident}"); println!(
"gen {gen_value}: collected={collected} persistent_resident={resident} \
snapshot_gc_interrupted={interrupted}"
);
max_resident = max_resident.max(resident); max_resident = max_resident.max(resident);
total_collected += collected; total_collected += collected;
interrupted_rounds += usize::from(interrupted);
} }
let no_gc_growth = baseline + (2 * WIDTH as usize) * (ROUNDS as usize); let no_gc_growth = baseline + (2 * WIDTH as usize) * (ROUNDS as usize);
println!( println!(
"baseline={baseline} max_resident={max_resident} total_collected={total_collected} \ "baseline={baseline} max_resident={max_resident} total_collected={total_collected} \
no_gc_growth_would_be={no_gc_growth}" interrupted_rounds={interrupted_rounds} no_gc_growth_would_be={no_gc_growth}"
); );
// Without GC the persistent resident set would climb by ~2*WIDTH per round (each generation's // Without GC the persistent resident set would climb by ~2*WIDTH per round (each generation's
// intermediates + leaves persisted forever), so a bound within a small constant of the baseline // intermediates + leaves persisted forever), so a bound within a small constant of the baseline
...@@ -129,7 +112,10 @@ async fn gc_re_rooting_stays_flat() { ...@@ -129,7 +112,10 @@ async fn gc_re_rooting_stays_flat() {
assert!( assert!(
total_collected >= expected_min_collected, total_collected >= expected_min_collected,
"GC collected too little ({total_collected}); expected >= {expected_min_collected} across \ "GC collected too little ({total_collected}); expected >= {expected_min_collected} across \
{ROUNDS} re-rootings — GC is not doing the collection" {ROUNDS} re-rootings — GC is not doing the collection. snapshot-pass interrupts: \
{interrupted_rounds}/{ROUNDS} (a non-zero count means the interruptible pass inside \
`snapshot_and_persist` wound down early under load, which is a flaky-environment signal \
rather than a GC defect)"
); );
// The live graph must still compute correctly after all the churn. // The live graph must still compute correctly after all the churn.
...@@ -137,7 +123,7 @@ async fn gc_re_rooting_stays_flat() { ...@@ -137,7 +123,7 @@ async fn gc_re_rooting_stays_flat() {
let result = turbo_tasks::run_once(tt.clone(), async move { let result = turbo_tasks::run_once(tt.clone(), async move {
let generation_op = create_generation(); let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?; let generation_vc = generation_op.resolve().strongly_consistent().await?;
let output = wide_root(generation_vc); let output = wide_root(generation_vc, WIDTH);
// generation is ROUNDS; sum = WIDTH intermediates each = 1 + leaf(ROUNDS, i). // generation is ROUNDS; sum = WIDTH intermediates each = 1 + leaf(ROUNDS, i).
let expected: u32 = (0..WIDTH) let expected: u32 = (0..WIDTH)
.map(|i| 1 + (ROUNDS.wrapping_mul(1000).wrapping_add(i))) .map(|i| 1 + (ROUNDS.wrapping_mul(1000).wrapping_add(i)))
......
...@@ -23,7 +23,7 @@ use turbo_tasks_backend::{ ...@@ -23,7 +23,7 @@ use turbo_tasks_backend::{
/// Reusing the same `path` (after the previous backend has been stopped) reopens the persisted /// Reusing the same `path` (after the previous backend has been stopped) reopens the persisted
/// database, which is how a test can assert that state survives a restart. /// database, which is how a test can assert that state survives a restart.
fn open_tt_at(path: &Path, num_workers: usize) -> Arc<TurboTasks<TurboTasksBackend>> { fn open_tt_at(path: &Path, num_workers: usize) -> Arc<TurboTasks<TurboTasksBackend>> {
open_tt_at_with_gc(path, num_workers, Some(true), None) open_tt_at_with_gc(path, num_workers, Some(true), None, None)
} }
/// Like [`open_tt_at`], but forces the GC on or off for this backend instead of deriving it from /// Like [`open_tt_at`], but forces the GC on or off for this backend instead of deriving it from
...@@ -37,6 +37,7 @@ fn open_tt_at_with_gc( ...@@ -37,6 +37,7 @@ fn open_tt_at_with_gc(
num_workers: usize, num_workers: usize,
gc: Option<bool>, gc: Option<bool>,
gc_root_ttl: Option<Duration>, gc_root_ttl: Option<Duration>,
gc_min_progress: Option<Duration>,
) -> Arc<TurboTasks<TurboTasksBackend>> { ) -> Arc<TurboTasks<TurboTasksBackend>> {
TurboTasks::new(TurboTasksBackend::new( TurboTasks::new(TurboTasksBackend::new(
BackendOptions { BackendOptions {
...@@ -48,6 +49,7 @@ fn open_tt_at_with_gc( ...@@ -48,6 +49,7 @@ fn open_tt_at_with_gc(
eviction_mode: EvictionMode::Full, eviction_mode: EvictionMode::Full,
gc, gc,
gc_root_ttl, gc_root_ttl,
gc_min_progress,
..Default::default() ..Default::default()
}, },
turbo_tasks_backend::turbo_backing_storage( turbo_tasks_backend::turbo_backing_storage(
...@@ -70,9 +72,11 @@ fn open_tt_at_with_gc( ...@@ -70,9 +72,11 @@ fn open_tt_at_with_gc(
/// A persistence directory that outlives the backends opened on it, so a test can stop one backend /// A persistence directory that outlives the backends opened on it, so a test can stop one backend
/// and open another on the same path to simulate a restart. /// and open another on the same path to simulate a restart.
pub fn create_persistence_dir(name: &str) -> tempfile::TempDir { pub fn create_persistence_dir(name: &str) -> tempfile::TempDir {
let parent = Path::new(env!("CARGO_TARGET_TMPDIR")).join(".cache");
std::fs::create_dir_all(&parent).unwrap();
tempfile::Builder::new() tempfile::Builder::new()
.prefix(&format!("{name}-")) .prefix(&format!("{name}-"))
.tempdir() .tempdir_in(&parent)
.unwrap() .unwrap()
} }
...@@ -80,7 +84,7 @@ pub fn create_persistence_dir(name: &str) -> tempfile::TempDir { ...@@ -80,7 +84,7 @@ pub fn create_persistence_dir(name: &str) -> tempfile::TempDir {
/// [`open_tt_at_with_gc`]). The previous backend must already be stopped so its shutdown snapshot /// [`open_tt_at_with_gc`]). The previous backend must already be stopped so its shutdown snapshot
/// has been flushed. /// has been flushed.
pub fn reopen_tt_with_gc(dir: &tempfile::TempDir) -> Arc<TurboTasks<TurboTasksBackend>> { pub fn reopen_tt_with_gc(dir: &tempfile::TempDir) -> Arc<TurboTasks<TurboTasksBackend>> {
open_tt_at_with_gc(dir.path(), 2, Some(true), None) open_tt_at_with_gc(dir.path(), 2, Some(true), None, None)
} }
/// [`reopen_tt_with_gc`] with the GC root TTL pinned, so a test can age a root out inside the test /// [`reopen_tt_with_gc`] with the GC root TTL pinned, so a test can age a root out inside the test
...@@ -89,7 +93,7 @@ pub fn reopen_tt_with_gc_ttl( ...@@ -89,7 +93,7 @@ pub fn reopen_tt_with_gc_ttl(
dir: &tempfile::TempDir, dir: &tempfile::TempDir,
gc_root_ttl: Duration, gc_root_ttl: Duration,
) -> Arc<TurboTasks<TurboTasksBackend>> { ) -> Arc<TurboTasks<TurboTasksBackend>> {
open_tt_at_with_gc(dir.path(), 2, Some(true), Some(gc_root_ttl)) open_tt_at_with_gc(dir.path(), 2, Some(true), Some(gc_root_ttl), None)
} }
/// A fresh persistent backend in its own temp directory, with `num_workers` workers. /// A fresh persistent backend in its own temp directory, with `num_workers` workers.
...@@ -106,3 +110,13 @@ pub fn create_tt_with_workers( ...@@ -106,3 +110,13 @@ pub fn create_tt_with_workers(
pub fn create_tt(name: &str) -> (Arc<TurboTasks<TurboTasksBackend>>, tempfile::TempDir) { pub fn create_tt(name: &str) -> (Arc<TurboTasks<TurboTasksBackend>>, tempfile::TempDir) {
create_tt_with_workers(name, 2) create_tt_with_workers(name, 2)
} }
/// [`create_tt`] with a custom GC min-progress floor
pub fn create_tt_with_gc_min_progress(
name: &str,
gc_min_progress: Duration,
) -> (Arc<TurboTasks<TurboTasksBackend>>, tempfile::TempDir) {
let dir = create_persistence_dir(name);
let tt = open_tt_at_with_gc(dir.path(), 2, Some(true), None, Some(gc_min_progress));
(tt, dir)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment