Unverified Commit 8ea76d64 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks-backend: parent_count-driven garbage collection (#97282)

Adds garbage collection, built on the `parent_count` reference counting

### How collection works

* we scan all storage shards for collectible tasks to seed the sweep
* For each collectible task we
    1. mark it as deleted
    2. remove all outgoing edges
    3. queue new tasks for deletion if the edge removals triggered it

For removing edges we rely on the existing `CleanupOldEdges` operation,
though it is enhanced to collect the tasks that become collectible as
edges are removed

Deletion markers are transformed into tombstones by persistence and then
dropped from memory by eviction.

### Coordinator changes

GC mutates the graph, so we exclude other tasks from running while it
works. When gc completes we hand off to persistence which switches to
the existing copy on write mode.

The subtle piece is `GcPhase::into_snapshot`: GC has to hand its
exclusion directly to the snapshot that tombstones what it collected.
Swapping the flags under one lock leaves no window where an operation
could start and resurrect a just-collected task.

### Enabling it

Off by default; `TURBO_ENGINE_GC` turns it on, also requires eviction to
be enabled (otherwise deleted tasks persist in RAM)

`prevent_gc()` becomes real, pinning the calling task through
`transient_ref_count`.
parent bcfa15d0
//! Benchmarks for the `parent_count` garbage-collection pass ([`TurboTasksBackend::gc_collect`],
//! driven here via the `gc_for_testing` hook).
//!
//! Gated behind `TURBOPACK_BENCH_GC` because the per-iteration setup is expensive. Run with:
//!
//! ```bash
//! TURBOPACK_BENCH_GC=1 cargo bench -p turbo-tasks-backend --bench mod -- gc
//! ```
use std::{sync::Arc, time::Duration};
use anyhow::Result;
use criterion::{BatchSize, BenchmarkId, Criterion, Throughput};
use tokio::runtime::Runtime;
use turbo_tasks::{
ResolvedVc, State, TurboTasks, Vc, unmark_top_level_task_may_leak_eventually_consistent_state,
};
use turbo_tasks_backend::{
BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, StorageMode,
TurboTasksBackend,
};
fn enabled() -> bool {
!matches!(
std::env::var("TURBOPACK_BENCH_GC").ok().as_deref(),
None | Some("") | Some("no") | Some("false")
)
}
/// A persistent backend with GC-relevant options
fn create_tt() -> (Arc<TurboTasks<TurboTasksBackend>>, tempfile::TempDir) {
let parent = std::path::PathBuf::from(format!("{}/.cache", env!("CARGO_TARGET_TMPDIR")));
std::fs::create_dir_all(&parent).unwrap();
let dir = tempfile::Builder::new()
.prefix("gc-bench-")
.tempdir_in(&parent)
.unwrap();
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
num_workers: None,
small_preallocation: false,
storage_mode: Some(StorageMode::ReadWriteOnShutdown),
eviction_mode: EvictionMode::Full,
gc: Some(true),
..Default::default()
},
turbo_tasks_backend::turbo_backing_storage(
dir.path(),
&GitVersionInfo {
describe: "bench-unversioned",
dirty: false,
},
BackingStorageOptions {
is_ci: false,
is_short_session: true,
skip_compaction: true,
},
)
.unwrap()
.0,
));
(tt, dir)
}
#[turbo_tasks::value(transparent)]
struct Generation(State<u32>);
#[turbo_tasks::function(operation, root)]
fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell()
}
// WIDE shape: root -> `width` intermediates, each -> a leaf (2 tasks per index). Bumping the
// generation disconnects the whole previous generation, so a collect tears down ~2*width tasks and
// exercises the per-task fan-out plus a one-level cascade (intermediate -> its leaf).
#[turbo_tasks::function]
fn wide_leaf(generation: u32, index: u32) -> Vc<u32> {
Vc::cell(generation.wrapping_mul(1_000_003).wrapping_add(index))
}
#[turbo_tasks::function]
async fn wide_intermediate(generation: u32, index: u32) -> Result<Vc<u32>> {
Ok(Vc::cell(1 + *wide_leaf(generation, index).await?))
}
#[turbo_tasks::function(operation, root)]
async fn wide_root(generation: ResolvedVc<Generation>, width: u32) -> Result<Vc<u32>> {
let generation = *generation.await?.get();
let mut sum = 0u32;
for index in 0..width {
sum = sum.wrapping_add(*wide_intermediate(generation, index).await?);
}
Ok(Vc::cell(sum))
}
/// Build generation 0 of the WIDE graph then bump to generation 1, leaving generation 0 fully
/// disconnected (garbage) and resident. Returns the backend ready for a timed collect.
fn setup_wide_garbage(
rt: &Runtime,
width: u32,
) -> (Arc<TurboTasks<TurboTasksBackend>>, tempfile::TempDir) {
rt.block_on(async move {
// `TurboTasks::new` builds a `PriorityRunner` that calls `Handle::current()`, so the
// backend must be constructed inside the runtime context.
let (tt, dir) = create_tt();
turbo_tasks::run_once(tt.clone(), async move {
unmark_top_level_task_may_leak_eventually_consistent_state();
let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?;
let generation = generation_op.read_strongly_consistent().await?;
wide_root(generation_vc, width)
.read_strongly_consistent()
.await?;
generation.set(1);
wide_root(generation_vc, width)
.read_strongly_consistent()
.await?;
anyhow::Ok(())
})
.await
.unwrap();
(tt, dir)
})
}
pub fn gc(c: &mut Criterion) {
if !enabled() {
return;
}
let mut group = c.benchmark_group("turbo_tasks_backend_gc");
group.measurement_time(Duration::from_secs(20));
group.sample_size(10);
for width in [5_000u32, 25_000, 100_000] {
// ~2 tasks per index become garbage (intermediate + leaf).
let garbage = (2 * width) as u64;
group.throughput(Throughput::Elements(garbage));
group.bench_with_input(BenchmarkId::new("wide", garbage), &width, |b, &width| {
// Must match `create_tt`'s `num_workers`.
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.worker_threads(std::thread::available_parallelism().map_or(4, |n| n.get()))
.build()
.unwrap();
// Each collect consumes its garbage, so every iteration needs a freshly
// built+disconnected graph. Setup is not timed.
b.iter_batched(
|| setup_wide_garbage(&rt, width),
|(tt, _dir)| {
let collected = rt.block_on(async { tt.backend().gc_for_testing(&tt) });
(collected, tt, _dir)
},
BatchSize::PerIteration,
);
});
}
group.finish();
}
......@@ -3,6 +3,7 @@
use criterion::{Criterion, criterion_group, criterion_main};
pub(crate) mod gc;
pub(crate) mod overhead;
pub(crate) mod scope_stress;
pub(crate) mod stress;
......@@ -10,6 +11,6 @@ pub(crate) mod stress;
criterion_group!(
name = turbo_tasks_backend_stress;
config = Criterion::default();
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc
);
criterion_main!(turbo_tasks_backend_stress);
//! Garbage collection for the persistent backend.
//!
//! GC identifies and tears down tasks that have no reverse references using the `parent_count` and
//! `transient_ref_count`. Tasks are marked `deleted` and then have their outgoing edges teared down
//! recursively.
//!
//! A collected task also has its cell data released immediately to deliver immediate memory wins.
//!
//! The pass runs under the coordinator's exclusion phase (see
//! [`SnapshotCoordinator::begin_exclusion`](crate::backend::snapshot_coordinator)), which excludes
//! normal operations. That exclusion is what lets a pass edit the graph without racing a mutation
//! that could resurrect a task mid-collect, and hand its decisions straight to persistence: the
//! same guard stays held across the snapshot that writes the tombstones.
//!
//! TODO: find a way to collect GC roots that go away between sessions. Right now they are
//! persisted forever and if a later session doesn't read them it is never deleted.
use std::{fmt::Display, ops::ControlFlow, sync::atomic::Ordering};
use turbo_tasks::{TaskId, TurboTasks, scope_unbounded::scope_unbounded_with};
use crate::backend::{
AnyOperation, TurboTasksBackend,
operation::{
AggregationUpdateQueue, CleanupOldEdgesOperation, ExecuteContext, ExecuteContextImpl,
TaskGuard, capture_all_outgoing_edges,
},
snapshot_coordinator::SnapshotPhase,
storage::{SpecificTaskDataCategory, TaskDataCategory},
storage_schema::TaskStorageAccessors,
};
/// One unit of GC work.
enum GcJob {
/// Scan one shard of the resident map (by index) and enqueue its candidates as
/// [`GcJob::Collect`].
ScanShard(usize),
/// Collect a single task
Collect(TaskId),
}
/// Observability counters for one [`TurboTasksBackend::gc_collect`] pass.
#[derive(Default)]
pub(crate) struct GcStats {
/// Tasks collected (marked soft-deleted).
pub collected: usize,
/// Edges torn down across all collected tasks (children + forward-dependency reverse edges).
pub edges_deleted: usize,
}
impl Display for GcStats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"collected: {collected}, edges_deleted: {edges_deleted}",
collected = self.collected,
edges_deleted = self.edges_deleted
)
}
}
impl GcStats {
fn merge(mut self, other: Self) -> Self {
self.collected += other.collected;
self.edges_deleted += other.edges_deleted;
self
}
}
impl TurboTasksBackend {
/// Collect all garbage from the task-cache
///
/// `phase` is the held exclusion; it is the caller's proof that no operation is running, which
/// is what makes it safe to mutate the graph here.
///
/// Returns [`GcStats`] for the pass.
pub(crate) fn gc_collect(
&self,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
phase: &SnapshotPhase<'_, AnyOperation>,
) -> GcStats {
// TODO(perf): recycle the task ids of collected tasks.
scope_unbounded_with(
(0..self.storage.shard_count()).map(GcJob::ScanShard),
GcStats::default,
|spawner, job, stats| {
let collector = |task_id| spawner.spawn(GcJob::Collect(task_id));
let task_id = match job {
GcJob::ScanShard(index) => {
self.storage.gc_scan_shard(index, collector);
return ControlFlow::Continue(());
}
GcJob::Collect(task_id) => task_id,
};
let mut ctx = ExecuteContextImpl::new_for_gc(self, turbo_tasks, phase, &collector);
// `All` restores Data so the edge capture below can read the Data-category dep
// sets.
let mut task = ctx.task(task_id, TaskDataCategory::All);
// Recheck under the guard, and note that this is the **authoritative** check:
// the shard scan that produced this candidate only had Meta, so it could not see
// dependency edges (see `TaskStorage::gc_maybe_collectible`). With `All` open the
// same predicate is exact. A racing teardown can also add uppers/followers that
// temporarily remove collectibility; such a task is re-enqueued by a later pass.
if !task.is_gc_collectible() {
return ControlFlow::Continue(());
}
let old_edges = capture_all_outgoing_edges(&task);
// Clear `immutable` defensively so `resurrect_deleted` can mark the task dirty if
// it needs to
task.set_immutable(false);
// Drop the whole cell payload. This recovers most of the RAM while persistence
// writes the tombstone.
drop(task.take_cell_data());
task.set_deleted(true);
if task.new_task() {
task.discard_modifications_for_gc_new_task();
} else {
// Persisted ensure it is marked modified so the next snapshot tombstones it.
// It is almost certainly already marked modified, so this is mostly a no-op.
let _ = task.track_modification(SpecificTaskDataCategory::Meta, "gc_deleted");
}
drop(task); // drop the lock so CleanupOldEdgesOperation can run
stats.collected += 1;
stats.edges_deleted += old_edges.len();
CleanupOldEdgesOperation::run(
task_id,
old_edges,
AggregationUpdateQueue::new(),
&mut ctx,
);
ControlFlow::Continue(())
},
GcStats::merge,
)
}
pub(super) fn pin_task_for_gc(
&self,
task: TaskId,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
) {
self.gc_update_pin(task, 1, "pin_task_for_gc", turbo_tasks);
}
pub(super) fn unpin_task_for_gc(
&self,
task: TaskId,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
) {
self.gc_update_pin(task, -1, "unpin_task_for_gc", turbo_tasks);
}
/// Applies `delta` to a task's `transient_ref_count`
fn gc_update_pin(
&self,
task: TaskId,
delta: i32,
op: &'static str,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
) {
// Once stopping, GC bookkeeping is irrelevant. This also keeps handles finalized during
// shutdown (after the map is dropped) from underflowing the count.
if self.stopping.load(Ordering::Acquire) {
return;
}
let mut ctx = self.execute_context(turbo_tasks);
// Technically we only need to manipulate transient data so meta is overkill. But the task
// must be resident if we are adding a pin so this isn't wasteful
let mut task = ctx.task(task, TaskDataCategory::Meta);
task.assert_not_deleted(op);
task.update_and_get_transient_ref_count(delta);
}
/// Runs a full GC pass under the GC phase and returns the number of tasks collected.
#[doc(hidden)]
pub fn gc_for_testing(&self, turbo_tasks: &TurboTasks<TurboTasksBackend>) -> usize {
// A pass sets `deleted` flags, and the persist path only knows how to tombstone those when
// GC is enabled. Running a pass on a GC-disabled backend would leave soft-deleted tasks
// that persistence refuses to handle, so require the backend to be configured for GC
// (`BackendOptions::gc` or `TURBO_ENGINE_GC`) rather than silently diverging from
// production.
assert!(
self.gc_enabled,
"gc_for_testing requires a GC-enabled backend: set `BackendOptions::gc = Some(true)`"
);
let _serialize = self.snapshot_in_progress.lock();
let phase = self.snapshot_coord.begin_snapshot();
self.gc_collect(turbo_tasks, &phase).collected
}
}
......@@ -54,6 +54,39 @@ pub enum OutdatedEdge {
CollectiblesDependency(CollectiblesRef),
}
/// Captures *all* of a task's outgoing edges as [`OutdatedEdge`]s
pub fn capture_all_outgoing_edges(task: &impl TaskStorageAccessors) -> Vec<OutdatedEdge> {
let mut old_edges: Vec<OutdatedEdge> = Vec::new();
old_edges.extend(task.iter_children().map(OutdatedEdge::Child));
old_edges.extend(
task.iter_output_dependencies()
.map(OutdatedEdge::OutputDependency),
);
old_edges.extend(
task.iter_cell_dependencies()
.map(OutdatedEdge::CellDependency),
);
old_edges.extend(
task.iter_cell_dependencies_hashed()
.map(|(r, k)| OutdatedEdge::HashedCellDependency(r, k)),
);
old_edges.extend(
task.iter_collectibles_dependencies()
.map(OutdatedEdge::CollectiblesDependency),
);
old_edges
}
/// The category to open a dependency *target* with when scrubbing its incoming edge.
fn dependent_scrub_category<'e, C: ExecuteContext<'e>>(ctx: &C) -> TaskDataCategory {
if ctx.collects_gc_candidates() {
// Under GC we need to query meta fields so be sure to recover Meta also
TaskDataCategory::All
} else {
TaskDataCategory::Data
}
}
#[cfg(feature = "trace_aggregation_update_stats")]
type Stats = super::aggregation_update::AggregationUpdateQueueStats;
#[cfg(not(feature = "trace_aggregation_update_stats"))]
......@@ -194,11 +227,15 @@ impl CleanupOldEdgesOperation {
cell,
} = forward;
{
let mut task = ctx.task(cell_task_id, TaskDataCategory::Data);
task.remove_cell_dependents(&CellRef {
let category = dependent_scrub_category(ctx);
let mut task = ctx.task(cell_task_id, category);
let removed = task.remove_cell_dependents(&CellRef {
task: task_id,
cell,
});
if removed && task.is_cell_dependents_empty() {
ctx.note_maybe_collectible(&task);
}
}
{
let mut task = ctx.task(task_id, TaskDataCategory::Data);
......@@ -212,14 +249,19 @@ impl CleanupOldEdgesOperation {
cell,
} = forward;
{
let mut task = ctx.task(cell_task_id, TaskDataCategory::Data);
task.remove_cell_dependents_hashed(&(
let category = dependent_scrub_category(ctx);
let mut task = ctx.task(cell_task_id, category);
let removed = task.remove_cell_dependents_hashed(&(
CellRef {
task: task_id,
cell,
},
key,
));
if removed && task.is_cell_dependents_hashed_empty() {
ctx.note_maybe_collectible(&task);
}
}
{
let mut task = ctx.task(task_id, TaskDataCategory::Data);
......@@ -235,8 +277,12 @@ impl CleanupOldEdgesOperation {
)
.entered();
{
let mut task = ctx.task(output_task_id, TaskDataCategory::Data);
task.remove_output_dependent(&task_id);
let category = dependent_scrub_category(ctx);
let mut task = ctx.task(output_task_id, category);
let removed = task.remove_output_dependent(&task_id);
if removed && task.is_output_dependent_empty() {
ctx.note_maybe_collectible(&task);
}
}
{
let mut task = ctx.task(task_id, TaskDataCategory::Data);
......@@ -248,12 +294,15 @@ impl CleanupOldEdgesOperation {
task: dependent_task_id,
}) => {
{
let mut task =
ctx.task(dependent_task_id, TaskDataCategory::Data);
task.remove_collectibles_dependents(&(
let category = dependent_scrub_category(ctx);
let mut task = ctx.task(dependent_task_id, category);
let removed = task.remove_collectibles_dependents(&(
collectible_type,
task_id,
));
if removed && task.collectibles_dependents_len() == 0 {
ctx.note_maybe_collectible(&task);
}
}
{
let mut task = ctx.task(task_id, TaskDataCategory::Data);
......
......@@ -9,12 +9,55 @@ use crate::{
aggregation_update::{
AggregationUpdateJob, AggregationUpdateQueue, get_aggregation_number, is_root_node,
},
invalidate::make_task_dirty_internal,
},
storage_schema::TaskStorageAccessors,
},
data::{InProgressState, InProgressStateInner},
};
/// Revive `task_id` if it was GC-soft-deleted, given a guard the caller already holds during the
/// connect handshake.
///
/// GC destructively mutates tasks so mark resurrected tasks as dirty to get them re-scheduled.
pub(super) fn resurrect_deleted<'e, C: ExecuteContext<'e>>(
guard: C::TaskGuardImpl,
task_id: TaskId,
queue: &mut AggregationUpdateQueue,
ctx: &mut C,
) -> C::TaskGuardImpl {
if !guard.deleted() {
return guard;
}
#[cfg(debug_assertions)]
let category = guard.access();
drop(guard);
let mut task = ctx.task(task_id, TaskDataCategory::All);
// Double-check under the re-acquired guard: a concurrent connect may have already done this
if task.deleted() {
task.set_deleted(false);
// Mark dirty so it is rescheduled, GC has already dropped its edges and data, so we need to
// re-execute them to bring it back
// NOTE: recovering from disk is technically sometimes possible but doesn't work for new
// tasks, and the snapshot may have already persisted a tombstone. So it would at
// best be an optimistic way to recover data that is in the process of being deleted. It
// shouldn't matter for resolving this rare race condition.
make_task_dirty_internal(
&mut task,
/* make_stale */ true,
#[cfg(feature = "task_dirty_cause")]
turbo_tasks::TaskDirtyCause::Resurrected,
queue,
ctx,
);
}
// Conditionally downgrade from All->category so we don't hide incorrect access patterns.
#[cfg(debug_assertions)]
task.downgrade_access(category);
task
}
#[derive(Encode, Decode, Clone, Default)]
#[allow(clippy::large_enum_variant)]
pub enum ConnectChildOperation {
......@@ -61,7 +104,7 @@ impl ConnectChildOperation {
let mut queue = AggregationUpdateQueue::new();
// Handle the transient to persistent boundary by making the persistent task a root task
// Handle the transient to persistent boundary by making the persistent task a root task.
let should_make_root =
parent_task_id.is_none_or(|id| id.is_transient() && !child_task_id.is_transient());
......@@ -79,8 +122,12 @@ impl ConnectChildOperation {
} else {
// First connect of this child: its id is minted but the storage entry may not exist
// yet, and concurrent connects race to be the one that first touches it.
let mut child_task =
ctx.open_or_create_task_storage(child_task_id, TaskDataCategory::Meta);
let child_task = ctx.open_or_create_task_storage(child_task_id, TaskDataCategory::Meta);
// Revive the child if GC soft-deleted it. This can happen in a rare race between a
// cache hit on a task and snapshotting actually performing the delete.
let mut child_task = resurrect_deleted(child_task, child_task_id, &mut queue, &mut ctx);
let has_output = child_task.has_output();
// An already constructed top-level task was made a root when it was first connected.
// It may still be dirty and need to run; this only avoids repeating the idempotent
......@@ -103,7 +150,7 @@ impl ConnectChildOperation {
EventDescription::new(|| child_task.get_task_desc_fn()),
)
{
ctx.schedule_task(child_task, ctx.get_current_task_priority());
ctx.schedule_task(&child_task, ctx.get_current_task_priority());
}
}
......
......@@ -79,10 +79,8 @@ pub fn connect_children(
}
}
if !child.has_output() {
let child_id = child.id();
make_task_dirty_internal(
child,
child_id,
&mut child,
false,
#[cfg(feature = "task_dirty_cause")]
TaskDirtyCause::InitialDirty,
......
......@@ -13,6 +13,7 @@ use crate::{
AggregationUpdateJob, AggregationUpdateQueue, ComputeDirtyAndCleanUpdate,
},
},
storage_schema::TaskStorageAccessors,
},
data::{Dirtyness, InProgressState, InProgressStateInner},
};
......@@ -93,10 +94,9 @@ pub fn make_task_dirty(
queue: &mut AggregationUpdateQueue,
ctx: &mut impl ExecuteContext<'_>,
) {
let task = ctx.task(task_id, TaskDataCategory::All);
let mut task = ctx.task(task_id, TaskDataCategory::All);
make_task_dirty_internal(
task,
task_id,
&mut task,
true,
#[cfg(feature = "task_dirty_cause")]
cause,
......@@ -105,13 +105,13 @@ pub fn make_task_dirty(
);
}
pub fn make_task_dirty_internal(
mut task: impl TaskGuard,
task_id: TaskId,
/// Requires the guard to be allocated with [TaskDataCategory::All]
pub fn make_task_dirty_internal<'e, E: ExecuteContext<'e>>(
task: &mut E::TaskGuardImpl,
make_stale: bool,
#[cfg(feature = "task_dirty_cause")] cause: TaskDirtyCause,
queue: &mut AggregationUpdateQueue,
ctx: &mut impl ExecuteContext<'_>,
ctx: &mut E,
) {
// There must be no way to invalidate immutable tasks. If there would be a way the task is not
// immutable.
......@@ -252,11 +252,8 @@ pub fn make_task_dirty_internal(
}
.compute();
if let Some(aggregated_update) = result.aggregated_update(task_id) {
queue.extend(AggregationUpdateJob::data_update(
&mut task,
aggregated_update,
));
if let Some(aggregated_update) = result.aggregated_update(task.id()) {
queue.extend(AggregationUpdateJob::data_update(task, aggregated_update));
}
let should_schedule = !ctx.should_track_activeness() || task.has_activeness();
......@@ -264,9 +261,7 @@ pub fn make_task_dirty_internal(
if should_schedule {
let description = EventDescription::new(|| task.get_task_desc_fn());
if task.add_scheduled(TaskExecutionReason::Invalidated, description) {
drop(task);
let task = ctx.task(task_id, TaskDataCategory::All);
ctx.schedule_task(task, parent_priority);
ctx.schedule_task(&*task, parent_priority);
}
}
}
......@@ -295,7 +295,7 @@ impl Operation for UpdateCellOperation {
} => {
if let Some((dependent_task_id, keys)) = dependent_tasks.pop() {
let mut make_stale = false;
let dependent = ctx.task(dependent_task_id, TaskDataCategory::All);
let mut dependent = ctx.task(dependent_task_id, TaskDataCategory::All);
for key in keys.iter().copied() {
let (in_outdated, in_current) = if let Some(k) = key {
let in_outdated = dependent
......@@ -327,8 +327,7 @@ impl Operation for UpdateCellOperation {
}
}
make_task_dirty_internal(
dependent,
dependent_task_id,
&mut dependent,
make_stale,
#[cfg(feature = "task_dirty_cause")]
TaskDirtyCause::CellChange {
......
//! Coordinator that gates concurrent operations against snapshotting.
//! Coordinator that gates concurrent operations against snapshotting and garbage collection.
//!
//! Backend operations and snapshot work share a single [`SnapshotCoordinator`]
//! that enforces the protocol:
//! Backend operations, snapshot work, and garbage collection share a single
//! [`SnapshotCoordinator`] that enforces the protocol:
//!
//! - When no snapshot is in flight, [`begin_operation`](SnapshotCoordinator::begin_operation) is a
//! single uncontended atomic increment.
//! - When a snapshot is requested, new operations block until the snapshot finishes, and operations
//! already in flight either complete or call
//! [`suspend_point`](SnapshotCoordinator::suspend_point) to suspend.
//! - The snapshotter waits for every in-flight operation to drain or suspend, takes its snapshot,
//! then wakes everyone.
//! - When no exclusive phase is in flight,
//! [`begin_operation`](SnapshotCoordinator::begin_operation) is a single uncontended atomic
//! increment.
//! - When a phase is requested, new operations block until it finishes, and operations already in
//! flight either complete or call [`suspend_point`](SnapshotCoordinator::suspend_point) to
//! suspend.
//! - The phase holder waits for every in-flight operation to drain or suspend, does its work, then
//! wakes everyone.
//!
//! Snapshotting and GC are the same kind of exclusion and share one [`ExclusionPhase`]: both need
//! every operation stopped, neither can overlap the other, and GC hands its work straight to the
//! snapshot that persists it. A single [`EXCLUSION_REQUESTED_BIT`] covers both, which is also what
//! lets a GC pass and the snapshot that commits it run under one uninterrupted guard.
use std::sync::{
Arc,
......@@ -18,6 +24,7 @@ use std::sync::{
use parking_lot::{Condvar, Mutex};
use rustc_hash::FxHashSet;
use tracing::info_span;
use crate::{backend::AnyOperation, utils::ptr_eq_arc::PtrEqArc};
......@@ -36,7 +43,7 @@ struct State<O> {
suspended_operations: FxHashSet<PtrEqArc<O>>,
}
/// Coordinates operation/snapshot interleaving.
/// Coordinates operation/snapshot/GC interleaving.
///
/// Generic over the operation type the caller wants to suspend. The
/// coordinator only requires `O: Send + Sync + 'static`; it never inspects
......@@ -106,8 +113,10 @@ impl<O> SnapshotCoordinator<O> {
if prev - 1 == SNAPSHOT_REQUESTED_BIT {
this.operations_drained.notify_all();
}
this.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested);
tokio::task::block_in_place(|| {
this.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested);
});
// Re-add now that the snapshot is done. Bit is cleared because
// we just observed `snapshot_requested == false` under the
// mutex.
......@@ -130,13 +139,13 @@ impl<O> SnapshotCoordinator<O> {
}
#[cold]
fn suspend_point_cold<O>(this: &SnapshotCoordinator<O>, suspend: impl FnOnce() -> O) {
let op = Arc::new(suspend());
let mut state = this.state.lock();
if !state.snapshot_requested {
// Race: snapshot finished between the `snapshot_pending` check
// and acquiring the mutex. Nothing to do.
return;
}
let op = Arc::new(suspend());
state
.suspended_operations
.insert(PtrEqArc::from(op.clone()));
......@@ -153,8 +162,11 @@ impl<O> SnapshotCoordinator<O> {
this.operations_drained.notify_all();
}
// Wait for the snapshot to finish.
this.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested);
tokio::task::block_in_place(|| {
this.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested);
});
// Resume: re-increment and remove ourselves from the suspended set.
this.in_progress_operations.fetch_add(1, Ordering::AcqRel);
state.suspended_operations.remove(&PtrEqArc::from(op));
......@@ -190,11 +202,19 @@ impl<O> SnapshotCoordinator<O> {
"snapshot bit was already set when begin_snapshot ran: {active:#x}"
);
if (active & !SNAPSHOT_REQUESTED_BIT) != 0 {
// Some operations are in flight. Wait for them to drain or
// suspend. The predicate is Acquire-loaded so we synchronize
// with the AcqRel decrement that woke us.
self.operations_drained.wait_while(&mut state, |_| {
self.in_progress_operations.load(Ordering::Acquire) != SNAPSHOT_REQUESTED_BIT
// The predicate is Acquire-loaded so we synchronize with the AcqRel decrement that woke
// us. This can block for a while under load (until every in-flight operation reaches a
// suspend point or finishes), so it gets its own span for latency attribution.
let num_operations = active & !SNAPSHOT_REQUESTED_BIT;
let _span = info_span!("await operations settle", num_operations).entered();
// Release our worker thread so pending operations have it available. This preserves
// liveness: this runs as a `tokio::spawn` background job, so parking here without
// handing the worker back would starve the very operations we are waiting on.
tokio::task::block_in_place(|| {
self.operations_drained.wait_while(&mut state, |_| {
(self.in_progress_operations.load(Ordering::Acquire) & !SNAPSHOT_REQUESTED_BIT)
!= 0
});
});
}
// Snapshot ranges that follow can read the suspended_operations
......@@ -254,7 +274,7 @@ impl<O> Drop for OperationGuard<'_, O> {
// (not under the user mutex), so a notifier that has never synchronized
// with the user mutex can racily observe stale null and drop the notify.
//
// It is generally a best practice to only notify under the loc
// It is generally a best practice to only notify under the lock
let _g = coord.state.lock();
coord.operations_drained.notify_all();
}
......@@ -293,7 +313,7 @@ impl<O> Drop for SnapshotPhase<'_, O> {
let prev = self
.coord
.in_progress_operations
.fetch_sub(SNAPSHOT_REQUESTED_BIT, Ordering::AcqRel);
.fetch_and(!SNAPSHOT_REQUESTED_BIT, Ordering::AcqRel);
assert!(
(prev & SNAPSHOT_REQUESTED_BIT) != 0,
"SnapshotPhase::drop: snapshot bit was already cleared (prev={prev:#x})"
......@@ -412,11 +432,19 @@ mod tests {
while arrived.load(Ordering::Acquire) == 0 {
thread::yield_now();
}
assert_eq!(started_op.load(Ordering::Acquire), 0);
assert_eq!(
started_op.load(Ordering::Acquire),
0,
"a new operation must block while the exclusion is held"
);
drop(phase);
op_thread.join().unwrap();
assert_eq!(started_op.load(Ordering::Acquire), 1);
assert_eq!(
started_op.load(Ordering::Acquire),
1,
"the operation must run once the exclusion ends"
);
}
#[test]
......@@ -431,7 +459,11 @@ mod tests {
let snapshotter_done = snapshotter_done.clone();
move || {
let phase = coord_snap.begin_snapshot();
assert_eq!(phase.suspended_operations().len(), 1);
assert_eq!(
phase.suspended_operations().len(),
1,
"must record the suspended operation for replay"
);
snapshotter_done.store(1, Ordering::Release);
// Hold the snapshot for a moment so the suspend_point thread
// observes `snapshot_requested == true` after waking.
......@@ -440,11 +472,26 @@ mod tests {
});
wait_for_snapshot_pending(&coord);
// Snapshotter is now waiting for our operation to drain. Calling
// suspend_point should let it proceed.
coord.suspend_point(|| 42u32);
// suspend_point returns once the snapshot is finished.
assert_eq!(snapshotter_done.load(Ordering::Acquire), 1);
// The phase is waiting for our operation to drain; suspending should let it proceed.
let recorded = Arc::new(AtomicUsize::new(0));
coord.suspend_point({
let recorded = recorded.clone();
move || {
recorded.fetch_add(1, Ordering::Release);
42u32
}
});
// `suspend_point` returns only once the phase has finished.
assert_eq!(
snapshotter_done.load(Ordering::Acquire),
1,
"suspend_point must not return before the phase completes"
);
assert_eq!(
recorded.load(Ordering::Acquire),
1,
"the suspend closure must run so the operation is recorded for a possible snapshot"
);
snap_thread.join().unwrap();
drop(g);
......@@ -595,4 +642,12 @@ mod tests {
"in_progress_operations should be 0 after all ops and snapshots done"
);
}
#[test]
#[should_panic(expected = "already in flight")]
fn overlapping_exclusions_panic() {
let coord = SnapshotCoordinator::<Op>::new();
let _first = coord.begin_snapshot();
let _second = coord.begin_snapshot();
}
}
......@@ -9,6 +9,7 @@ use std::{
},
};
use crossbeam_utils::CachePadded;
use hashbrown::hash_table;
use thread_local::ThreadLocal;
use tracing::span::Id;
......@@ -34,6 +35,20 @@ pub enum TaskDataCategory {
Data,
All,
}
impl PartialOrd for TaskDataCategory {
/// `All` is greater than both `Meta` and `Data`; `Meta` and `Data` are unordered.
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
use std::cmp::Ordering::*;
use TaskDataCategory::All;
match (self, other) {
_ if self == other => Some(Equal),
(All, _) => Some(Greater),
(_, All) => Some(Less),
_ => None,
}
}
}
/// Counts of tasks evicted at each level.
#[derive(Debug, Default)]
......@@ -161,7 +176,7 @@ pub struct Storage {
/// that `shard_modified_counts.len()==map.shards().len()`
///
/// Should only be modified while holding the corresponding dashmap shard lock.
shard_modified_counts: Box<[AtomicU64]>,
shard_modified_counts: Box<[CachePadded<AtomicU64>]>,
/// Stores snapshots of task state for tasks accessed during snapshot mode.
/// - `Some(snapshot)`: Task was modified before snapshot mode and accessed again during it.
/// Contains a copy of the pre-snapshot state that needs to be persisted.
......@@ -218,7 +233,7 @@ impl Storage {
shard_amount,
);
let shard_modified_counts = (0..shard_amount)
.map(|_| AtomicU64::new(0))
.map(|_| CachePadded::new(AtomicU64::new(0)))
.collect::<Vec<_>>()
.into_boxed_slice();
Self {
......@@ -512,6 +527,41 @@ impl Storage {
Some(f(task.value()))
}
/// The number of **persistent** (non-transient) tasks resident in the map. Use this to assert
/// GC returns to a flat baseline across re-rooting: GC never collects transient tasks (e.g.
/// `run_once`/Once roots), so their count is not expected to settle.
#[doc(hidden)]
pub fn resident_persistent_task_count_for_testing(&self) -> usize {
let mut persistent = 0;
for shard in self.map.shards() {
let shard = shard.read();
for (task_id, _) in shard.iter() {
if !task_id.is_transient() {
persistent += 1;
}
}
}
persistent
}
/// The number of shards in the resident map. GC seeds one `ScanShard` job per index; the slice
/// returned by `map.shards()` is fixed for the map's lifetime, so an index is a stable handle
/// to one shard.
pub fn shard_count(&self) -> usize {
self.map.shards().len()
}
/// Scans a **single** shard by index, invoking `on_candidate` for each resident, non-transient
/// task whose storage passes the cheap [`TaskStorage::gc_maybe_collectible`] pre-filter.
pub fn gc_scan_shard(&self, index: usize, mut on_candidate: impl FnMut(TaskId)) {
let shard = self.map.shards()[index].read();
for (task_id, task) in shard.iter() {
if !task_id.is_transient() && task.gc_maybe_collectible() {
on_candidate(*task_id);
}
}
}
pub fn access_pair_mut(
&self,
key1: TaskId,
......@@ -573,11 +623,44 @@ impl Storage {
// avoid a lock cycle with get_or_create_persistent_task, which takes task_cache
// before map. Allocated lazily on first conflict.
let mut deferred_task_cache_removals: Vec<CachedTaskTypeArc> = Vec::new();
// Remove a task type from `task_cache`, deferring on contention. Shared by the
// GC-deleted path below and the ordinary key eviction.
let remove_from_task_cache =
|evicted: &mut EvictionCounts,
deferred: &mut Vec<CachedTaskTypeArc>,
task_type: &CachedTaskTypeArc| {
match try_lock_and_remove(&self.task_cache, task_type.as_ref()) {
TryLockAndRemove::Removed => {
evicted.key_evictions += 1;
}
TryLockAndRemove::NotFound => {
// Generally this should be rare, it more or less implies something
// else is concurrently holding the Arc
}
TryLockAndRemove::WouldBlock => {
// Contention, to avoid a deadlock just defer
deferred.push(task_type.clone());
}
}
};
shard.retain(|(task_id, task)| {
if task_id.is_transient() {
evicted.unevictable_reasons[UnevictableReason::Transient.index()] += 1;
return true;
}
// GC'd tasks were tombstoned during the snapshot so we can drop them fully now.
if task.flags.deleted() {
let task_type = task
.get_persistent_task_type()
.expect("GC deleted tasks must have a task type");
remove_from_task_cache(
&mut evicted,
&mut deferred_task_cache_removals,
task_type,
);
evicted.full += 1;
return false;
}
let (key_evictability, value_evictability) = task.evictability();
match key_evictability {
KeyEvictability::Evictable => {
......@@ -589,19 +672,11 @@ impl Storage {
// Because `get_or_create_task` acquires 'task_cache' then `storage.map` and
// we do the opposite we need to be defensive here. Attempting here is just
// an optimization to avoid pushing into `deferred_task_cache_removals`
match try_lock_and_remove(&self.task_cache, task_type.as_ref()) {
TryLockAndRemove::Removed => {
evicted.key_evictions += 1;
}
TryLockAndRemove::NotFound => {
// Generally this should be rare, it more or less implies something
// else is concurrently holding the Arc
}
TryLockAndRemove::WouldBlock => {
// Contention, to avoid a deadlock just defer
deferred_task_cache_removals.push(task_type.clone());
}
}
remove_from_task_cache(
&mut evicted,
&mut deferred_task_cache_removals,
task_type,
);
}
KeyEvictability::AlreadyEvicted | KeyEvictability::Unevictable => {}
}
......@@ -810,6 +885,26 @@ impl StorageWriteGuard<'_> {
}
}
}
/// Clears all modified/new flags for a GC-collected task that was **never persisted**
/// (`new_task`).
pub fn discard_modifications_for_gc_new_task(&mut self) {
debug_assert!(
!self.storage.snapshot_mode(),
"discard_modifications_for_gc_new_task must run before the snapshot starts"
);
debug_assert!(
self.inner.flags.new_task(),
"only a never-persisted (new_task) collected task may be discarded this way"
);
if self.inner.flags.any_modified() {
let shard_idx = self.storage.shard_index(self.inner.key());
self.storage.shard_modified_counts[shard_idx].fetch_sub(1, Ordering::Relaxed);
}
self.inner.flags.set_meta_modified(false);
self.inner.flags.set_data_modified(false);
self.inner.flags.set_new_task(false);
}
}
impl Deref for StorageWriteGuard<'_> {
......
......@@ -230,6 +230,10 @@ struct TaskStorageSchema {
#[field(storage = "flag", category = "transient")]
pub new_task: bool,
/// GC soft-deletion marker. Set by the garbage collector when a task is marked for deletion.
#[field(storage = "flag", category = "transient")]
deleted: bool,
// =========================================================================
// CHILDREN & AGGREGATION (meta)
// =========================================================================
......@@ -268,7 +272,7 @@ struct TaskStorageSchema {
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies: AutoSet<CellRef, 2>,
cell_dependencies: AutoSet<CellRef, 3>,
/// Cells this task depends on, narrowed to a hashed sub-value (`CellDependency::Hash`). Rare.
#[field(
......@@ -296,7 +300,7 @@ struct TaskStorageSchema {
/// Outdated keyless cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_cell_dependencies: AutoSet<CellRef, 2>,
outdated_cell_dependencies: AutoSet<CellRef, 3>,
/// Outdated hashed cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
......@@ -318,7 +322,7 @@ struct TaskStorageSchema {
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents: AutoSet<CellRef, 2>,
cell_dependents: AutoSet<CellRef, 3>,
/// Tasks that depend on a hashed sub-value of this task's cells. Reverse of
/// `cell_dependencies_hashed`.
......@@ -853,6 +857,47 @@ impl TaskStorage {
pub fn gc_transient_ref_count(&self) -> u32 {
self.get_transient_ref_count().copied().unwrap_or(0)
}
/// Whether a GC pass may collect this task: nothing references it, via parents, transient
/// pins, aggregation edges, or dependency edges.
///
/// Precision depends on what the caller restored. Meta alone cannot see the three Data-category
/// dependent sets, so the answer is a sound *pre-filter*: a `false` is definitive, a `true` may
/// still have dependents. With Meta + Data it is the full predicate. That one-directional
/// conservatism lets the cheap Meta-only shard scan and the authoritative under-guard recheck
/// (which opens `TaskDataCategory::All`, and is what actually gates collection) share this
/// single predicate.
///
/// Refusing to collect a task with dependents is what makes task-id reuse safe: a hard-deleted
/// id can be handed out again, so a surviving dependent edge would silently resolve to an
/// unrelated live task instead of tripping `MustExist`.
pub fn gc_maybe_collectible(&self) -> bool {
// None of the predicates below are correct without this.
self.flags.is_restored(TaskDataCategory::Meta)
// Already collected this session (soft-deleted, awaiting tombstone + hard-delete):
// don't re-select it, or a second pass would collect it again while it is still
// resident.
&& !self.flags.deleted()
&& self.gc_parent_count() == 0
&& self.gc_transient_ref_count() == 0
&& self.get_activeness().is_none()
&& self.get_in_progress().is_none()
// It is rare for upper/followers to be present when the ref counts are 0 but it can happen transiently during a concurrent GC pass as uppers are moved around during the cascade.
&& self.upper().is_empty()
&& self.followers().is_none_or(|f| f.is_empty())
// `collectibles_dependents` is Meta, so it is always checkable here.
&& self
.collectibles_dependents()
.is_none_or(|d| d.is_empty())
// The remaining dependent sets are Data; skipped (leaving this a pre-filter) when Data
// is not restored.
&& (!self.flags.is_restored(TaskDataCategory::Data)
|| (self.output_dependent().is_empty()
&& self.cell_dependents().is_none_or(|d| d.is_empty())
&& self
.cell_dependents_hashed()
.is_none_or(|d| d.is_empty())))
}
}
/// Counts for aggregation tree and collectibles fields.
......
......@@ -22,9 +22,6 @@ pub enum SnapshotItem {
/// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>,
},
// Constructed by the GC pass that emits `Delete` for soft-deleted tasks, which lands in a
// later PR in the stack.
#[allow(dead_code)]
Delete {
task_id: TaskId,
/// The deleted task's `TaskCache` key. Always present: only persistent tasks are
......
// `ValueDebug` only carries `dbg()` under `debug_assertions`
#![cfg(debug_assertions)]
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
......
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
mod util;
use anyhow::Result;
use turbo_tasks::{ResolvedVc, State, TaskId, Vc, prevent_gc};
use crate::util::create_tt;
/// The `TaskId` backing a resolved `Vc` (its `TaskOutput` node).
fn task_id_of<T>(vc: Vc<T>) -> TaskId {
Vc::into_raw(vc)
.try_get_task_id()
.expect("a resolved Vc should be backed by a task")
}
#[turbo_tasks::value(transparent)]
struct Selector(State<bool>);
#[turbo_tasks::function(operation, root)]
fn create_selector(initial: bool) -> Vc<Selector> {
Selector(State::new(initial)).cell()
}
#[turbo_tasks::function]
fn leaf(n: u32) -> Vc<u32> {
Vc::cell(n)
}
/// Resolves `leaf(n)` and returns the raw `TaskId` backing it, as a `u32`.
///
/// This runs inside a tracked task rather than at the top level of `run_once`, so the plain
/// (eventually consistent) `.resolve()` read below is legal and deterministic — the read is
/// ordered by the task graph instead of racing whatever the session happens to be doing.
#[turbo_tasks::function(operation, root)]
async fn leaf_task_id(n: u32) -> Result<Vc<u32>> {
Ok(Vc::cell(*task_id_of(leaf(n).resolve().await?)))
}
#[turbo_tasks::function]
async fn branch_a() -> Result<Vc<u32>> {
Ok(Vc::cell(1 + *leaf(10).await?))
}
#[turbo_tasks::function]
async fn branch_b() -> Result<Vc<u32>> {
Ok(Vc::cell(2 + *leaf(20).await?))
}
/// A task that pins itself against GC while executing. Once pinned it must survive collection even
/// after it is disconnected.
#[turbo_tasks::function]
async fn pinned_branch() -> Result<Vc<u32>> {
prevent_gc();
Ok(Vc::cell(99))
}
/// Reads exactly one branch depending on the selector; flipping it re-executes and disconnects the
/// previously-read branch (and its subtree), which should drop that branch's `parent_count` to 0.
#[turbo_tasks::function(operation, root)]
async fn select(selector: ResolvedVc<Selector>) -> Result<Vc<u32>> {
let use_b = *selector.await?.get();
let value = if use_b {
*branch_b().await?
} else {
*branch_a().await?
};
Ok(Vc::cell(value))
}
/// Like `select`, but reads `pinned_branch` instead of `branch_a` when the selector is false.
#[turbo_tasks::function(operation, root)]
async fn select_pinned(selector: ResolvedVc<Selector>) -> Result<Vc<u32>> {
let use_b = *selector.await?.get();
let value = if use_b {
*branch_b().await?
} else {
*pinned_branch().await?
};
Ok(Vc::cell(value))
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn gc_collects_disconnected_subtree() {
let (tt, _persistence_dir) = create_tt("gc_collects_disconnected_subtree");
let tt2 = tt.clone();
let result = turbo_tasks::run_once(tt.clone(), async move {
let selector_op = create_selector(false);
let selector_vc = selector_op.resolve().strongly_consistent().await?;
let selector = selector_op.read_strongly_consistent().await?;
let output = select(selector_vc);
assert_eq!(*output.read_strongly_consistent().await?, 11);
// Flip: select drops branch_a; branch_a (parent_count 0) becomes a candidate.
selector.set(true);
assert_eq!(*output.read_strongly_consistent().await?, 22);
anyhow::Ok(())
})
.await;
result.unwrap();
let collected = tt2.backend().gc_for_testing(&tt2);
assert_eq!(
collected, 2,
"branch_a and its cascaded child leaf(10) should both be collected"
);
assert_eq!(
tt2.backend().gc_for_testing(&tt2),
0,
"a second GC pass must collect nothing"
);
// Flipping back must recompute branch_a fresh, since it was collected.
let tt3 = tt.clone();
let result = turbo_tasks::run_once(tt.clone(), async move {
let selector_op = create_selector(true);
let selector_vc = selector_op.resolve().strongly_consistent().await?;
let selector = selector_op.read_strongly_consistent().await?;
let output = select(selector_vc);
assert_eq!(*output.read_strongly_consistent().await?, 22);
selector.set(false);
assert_eq!(*output.read_strongly_consistent().await?, 11);
let _ = &tt3;
anyhow::Ok(())
})
.await;
result.unwrap();
tt.stop_and_wait().await;
}
/// A task that pins itself via `prevent_gc()` must survive collection even after it is disconnected
/// from the live graph, because the pin makes it a GC root.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn gc_does_not_collect_pinned_task() {
let (tt, _persistence_dir) = create_tt("gc_does_not_collect_pinned_task");
let tt2 = tt.clone();
let result = turbo_tasks::run_once(tt.clone(), async move {
let selector_op = create_selector(false);
let selector_vc = selector_op.resolve().strongly_consistent().await?;
let selector = selector_op.read_strongly_consistent().await?;
let output = select_pinned(selector_vc);
assert_eq!(*output.read_strongly_consistent().await?, 99);
// Flip: select_pinned re-executes, reads branch_b, and disconnects pinned_branch.
selector.set(true);
assert_eq!(*output.read_strongly_consistent().await?, 22);
anyhow::Ok(())
})
.await;
result.unwrap();
// GC runs after the run has released activeness, so pinned_branch is disconnected
// (parent_count 0) and otherwise collectible.
let collected = tt2.backend().gc_for_testing(&tt2);
assert_eq!(
collected, 0,
"a pinned task must not be collected even when disconnected"
);
// A snapshot + evict must not lose the (transient) pin. A pinned task is not forced fully
// resident — its Meta/Data may be partially evicted — but the session-only
// `transient_ref_count` is retained as residue (the map entry is kept), so the task stays
// uncollectible and a subsequent GC still collects nothing.
tt2.backend().snapshot_and_evict_for_testing(&tt2);
assert_eq!(
tt2.backend().gc_for_testing(&tt2),
0,
"pinned task must survive eviction and not be collected"
);
tt.stop_and_wait().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unpin_after_stop_does_not_panic() {
let (tt, _persistence_dir) = create_tt("unpin_after_stop_does_not_panic");
// Pin a real task inside a session (as `prevent_gc` / `DetachedVc::new` would).
let tt2 = tt.clone();
let leaf_id = turbo_tasks::run_once(tt.clone(), async move {
let id = TaskId::try_from(*leaf_task_id(7).read_strongly_consistent().await?)?;
tt2.pin_task_for_gc(id);
anyhow::Ok(id)
})
.await
.unwrap();
// Stop the backend — this drops the in-memory task map, so the pinned task is no longer
// resident (exactly as at `next build` shutdown).
tt.stop_and_wait().await;
tt.unpin_task_for_gc(leaf_id);
}
This diff is collapsed.
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
mod util;
use std::sync::Arc;
use anyhow::Result;
use turbo_tasks::{ResolvedVc, State, TurboTasks, Vc};
use turbo_tasks_backend::TurboTasksBackend;
use crate::util::create_tt;
#[turbo_tasks::value(transparent)]
struct Generation(State<u32>);
#[turbo_tasks::function(operation, root)]
fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell()
}
/// A leaf keyed by (generation, index). Bumping the generation makes `wide_root` read an entirely
/// fresh set of these, disconnecting the whole previous generation.
#[turbo_tasks::function]
fn leaf(generation: u32, index: u32) -> Vc<u32> {
Vc::cell(generation.wrapping_mul(1000).wrapping_add(index))
}
/// A shared-dependency layer, so the disconnected garbage is a subtree (intermediate + leaf) and
/// the test exercises the cascade rather than single-node collection.
#[turbo_tasks::function]
async fn intermediate(generation: u32, index: u32) -> Result<Vc<u32>> {
Ok(Vc::cell(1 + *leaf(generation, index).await?))
}
const WIDTH: u32 = 24;
/// Bumping the generation re-executes this and connects a fresh generation's worth of tasks,
/// disconnecting the entire previous generation (2*WIDTH tasks) as garbage.
#[turbo_tasks::function(operation, root)]
async fn wide_root(generation: ResolvedVc<Generation>) -> Result<Vc<u32>> {
let generation = *generation.await?.get();
let mut sum = 0u32;
for index in 0..WIDTH {
sum = sum.wrapping_add(*intermediate(generation, index).await?);
}
Ok(Vc::cell(sum))
}
/// Repeatedly swapping a wide set of common dependencies (as happens when a project is re-rooted or
/// its dependencies churn) must NOT grow the resident task set monotonically. Each round bumps the
/// generation (disconnecting the previous generation's 2*WIDTH tasks), then snapshots + GCs +
/// evicts, and the resident count must return to a flat baseline.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn gc_re_rooting_stays_flat() {
let (tt, _persistence_dir) = create_tt("gc_re_rooting_stays_flat");
let tt2 = tt.clone();
const ROUNDS: u32 = 20;
// Each round runs in its own `run_once` so the root's activeness is released before GC (a
// `run_once` root keeps everything it touched active until it returns).
async fn round(tt: &Arc<TurboTasks<TurboTasksBackend>>, gen_value: u32) -> (usize, usize) {
let tt_inner = tt.clone();
turbo_tasks::run_once(tt.clone(), async move {
// `create_generation` is a cached root operation, so this returns the same task each
// round.
let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?;
if gen_value > 0 {
let generation = generation_op.read_strongly_consistent().await?;
generation.set(gen_value);
}
let output = wide_root(generation_vc);
output.read_strongly_consistent().await?;
let _ = &tt_inner;
anyhow::Ok(())
})
.await
.unwrap();
// GC runs BEFORE eviction (matching the production background cycle: snapshot -> GC ->
// evict), so the disconnected garbage is still resident when GC scans it. Running eviction
// first would drop the garbage to disk-only where the in-memory GC can't collect or
// tombstone it.
let collected = tt.backend().gc_for_testing(tt);
tt.backend().snapshot_and_evict_for_testing(tt);
// Measure the *persistent* resident count: GC only collects persistent tasks. Transient
// roots (each round's `run_once`/Once task) are never collected and accumulate
// independently of GC — including them would mask the real signal.
(
collected,
tt.backend().resident_persistent_task_count_for_testing(),
)
}
// Warm up: build generation 0, then measure the steady-state baseline after one full cycle.
let (_, baseline) = round(&tt2, 0).await;
println!("baseline persistent resident after gen 0: {baseline}");
// Churn: swap the whole dependency set many times.
let mut max_resident = baseline;
let mut total_collected = 0usize;
for gen_value in 1..=ROUNDS {
let (collected, resident) = round(&tt2, gen_value).await;
println!("gen {gen_value}: collected={collected} persistent_resident={resident}");
max_resident = max_resident.max(resident);
total_collected += collected;
}
let no_gc_growth = baseline + (2 * WIDTH as usize) * (ROUNDS as usize);
println!(
"baseline={baseline} max_resident={max_resident} total_collected={total_collected} \
no_gc_growth_would_be={no_gc_growth}"
);
// Without GC the persistent resident set would climb by ~2*WIDTH per round (each generation's
// intermediates + leaves persisted forever), so a bound within a small constant of the baseline
// is only satisfiable if each generation's garbage subtree is actually collected.
assert!(
max_resident <= baseline + 2 * WIDTH as usize,
"persistent resident set grew too much across re-rooting (max={max_resident}, \
baseline={baseline}); GC is not returning to a flat baseline"
);
// Assert GC collected at least half of a full generation per round (slack for tasks that settle
// across passes), proving the flat baseline comes from GC collecting garbage — not merely from
// eviction hiding it on disk.
let expected_min_collected = (2 * WIDTH as usize) * (ROUNDS as usize) / 2;
assert!(
total_collected >= expected_min_collected,
"GC collected too little ({total_collected}); expected >= {expected_min_collected} across \
{ROUNDS} re-rootings — GC is not doing the collection"
);
// The live graph must still compute correctly after all the churn.
let tt3 = tt.clone();
let result = turbo_tasks::run_once(tt.clone(), async move {
let generation_op = create_generation();
let generation_vc = generation_op.resolve().strongly_consistent().await?;
let output = wide_root(generation_vc);
// generation is ROUNDS; sum = WIDTH intermediates each = 1 + leaf(ROUNDS, i).
let expected: u32 = (0..WIDTH)
.map(|i| 1 + (ROUNDS.wrapping_mul(1000).wrapping_add(i)))
.fold(0u32, |a, b| a.wrapping_add(b));
assert_eq!(*output.read_strongly_consistent().await?, expected);
let _ = &tt3;
anyhow::Ok(())
})
.await;
result.unwrap();
tt.stop_and_wait().await;
}
......@@ -27,6 +27,9 @@ fn open_tt_at(path: &Path, num_workers: usize) -> Arc<TurboTasks<TurboTasksBacke
// snapshot_and_evict_for_testing manually.
storage_mode: Some(turbo_tasks_backend::StorageMode::ReadWriteOnShutdown),
eviction_mode: EvictionMode::Full,
// `gc_for_testing` requires a GC-enabled backend. Set per-backend rather than via
// `TURBO_ENGINE_GC` so parallel tests in the same binary don't share the setting.
gc: Some(true),
..Default::default()
},
turbo_tasks_backend::turbo_backing_storage(
......
......@@ -734,6 +734,14 @@ pub trait Backend: Sized + Sync + Send {
// Do nothing by default
}
/// Pin a task against garbage collection (via [`prevent_gc`](crate::prevent_gc)). A pinned task
/// is treated as a GC root, keeping alive references that escape the tracked task graph. The
/// default is a no-op for backends without GC.
fn pin_task_for_gc(&self, _task: TaskId, _turbo_tasks: &TurboTasks<Self>);
/// Removes a pin added by [`pin_task_for_gc`](Backend::pin_task_for_gc).
fn unpin_task_for_gc(&self, _task: TaskId, _turbo_tasks: &TurboTasks<Self>);
fn create_transient_task(
&self,
task_type: TransientTaskType,
......
......@@ -199,6 +199,13 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send {
);
fn mark_own_task_as_finished(&self, task: TaskId);
/// Pin a task against garbage collection. Delegates to
/// [`Backend::pin_task_for_gc`](crate::backend::Backend::pin_task_for_gc).
fn pin_task_for_gc(&self, task: TaskId);
/// Removes a pin added by [`pin_task_for_gc`](TurboTasksApi::pin_task_for_gc).
fn unpin_task_for_gc(&self, task: TaskId);
fn connect_task(&self, task: TaskId);
/// Wraps the given future in the current task.
......@@ -925,6 +932,18 @@ impl<B: Backend + 'static> TurboTasks<B> {
self.backend.dispose_root_task(task_id, self);
}
/// Pins a task against garbage collection (a transient, session-only reference). Balanced by
/// [`unpin_task_for_gc`](Self::unpin_task_for_gc). Used for references that escape the tracked
/// task graph — e.g. a `DetachedVc` holding an `OperationVc` across the NAPI boundary.
pub fn pin_task_for_gc(&self, task_id: TaskId) {
self.backend.pin_task_for_gc(task_id, self);
}
/// Releases a pin added by [`pin_task_for_gc`](Self::pin_task_for_gc).
pub fn unpin_task_for_gc(&self, task_id: TaskId) {
self.backend.unpin_task_for_gc(task_id, self);
}
// TODO make sure that all dependencies settle before reading them
/// Creates a new root task, that is only executed once.
/// Dependencies will not invalidate the task.
......@@ -1898,6 +1917,14 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> {
self.backend.mark_own_task_as_finished(task, self);
}
fn pin_task_for_gc(&self, task: TaskId) {
self.backend.pin_task_for_gc(task, self);
}
fn unpin_task_for_gc(&self, task: TaskId) {
self.backend.unpin_task_for_gc(task, self);
}
/// Creates a future that inherits the current task id and task state. The current global task
/// will wait for this future to be dropped before exiting.
fn spawn_detached_for_testing(&self, fut: Pin<Box<dyn Future<Output = ()> + Send + 'static>>) {
......@@ -2222,8 +2249,17 @@ pub fn unmark_top_level_task_may_leak_eventually_consistent_state() {
}
}
/// Pins the current task against garbage collection for the rest of the session, keeping it (and,
/// via the reachability it anchors, the values it produced) alive even if it becomes disconnected
/// from the live task graph. Use this when a value escapes the tracked graph — e.g. a `Vc` sent out
/// of a `spawn_detached` future across a channel, or handed across the NAPI boundary — so no
/// persistent parent lists it as a child and it would otherwise be collected.
///
/// No-op outside a task context, and on backends without garbage collection.
pub fn prevent_gc() {
// TODO implement garbage collection
if let Some(task) = current_task_if_available("prevent_gc") {
with_turbo_tasks(|tt| tt.pin_task_for_gc(task));
}
}
pub fn emit<T: VcValueTrait + ?Sized>(collectible: ResolvedVc<T>) {
......
......@@ -21,6 +21,9 @@ pub enum TaskDirtyCause {
collectible_type: TraitTypeId,
},
Invalidator,
/// Re-dirtied because a GC-soft-deleted task was resurrected by a new connection before its
/// hard-delete: its edges were scrubbed, so it must re-execute to rebuild them.
Resurrected,
Unknown,
}
......@@ -83,6 +86,7 @@ impl std::fmt::Display for TaskDirtyCause {
)
}
TaskDirtyCause::Invalidator => write!(f, "invalidator"),
TaskDirtyCause::Resurrected => write!(f, "resurrected"),
TaskDirtyCause::Unknown => write!(f, "unknown"),
}
}
......
......@@ -192,6 +192,10 @@ impl<T: ?Sized> OperationVc<T> {
{
self.connect().into_trait_ref().strongly_consistent()
}
pub fn task_id(self) -> TaskId {
self.task
}
}
impl<T> Copy for OperationVc<T> where T: ?Sized {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment