Unverified Commit cb95ca37 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Fix root predicates (#98942)

Fix the definition of GC roots to account for all transient references

There was a categorical mistake in how `uppers` and
`collectible_dependents` were interpreted. Both of these refer to a task
above us and this is it is a reference but the reference might be
comming from transient or persistent tasks. This is what kept confusing
the 'root' definition

So the new definition is simply

* You are collectible if there are no references (persistent or
transient)
* You are a root if there are only transient references, which includes
transient uppers and collectible_dependents


The old definition expected roots to have no uppers at all, which isn't
correct. A `run_once` task becomes an aggregation root and so the nested
persistent tasks reference it as an upper. Those tasks should become gc
roots since they are on the transient <--> persistent boundary and the
only thing retaining them is references from transient tasks.
parent c947937b
...@@ -1138,7 +1138,7 @@ impl TurboTasksBackend { ...@@ -1138,7 +1138,7 @@ impl TurboTasksBackend {
parent_span: Option<tracing::Id>, parent_span: Option<tracing::Id>,
reason: SnapshotReason, reason: SnapshotReason,
turbo_tasks: &TurboTasks<TurboTasksBackend>, turbo_tasks: &TurboTasks<TurboTasksBackend>,
) -> Result<(Instant, bool, Option<(GcStats, GcPassResult)>), anyhow::Error> { ) -> Result<(Instant, bool, Option<(GcStats, GcPassResult)>)> {
let snapshot_span = let snapshot_span =
tracing::trace_span!(parent: parent_span.clone(), "snapshot", reason = reason.as_str()) tracing::trace_span!(parent: parent_span.clone(), "snapshot", reason = reason.as_str())
.entered(); .entered();
...@@ -1158,7 +1158,6 @@ impl TurboTasksBackend { ...@@ -1158,7 +1158,6 @@ impl TurboTasksBackend {
let mut snapshot_phase = self.snapshot_coord.begin_snapshot(); let mut snapshot_phase = self.snapshot_coord.begin_snapshot();
let (gc_elapsed, gc_roots_to_persist, gc_outcome) = if self.gc_enabled { let (gc_elapsed, gc_roots_to_persist, gc_outcome) = if self.gc_enabled {
let gc_span = tracing::info_span!( let gc_span = tracing::info_span!(
parent: parent_span.clone(),
"gc", "gc",
stats = tracing::field::Empty, stats = tracing::field::Empty,
interrupted = tracing::field::Empty interrupted = tracing::field::Empty
...@@ -1415,7 +1414,7 @@ impl TurboTasksBackend { ...@@ -1415,7 +1414,7 @@ impl TurboTasksBackend {
}; };
} else { } else {
debug_assert!( debug_assert!(
!inner.gc_maybe_collectible(), !inner.gc_collectible(),
"tasks scheduled for persistent must not be collectible, this implies a \ "tasks scheduled for persistent must not be collectible, this implies a \
missed task during GC" missed task during GC"
); );
......
...@@ -1585,7 +1585,7 @@ pub trait TaskGuard: Debug + TaskStorageAccessors { ...@@ -1585,7 +1585,7 @@ pub trait TaskGuard: Debug + TaskStorageAccessors {
/// Whether a GC pass may collect this task: nothing references it. /// Whether a GC pass may collect this task: nothing references it.
fn is_gc_collectible(&self) -> bool { fn is_gc_collectible(&self) -> bool {
self.check_access(SpecificTaskDataCategory::Meta); self.check_access(SpecificTaskDataCategory::Meta);
self.typed().gc_maybe_collectible() self.typed().gc_collectible()
} }
fn invalidate_serialization(&mut self); fn invalidate_serialization(&mut self);
......
...@@ -16,8 +16,6 @@ use tracing::span::Id; ...@@ -16,8 +16,6 @@ use tracing::span::Id;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, parallel}; use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, parallel};
#[cfg(debug_assertions)]
use crate::backend::storage_schema::GcRootHolders;
use crate::{ use crate::{
backend::storage_schema::{ backend::storage_schema::{
DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability, DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability,
...@@ -558,11 +556,11 @@ impl Storage { ...@@ -558,11 +556,11 @@ impl Storage {
} }
/// Scans a **single** shard by index, invoking `on_candidate` for each resident task whose /// Scans a **single** shard by index, invoking `on_candidate` for each resident task whose
/// storage passes [`TaskStorage::gc_maybe_collectible`]. /// storage passes [`TaskStorage::gc_collectible`].
pub fn gc_scan_shard(&self, index: usize, mut on_candidate: impl FnMut(TaskId)) { pub fn gc_scan_shard(&self, index: usize, mut on_candidate: impl FnMut(TaskId)) {
let shard = self.map.shards()[index].read(); let shard = self.map.shards()[index].read();
for (task_id, task) in shard.iter() { for (task_id, task) in shard.iter() {
if task.gc_maybe_collectible() { if task.gc_collectible() {
on_candidate(*task_id); on_candidate(*task_id);
} }
} }
...@@ -570,69 +568,18 @@ impl Storage { ...@@ -570,69 +568,18 @@ impl Storage {
/// Return the set of all known live roots. /// Return the set of all known live roots.
pub fn gc_scan_roots(&self) -> impl Iterator<Item = TaskId> { pub fn gc_scan_roots(&self) -> impl Iterator<Item = TaskId> {
// Roots that failed the "held by a transient pin" expectation, with the referencing tasks
// that kept them un-collectible. Collected during the scan and reported *after* it: naming
// a dependent means reading its storage, and the shard locks are held inside the closure.
#[cfg(debug_assertions)]
let unexpected = std::sync::Mutex::new(Vec::<(TaskId, GcRootHolders)>::new());
let per_shard: Vec<Vec<TaskId>> = let per_shard: Vec<Vec<TaskId>> =
parallel::map_collect(&(0..self.shard_count()).collect::<Vec<_>>(), |&index| { parallel::map_collect(&(0..self.shard_count()).collect::<Vec<_>>(), |&index| {
let mut roots = Vec::new(); let mut roots = Vec::new();
let shard = self.map.shards()[index].read(); let shard = self.map.shards()[index].read();
for (task_id, task) in shard.iter() { for (task_id, task) in shard.iter() {
if !task_id.is_transient() && task.gc_is_root() { if !task_id.is_transient() && task.gc_is_root() {
// The `is_root` criteria is conservative, in debug assert that we
// aren't marking things as roots for surprising reasons
#[cfg(debug_assertions)]
if !task.gc_is_held_by_transient_pin() {
unexpected
.lock()
.unwrap()
.push((*task_id, task.gc_root_holders()));
}
roots.push(*task_id); roots.push(*task_id);
} }
} }
roots roots
}); });
#[cfg(debug_assertions)]
{
use std::fmt::Write as _;
let unexpected = unexpected.into_inner().unwrap();
if !unexpected.is_empty() {
let mut report = String::new();
fn describe_task(storage: &Storage, task_id: TaskId) -> String {
storage
.access_mut(task_id)
.get_persistent_task_type()
// `NativeFunction`'s `Debug` is the public view of its name fields.
.map(|t| format!("{:?}", t.native_fn))
.unwrap_or_else(|| "<unknown>".to_string())
}
for (task_id, holders) in &unexpected {
let _ = writeln!(
report,
" {} ({task_id:?}) is held by:",
describe_task(self, *task_id)
);
for (kind, holder) in holders.iter() {
let _ = writeln!(
report,
" via {kind}: {} ({holder:?})",
describe_task(self, *holder)
);
}
}
panic!(
"{} GC root(s) held by a non-transient pin.\nBeing held by another kind of \
reference implies a bug in GC or the aggregation graph.\n{report}",
unexpected.len()
);
}
}
per_shard.into_iter().flatten() per_shard.into_iter().flatten()
} }
...@@ -1244,7 +1191,7 @@ mod tests { ...@@ -1244,7 +1191,7 @@ mod tests {
let task = storage.access_mut(task_id); let task = storage.access_mut(task_id);
assert_eq!(task.gc_transient_ref_count(), 1); assert_eq!(task.gc_transient_ref_count(), 1);
assert!(!task.gc_maybe_collectible()); assert!(!task.gc_collectible());
} }
/// A process fn that returns a non-empty SnapshotItem so the iterator doesn't /// A process fn that returns a non-empty SnapshotItem so the iterator doesn't
......
...@@ -867,7 +867,7 @@ impl TaskStorage { ...@@ -867,7 +867,7 @@ impl TaskStorage {
new_value new_value
} }
/// Whether a GC pass may collect this task: nothing references it, via parents, transient /// Whether a GC pass can collect this task: nothing references it, via parents, transient
/// pins, or aggregation edges. /// pins, or aggregation edges.
/// ///
/// Only reads `Meta`, so the shard scan and the under-guard recheck get the same answer -- it /// Only reads `Meta`, so the shard scan and the under-guard recheck get the same answer -- it
...@@ -888,95 +888,67 @@ impl TaskStorage { ...@@ -888,95 +888,67 @@ impl TaskStorage {
/// Removing the cell sets exposed a race in the GC cascade -- rebalancing running while other /// Removing the cell sets exposed a race in the GC cascade -- rebalancing running while other
/// workers were still collecting -- which `gc_collect` now avoids by deferring all rebalance /// workers were still collecting -- which `gc_collect` now avoids by deferring all rebalance
/// work until the parallel phase is quiescent. /// work until the parallel phase is quiescent.
pub fn gc_maybe_collectible(&self) -> bool { pub fn gc_collectible(&self) -> bool {
self.gc_unreferenced(ReferenceScope::All)
}
/// Whether nothing in `scope` refers to this task.
fn gc_unreferenced(&self, scope: ReferenceScope) -> bool {
// None of the predicates below are correct without this. // None of the predicates below are correct without this.
self.flags.is_restored(TaskDataCategory::Meta) if !self.flags.is_restored(TaskDataCategory::Meta)
// Already collected this session (soft-deleted, awaiting tombstone + hard-delete): // Already collected this session (soft-deleted, awaiting tombstone + hard-delete):
// don't re-select it, or a second pass would collect it again while it is still // don't re-select it, or a second pass would collect it again while it is still
// resident. // resident.
&& !self.flags.deleted() || self.flags.deleted()
&& self.gc_parent_count() == 0 || self.gc_parent_count() != 0
&& self.gc_transient_ref_count() == 0 {
&& self.get_activeness().is_none() return false;
&& self.get_in_progress().is_none()
// It is rare for an upper to be present when the ref counts are 0, but it
// happens transiently during a concurrent GC pass as uppers move around in the cascade.
&& self.upper().is_empty()
// It would be rare for a collectibles dependent to be the only thing holding a task,
// but the invalidation that disconnected the task may not have bubbled all the way up
// yet.
&& self
.collectibles_dependents()
.is_none_or(|d| d.is_empty())
}
/// Whether this task is a GC **root**: parent-less, but pinned for some reason
///
/// NOTE: this is a conservative classification. The typical reason is that there is a
/// [`TaskStorage::transient_ref`] live, but this will return true if there is merely an
/// `upper`.
pub fn gc_is_root(&self) -> bool {
self.flags.is_restored(TaskDataCategory::Meta)
&& !self.flags.deleted()
&& self.gc_parent_count() == 0
&& !self.gc_maybe_collectible()
}
/// Whether this task is held by a pin that eviction cannot drop, which is what a task
/// classified by [`TaskStorage::gc_is_root`] is expected to be held by.
#[cfg(debug_assertions)]
pub fn gc_is_held_by_transient_pin(&self) -> bool {
self.gc_transient_ref_count() > 0
|| self.get_in_progress().is_some()
|| self.get_activeness().is_some()
}
/// The concrete references keeping this task un-collectible, as `(edge kind, holder task)`
/// pairs.
#[cfg(debug_assertions)]
pub fn gc_root_holders(&self) -> GcRootHolders {
let mut holders = GcRootHolders::default();
for (&upper, _) in self.upper().iter() {
holders.push("upper", upper);
}
if let Some(deps) = self.collectibles_dependents() {
for &(_, task) in deps.iter() {
holders.push("collectibles_dependent", task);
}
}
for &task in self.output_dependent().iter() {
holders.push("output_dependent", task);
} }
if let Some(deps) = self.cell_dependents() { match scope {
// In a `cell_dependents` entry `CellRef.task` is the DEPENDENT's id, not this task's. ReferenceScope::All => {
for cell_ref in deps.iter() { // It is rare for an upper to be present when the ref counts are 0, but it happens
holders.push("cell_dependent", cell_ref.task); // transiently during a concurrent GC pass as uppers move around in the cascade.
self.upper().is_empty()
// It would be rare for a collectibles dependent to be the only thing holding a
// task, but the invalidation that disconnected the task may not have bubbled
// all the way up yet.
&& self.collectibles_dependents().is_none_or(|d| d.is_empty())
&& self.gc_transient_ref_count() == 0
&& self.get_in_progress().is_none()
&& self.get_activeness().is_none()
} }
} // Same two edge sets, minus the entries that die with the session. The pins above are
if let Some(deps) = self.cell_dependents_hashed() { // skipped entirely: they are `category = "transient"` and never reach disk.
for (cell_ref, _) in deps.iter() { ReferenceScope::Persistent => {
holders.push("cell_dependent_hashed", cell_ref.task); self.upper().iter().all(|(u, _)| u.is_transient())
&& self
.collectibles_dependents()
.is_none_or(|d| d.iter().all(|(_, t)| t.is_transient()))
} }
} }
holders
} }
}
/// `(edge kind, holder task)` pairs explaining why a task is not collectible. /// Whether this task is a GC **root**: nothing *persistent* refers to it, so only this session
/// See [`TaskStorage::gc_root_holders`]. /// is keeping it alive -- a `transient_ref` pin, an in-progress execution, activeness, or an
#[cfg(debug_assertions)] /// `upper` / collectibles edge from a transient task.
#[derive(Default, Debug)] pub fn gc_is_root(&self) -> bool {
pub struct GcRootHolders(Vec<(&'static str, TaskId)>); self.gc_unreferenced(ReferenceScope::Persistent) && !self.gc_collectible()
#[cfg(debug_assertions)]
impl GcRootHolders {
fn push(&mut self, kind: &'static str, task: TaskId) {
self.0.push((kind, task));
} }
}
pub fn iter(&self) -> impl Iterator<Item = &(&'static str, TaskId)> { /// Which references [`TaskStorage::gc_unreferenced`] counts.
self.0.iter() ///
} /// A task can be held by references that outlive the session and by references that do not -- the
/// transient entries of `upper` / `collectibles_dependents`, and the `transient_ref_count`,
/// `in_progress` and `activeness` pins. The two GC predicates care about different subsets:
/// collectibility about everything currently holding the task, rootness about only what would
/// survive a restart.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum ReferenceScope {
/// Every referrer, transient ones included.
All,
/// Only referrers that outlive the session.
Persistent,
} }
/// Counts for aggregation tree and collectibles fields. /// Counts for aggregation tree and collectibles fields.
......
...@@ -87,7 +87,7 @@ async fn select_diamond( ...@@ -87,7 +87,7 @@ async fn select_diamond(
/// The **aggregation-graph rebalance** in GC: when the `reader` subtree is disconnected cleanly and /// The **aggregation-graph rebalance** in GC: when the `reader` subtree is disconnected cleanly and
/// collected, GC must remove `reader` from each `sd_leaf`'s `upper` set so the leaves — now /// collected, GC must remove `reader` from each `sd_leaf`'s `upper` set so the leaves — now
/// parentless *and* upper-less — cascade-collect in the same pass. Without the rebalance a leaf /// parentless *and* upper-less — cascade-collect in the same pass. Without the rebalance a leaf
/// keeps a dangling `upper` edge to the deleted `reader`, fails `gc_maybe_collectible`, and leaks /// keeps a dangling `upper` edge to the deleted `reader`, fails `gc_collectible`, and leaks
/// until eviction hides it. /// until eviction hides it.
#[tokio::test(flavor = "multi_thread", worker_threads = 8)] #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn gc_rebalances_aggregation_and_cascades_in_one_pass() { async fn gc_rebalances_aggregation_and_cascades_in_one_pass() {
...@@ -132,10 +132,10 @@ async fn gc_rebalances_aggregation_and_cascades_in_one_pass() { ...@@ -132,10 +132,10 @@ async fn gc_rebalances_aggregation_and_cascades_in_one_pass() {
"resident count must drop by exactly the collected subtree" "resident count must drop by exactly the collected subtree"
); );
// Only the three top-level `(operation, root)` tasks may be tracked as roots. A leaf that // Only the three top-level `(operation, root)` tasks may be tracked as roots -- they are the
// reached the post-drain scan still holding a dangling `upper` edge to the deleted `reader` // ones held by a transient pin. A leaf still holding a dangling `upper` edge to the deleted
// would land in the map as `MostRecent`, which never ages out. If this fires it is a finding // `reader` is not a root (it fails `gc_unreferenced`), so it silently leaks rather than being
// about the aggregation graph, not a reason to narrow `gc_is_root`. // tracked; the resident-count assertions above are what catch that.
let roots = tt2.backend().persisted_gc_roots_for_testing(); let roots = tt2.backend().persisted_gc_roots_for_testing();
assert_eq!(roots.len(), 3, "unexpected roots tracked: {roots:?}"); assert_eq!(roots.len(), 3, "unexpected roots tracked: {roots:?}");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment