Unverified Commit cb95ca37 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Fix root predicates (#98942)

Fix the definition of GC roots to account for all transient references

There was a categorical mistake in how `uppers` and
`collectible_dependents` were interpreted. Both of these refer to a task
above us and this is it is a reference but the reference might be
comming from transient or persistent tasks. This is what kept confusing
the 'root' definition

So the new definition is simply

* You are collectible if there are no references (persistent or
transient)
* You are a root if there are only transient references, which includes
transient uppers and collectible_dependents


The old definition expected roots to have no uppers at all, which isn't
correct. A `run_once` task becomes an aggregation root and so the nested
persistent tasks reference it as an upper. Those tasks should become gc
roots since they are on the transient <--> persistent boundary and the
only thing retaining them is references from transient tasks.
parent c947937b
......@@ -1138,7 +1138,7 @@ impl TurboTasksBackend {
parent_span: Option<tracing::Id>,
reason: SnapshotReason,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
) -> Result<(Instant, bool, Option<(GcStats, GcPassResult)>), anyhow::Error> {
) -> Result<(Instant, bool, Option<(GcStats, GcPassResult)>)> {
let snapshot_span =
tracing::trace_span!(parent: parent_span.clone(), "snapshot", reason = reason.as_str())
.entered();
......@@ -1158,7 +1158,6 @@ impl TurboTasksBackend {
let mut snapshot_phase = self.snapshot_coord.begin_snapshot();
let (gc_elapsed, gc_roots_to_persist, gc_outcome) = if self.gc_enabled {
let gc_span = tracing::info_span!(
parent: parent_span.clone(),
"gc",
stats = tracing::field::Empty,
interrupted = tracing::field::Empty
......@@ -1415,7 +1414,7 @@ impl TurboTasksBackend {
};
} else {
debug_assert!(
!inner.gc_maybe_collectible(),
!inner.gc_collectible(),
"tasks scheduled for persistent must not be collectible, this implies a \
missed task during GC"
);
......
......@@ -1585,7 +1585,7 @@ pub trait TaskGuard: Debug + TaskStorageAccessors {
/// Whether a GC pass may collect this task: nothing references it.
fn is_gc_collectible(&self) -> bool {
self.check_access(SpecificTaskDataCategory::Meta);
self.typed().gc_maybe_collectible()
self.typed().gc_collectible()
}
fn invalidate_serialization(&mut self);
......
......@@ -16,8 +16,6 @@ use tracing::span::Id;
use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, parallel};
#[cfg(debug_assertions)]
use crate::backend::storage_schema::GcRootHolders;
use crate::{
backend::storage_schema::{
DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability,
......@@ -558,11 +556,11 @@ impl Storage {
}
/// Scans a **single** shard by index, invoking `on_candidate` for each resident task whose
/// storage passes [`TaskStorage::gc_maybe_collectible`].
/// storage passes [`TaskStorage::gc_collectible`].
pub fn gc_scan_shard(&self, index: usize, mut on_candidate: impl FnMut(TaskId)) {
let shard = self.map.shards()[index].read();
for (task_id, task) in shard.iter() {
if task.gc_maybe_collectible() {
if task.gc_collectible() {
on_candidate(*task_id);
}
}
......@@ -570,69 +568,18 @@ impl Storage {
/// Return the set of all known live roots.
pub fn gc_scan_roots(&self) -> impl Iterator<Item = TaskId> {
// Roots that failed the "held by a transient pin" expectation, with the referencing tasks
// that kept them un-collectible. Collected during the scan and reported *after* it: naming
// a dependent means reading its storage, and the shard locks are held inside the closure.
#[cfg(debug_assertions)]
let unexpected = std::sync::Mutex::new(Vec::<(TaskId, GcRootHolders)>::new());
let per_shard: Vec<Vec<TaskId>> =
parallel::map_collect(&(0..self.shard_count()).collect::<Vec<_>>(), |&index| {
let mut roots = Vec::new();
let shard = self.map.shards()[index].read();
for (task_id, task) in shard.iter() {
if !task_id.is_transient() && task.gc_is_root() {
// The `is_root` criteria is conservative, in debug assert that we
// aren't marking things as roots for surprising reasons
#[cfg(debug_assertions)]
if !task.gc_is_held_by_transient_pin() {
unexpected
.lock()
.unwrap()
.push((*task_id, task.gc_root_holders()));
}
roots.push(*task_id);
}
}
roots
});
#[cfg(debug_assertions)]
{
use std::fmt::Write as _;
let unexpected = unexpected.into_inner().unwrap();
if !unexpected.is_empty() {
let mut report = String::new();
fn describe_task(storage: &Storage, task_id: TaskId) -> String {
storage
.access_mut(task_id)
.get_persistent_task_type()
// `NativeFunction`'s `Debug` is the public view of its name fields.
.map(|t| format!("{:?}", t.native_fn))
.unwrap_or_else(|| "<unknown>".to_string())
}
for (task_id, holders) in &unexpected {
let _ = writeln!(
report,
" {} ({task_id:?}) is held by:",
describe_task(self, *task_id)
);
for (kind, holder) in holders.iter() {
let _ = writeln!(
report,
" via {kind}: {} ({holder:?})",
describe_task(self, *holder)
);
}
}
panic!(
"{} GC root(s) held by a non-transient pin.\nBeing held by another kind of \
reference implies a bug in GC or the aggregation graph.\n{report}",
unexpected.len()
);
}
}
per_shard.into_iter().flatten()
}
......@@ -1244,7 +1191,7 @@ mod tests {
let task = storage.access_mut(task_id);
assert_eq!(task.gc_transient_ref_count(), 1);
assert!(!task.gc_maybe_collectible());
assert!(!task.gc_collectible());
}
/// A process fn that returns a non-empty SnapshotItem so the iterator doesn't
......
......@@ -867,7 +867,7 @@ impl TaskStorage {
new_value
}
/// Whether a GC pass may collect this task: nothing references it, via parents, transient
/// Whether a GC pass can collect this task: nothing references it, via parents, transient
/// pins, or aggregation edges.
///
/// Only reads `Meta`, so the shard scan and the under-guard recheck get the same answer -- it
......@@ -888,95 +888,67 @@ impl TaskStorage {
/// Removing the cell sets exposed a race in the GC cascade -- rebalancing running while other
/// workers were still collecting -- which `gc_collect` now avoids by deferring all rebalance
/// work until the parallel phase is quiescent.
pub fn gc_maybe_collectible(&self) -> bool {
pub fn gc_collectible(&self) -> bool {
self.gc_unreferenced(ReferenceScope::All)
}
/// Whether nothing in `scope` refers to this task.
fn gc_unreferenced(&self, scope: ReferenceScope) -> bool {
// None of the predicates below are correct without this.
self.flags.is_restored(TaskDataCategory::Meta)
if !self.flags.is_restored(TaskDataCategory::Meta)
// Already collected this session (soft-deleted, awaiting tombstone + hard-delete):
// don't re-select it, or a second pass would collect it again while it is still
// resident.
&& !self.flags.deleted()
&& self.gc_parent_count() == 0
&& self.gc_transient_ref_count() == 0
&& self.get_activeness().is_none()
&& self.get_in_progress().is_none()
// It is rare for an upper to be present when the ref counts are 0, but it
// happens transiently during a concurrent GC pass as uppers move around in the cascade.
&& self.upper().is_empty()
// It would be rare for a collectibles dependent to be the only thing holding a task,
// but the invalidation that disconnected the task may not have bubbled all the way up
// yet.
&& self
.collectibles_dependents()
.is_none_or(|d| d.is_empty())
}
/// Whether this task is a GC **root**: parent-less, but pinned for some reason
///
/// NOTE: this is a conservative classification. The typical reason is that there is a
/// [`TaskStorage::transient_ref`] live, but this will return true if there is merely an
/// `upper`.
pub fn gc_is_root(&self) -> bool {
self.flags.is_restored(TaskDataCategory::Meta)
&& !self.flags.deleted()
&& self.gc_parent_count() == 0
&& !self.gc_maybe_collectible()
}
/// Whether this task is held by a pin that eviction cannot drop, which is what a task
/// classified by [`TaskStorage::gc_is_root`] is expected to be held by.
#[cfg(debug_assertions)]
pub fn gc_is_held_by_transient_pin(&self) -> bool {
self.gc_transient_ref_count() > 0
|| self.get_in_progress().is_some()
|| self.get_activeness().is_some()
}
/// The concrete references keeping this task un-collectible, as `(edge kind, holder task)`
/// pairs.
#[cfg(debug_assertions)]
pub fn gc_root_holders(&self) -> GcRootHolders {
let mut holders = GcRootHolders::default();
for (&upper, _) in self.upper().iter() {
holders.push("upper", upper);
}
if let Some(deps) = self.collectibles_dependents() {
for &(_, task) in deps.iter() {
holders.push("collectibles_dependent", task);
}
}
for &task in self.output_dependent().iter() {
holders.push("output_dependent", task);
|| self.flags.deleted()
|| self.gc_parent_count() != 0
{
return false;
}
if let Some(deps) = self.cell_dependents() {
// In a `cell_dependents` entry `CellRef.task` is the DEPENDENT's id, not this task's.
for cell_ref in deps.iter() {
holders.push("cell_dependent", cell_ref.task);
match scope {
ReferenceScope::All => {
// It is rare for an upper to be present when the ref counts are 0, but it happens
// transiently during a concurrent GC pass as uppers move around in the cascade.
self.upper().is_empty()
// It would be rare for a collectibles dependent to be the only thing holding a
// task, but the invalidation that disconnected the task may not have bubbled
// all the way up yet.
&& self.collectibles_dependents().is_none_or(|d| d.is_empty())
&& self.gc_transient_ref_count() == 0
&& self.get_in_progress().is_none()
&& self.get_activeness().is_none()
}
}
if let Some(deps) = self.cell_dependents_hashed() {
for (cell_ref, _) in deps.iter() {
holders.push("cell_dependent_hashed", cell_ref.task);
// Same two edge sets, minus the entries that die with the session. The pins above are
// skipped entirely: they are `category = "transient"` and never reach disk.
ReferenceScope::Persistent => {
self.upper().iter().all(|(u, _)| u.is_transient())
&& self
.collectibles_dependents()
.is_none_or(|d| d.iter().all(|(_, t)| t.is_transient()))
}
}
holders
}
}
/// `(edge kind, holder task)` pairs explaining why a task is not collectible.
/// See [`TaskStorage::gc_root_holders`].
#[cfg(debug_assertions)]
#[derive(Default, Debug)]
pub struct GcRootHolders(Vec<(&'static str, TaskId)>);
#[cfg(debug_assertions)]
impl GcRootHolders {
fn push(&mut self, kind: &'static str, task: TaskId) {
self.0.push((kind, task));
/// Whether this task is a GC **root**: nothing *persistent* refers to it, so only this session
/// is keeping it alive -- a `transient_ref` pin, an in-progress execution, activeness, or an
/// `upper` / collectibles edge from a transient task.
pub fn gc_is_root(&self) -> bool {
self.gc_unreferenced(ReferenceScope::Persistent) && !self.gc_collectible()
}
}
pub fn iter(&self) -> impl Iterator<Item = &(&'static str, TaskId)> {
self.0.iter()
}
/// Which references [`TaskStorage::gc_unreferenced`] counts.
///
/// A task can be held by references that outlive the session and by references that do not -- the
/// transient entries of `upper` / `collectibles_dependents`, and the `transient_ref_count`,
/// `in_progress` and `activeness` pins. The two GC predicates care about different subsets:
/// collectibility about everything currently holding the task, rootness about only what would
/// survive a restart.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum ReferenceScope {
/// Every referrer, transient ones included.
All,
/// Only referrers that outlive the session.
Persistent,
}
/// Counts for aggregation tree and collectibles fields.
......
......@@ -87,7 +87,7 @@ async fn select_diamond(
/// The **aggregation-graph rebalance** in GC: when the `reader` subtree is disconnected cleanly and
/// collected, GC must remove `reader` from each `sd_leaf`'s `upper` set so the leaves — now
/// parentless *and* upper-less — cascade-collect in the same pass. Without the rebalance a leaf
/// keeps a dangling `upper` edge to the deleted `reader`, fails `gc_maybe_collectible`, and leaks
/// keeps a dangling `upper` edge to the deleted `reader`, fails `gc_collectible`, and leaks
/// until eviction hides it.
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn gc_rebalances_aggregation_and_cascades_in_one_pass() {
......@@ -132,10 +132,10 @@ async fn gc_rebalances_aggregation_and_cascades_in_one_pass() {
"resident count must drop by exactly the collected subtree"
);
// Only the three top-level `(operation, root)` tasks may be tracked as roots. A leaf that
// reached the post-drain scan still holding a dangling `upper` edge to the deleted `reader`
// would land in the map as `MostRecent`, which never ages out. If this fires it is a finding
// about the aggregation graph, not a reason to narrow `gc_is_root`.
// Only the three top-level `(operation, root)` tasks may be tracked as roots -- they are the
// ones held by a transient pin. A leaf still holding a dangling `upper` edge to the deleted
// `reader` is not a root (it fails `gc_unreferenced`), so it silently leaks rather than being
// tracked; the resident-count assertions above are what catch that.
let roots = tt2.backend().persisted_gc_roots_for_testing();
assert_eq!(roots.len(), 3, "unexpected roots tracked: {roots:?}");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment