Unverified Commit 82dd6d6d authored by Jimmy Miller's avatar Jimmy Miller Committed by GitHub

Move suspension into operation guard (#99053)

So in some testing I found a rare error where a child task could take
the suspend point, causing it to decrement the counter even though it
hadn't incremented it. Move this operation to the guard makes this not
impossible (you can still just grab the coordinator) but at least in
this code path makes it unlikely you do this. It fixes the bug I found.
Discussed this with Luke. We agreed trying to recreate the case in a
unit test would probably be a bit overkill.
parent 0c129d01
......@@ -388,17 +388,11 @@ impl TurboTasksBackend {
))
}
fn operation_suspend_point(&self, suspend: impl FnOnce() -> AnyOperation) {
if self.should_persist() {
self.snapshot_coord.suspend_point(suspend);
}
}
pub(crate) fn start_operation(&self) -> OperationGuard<'_, AnyOperation> {
pub(crate) fn start_operation(&self) -> Option<OperationGuard<'_, AnyOperation>> {
if !self.should_persist() {
return OperationGuard::noop();
return None;
}
self.snapshot_coord.begin_operation()
Some(self.snapshot_coord.begin_operation())
}
fn should_persist(&self) -> bool {
......
......@@ -222,7 +222,7 @@ impl TaskLockCounter {
enum ExecutePhase<'e> {
Normal {
_guard: OperationGuard<'e, AnyOperation>,
guard: Option<OperationGuard<'e, AnyOperation>>,
},
Child,
Gc(&'e dyn Fn(TaskId)),
......@@ -247,7 +247,7 @@ impl<'e> ExecuteContextImpl<'e> {
backend,
turbo_tasks,
phase: ExecutePhase::Normal {
_guard: backend.start_operation(),
guard: backend.start_operation(),
},
_shutdown_guard: None,
task_lock_counter: TaskLockCounter::new(),
......@@ -266,7 +266,7 @@ impl<'e> ExecuteContextImpl<'e> {
backend,
turbo_tasks,
phase: ExecutePhase::Normal {
_guard: backend.start_operation(),
guard: backend.start_operation(),
},
_shutdown_guard: Some(shutdown_guard),
task_lock_counter: TaskLockCounter::new(),
......@@ -1274,11 +1274,10 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
}
fn operation_suspend_point<T: Clone + Into<AnyOperation>>(&mut self, op: &T) {
// suspend guards become no-ops under GC
if matches!(self.phase, ExecutePhase::Gc(_)) {
let ExecutePhase::Normal { guard: Some(guard) } = &mut self.phase else {
return;
}
self.backend.operation_suspend_point(|| op.clone().into());
};
guard.suspend_point(|| op.clone().into());
}
fn note_maybe_collectible(&mut self, task: &impl TaskGuard) {
......
......@@ -7,8 +7,7 @@
//! [`begin_operation`](SnapshotCoordinator::begin_operation) is a single uncontended atomic
//! increment.
//! - When a phase is requested, new operations block until it finishes, and operations already in
//! flight either complete or call [`suspend_point`](SnapshotCoordinator::suspend_point) to
//! suspend.
//! flight either complete or call [`suspend_point`](OperationGuard::suspend_point) to suspend.
//! - The phase holder waits for every in-flight operation to drain or suspend, does its work, then
//! wakes everyone.
//!
......@@ -36,7 +35,7 @@ const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1);
struct State<O> {
/// `true` between `begin_snapshot` and `SnapshotPhase::drop`.
snapshot_requested: bool,
/// Operations that called [`SnapshotCoordinator::suspend_point`] and have
/// Operations that called [`OperationGuard::suspend_point`] and have
/// not yet resumed. Returned to the snapshotter via
/// [`SnapshotPhase::suspended_operations`] so it can persist them in the
/// uncompleted-operations log.
......@@ -84,7 +83,7 @@ impl<O> SnapshotCoordinator<O> {
/// Cheap check used by hot paths. Returns `true` while a snapshot is in
/// flight (or being requested). May return `false` racily if a snapshot
/// is just about to start; the actual coordination happens in
/// [`suspend_point`](Self::suspend_point) and [`begin_operation`](Self::begin_operation).
/// [`OperationGuard::suspend_point`] and [`begin_operation`](Self::begin_operation).
pub fn snapshot_pending(&self) -> bool {
// Acquire so that observing the bit synchronizes with anything the
// snapshotter wrote before setting it.
......@@ -99,7 +98,7 @@ impl<O> SnapshotCoordinator<O> {
// Fast path: no snapshot in flight, single atomic increment.
let prev = self.in_progress_operations.fetch_add(1, Ordering::AcqRel);
if (prev & SNAPSHOT_REQUESTED_BIT) == 0 {
return OperationGuard { coord: Some(self) };
return OperationGuard { coord: self };
}
#[cold]
fn wait_for_snapshot_to_complete<O>(this: &SnapshotCoordinator<O>) {
......@@ -130,53 +129,7 @@ impl<O> SnapshotCoordinator<O> {
// Slow path: a snapshot is in flight (or just requested). Back out
// the increment, wait for the snapshot to complete, then re-increment.
wait_for_snapshot_to_complete(self);
OperationGuard { coord: Some(self) }
}
/// Suspend the current operation if a snapshot is requested. Otherwise a
/// no-op. The closure is called only when actually suspending — it must
/// produce a handle to this operation so the snapshotter can persist it
/// for replay on the next startup.
pub fn suspend_point(&self, suspend: impl FnOnce() -> O) {
if !self.snapshot_pending() {
return;
}
#[cold]
fn suspend_point_cold<O>(this: &SnapshotCoordinator<O>, suspend: impl FnOnce() -> O) {
let mut state = this.state.lock();
if !state.snapshot_requested {
// Race: snapshot finished between the `snapshot_pending` check
// and acquiring the mutex. Nothing to do.
return;
}
let op = Arc::new(suspend());
state
.suspended_operations
.insert(PtrEqArc::from(op.clone()));
// Decrement the count so the snapshotter can drain.
let prev = this.in_progress_operations.fetch_sub(1, Ordering::AcqRel);
// Protocol violation if either invariant fails. Keep as a regular
// `assert!` so production builds also catch it: the alternative is
// a corrupted counter that hangs the next snapshot indefinitely.
assert!(
(prev & SNAPSHOT_REQUESTED_BIT) != 0 && (prev & !SNAPSHOT_REQUESTED_BIT) > 0,
"suspend_point called without a live operation: prev={prev:#x}"
);
if prev - 1 == SNAPSHOT_REQUESTED_BIT {
this.operations_drained.notify_all();
}
this.operations_waiting.store(true, Ordering::Relaxed);
// Wait for the snapshot to finish.
tokio::task::block_in_place(|| {
this.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested);
});
// Resume: re-increment and remove ourselves from the suspended set.
this.in_progress_operations.fetch_add(1, Ordering::AcqRel);
state.suspended_operations.remove(&PtrEqArc::from(op));
}
suspend_point_cold(self, suspend);
OperationGuard { coord: self }
}
/// Begin a snapshot. Sets the snapshot bit, blocks until all in-flight
......@@ -244,23 +197,61 @@ impl<O> SnapshotCoordinator<O> {
/// Guard returned by [`SnapshotCoordinator::begin_operation`]. Decrements the
/// in-progress count on drop and notifies the snapshotter if it is waiting.
pub struct OperationGuard<'a, O> {
coord: Option<&'a SnapshotCoordinator<O>>,
coord: &'a SnapshotCoordinator<O>,
}
impl<O> OperationGuard<'_, O> {
/// A guard that does nothing on drop. Useful for backends that don't
/// participate in the snapshot protocol (e.g. when persistence is
/// disabled).
pub fn noop() -> Self {
Self { coord: None }
/// Suspend this operation if a snapshot is requested. Otherwise a no-op. The closure is called
/// only when actually suspending and must produce a handle to this operation so the
/// snapshotter can persist it for replay on the next startup.
pub fn suspend_point(&mut self, suspend: impl FnOnce() -> O) {
let coord = self.coord;
if !coord.snapshot_pending() {
return;
}
#[cold]
fn suspend_point_cold<O>(coord: &SnapshotCoordinator<O>, suspend: impl FnOnce() -> O) {
let mut state = coord.state.lock();
if !state.snapshot_requested {
// Race: snapshot finished between the `snapshot_pending` check
// and acquiring the mutex. Nothing to do.
return;
}
let op = Arc::new(suspend());
state
.suspended_operations
.insert(PtrEqArc::from(op.clone()));
// Decrement the count so the snapshotter can drain.
let prev = coord.in_progress_operations.fetch_sub(1, Ordering::AcqRel);
// Protocol violation if either invariant fails. Keep as a regular
// `assert!` so production builds also catch it: the alternative is
// a corrupted counter that hangs the next snapshot indefinitely.
assert!(
(prev & SNAPSHOT_REQUESTED_BIT) != 0 && (prev & !SNAPSHOT_REQUESTED_BIT) > 0,
"suspend_point called without a live operation: prev={prev:#x}"
);
if prev - 1 == SNAPSHOT_REQUESTED_BIT {
coord.operations_drained.notify_all();
}
coord.operations_waiting.store(true, Ordering::Relaxed);
// Wait for the snapshot to finish.
tokio::task::block_in_place(|| {
coord
.snapshot_completed
.wait_while(&mut state, |s| s.snapshot_requested);
});
// Resume: re-increment and remove ourselves from the suspended set.
coord.in_progress_operations.fetch_add(1, Ordering::AcqRel);
state.suspended_operations.remove(&PtrEqArc::from(op));
}
suspend_point_cold(coord, suspend);
}
}
impl<O> Drop for OperationGuard<'_, O> {
fn drop(&mut self) {
let Some(coord) = self.coord else {
return;
};
let coord = self.coord;
let prev = coord.in_progress_operations.fetch_sub(1, Ordering::AcqRel);
// Underflow means a guard was dropped without a matching increment;
// promoted from debug_assert because the alternative is silently
......@@ -465,7 +456,7 @@ mod tests {
#[test]
fn suspend_point_lets_snapshot_proceed() {
let coord = Arc::new(SnapshotCoordinator::<Op>::new());
let g = coord.begin_operation();
let mut g = coord.begin_operation();
let snapshotter_done = Arc::new(AtomicUsize::new(0));
let coord_snap = coord.clone();
......@@ -489,7 +480,7 @@ mod tests {
wait_for_snapshot_pending(&coord);
// The phase is waiting for our operation to drain; suspending should let it proceed.
let recorded = Arc::new(AtomicUsize::new(0));
coord.suspend_point({
g.suspend_point({
let recorded = recorded.clone();
move || {
recorded.fetch_add(1, Ordering::Release);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment