Unverified Commit 2e973ccf authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Track the last persistence status with an enum

Use a sticky LastPersistenceStatus instead of a bool, and skip a disabled persist with Ok(None) so shutdown does not need a separate check.
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 88b16f6d
...@@ -229,6 +229,17 @@ impl SnapshotReason { ...@@ -229,6 +229,17 @@ impl SnapshotReason {
} }
} }
/// The outcome of the last persistence cycle. The backend starts at `Ok` in its constructor.
///
/// `Failed` is sticky: a failed persist (an error or a panic) leaves the task graph and the
/// database out of sync (e.g. `save_snapshot` already consumed `persisted_task_cache_log`
/// entries), so persisting stays disabled for the rest of the session.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum LastPersistenceStatus {
Ok,
Failed,
}
pub struct TurboTasksBackend { pub struct TurboTasksBackend {
options: BackendOptions, options: BackendOptions,
...@@ -247,11 +258,8 @@ pub struct TurboTasksBackend { ...@@ -247,11 +258,8 @@ pub struct TurboTasksBackend {
/// enforces that contract for our callers (background loop and /// enforces that contract for our callers (background loop and
/// `stop_and_wait`). /// `stop_and_wait`).
/// ///
/// Holds whether persisting has failed (returned an error or panicked). A failed persist /// Also holds the [`LastPersistenceStatus`], which disables persisting after a failure.
/// leaves the task graph and the database out of sync (e.g. `save_snapshot` already consumed snapshot_in_progress: Mutex<LastPersistenceStatus>,
/// `persisted_task_cache_log` entries), so persisting stays disabled for the rest of the
/// session.
snapshot_in_progress: Mutex<bool>,
/// Experimental feature to enable dead tasks to be deleted from storage and ram. /// Experimental feature to enable dead tasks to be deleted from storage and ram.
gc_enabled: bool, gc_enabled: bool,
...@@ -359,7 +367,7 @@ impl TurboTasksBackend { ...@@ -359,7 +367,7 @@ impl TurboTasksBackend {
small_preallocation, small_preallocation,
}), }),
snapshot_coord: SnapshotCoordinator::new(), snapshot_coord: SnapshotCoordinator::new(),
snapshot_in_progress: Mutex::new(false), snapshot_in_progress: Mutex::new(LastPersistenceStatus::Ok),
stopping: RwLock::new(false), stopping: RwLock::new(false),
stopping_event: Event::new(|| || "TurboTasksBackend::stopping_event".to_string()), stopping_event: Event::new(|| || "TurboTasksBackend::stopping_event".to_string()),
idle_start_event: Event::new(|| || "TurboTasksBackend::idle_start_event".to_string()), idle_start_event: Event::new(|| || "TurboTasksBackend::idle_start_event".to_string()),
...@@ -434,8 +442,9 @@ impl TurboTasksBackend { ...@@ -434,8 +442,9 @@ impl TurboTasksBackend {
let snapshot_result = self.snapshot_and_persist(None, SnapshotReason::Test, turbo_tasks); let snapshot_result = self.snapshot_and_persist(None, SnapshotReason::Test, turbo_tasks);
let (had_new_data, gc_outcome) = match snapshot_result { let (had_new_data, gc_outcome) = match snapshot_result {
Ok(Some((_, new_data, gc_outcome))) => (new_data, gc_outcome), Ok(Some((_, new_data, gc_outcome))) => (new_data, gc_outcome),
Ok(None) => unreachable!("test snapshots wait for operations to settle"), // Test snapshots wait for operations to settle, so `None` means persisting is
Err(_) => { // disabled after an earlier failure.
Ok(None) | Err(_) => {
// Snapshot/persist failed — skip eviction since the data may not // Snapshot/persist failed — skip eviction since the data may not
// be on disk yet. Evicting now could lose in-memory state that // be on disk yet. Evicting now could lose in-memory state that
// can't be restored. // can't be restored.
...@@ -1143,15 +1152,16 @@ impl TurboTasksBackend { ...@@ -1143,15 +1152,16 @@ impl TurboTasksBackend {
/// Runs a persistence cycle /// Runs a persistence cycle
/// ///
/// Returns `None` if the cycle was skipped because operations were active and the reason /// Returns `None` if the cycle was skipped: persisting is disabled after an earlier failure, or
/// doesn't [wait for them](SnapshotReason::waits_for_operations). Otherwise returns /// operations were active and the reason doesn't
/// [wait for them](SnapshotReason::waits_for_operations). Otherwise returns
/// `(snapshot_start, had_new_data, gc_outcome)`. `gc_outcome` is `None` when GC is /// `(snapshot_start, had_new_data, gc_outcome)`. `gc_outcome` is `None` when GC is
/// disabled; it is returned rather than stashed on `self` so a test can inspect the pass it /// disabled; it is returned rather than stashed on `self` so a test can inspect the pass it
/// just triggered without the backend carrying test-only state. Production reads the same /// just triggered without the backend carrying test-only state. Production reads the same
/// numbers off the `gc` span. /// numbers off the `gc` span.
/// ///
/// A failure (an error or a panic) disables persisting for the rest of the session: later /// A failure (an error or a panic) disables persisting for the rest of the session: later
/// calls return an error without starting a snapshot or GC. /// calls return `None` without starting a snapshot or GC.
#[allow(clippy::type_complexity, reason = "only used for tests")] #[allow(clippy::type_complexity, reason = "only used for tests")]
fn snapshot_and_persist( fn snapshot_and_persist(
&self, &self,
...@@ -1160,25 +1170,20 @@ impl TurboTasksBackend { ...@@ -1160,25 +1170,20 @@ impl TurboTasksBackend {
turbo_tasks: &TurboTasks<TurboTasksBackend>, turbo_tasks: &TurboTasks<TurboTasksBackend>,
) -> Result<Option<(Instant, bool, Option<(GcStats, GcPassResult)>)>> { ) -> Result<Option<(Instant, bool, Option<(GcStats, GcPassResult)>)>> {
// Serialize snapshots and GC for the entire persistence cycle. // Serialize snapshots and GC for the entire persistence cycle.
let mut persist_failed = self.snapshot_in_progress.lock(); let mut status = self.snapshot_in_progress.lock();
if *persist_failed { if *status == LastPersistenceStatus::Failed {
bail!("Persisting is disabled after an earlier failure"); return Ok(None);
} }
// Set the flag up front and only clear it on success: encoding failures panic, and a // Mark the cycle failed up front and only reset it on success: encoding failures panic,
// panic unwinds past any post-call check (`parking_lot` mutexes don't poison). // and a panic unwinds past any post-call check (`parking_lot` mutexes don't poison).
*persist_failed = true; *status = LastPersistenceStatus::Failed;
let result = self.snapshot_and_persist_locked(parent_span, reason, turbo_tasks); let result = self.snapshot_and_persist_locked(parent_span, reason, turbo_tasks);
if result.is_ok() { if result.is_ok() {
*persist_failed = false; *status = LastPersistenceStatus::Ok;
} }
result result
} }
/// Whether a persist failed earlier in this session, which disables persisting.
fn persisting_failed(&self) -> bool {
*self.snapshot_in_progress.lock()
}
/// [`Self::snapshot_and_persist`], called while holding `snapshot_in_progress`. /// [`Self::snapshot_and_persist`], called while holding `snapshot_in_progress`.
#[allow(clippy::type_complexity, reason = "only used for tests")] #[allow(clippy::type_complexity, reason = "only used for tests")]
fn snapshot_and_persist_locked( fn snapshot_and_persist_locked(
...@@ -1616,7 +1621,6 @@ impl TurboTasksBackend { ...@@ -1616,7 +1621,6 @@ impl TurboTasksBackend {
// eagerly drop the task cache before persisting // eagerly drop the task cache before persisting
self.storage.drop_task_cache(); self.storage.drop_task_cache();
if self.should_persist() if self.should_persist()
&& !self.persisting_failed()
&& let Err(err) = && let Err(err) =
self.snapshot_and_persist(Span::current().into(), SnapshotReason::Stop, turbo_tasks) self.snapshot_and_persist(Span::current().into(), SnapshotReason::Stop, turbo_tasks)
{ {
...@@ -3910,7 +3914,7 @@ mod persist_failure_tests { ...@@ -3910,7 +3914,7 @@ mod persist_failure_tests {
use crate::{ use crate::{
backend::{ backend::{
BackendOptions, SnapshotReason, StorageMode, TurboTasksBackend, BackendOptions, LastPersistenceStatus, SnapshotReason, StorageMode, TurboTasksBackend,
storage::SpecificTaskDataCategory, storage::SpecificTaskDataCategory,
}, },
noop_backing_storage, noop_backing_storage,
...@@ -3937,21 +3941,19 @@ mod persist_failure_tests { ...@@ -3937,21 +3941,19 @@ mod persist_failure_tests {
task_id task_id
}; };
let status = || *backend.snapshot_in_progress.lock();
modify(1); modify(1);
assert!(!backend.persisting_failed()); assert_eq!(status(), LastPersistenceStatus::Ok);
let _ = backend.snapshot_and_evict_for_testing(&tt); let _ = backend.snapshot_and_evict_for_testing(&tt);
assert!(backend.persisting_failed()); assert_eq!(status(), LastPersistenceStatus::Failed);
let task_id = modify(2); let task_id = modify(2);
let err = backend let result = backend
.snapshot_and_persist(None, SnapshotReason::Test, &tt) .snapshot_and_persist(None, SnapshotReason::Test, &tt)
.err() .expect("a disabled persist is skipped, not an error");
.expect("persisting must stay disabled"); assert!(result.is_none(), "persisting must stay disabled");
assert!( assert_eq!(status(), LastPersistenceStatus::Failed);
err.to_string()
.contains("Persisting is disabled after an earlier failure"),
"{err:?}"
);
let task = backend.storage.access_mut(task_id); let task = backend.storage.access_mut(task_id);
assert!( assert!(
task.flags.data_modified(), task.flags.data_modified(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment