Unverified Commit 4250c5b3 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

feat(turbopack): Invalidate the persistent cache upon panic (#80225)

This is intended as a band-aid "fix" for persistent caching where issues stick around after restarts of the development server. It builds on top of the same codepath we're using for the explicit cache clearing button in the devtools.

As part of this work, this adds a centralized registry for dynamically-registered panic hooks.

## Notes & Next Steps

- I'll disable this behavior for Vercel employees in front, so that we continue to get valuable bug reports.
- I'll follow up with some warning message in the terminal so that the subsequent restart of `next dev` will warn the user that the persistent cache was cleared and that's why build may be slow.
- We've got a some in-progress work handed to me from @wbinnssmith for better logging of anonymized (uncaught) panics to telemetry, so that should be useful in cases like this.
- Yes, this will still invalidate the cache on caught panics. I'm not 100% sure that's the behavior we want, but I think it is.

## Test Plan

Built with:

```
pnpm pack-next -p ~/front/apps/vercel-site
cd ~/front
pnpm i
pnpm turbo --filter=vercel-site dev
```

Restarted a few times and looked at the persistent cache files and saw that they keep accumulating.

Then, deleted a random `.sst` file, restarted, and observed a bunch of panics. Saw that the invalidation file was written. Restarted the server again and saw that it recovered itself by clearing the cache.
parent 9be85fad
...@@ -72,7 +72,7 @@ fn init() { ...@@ -72,7 +72,7 @@ fn init() {
use std::panic::{set_hook, take_hook}; use std::panic::{set_hook, take_hook};
use tokio::runtime::Builder; use tokio::runtime::Builder;
use turbo_tasks::handle_panic; use turbo_tasks::panic_hooks::handle_panic;
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
let prev_hook = take_hook(); let prev_hook = take_hook();
......
use core::panic; use core::panic;
use std::{ use std::{
panic::{set_hook, take_hook}, panic::{set_hook, take_hook},
sync::LazyLock, sync::{
Arc, LazyLock,
atomic::{AtomicBool, Ordering},
},
}; };
use anyhow::Result; use anyhow::Result;
use regex::Regex; use regex::Regex;
use turbo_tasks::{Vc, backend::TurboTasksExecutionError, handle_panic}; use turbo_tasks::{
Vc,
backend::TurboTasksExecutionError,
panic_hooks::{handle_panic, register_panic_hook},
};
use turbo_tasks_testing::{Registration, register, run_without_cache_check}; use turbo_tasks_testing::{Registration, register, run_without_cache_check};
static REGISTRATION: Registration = register!(); static REGISTRATION: Registration = register!();
...@@ -14,17 +21,29 @@ static REGISTRATION: Registration = register!(); ...@@ -14,17 +21,29 @@ static REGISTRATION: Registration = register!();
static FILE_PATH_REGEX: LazyLock<Regex> = static FILE_PATH_REGEX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"panics\.rs:\d+:\d+$").unwrap()); LazyLock::new(|| Regex::new(r"panics\.rs:\d+:\d+$").unwrap());
// DO NOT ADD MORE TESTS TO THIS FILE!
//
// This test depends on the process-wide global panic handler. This test must be run in its own
// process in isolation of any other tests.
#[tokio::test] #[tokio::test]
async fn test_panics_include_location() { async fn test_panic_hook() {
let prev_hook = take_hook(); let prev_hook = take_hook();
set_hook(Box::new(move |info| { set_hook(Box::new(move |info| {
handle_panic(info); handle_panic(info);
prev_hook(info); prev_hook(info);
})); }));
let hook_was_called = Arc::new(AtomicBool::new(false));
let _hook_guard = register_panic_hook({
let hook_was_called = hook_was_called.clone();
Box::new(move |_| hook_was_called.store(true, Ordering::SeqCst))
});
let result = let result =
run_without_cache_check(&REGISTRATION, async move { anyhow::Ok(*double(3).await?) }).await; run_without_cache_check(&REGISTRATION, async move { anyhow::Ok(*double(3).await?) }).await;
assert!(hook_was_called.load(Ordering::SeqCst));
let error = result.unwrap_err(); let error = result.unwrap_err();
let root_cause = error.root_cause(); let root_cause = error.root_cause();
......
...@@ -14,7 +14,7 @@ use pin_project_lite::pin_project; ...@@ -14,7 +14,7 @@ use pin_project_lite::pin_project;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_tasks_malloc::{AllocationInfo, TurboMalloc}; use turbo_tasks_malloc::{AllocationInfo, TurboMalloc};
use crate::{LAST_ERROR_LOCATION, backend::TurboTasksExecutionErrorMessage}; use crate::{backend::TurboTasksExecutionErrorMessage, panic_hooks::LAST_ERROR_LOCATION};
struct ThreadLocalData { struct ThreadLocalData {
duration: Duration, duration: Duration,
......
...@@ -28,7 +28,6 @@ ...@@ -28,7 +28,6 @@
#![feature(trivial_bounds)] #![feature(trivial_bounds)]
#![feature(min_specialization)] #![feature(min_specialization)]
#![feature(thread_local)]
#![feature(try_trait_v2)] #![feature(try_trait_v2)]
#![deny(unsafe_op_in_unsafe_fn)] #![deny(unsafe_op_in_unsafe_fn)]
#![feature(error_generic_member_access)] #![feature(error_generic_member_access)]
...@@ -63,6 +62,7 @@ mod native_function; ...@@ -63,6 +62,7 @@ mod native_function;
mod no_move_vec; mod no_move_vec;
mod once_map; mod once_map;
mod output; mod output;
pub mod panic_hooks;
pub mod persisted_graph; pub mod persisted_graph;
pub mod primitives; pub mod primitives;
mod raw_vc; mod raw_vc;
...@@ -84,7 +84,7 @@ mod value; ...@@ -84,7 +84,7 @@ mod value;
mod value_type; mod value_type;
mod vc; mod vc;
use std::{cell::RefCell, hash::BuildHasherDefault, panic}; use std::hash::BuildHasherDefault;
pub use anyhow::{Error, Result}; pub use anyhow::{Error, Result};
use auto_hash_map::AutoSet; use auto_hash_map::AutoSet;
...@@ -302,19 +302,6 @@ pub mod test_helpers { ...@@ -302,19 +302,6 @@ pub mod test_helpers {
pub use super::manager::{current_task_for_testing, with_turbo_tasks_for_testing}; pub use super::manager::{current_task_for_testing, with_turbo_tasks_for_testing};
} }
thread_local! {
/// The location of the last error that occurred in the current thread.
///
/// Used for debugging when errors are sent to telemetry
pub(crate) static LAST_ERROR_LOCATION: RefCell<Option<String>> = const { RefCell::new(None) };
}
pub fn handle_panic(info: &panic::PanicHookInfo<'_>) {
LAST_ERROR_LOCATION.with_borrow_mut(|loc| {
*loc = info.location().map(|l| l.to_string());
});
}
pub fn register() { pub fn register() {
include!(concat!(env!("OUT_DIR"), "/register.rs")); include!(concat!(env!("OUT_DIR"), "/register.rs"));
} }
//! Provides a central registry for safe runtime registration and de-registration of panic hooks.
//!
//! Registered hooks are called in an arbitrary order.
//!
//! This is used inside `turbo-tasks-backend` to invalidate the persistent cache if a panic occurs
//! anywhere inside of Turbopack. That panic hook must be dynamically registered as it contains a
//! reference to the database.
//!
//! The program using turbo-tasks must call [`std::panic::set_hook`] with [`handle_panic`] exactly
//! once for these registered panic handlers to function. Short-lived programs or code that does not
//! fully control its execution environment (like unit tests) may choose not to do this, so these
//! panic hooks are best-effort.
//!
//! It's recommended that when adding this global panic handler (or any other panic handler) that:
//! - You call it as early in the program as possible, to avoid race conditions with other threads.
//! - The new panic handler should call any existing panic handler.
//!
//! ```
//! use std::panic::{set_hook, take_hook};
//! use turbo_tasks::panic_hooks::handle_panic;
//!
//! let prev_hook = take_hook();
//! set_hook(Box::new(move |info| {
//! handle_panic(info);
//! prev_hook(info);
//! }));
//! ```
//!
//! This code is not particularly well-optimized under the assumption that panics are a rare
//! occurrence.
use std::{
cell::RefCell,
collections::HashMap,
hash::{BuildHasherDefault, DefaultHasher},
num::NonZeroU64,
panic::PanicHookInfo,
sync::{Arc, RwLock},
};
use crate::util::IdFactory;
thread_local! {
/// The location of the last error that occurred in the current thread.
///
/// Used for debugging when errors are sent to telemetry.
pub(crate) static LAST_ERROR_LOCATION: RefCell<Option<String>> = const { RefCell::new(None) };
}
static HOOK_ID_FACTORY: IdFactory<NonZeroU64> =
IdFactory::new_const(NonZeroU64::MIN, NonZeroU64::MAX);
// We could use a `DashMap` or the `slab` crate, but we anticipate that setting up and tearing down
// hooks is rare.
static PANIC_HOOKS: RwLock<HashMap<NonZeroU64, ArcPanicHook, BuildHasherDefault<DefaultHasher>>> =
RwLock::new(HashMap::with_hasher(BuildHasherDefault::new()));
pub type PanicHook = Box<dyn Fn(&PanicHookInfo<'_>) + Sync + Send + 'static>;
pub type ArcPanicHook = Arc<dyn Fn(&PanicHookInfo<'_>) + Sync + Send + 'static>;
/// This function should be registered as the global panic handler using [`std::panic::set_hook`].
/// See [the module-level documentation][self] for usage examples.
pub fn handle_panic(info: &PanicHookInfo<'_>) {
// we only want to do this once-per-process, so hard-code it here instead of using a dynamically
// registered panic hook
LAST_ERROR_LOCATION.with_borrow_mut(|loc| {
*loc = info.location().map(|l| l.to_string());
});
// Collect and clone all the hooks and drop the lock guard so that we can avoid risks of
// deadlocks due to potentially re-entrant calls to `register_panic_hook` or
// `PanicHookGuard::drop`. This is expensive, but this should be a cold codepath.
let hooks: Vec<ArcPanicHook> = PANIC_HOOKS.read().unwrap().values().cloned().collect();
for hook in hooks {
(hook)(info);
}
}
/// Registers a hook to be called when a panic occurs. Panic hooks are called in the order that they
/// are registered. Dropping the returned [`PanicHookGuard`] removes the registered hook.
///
/// In the case that the panic hook refers to the object that contains the [`PanicHookGuard`], make
/// sure to use [`std::sync::Weak`] to avoid leaks. [`Arc::new_cyclic`] may be useful in
/// constructing such an object.
pub fn register_panic_hook(hook: PanicHook) -> PanicHookGuard {
let id = HOOK_ID_FACTORY.get();
PANIC_HOOKS.write().unwrap().insert(id, Arc::from(hook));
PanicHookGuard { id }
}
/// A guard returned from [`register_panic_hook`] that cleans up the panic hook when dropped.
#[must_use = "If the guard is not stored somewhere, it will be immediately dropped and the panic \
hook will be immediately cleaned up"]
pub struct PanicHookGuard {
id: NonZeroU64,
}
impl Drop for PanicHookGuard {
fn drop(&mut self) {
PANIC_HOOKS.write().unwrap().remove(&self.id);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment