Unverified Commit 2e1e5958 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks-backend: fix snapshot coordination races + extract SnapshotCoordinator (#93416)

Hold a lock while persisting so two snapshots cannot execute
concurrently

* Currently if `stop` is called while an idle snapshot is running, then
snapshotting can race with itself, this can corrupt the use of the
`in_progress_operations` parameter since two threads will `fetch_or`
with it and wait for the bit to be cleared

Abort the process if `panic!` occurs during task spawning
* Currently if `try_start_task_execution` panics it ends up hanging a
task which can deadlock the process, in this case we have no better
option than to just log and abort.
* I considered strategies that would 'poison' the task or possibly all
of turbo-tasks and this is attractive but i believe fundamentally
unsafe, the most likely cause of these panics is something wrong with
state tracking in the backend, so exiting is all we can do
parent 818aa245
...@@ -26,8 +26,8 @@ use turbo_tasks::{ ...@@ -26,8 +26,8 @@ use turbo_tasks::{
use self::aggregation_update::ComputeDirtyAndCleanUpdate; use self::aggregation_update::ComputeDirtyAndCleanUpdate;
use crate::{ use crate::{
backend::{ backend::{
EventDescription, OperationGuard, TaskDataCategory, TurboTasksBackend, EventDescription, TaskDataCategory, TurboTasksBackend, TurboTasksBackendInner,
TurboTasksBackendInner, snapshot_coordinator::OperationGuard,
storage::{SpecificTaskDataCategory, StorageWriteGuard}, storage::{SpecificTaskDataCategory, StorageWriteGuard},
storage_schema::{TaskStorage, TaskStorageAccessors}, storage_schema::{TaskStorage, TaskStorageAccessors},
}, },
...@@ -168,7 +168,7 @@ impl TaskLockCounter { ...@@ -168,7 +168,7 @@ impl TaskLockCounter {
pub struct ExecuteContextImpl<'e, B: BackingStorage> { pub struct ExecuteContextImpl<'e, B: BackingStorage> {
backend: &'e TurboTasksBackendInner<B>, backend: &'e TurboTasksBackendInner<B>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>, turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
_operation_guard: Option<OperationGuard<'e, B>>, _operation_guard: Option<OperationGuard<'e, AnyOperation>>,
task_lock_counter: TaskLockCounter, task_lock_counter: TaskLockCounter,
} }
......
...@@ -4,7 +4,9 @@ use std::{ ...@@ -4,7 +4,9 @@ use std::{
future::Future, future::Future,
hash::{BuildHasher, BuildHasherDefault}, hash::{BuildHasher, BuildHasherDefault},
mem::take, mem::take,
panic::AssertUnwindSafe,
pin::Pin, pin::Pin,
process::abort,
sync::{ sync::{
Arc, Mutex, RwLock, Weak, Arc, Mutex, RwLock, Weak,
atomic::{AtomicBool, AtomicUsize, Ordering}, atomic::{AtomicBool, AtomicUsize, Ordering},
...@@ -16,7 +18,7 @@ use anyhow::{Result, anyhow}; ...@@ -16,7 +18,7 @@ use anyhow::{Result, anyhow};
use auto_hash_map::AutoMap; use auto_hash_map::AutoMap;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use either::Either; use either::Either;
use futures::stream::FuturesUnordered; use futures::{FutureExt, stream::FuturesUnordered};
use rustc_hash::{FxBuildHasher, FxHasher}; use rustc_hash::{FxBuildHasher, FxHasher};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use smallvec::SmallVec; use smallvec::SmallVec;
...@@ -1186,6 +1188,25 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -1186,6 +1188,25 @@ impl<B: Backend + 'static> TurboTasks<B> {
struct TurboTasksExecutor; struct TurboTasksExecutor;
/// Run a future and abort the process if a panic is reported
///
/// Turbtasks catches panics from user code and propagates throught the task tree, but if it happens
/// as part of state management we have to abort
async fn abort_on_panic<F: Future>(f: F) -> F::Output {
match AssertUnwindSafe(f).catch_unwind().await {
Ok(r) => r,
Err(_) => {
eprintln!(
"\nturbo-tasks: an internal panic occurred outside the per-task panic \
boundary. This is a bug in turbo-tasks/Turbopack — please report it at \
https://github.com/vercel/next.js/discussions and include the panic message \
and stack trace above.\n\nAborting."
);
abort();
}
}
}
impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboTasksExecutor { impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboTasksExecutor {
type Future = impl Future<Output = ()> + Send + 'static; type Future = impl Future<Output = ()> + Send + 'static;
...@@ -1200,10 +1221,11 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT ...@@ -1200,10 +1221,11 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT
let this2 = this.clone(); let this2 = this.clone();
let this = this.clone(); let this = this.clone();
let future = async move { let future = async move {
abort_on_panic(async {
let mut schedule_again = true; let mut schedule_again = true;
while schedule_again { while schedule_again {
// it's okay for execution ids to overflow and wrap, they're just used for // it's okay for execution ids to overflow and wrap, they're just used
// an assert // for an assert
let execution_id = this.execution_id_factory.wrapping_get(); let execution_id = this.execution_id_factory.wrapping_get();
let current_task_state = Arc::new(RwLock::new(CurrentTaskState::new( let current_task_state = Arc::new(RwLock::new(CurrentTaskState::new(
task_id, task_id,
...@@ -1233,12 +1255,15 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT ...@@ -1233,12 +1255,15 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT
let result = match result { let result = match result {
Ok(Ok(raw_vc)) => Ok(raw_vc), Ok(Ok(raw_vc)) => Ok(raw_vc),
Ok(Err(err)) => Err(err.into()), Ok(Err(err)) => Err(err.into()),
Err(err) => Err(TurboTasksExecutionError::Panic(Arc::new(err))), Err(err) => {
Err(TurboTasksExecutionError::Panic(Arc::new(err)))
}
}; };
let finished_state = this.finish_current_task_state(); let finished_state = this.finish_current_task_state();
let cell_counters = CURRENT_TASK_STATE let cell_counters = CURRENT_TASK_STATE.with(|ts| {
.with(|ts| ts.write().unwrap().cell_counters.take().unwrap()); ts.write().unwrap().cell_counters.take().unwrap()
});
this.backend.task_execution_completed( this.backend.task_execution_completed(
task_id, task_id,
result, result,
...@@ -1257,6 +1282,8 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT ...@@ -1257,6 +1282,8 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT
.await; .await;
} }
this.finish_foreground_job(); this.finish_foreground_job();
})
.await
}; };
Either::Left(TURBO_TASKS.scope(this2, future).instrument(span)) Either::Left(TURBO_TASKS.scope(this2, future).instrument(span))
...@@ -1280,6 +1307,7 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT ...@@ -1280,6 +1307,7 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT
trait_method.resolve_span(priority) trait_method.resolve_span(priority)
} }
}; };
abort_on_panic(
async move { async move {
let result = match ty.task_type { let result = match ty.task_type {
LocalTaskType::ResolveNative { native_fn } => { LocalTaskType::ResolveNative { native_fn } => {
...@@ -1321,13 +1349,16 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT ...@@ -1321,13 +1349,16 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT
local_task, local_task,
); );
let LocalTask::Scheduled { done_event } = scheduled_task else { let LocalTask::Scheduled { done_event } = scheduled_task else {
panic!("local task finished, but was not in the scheduled state?"); panic!(
"local task finished, but was not in the scheduled state?"
);
}; };
done_event done_event
}); });
done_event.notify(usize::MAX) done_event.notify(usize::MAX)
} }
.instrument(span) .instrument(span),
)
.await .await
}; };
let future = CURRENT_TASK_STATE.scope(global_task_state, future); let future = CURRENT_TASK_STATE.scope(global_task_state, future);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment