Unverified Commit e2af4d88 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Implement TraceRawVcs and NonLocalValue correctly for Effects (#89133)

In subsequent PRs, I want to store a `ResolvedVc` (specifically, a `FileSystemPath`, which contains a `Vc<Box<dyn FileSystem>>`) inside of an `Effect`.

To do this in a way that wouldn't break a hypothetical future tracing garbage collector, we must implement `TraceRawVcs` on `Effect`.

The previous PR changed the `Effect` type from a closure to a trait in order to enable this.
parent a7cf5eee
...@@ -976,6 +976,7 @@ impl FileSystem for DiskFileSystem { ...@@ -976,6 +976,7 @@ impl FileSystem for DiskFileSystem {
let inner = self.inner.clone(); let inner = self.inner.clone();
let invalidator = turbo_tasks::get_invalidator(); let invalidator = turbo_tasks::get_invalidator();
#[derive(TraceRawVcs, NonLocalValue)]
struct WriteEffect { struct WriteEffect {
full_path: PathBuf, full_path: PathBuf,
inner: Arc<DiskFileSystemInner>, inner: Arc<DiskFileSystemInner>,
...@@ -986,7 +987,7 @@ impl FileSystem for DiskFileSystem { ...@@ -986,7 +987,7 @@ impl FileSystem for DiskFileSystem {
impl Effect for WriteEffect { impl Effect for WriteEffect {
type Error = AnyhowWrapper; type Error = AnyhowWrapper;
async fn apply(self) -> Result<(), Self::Error> { async fn apply(&self) -> Result<(), Self::Error> {
let full_path = validate_path_length(&self.full_path)?; let full_path = validate_path_length(&self.full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
...@@ -1128,6 +1129,7 @@ impl FileSystem for DiskFileSystem { ...@@ -1128,6 +1129,7 @@ impl FileSystem for DiskFileSystem {
let inner = self.inner.clone(); let inner = self.inner.clone();
let invalidator = turbo_tasks::get_invalidator(); let invalidator = turbo_tasks::get_invalidator();
#[derive(TraceRawVcs, NonLocalValue)]
struct WriteLinkEffect { struct WriteLinkEffect {
full_path: PathBuf, full_path: PathBuf,
inner: Arc<DiskFileSystemInner>, inner: Arc<DiskFileSystemInner>,
...@@ -1138,7 +1140,7 @@ impl FileSystem for DiskFileSystem { ...@@ -1138,7 +1140,7 @@ impl FileSystem for DiskFileSystem {
impl Effect for WriteLinkEffect { impl Effect for WriteLinkEffect {
type Error = AnyhowWrapper; type Error = AnyhowWrapper;
async fn apply(self) -> Result<(), Self::Error> { async fn apply(&self) -> Result<(), Self::Error> {
let full_path = validate_path_length(&self.full_path)?; let full_path = validate_path_length(&self.full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
...@@ -2863,6 +2865,7 @@ async fn realpath_with_links(path: FileSystemPath) -> Result<Vc<RealPathResult>> ...@@ -2863,6 +2865,7 @@ async fn realpath_with_links(path: FileSystemPath) -> Result<Vc<RealPathResult>>
/// Wrapper to convert `anyhow::Error` to `impl std::error::Error` for use in `Effect::apply`. /// Wrapper to convert `anyhow::Error` to `impl std::error::Error` for use in `Effect::apply`.
// TODO(bgw): use a structured error type instead of anyhow for write/write_link // TODO(bgw): use a structured error type instead of anyhow for write/write_link
#[derive(TraceRawVcs, NonLocalValue)]
struct AnyhowWrapper(anyhow::Error); struct AnyhowWrapper(anyhow::Error);
impl std::fmt::Display for AnyhowWrapper { impl std::fmt::Display for AnyhowWrapper {
......
...@@ -17,15 +17,17 @@ use tokio::task_local; ...@@ -17,15 +17,17 @@ use tokio::task_local;
use tracing::Instrument; use tracing::Instrument;
use crate::{ use crate::{
self as turbo_tasks, CollectiblesSource, ReadRef, ResolvedVc, TryJoinIterExt, emit, self as turbo_tasks, CollectiblesSource, NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt,
emit,
event::{Event, EventListener}, event::{Event, EventListener},
spawn, spawn,
trace::TraceRawVcs,
}; };
const APPLY_EFFECTS_CONCURRENCY_LIMIT: usize = 1024; const APPLY_EFFECTS_CONCURRENCY_LIMIT: usize = 1024;
pub trait Effect: Send + Sync + 'static { pub trait Effect: TraceRawVcs + NonLocalValue + Send + Sync + 'static {
type Error: StdError + Send + Sync + 'static; type Error: EffectError;
/// A function that is called once at the top level of the program's execution after everything /// A function that is called once at the top level of the program's execution after everything
/// has "settled". /// has "settled".
...@@ -33,7 +35,7 @@ pub trait Effect: Send + Sync + 'static { ...@@ -33,7 +35,7 @@ pub trait Effect: Send + Sync + 'static {
/// This function is executed outside of the turbo-tasks context, and therefore cannot read any /// This function is executed outside of the turbo-tasks context, and therefore cannot read any
/// `Vc`s or call any turbo-task functions. The effect can store [`ResolvedVc`]s (or any other /// `Vc`s or call any turbo-task functions. The effect can store [`ResolvedVc`]s (or any other
/// `Vc` type), but should not read or resolve their contents. /// `Vc` type), but should not read or resolve their contents.
fn apply(self) -> impl Future<Output = Result<(), Self::Error>> + Send; fn apply(&self) -> impl Future<Output = Result<(), Self::Error>> + Send;
} }
/// The error type that an effect can return. We use `dyn std::error::Error` (instead of /// The error type that an effect can return. We use `dyn std::error::Error` (instead of
...@@ -49,43 +51,57 @@ pub trait Effect: Send + Sync + 'static { ...@@ -49,43 +51,57 @@ pub trait Effect: Send + Sync + 'static {
/// So instead, we leave it up to the caller to figure out how to downcast these errors themselves. /// So instead, we leave it up to the caller to figure out how to downcast these errors themselves.
/// ///
/// [`SharedError`]: crate::util::SharedError /// [`SharedError`]: crate::util::SharedError
type EffectError = Arc<dyn StdError + Send + Sync + 'static>; pub trait EffectError: StdError + TraceRawVcs + NonLocalValue + Send + Sync + 'static {}
impl<T> EffectError for T where T: StdError + TraceRawVcs + NonLocalValue + Send + Sync + 'static {}
// Private wrapper trait to allow dynamic dispatch of an `Effect`. This is similar to the pattern // Private wrapper trait to allow dynamic dispatch of an `Effect`. This is similar to the pattern
// that the dynosaur crate uses: https://github.com/spastorino/dynosaur // that the dynosaur crate uses: https://github.com/spastorino/dynosaur
trait DynEffect: Send + Sync + 'static { trait DynEffect: TraceRawVcs + NonLocalValue + Send + Sync + 'static {
fn dyn_apply(self: Box<Self>) -> Pin<Box<dyn Future<Output = Result<(), EffectError>> + Send>>; fn dyn_apply<'a>(&'a self) -> DynEffectApplyFuture<'a>;
} }
impl<T> DynEffect for T impl<T> DynEffect for T
where where
T: Effect, T: Effect,
{ {
fn dyn_apply(self: Box<Self>) -> Pin<Box<dyn Future<Output = Result<(), EffectError>> + Send>> { fn dyn_apply<'a>(&'a self) -> DynEffectApplyFuture<'a> {
Box::pin(async move { Box::pin(async move {
self.apply() self.apply()
.await .await
.map_err(|err| Arc::new(err) as EffectError) .map_err(|err| Arc::new(err) as Arc<dyn EffectError>)
}) })
} }
} }
type DynEffectApplyFuture<'a> =
Pin<Box<dyn Future<Output = Result<(), Arc<dyn EffectError>>> + Send + 'a>>;
/// A trait to emit a task effect as collectible. This trait only has one implementation, /// A trait to emit a task effect as collectible. This trait only has one implementation,
/// `EffectInstance` and no other implementation is allowed. The trait is private to this module so /// `EffectInstance` and no other implementation is allowed. The trait is private to this module so
/// that no other implementation can be added. /// that no other implementation can be added.
#[turbo_tasks::value_trait] #[turbo_tasks::value_trait]
trait EffectCollectible {} trait EffectCollectible {}
#[derive(TraceRawVcs, NonLocalValue)]
enum EffectState { enum EffectState {
NotStarted(Box<dyn DynEffect>), NotStarted(Box<dyn DynEffect>),
Started(Event), /// The `Effect` has already begun execution in another thread. The `DynEffect` is moved here so
Finished(Result<(), EffectError>), /// that `TraceRawVcs` works as expected. An alternative is that we could always run
/// `TraceRawVcs` before starting execution and just store a `Vec` of `Vc`s here, but
/// `TraceRawVcs` is potentially slow.
Started(Arc<dyn DynEffect>, Event),
Finished(Result<(), Arc<dyn EffectError>>),
/// Can occur if we paniced while constructing the Started state
Invalid,
} }
/// The Effect instance collectible that is emitted for effects. /// The Effect instance collectible that is emitted for effects.
#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] #[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")]
struct EffectInstance { struct EffectInstance {
#[turbo_tasks(trace_ignore, debug_ignore)] // Internal mutability: It's important that if `EffectInstance::apply` is called multiple
// times, the caller sees the same return value.
#[turbo_tasks(debug_ignore)]
inner: Mutex<EffectState>, inner: Mutex<EffectState>,
} }
...@@ -102,12 +118,12 @@ impl EffectInstance { ...@@ -102,12 +118,12 @@ impl EffectInstance {
loop { loop {
enum State { enum State {
Started(EventListener), Started(EventListener),
NotStarted(Box<dyn DynEffect>), NotStarted(Arc<dyn DynEffect>),
} }
let state = { let state = {
let mut guard = self.inner.lock(); let mut guard = self.inner.lock();
match &*guard { match &*guard {
EffectState::Started(event) => { EffectState::Started(_, event) => {
let listener = event.listen(); let listener = event.listen();
State::Started(listener) State::Started(listener)
} }
...@@ -115,30 +131,37 @@ impl EffectInstance { ...@@ -115,30 +131,37 @@ impl EffectInstance {
return result.clone().map_err(Into::into); return result.clone().map_err(Into::into);
} }
EffectState::NotStarted(_) => { EffectState::NotStarted(_) => {
let EffectState::NotStarted(inner) = std::mem::replace( let EffectState::NotStarted(effect) =
&mut *guard, std::mem::replace(&mut *guard, EffectState::Invalid)
EffectState::Started(Event::new(|| || "Effect".to_string())), else {
) else { unreachable!()
unreachable!();
}; };
State::NotStarted(inner) let effect: Arc<dyn DynEffect> = Arc::from(effect);
*guard = EffectState::Started(
effect.clone(),
Event::new(|| || "Effect".to_string()),
);
State::NotStarted(effect)
} }
EffectState::Invalid => unreachable!(),
} }
}; };
match state { match state {
State::Started(listener) => { State::Started(listener) => listener.await,
listener.await;
}
State::NotStarted(effect) => { State::NotStarted(effect) => {
let join_handle = // This spawn prevents the effect from running within a turbo_tasks context.
spawn(ApplyEffectsContext::in_current_scope(effect.dyn_apply())); // This is important because if we read a `Vc`, we want it to fail (panic). If
// it didn't, we'd assign the dependency to the wrong task.
let join_handle = spawn(ApplyEffectsContext::in_current_scope(async move {
effect.dyn_apply().await
}));
let result = match join_handle.await { let result = match join_handle.await {
Err(err) => Err(err), Err(err) => Err(err),
Ok(()) => Ok(()), Ok(()) => Ok(()),
}; };
let event = { let event = {
let mut guard = self.inner.lock(); let mut guard = self.inner.lock();
let EffectState::Started(event) = let EffectState::Started(_, event) =
replace(&mut *guard, EffectState::Finished(result.clone())) replace(&mut *guard, EffectState::Finished(result.clone()))
else { else {
unreachable!(); unreachable!();
...@@ -251,7 +274,7 @@ pub async fn get_effects(source: impl CollectiblesSource) -> Result<Effects> { ...@@ -251,7 +274,7 @@ pub async fn get_effects(source: impl CollectiblesSource) -> Result<Effects> {
#[derive(Default)] #[derive(Default)]
#[turbo_tasks::value(shared, eq = "manual", serialization = "none")] #[turbo_tasks::value(shared, eq = "manual", serialization = "none")]
pub struct Effects { pub struct Effects {
#[turbo_tasks(trace_ignore, debug_ignore)] #[turbo_tasks(debug_ignore)]
effects: Vec<ReadRef<EffectInstance>>, effects: Vec<ReadRef<EffectInstance>>,
} }
......
...@@ -7,7 +7,7 @@ use turbo_dyn_eq_hash::{ ...@@ -7,7 +7,7 @@ use turbo_dyn_eq_hash::{
}; };
use crate::{ use crate::{
FxIndexMap, FxIndexSet, TaskId, TurboTasksApi, FxIndexMap, FxIndexSet, NonLocalValue, OperationValue, TaskId, TurboTasksApi,
manager::{current_task_if_available, mark_invalidator}, manager::{current_task_if_available, mark_invalidator},
trace::TraceRawVcs, trace::TraceRawVcs,
util::StaticOrArc, util::StaticOrArc,
...@@ -55,6 +55,9 @@ impl TraceRawVcs for Invalidator { ...@@ -55,6 +55,9 @@ impl TraceRawVcs for Invalidator {
} }
} }
unsafe impl NonLocalValue for Invalidator {}
unsafe impl OperationValue for Invalidator {}
/// A user-facing reason why a task was invalidated. This should only be used /// A user-facing reason why a task was invalidated. This should only be used
/// for invalidation that were triggered by the user. /// for invalidation that were triggered by the user.
/// ///
......
...@@ -78,7 +78,9 @@ pub use crate::{ ...@@ -78,7 +78,9 @@ pub use crate::{
collectibles::CollectiblesSource, collectibles::CollectiblesSource,
completion::{Completion, Completions}, completion::{Completion, Completions},
display::{ValueToString, ValueToStringRef}, display::{ValueToString, ValueToStringRef},
effect::{ApplyEffectsContext, Effect, Effects, apply_effects, emit_effect, get_effects}, effect::{
ApplyEffectsContext, Effect, EffectError, Effects, apply_effects, emit_effect, get_effects,
},
error::PrettyPrintError, error::PrettyPrintError,
id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId}, id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId},
invalidation::{ invalidation::{
......
...@@ -62,12 +62,15 @@ macro_rules! impl_auto_marker_trait { ...@@ -62,12 +62,15 @@ macro_rules! impl_auto_marker_trait {
unsafe impl<K: $trait, V: $trait> $trait for ::std::collections::BTreeMap<K, V> {} unsafe impl<K: $trait, V: $trait> $trait for ::std::collections::BTreeMap<K, V> {}
unsafe impl<K: $trait, V: $trait, S> $trait for ::indexmap::IndexMap<K, V, S> {} unsafe impl<K: $trait, V: $trait, S> $trait for ::indexmap::IndexMap<K, V, S> {}
unsafe impl<K: $trait, V: $trait> $trait for ::turbo_frozenmap::FrozenMap<K, V> {} unsafe impl<K: $trait, V: $trait> $trait for ::turbo_frozenmap::FrozenMap<K, V> {}
unsafe impl<T> $trait for ::std::pin::Pin<T>
where T: ::std::ops::Deref, <T as ::std::ops::Deref>::Target: $trait {}
unsafe impl<T: $trait + ?Sized> $trait for ::std::boxed::Box<T> {} unsafe impl<T: $trait + ?Sized> $trait for ::std::boxed::Box<T> {}
unsafe impl<T: $trait + ?Sized> $trait for ::std::sync::Arc<T> {} unsafe impl<T: $trait + ?Sized> $trait for ::std::sync::Arc<T> {}
unsafe impl<B: $trait + ::std::borrow::ToOwned + ?Sized> $trait unsafe impl<B: $trait + ::std::borrow::ToOwned + ?Sized> $trait
for ::std::borrow::Cow<'_, B> {} for ::std::borrow::Cow<'_, B> {}
unsafe impl<T: $trait, E: $trait> $trait for ::std::result::Result<T, E> {} unsafe impl<T: $trait, E: $trait> $trait for ::std::result::Result<T, E> {}
unsafe impl<T: $trait + ?Sized> $trait for ::std::sync::Mutex<T> {} unsafe impl<T: $trait + ?Sized> $trait for ::std::sync::Mutex<T> {}
unsafe impl<T: $trait + ?Sized> $trait for ::parking_lot::Mutex<T> {}
unsafe impl<T: $trait + ?Sized> $trait for ::std::cell::RefCell<T> {} unsafe impl<T: $trait + ?Sized> $trait for ::std::cell::RefCell<T> {}
unsafe impl<T: ?Sized> $trait for ::std::marker::PhantomData<T> {} unsafe impl<T: ?Sized> $trait for ::std::marker::PhantomData<T> {}
unsafe impl<L: $trait, R: $trait> $trait for ::either::Either<L, R> {} unsafe impl<L: $trait, R: $trait> $trait for ::either::Either<L, R> {}
...@@ -82,6 +85,7 @@ macro_rules! impl_auto_marker_trait { ...@@ -82,6 +85,7 @@ macro_rules! impl_auto_marker_trait {
unsafe impl<T: $trait> $trait for $crate::TransientState<T> {} unsafe impl<T: $trait> $trait for $crate::TransientState<T> {}
unsafe impl<T: $trait> $trait for $crate::TransientValue<T> {} unsafe impl<T: $trait> $trait for $crate::TransientValue<T> {}
unsafe impl<T: $trait> $trait for $crate::TransientInstance<T> {} unsafe impl<T: $trait> $trait for $crate::TransientInstance<T> {}
unsafe impl $trait for $crate::event::Event {}
unsafe impl<T: $trait + ?Sized> $trait for &T {} unsafe impl<T: $trait + ?Sized> $trait for &T {}
unsafe impl<T: $trait + ?Sized> $trait for &mut T {} unsafe impl<T: $trait + ?Sized> $trait for &mut T {}
......
...@@ -3,8 +3,10 @@ use std::{ ...@@ -3,8 +3,10 @@ use std::{
cell::RefCell, cell::RefCell,
collections::{BTreeMap, BTreeSet, HashMap, HashSet}, collections::{BTreeMap, BTreeSet, HashMap, HashSet},
marker::PhantomData, marker::PhantomData,
ops::Deref,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::{Arc, Mutex, atomic::*}, pin::Pin,
sync::{Arc, Mutex as StdMutex, atomic::*},
time::Duration, time::Duration,
}; };
...@@ -16,7 +18,7 @@ use turbo_frozenmap::{FrozenMap, FrozenSet}; ...@@ -16,7 +18,7 @@ use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks_hash::HashAlgorithm; use turbo_tasks_hash::HashAlgorithm;
use crate::RawVc; use crate::{RawVc, event::Event};
pub struct TraceRawVcsContext { pub struct TraceRawVcsContext {
list: Vec<RawVc>, list: Vec<RawVc>,
...@@ -79,10 +81,18 @@ ignore!( ...@@ -79,10 +81,18 @@ ignore!(
AtomicBool, AtomicBool,
AtomicUsize AtomicUsize
); );
ignore!((), str, String, Duration, anyhow::Error, RcStr); ignore!((), str, String, Duration, RcStr);
ignore!(Path, PathBuf); ignore!(Path, PathBuf);
ignore!(serde_json::Value, serde_json::Map<String, serde_json::Value>); ignore!(serde_json::Value, serde_json::Map<String, serde_json::Value>);
ignore!(HashAlgorithm); ignore!(HashAlgorithm);
ignore!(Event);
// This is not technically correct, as `anyhow::Error` contains a boxed source error, which could
// contain a `Vc`, but most errors don't do this, and implementing `TraceRawVcs` over every possible
// error type is impractical.
//
// For serialized errors: This is fine, we throw away the anyhow source upon serialization.
ignore!(anyhow::Error);
impl<T: ?Sized> TraceRawVcs for PhantomData<T> { impl<T: ?Sized> TraceRawVcs for PhantomData<T> {
fn trace_raw_vcs(&self, _trace_context: &mut TraceRawVcsContext) {} fn trace_raw_vcs(&self, _trace_context: &mut TraceRawVcsContext) {}
...@@ -256,6 +266,16 @@ impl<K: TraceRawVcs, V: TraceRawVcs> TraceRawVcs for FrozenMap<K, V> { ...@@ -256,6 +266,16 @@ impl<K: TraceRawVcs, V: TraceRawVcs> TraceRawVcs for FrozenMap<K, V> {
} }
} }
impl<T> TraceRawVcs for Pin<T>
where
T: Deref,
<T as Deref>::Target: TraceRawVcs,
{
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
TraceRawVcs::trace_raw_vcs(&**self, trace_context);
}
}
impl<T: TraceRawVcs + ?Sized> TraceRawVcs for Box<T> { impl<T: TraceRawVcs + ?Sized> TraceRawVcs for Box<T> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
TraceRawVcs::trace_raw_vcs(&**self, trace_context); TraceRawVcs::trace_raw_vcs(&**self, trace_context);
...@@ -289,12 +309,27 @@ impl<T: TraceRawVcs, E: TraceRawVcs> TraceRawVcs for Result<T, E> { ...@@ -289,12 +309,27 @@ impl<T: TraceRawVcs, E: TraceRawVcs> TraceRawVcs for Result<T, E> {
} }
} }
impl<T: TraceRawVcs + ?Sized> TraceRawVcs for Mutex<T> { // NOTE: Implementing `TraceRawVcs` on a synchronous lock with guards that do not implement `Send`
// is okay, as it won't be held across await points. We shouldn't attempt to implement it on an
// asynchronous locks (e.g. via `tokio::sync::Mutex::blocking_lock`).
//
// If a future garbage collection tracing implementation holds a global lock on execution, we could
// end up deadlocked if a currently executing task is holding a lock across an await point.
//
// Similarly, if we ever implement synchronous tasks with synchronous reads of `Vc`s, we need to add
// a mechanism to loudly fail if we're holding a traced lock across a task read.
impl<T: TraceRawVcs + ?Sized> TraceRawVcs for StdMutex<T> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
self.lock().unwrap().trace_raw_vcs(trace_context); self.lock().unwrap().trace_raw_vcs(trace_context);
} }
} }
impl<T: TraceRawVcs + ?Sized> TraceRawVcs for parking_lot::Mutex<T> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
self.lock().trace_raw_vcs(trace_context);
}
}
impl<T: TraceRawVcs + ?Sized> TraceRawVcs for RefCell<T> { impl<T: TraceRawVcs + ?Sized> TraceRawVcs for RefCell<T> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
self.borrow().trace_raw_vcs(trace_context); self.borrow().trace_raw_vcs(trace_context);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment