Unverified Commit df6cd599 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Remove `turbo_tasks::apply_effects`, use `Effects::apply` instead (#91858)

Having both `apply_effects` and `Effects::apply` is confusing.

`apply_effect`'s API seems a bit bad/dangerous because it combines work that should be done inside of a strongly-consistent operation (collecting the effects and reading their `Vc`s) with work that should be done at the top-level outside of any operation or turbo-task function (applying the effects).

The first commit in this PR removes the `apply_effect` API and tries to improve documentation, including adding a doctest that compiles.

The second commit in this PR is an LLM-driven refactor to remove the `apply_effect` callsites.
parent b6ff360b
......@@ -3,7 +3,7 @@ use bincode::{Decode, Encode};
use turbo_rcstr::RcStr;
use turbo_tasks::{
CollectiblesSource, FxIndexMap, NonLocalValue, OperationValue, OperationVc, ResolvedVc,
TaskInput, Vc, debug::ValueDebugFormat, get_effects, trace::TraceRawVcs,
TaskInput, Vc, debug::ValueDebugFormat, take_effects, trace::TraceRawVcs,
};
use turbopack_core::{diagnostics::Diagnostic, issue::CollectibleIssuesExt};
......@@ -40,7 +40,7 @@ async fn entrypoints_without_collectibles_operation(
let _ = entrypoints.resolve().strongly_consistent().await?;
entrypoints.drop_collectibles::<Box<dyn Diagnostic>>();
entrypoints.drop_issues();
let _ = get_effects(entrypoints).await?;
let _ = take_effects(entrypoints).await?;
Ok(entrypoints.connect())
}
......
......@@ -12,7 +12,9 @@ use next_api::{
route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ReadConsistency, ResolvedVc, TransientInstance, TurboTasks, Vc, get_effects};
use turbo_tasks::{
Effects, ReadConsistency, ReadRef, ResolvedVc, TransientInstance, TurboTasks, Vc, take_effects,
};
use turbo_tasks_backend::{NoopBackingStorage, TurboTasksBackend};
use turbo_tasks_malloc::TurboMalloc;
......@@ -186,21 +188,21 @@ pub async fn render_routes(
html_endpoint,
data_endpoint: _,
} => {
endpoint_write_to_disk_with_effects(*html_endpoint).await?;
endpoint_write_to_disk_with_apply(html_endpoint).await?;
}
Route::PageApi { endpoint } => {
endpoint_write_to_disk_with_effects(*endpoint).await?;
endpoint_write_to_disk_with_apply(endpoint).await?;
}
Route::AppPage(routes) => {
for route in routes {
endpoint_write_to_disk_with_effects(*route.html_endpoint).await?;
endpoint_write_to_disk_with_apply(route.html_endpoint).await?;
}
}
Route::AppRoute {
original_name: _,
endpoint,
} => {
endpoint_write_to_disk_with_effects(*endpoint).await?;
endpoint_write_to_disk_with_apply(endpoint).await?;
}
Route::Conflict => {
tracing::info!("WARN: conflict {}", name);
......@@ -243,21 +245,43 @@ pub async fn render_routes(
Ok(stream.len())
}
#[turbo_tasks::function]
async fn endpoint_write_to_disk_with_effects(
async fn endpoint_write_to_disk_with_apply(
endpoint: ResolvedVc<Box<dyn Endpoint>>,
) -> Result<Vc<EndpointOutputPaths>> {
let op = endpoint_write_to_disk_operation(endpoint);
let result = op.resolve().strongly_consistent().await?;
get_effects(op).await?.apply().await?;
Ok(*result)
}
) -> Result<ReadRef<EndpointOutputPaths>> {
#[turbo_tasks::function(operation)]
fn inner_operation(endpoint: ResolvedVc<Box<dyn Endpoint>>) -> Vc<EndpointOutputPaths> {
// we must wrap this in an operation so we can get the Effects collectibles
endpoint_write_to_disk(*endpoint)
}
#[turbo_tasks::function(operation)]
pub fn endpoint_write_to_disk_operation(
endpoint: ResolvedVc<Box<dyn Endpoint>>,
) -> Vc<EndpointOutputPaths> {
endpoint_write_to_disk(*endpoint)
#[turbo_tasks::value(serialization = "none")]
struct WithEffects {
output_paths: ReadRef<EndpointOutputPaths>,
effects: Effects,
}
#[turbo_tasks::function(operation)]
pub async fn inner_operation_with_effects(
endpoint: ResolvedVc<Box<dyn Endpoint>>,
) -> Result<Vc<WithEffects>> {
let op = inner_operation(endpoint);
let output_paths = op.read_strongly_consistent().await?;
let effects = take_effects(op).await?;
Ok(WithEffects {
output_paths,
effects,
}
.cell())
}
let op = inner_operation_with_effects(endpoint);
let WithEffects {
output_paths,
effects,
} = &*op.read_strongly_consistent().await?;
effects.apply().await?;
Ok(output_paths.clone())
}
async fn hmr(
......
......@@ -49,9 +49,11 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Effects, FxIndexSet, NonLocalValue, OperationValue, OperationVc, PrettyPrintError, ReadRef,
ResolvedVc, TaskInput, TransientInstance, TryJoinIterExt, TurboTasksApi, TurboTasksCallApi,
UpdateInfo, Vc, get_effects,
UpdateInfo, Vc, mark_top_level_task,
message_queue::{CompilationEvent, Severity},
take_effects,
trace::TraceRawVcs,
unmark_top_level_task_may_leak_eventually_consistent_state,
};
use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
use turbo_tasks_fs::{
......@@ -1840,7 +1842,7 @@ async fn hmr_update_with_issues_operation(
let filter = project.issue_filter();
let issues = get_issues(update_op, filter).await?;
let diagnostics = get_diagnostics(update_op).await?;
let effects = Arc::new(get_effects(update_op).await?);
let effects = Arc::new(take_effects(update_op).await?);
Ok(HmrUpdateWithIssues {
update,
issues,
......@@ -1874,6 +1876,8 @@ pub fn project_hmr_events(
let chunk_name: RcStr = outer_chunk_name.clone();
let session = session.clone();
async move {
// HACK(bgw): Remove this unmark call
unmark_top_level_task_may_leak_eventually_consistent_state();
let project = container.project().to_resolved().await?;
let state = project
.hmr_version_state(chunk_name.clone(), hmr_target, session)
......@@ -1893,7 +1897,11 @@ pub fn project_hmr_events(
diagnostics,
effects,
} = &*update;
// HACK(bgw): Remove this mark call
mark_top_level_task();
effects.apply().await?;
// HACK(bgw): Remove this unmark call
unmark_top_level_task_may_leak_eventually_consistent_state();
match &**update {
Update::Missing | Update::None => {}
Update::Total(TotalUpdate { to }) => {
......@@ -1975,7 +1983,7 @@ async fn get_hmr_chunk_names_with_issues_operation(
let filter = issue_filter_from_container(container);
let issues = get_issues(hmr_chunk_names_op, filter).await?;
let diagnostics = get_diagnostics(hmr_chunk_names_op).await?;
let effects = Arc::new(get_effects(hmr_chunk_names_op).await?);
let effects = Arc::new(take_effects(hmr_chunk_names_op).await?);
Ok(HmrChunkNamesWithIssues {
chunk_names: hmr_chunk_names,
issues,
......
......@@ -15,7 +15,7 @@ use rustc_hash::FxHashMap;
use serde::Serialize;
use turbo_rcstr::RcStr;
use turbo_tasks::{
Effects, OperationVc, ReadRef, TaskId, TryJoinIterExt, Vc, VcValueType, get_effects,
Effects, OperationVc, ReadRef, TaskId, TryJoinIterExt, Vc, VcValueType, take_effects,
};
use turbo_tasks_fs::FileContent;
use turbopack_core::{
......@@ -486,7 +486,7 @@ pub async fn strongly_consistent_catch_collectables<R: VcValueType + Send>(
let result = source_op.read_strongly_consistent().await;
let issues = get_issues(source_op, filter).await?;
let diagnostics = get_diagnostics(source_op).await?;
let effects = Arc::new(get_effects(source_op).await?);
let effects = Arc::new(take_effects(source_op).await?);
let result = if result.is_err() && issues.iter().any(|i| i.severity <= IssueSeverity::Error) {
None
......
This diff is collapsed.
......@@ -232,7 +232,7 @@ pub mod tests {
};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Completion, OperationVc, ReadRef, Vc, apply_effects};
use turbo_tasks::{Completion, Effects, OperationVc, ReadRef, Vc, take_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use crate::{
......@@ -486,10 +486,9 @@ pub mod tests {
}
#[turbo_tasks::function(operation)]
async fn apply_effects_operation(op: OperationVc<()>) -> anyhow::Result<()> {
op.read_strongly_consistent().await?;
apply_effects(op).await?;
Ok(())
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
#[turbo_tasks::function(operation)]
......@@ -559,8 +558,10 @@ pub mod tests {
.await?;
// Delete a file that we shouldn't be tracking
apply_effects_operation(delete(root.join("dir/sub/.vim/.gitignore")?))
extract_effects_operation(delete(root.join("dir/sub/.vim/.gitignore")?))
.read_strongly_consistent()
.await?
.apply()
.await?;
let read_dir2 = track_star_star_glob(dir.clone())
......@@ -569,8 +570,10 @@ pub mod tests {
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
// Delete a file that we should be tracking
apply_effects_operation(delete(root.join("dir/foo")?))
extract_effects_operation(delete(root.join("dir/foo")?))
.read_strongly_consistent()
.await?
.apply()
.await?;
let read_dir2 = track_star_star_glob(dir.clone())
......@@ -580,8 +583,10 @@ pub mod tests {
assert!(!ReadRef::ptr_eq(&read_dir, &read_dir2));
// Modify a symlink target file
apply_effects_operation(write(root.join("link_target.js")?, rcstr!("new_contents")))
extract_effects_operation(write(root.join("link_target.js")?, rcstr!("new_contents")))
.read_strongly_consistent()
.await?
.apply()
.await?;
let read_dir3 = track_star_star_glob(dir.clone())
.read_strongly_consistent()
......
......@@ -15,7 +15,8 @@ use rustc_hash::FxHashSet;
use tokio::time::sleep;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
NonLocalValue, ResolvedVc, TransientInstance, Vc, apply_effects, trace::TraceRawVcs,
Effects, NonLocalValue, OperationVc, ResolvedVc, TransientInstance, Vc, take_effects,
trace::TraceRawVcs,
};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{
......@@ -88,6 +89,12 @@ impl SymlinkMode {
#[derive(Default, NonLocalValue, TraceRawVcs)]
struct PathInvalidations(#[turbo_tasks(trace_ignore)] Arc<Mutex<FxHashSet<RcStr>>>);
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
std::fs::create_dir(&args.fs_root)?;
let fs_root = args.fs_root.canonicalize()?;
......@@ -135,7 +142,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
let symlink_is_directory =
symlink_mode.map(|m| m.to_link_type().contains(LinkType::DIRECTORY));
let initial_op = read_or_write_all_paths_operation(
let effects = extract_effects_operation(read_or_write_all_paths_operation(
invalidations.clone(),
project_root.clone(),
args.depth,
......@@ -143,10 +150,11 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
symlink_count,
symlink_is_directory,
track_writes,
);
initial_op.read_strongly_consistent().await?;
))
.read_strongly_consistent()
.await?;
if track_writes {
apply_effects(initial_op).await?;
effects.apply().await?;
let (total, mismatched) = verify_written_files(
&fs_root,
args.depth,
......@@ -220,7 +228,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
// there's no way to know when we've received all the pending events from the operating
// system, so just sleep and pray
sleep(Duration::from_millis(args.notify_timeout_ms)).await;
let read_or_write_op = read_or_write_all_paths_operation(
let effects = extract_effects_operation(read_or_write_all_paths_operation(
invalidations.clone(),
project_root.clone(),
args.depth,
......@@ -228,15 +236,16 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
symlink_count,
symlink_is_directory,
track_writes,
);
read_or_write_op.read_strongly_consistent().await?;
))
.read_strongly_consistent()
.await?;
let symlink_info = if args.symlinks.is_some() {
" and symlinks"
} else {
""
};
if track_writes {
apply_effects(read_or_write_op).await?;
effects.apply().await?;
let (total, mismatched) = verify_written_files(
&fs_root,
args.depth,
......
......@@ -8,7 +8,7 @@ const PROGRESS_INTERVAL: Duration = Duration::from_secs(1);
use clap::Args;
use rand::{RngExt, SeedableRng};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, apply_effects};
use turbo_tasks::{Effects, OperationVc, ResolvedVc, TryJoinIterExt, Vc, take_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, LinkContent, LinkType};
......@@ -30,6 +30,12 @@ pub struct SymlinkStress {
duration_secs: u64,
}
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
pub async fn run(args: SymlinkStress) -> anyhow::Result<()> {
std::fs::create_dir(&args.fs_root)?;
let fs_root = args.fs_root.canonicalize()?;
......@@ -76,10 +82,15 @@ pub async fn run(args: SymlinkStress) -> anyhow::Result<()> {
println!("creating {symlink_count} initial symlinks...");
let initial_op =
create_initial_symlinks_operation(symlinks_path.clone(), symlink_count, initial_target);
initial_op.read_strongly_consistent().await?;
apply_effects(initial_op).await?;
extract_effects_operation(create_initial_symlinks_operation(
symlinks_path.clone(),
symlink_count,
initial_target,
))
.read_strongly_consistent()
.await?
.apply()
.await?;
println!(
"starting stress test with parallelism={} for {}s...",
......@@ -109,9 +120,14 @@ pub async fn run(args: SymlinkStress) -> anyhow::Result<()> {
.collect();
// Execute writes in parallel via turbo-tasks
let write_op = write_symlinks_batch_operation(symlinks_path.clone(), updates);
write_op.read_strongly_consistent().await?;
apply_effects(write_op).await?;
extract_effects_operation(write_symlinks_batch_operation(
symlinks_path.clone(),
updates,
))
.read_strongly_consistent()
.await?
.apply()
.await?;
total_writes += parallelism as u64;
......
......@@ -3,13 +3,11 @@ use std::{
error::Error as StdError,
future::Future,
mem::replace,
panic,
pin::Pin,
sync::Arc,
};
use anyhow::Result;
use auto_hash_map::AutoSet;
use futures::{StreamExt, TryStreamExt};
use parking_lot::Mutex;
use rustc_hash::{FxHashMap, FxHashSet};
......@@ -20,6 +18,7 @@ use crate::{
self as turbo_tasks, CollectiblesSource, NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt,
emit,
event::{Event, EventListener},
manager::{debug_assert_in_top_level_task, debug_assert_not_in_top_level_task},
spawn,
trace::TraceRawVcs,
};
......@@ -179,7 +178,8 @@ impl EffectInstance {
#[turbo_tasks::value_impl]
impl EffectCollectible for EffectInstance {}
/// Emits an effect to be applied. The effect is executed once `apply_effects` is called.
/// Emits an effect to be applied. The effect is executed once [`Effects::apply`] is called (see
/// [`take_effects`]).
///
/// The effect will only executed once. The effect is executed outside of the current task
/// and can't read any Vcs. These need to be read before. ReadRefs can be passed into the effect.
......@@ -193,75 +193,67 @@ pub fn emit_effect(effect: impl Effect) {
));
}
/// Applies all effects that have been emitted by an operations.
/// Capture effects. Call this from within a [turbo-tasks operation][crate::OperationVc].
///
/// Usually it's important that effects are strongly consistent, so one want to use `apply_effects`
/// only on operations that have been strongly consistently read before.
/// Collectibles are read from `ResolvedVc`s, so this function, and the return value of this
/// function should be applied with [`Effect::apply`].
///
/// The order of execution is not defined and effects are executed in parallel.
///
/// `apply_effects` must only be used in a "once" task. When used in a "root" task, a
/// combination of `get_effects` and `Effects::apply` must be used.
/// It's important to wrap calls to this function in an [operation with a strongly consistent
/// read][crate::OperationVc::read_strongly_consistent] before applying the effects outside of the
/// operation at the top-level (e.g. in a `run_once` closure) with [`Effects::apply`].
///
/// # Example
///
/// ```rust
/// let operation = some_turbo_tasks_function(args);
/// let result = operation.strongly_consistent().await?;
/// apply_effects(operation).await?;
/// ```
pub async fn apply_effects(source: impl CollectiblesSource) -> Result<()> {
let effects: AutoSet<ResolvedVc<Box<dyn EffectCollectible>>> = source.take_collectibles();
if effects.is_empty() {
return Ok(());
}
let span = tracing::info_span!("apply effects", count = effects.len());
APPLY_EFFECTS_CONTEXT
.scope(Default::default(), async move {
// Limit the concurrency of effects
futures::stream::iter(effects)
.map(Ok)
.try_for_each_concurrent(APPLY_EFFECTS_CONCURRENCY_LIMIT, async |effect| {
let Some(effect) = ResolvedVc::try_downcast_type::<EffectInstance>(effect)
else {
panic!("Effect must only be implemented by EffectInstance");
};
effect.await?.apply().await
})
.await
})
.instrument(span)
.await
}
/// Capture effects from an turbo-tasks operation. Since this captures collectibles it might
/// invalidate the current task when effects are changing or even temporary change.
///
/// Therefore it's important to wrap this in a strongly consistent read before applying the effects
/// with `Effects::apply`.
///
/// # Example
/// # #![feature(arbitrary_self_types_pointers)]
/// #
/// # use anyhow::Result;
/// # use turbo_tasks::{Effects, ReadRef, Vc, run_once, take_effects};
/// #
/// # async fn _wrapper() -> Result<()> {
/// # type Example = ();
/// # type Args = ();
/// # let args = ();
/// # #[turbo_tasks::function(operation)]
/// # fn some_turbo_tasks_operation(_args: Args) {}
/// #
/// #[turbo_tasks::value(serialization = "none")]
/// struct OutputWithEffects {
/// output: ReadRef<Example>,
/// effects: Effects,
/// }
///
/// ```rust
/// async fn some_turbo_tasks_function_with_effects(args: Args) -> Result<ResultWithEffects> {
/// let operation = some_turbo_tasks_function(args);
/// let result = operation.strongly_consistent().await?;
/// let effects = get_effects(operation).await?;
/// Ok(ResultWithEffects { result, effects })
/// // ensure the return value and the collectibles match by using a single operation for both
/// #[turbo_tasks::function(operation)]
/// async fn some_turbo_tasks_operation_with_effects(args: Args) -> Result<Vc<OutputWithEffects>> {
/// let operation = some_turbo_tasks_operation(args);
/// // we must first read the operation to populate the collectibles
/// let output = operation.connect().await?;
/// // read the effects from the collectibles
/// let effects = take_effects(operation).await?;
/// Ok(OutputWithEffects { output, effects }.cell())
/// }
///
/// let result_with_effects = some_turbo_tasks_function_with_effects(args).strongly_consistent().await?;
/// // every operation must be read with strong consistency at the top-level
/// let result_with_effects = some_turbo_tasks_operation_with_effects(args)
/// .read_strongly_consistent()
/// .await?;
///
/// // apply the effects once outside of a turbo_tasks::function at the top-level (e.g. `run_once`)
/// result_with_effects.effects.apply().await?;
/// # Ok(())
/// # }
/// ```
pub async fn get_effects(source: impl CollectiblesSource) -> Result<Effects> {
let effects: AutoSet<ResolvedVc<Box<dyn EffectCollectible>>> = source.take_collectibles();
let effects = effects
pub async fn take_effects(source: impl CollectiblesSource) -> Result<Effects> {
debug_assert_not_in_top_level_task("take_effects");
let effects = source
.take_collectibles::<Box<dyn EffectCollectible>>()
.into_iter()
.map(|effect| async move {
.map(|effect| {
if let Some(effect) = ResolvedVc::try_downcast_type::<EffectInstance>(effect) {
Ok(effect.await?)
effect
} else {
panic!("Effect must only be implemented by EffectInstance");
unreachable!("EffectCollectible must only be implemented by EffectInstance");
}
})
.try_join()
......@@ -299,12 +291,27 @@ impl Eq for Effects {}
impl Effects {
/// Applies all effects that have been captured by this struct.
///
/// The order of execution is not defined and effects are executed in parallel.
///
/// `apply` must only be used in a "top-level" task (e.g. [`run_once`][crate::run_once]), after
/// [`take_effects`] is called from an [operation read with strong
/// consistency][crate::OperationVc::read_strongly_consistent].
///
/// See [`take_effects`] for example usage.
pub async fn apply(&self) -> Result<()> {
debug_assert_in_top_level_task(
"Effects::apply must be called from a top-level task to avoid unintended \
re-executions due to eventual consistency",
);
let span = tracing::info_span!("apply effects", count = self.effects.len());
if self.effects.is_empty() {
return Ok(());
}
APPLY_EFFECTS_CONTEXT
.scope(Default::default(), async move {
// Limit the concurrency of effects
futures::stream::iter(self.effects.iter())
futures::stream::iter(&self.effects)
.map(Ok)
.try_for_each_concurrent(APPLY_EFFECTS_CONCURRENCY_LIMIT, async |effect| {
effect.apply().await
......@@ -381,17 +388,22 @@ impl ApplyEffectsContext {
#[cfg(test)]
mod tests {
use crate::{CollectiblesSource, apply_effects, get_effects};
use crate::{CollectiblesSource, Effects, take_effects};
#[test]
#[allow(dead_code)]
fn is_sync_and_send() {
fn assert_sync<T: Sync + Send>(_: T) {}
fn check_apply_effects<T: CollectiblesSource + Send + Sync>(t: T) {
assert_sync(apply_effects(t));
fn check_effects_apply() {
assert_sync(
Effects {
effects: Vec::new(),
}
.apply(),
);
}
fn check_get_effects<T: CollectiblesSource + Send + Sync>(t: T) {
assert_sync(get_effects(t));
fn check_take_effects<T: CollectiblesSource + Send + Sync>(t: T) {
assert_sync(take_effects(t));
}
}
}
......@@ -76,9 +76,7 @@ pub use crate::{
collectibles::CollectiblesSource,
completion::{Completion, Completions},
display::{ValueToString, ValueToStringRef},
effect::{
ApplyEffectsContext, Effect, EffectError, Effects, apply_effects, emit_effect, get_effects,
},
effect::{ApplyEffectsContext, Effect, EffectError, Effects, emit_effect, take_effects},
error::PrettyPrintError,
id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId},
invalidation::{
......
......@@ -684,6 +684,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
TransientTaskType::Root(Box::new(move || {
let functor = functor.clone();
Box::pin(async move {
mark_top_level_task();
let raw_vc = functor().await?.node;
raw_vc.to_non_local().await
})
......@@ -709,6 +710,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
{
let id = self.backend.create_transient_task(
TransientTaskType::Once(Box::pin(async move {
mark_top_level_task();
let raw_vc = future.await?.node;
raw_vc.to_non_local().await
})),
......@@ -1730,8 +1732,24 @@ pub(crate) fn current_task(from: &str) -> TaskId {
}
}
/// Panics if we're not in a top-level task (e.g. [`run_once`]). Some function calls should only
/// happen in a top-level task (e.g. [`Effects::apply`][crate::Effects::apply]).
#[track_caller]
fn debug_assert_not_in_top_level_task(operation: &str) {
pub(crate) fn debug_assert_in_top_level_task(message: &str) {
if !cfg!(debug_assertions) {
return;
}
let in_top_level = CURRENT_TASK_STATE
.try_with(|ts| ts.read().unwrap().in_top_level_task)
.unwrap_or(true);
if !in_top_level {
panic!("{message}");
}
}
#[track_caller]
pub(crate) fn debug_assert_not_in_top_level_task(operation: &str) {
if !cfg!(debug_assertions) {
return;
}
......
......@@ -10,7 +10,10 @@ use either::Either;
use rustc_hash::FxHashSet;
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasks, Vc, apply_effects};
use turbo_tasks::{
Effects, OperationVc, ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasks, Vc,
take_effects,
};
use turbo_tasks_backend::{
BackendOptions, GitVersionInfo, NoopBackingStorage, StartupCacheState, StorageMode,
TurboBackingStorage, TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
......@@ -144,7 +147,7 @@ impl TurbopackBuildBuilder {
pub async fn build(self) -> Result<()> {
self.turbo_tasks
.run_once(async move {
let build_result_op = build_internal(
let wrapper_op = extract_effects_operation(build_internal(
self.project_dir.clone(),
self.root_dir,
self.entry_requests.clone(),
......@@ -153,12 +156,12 @@ impl TurbopackBuildBuilder {
self.minify_type,
self.target,
self.scope_hoist,
);
));
// Await the result to propagate any errors.
build_result_op.read_strongly_consistent().await?;
// Await the result to propagate any errors and capture effects.
let effects = wrapper_op.read_strongly_consistent().await?;
apply_effects(build_result_op).await?;
effects.apply().await?;
let issue_reporter: Vc<Box<dyn IssueReporter>> =
Vc::upcast(ConsoleUi::new(TransientInstance::new(LogOptions {
......@@ -169,14 +172,7 @@ impl TurbopackBuildBuilder {
log_level: self.log_level,
})));
handle_issues(
build_result_op,
issue_reporter,
IssueSeverity::Error,
None,
None,
)
.await?;
handle_issues(wrapper_op, issue_reporter, IssueSeverity::Error, None, None).await?;
Ok(())
})
......@@ -184,6 +180,12 @@ impl TurbopackBuildBuilder {
}
}
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
#[turbo_tasks::function(operation)]
async fn build_internal(
project_dir: RcStr,
......
......@@ -11,8 +11,8 @@ use hyper::{
};
use mime::Mime;
use turbo_tasks::{
CollectiblesSource, OperationVc, ReadRef, ResolvedVc, TransientInstance, Vc, apply_effects,
util::SharedError,
CollectiblesSource, Effects, OperationVc, ReadRef, ResolvedVc, TransientInstance, Vc,
take_effects, util::SharedError,
};
use turbo_tasks_bytes::Bytes;
use turbo_tasks_fs::FileContent;
......@@ -71,8 +71,29 @@ async fn get_from_source_operation(
)
}
/// Processes an HTTP request within a given content source and returns the
/// response.
#[turbo_tasks::value(serialization = "none")]
struct GetFromSourceResultWithCollectibles {
result: ReadRef<GetFromSourceResult>,
effects: Effects,
content_source_side_effects: AutoSet<ResolvedVc<Box<dyn ContentSourceSideEffect>>>,
}
#[turbo_tasks::function(operation)]
async fn get_from_source_with_collectibles_operation(
source_op: OperationVc<Box<dyn ContentSource>>,
request: TransientInstance<SourceRequest>,
) -> Result<Vc<GetFromSourceResultWithCollectibles>> {
let op = get_from_source_operation(source_op, request);
let result = op.read_strongly_consistent().await?;
Ok(GetFromSourceResultWithCollectibles {
result,
effects: take_effects(op).await?,
content_source_side_effects: op.peek_collectibles(),
}
.cell())
}
/// Processes an HTTP request within a given content source and returns the response.
pub async fn process_request_with_content_source(
source: OperationVc<Box<dyn ContentSource>>,
request: Request<hyper::Body>,
......@@ -83,20 +104,23 @@ pub async fn process_request_with_content_source(
)> {
let original_path = request.uri().path().to_string();
let request = http_request_to_source_request(request).await?;
let result_op = get_from_source_operation(source, TransientInstance::new(request));
let resolved_result = result_op.resolve().strongly_consistent().await?;
apply_effects(result_op).await?;
let side_effects: AutoSet<ResolvedVc<Box<dyn ContentSourceSideEffect>>> =
result_op.peek_collectibles();
let wrapper_op =
get_from_source_with_collectibles_operation(source, TransientInstance::new(request));
let GetFromSourceResultWithCollectibles {
result,
effects,
content_source_side_effects,
} = &*wrapper_op.read_strongly_consistent().await?;
effects.apply().await?;
handle_issues(
result_op,
wrapper_op,
issue_reporter,
IssueSeverity::Fatal,
Some(&original_path),
Some("get_from_source_operation"),
)
.await?;
match &*resolved_result.await? {
match &**result {
GetFromSourceResult::Static {
content,
status_code,
......@@ -203,7 +227,7 @@ pub async fn process_request_with_content_source(
response.body(hyper::Body::wrap_stream(stream::iter(owned_chunks)))?
};
return Ok((response, side_effects));
return Ok((response, content_source_side_effects.clone()));
}
}
GetFromSourceResult::HttpProxy(proxy_result) => {
......@@ -219,7 +243,7 @@ pub async fn process_request_with_content_source(
return Ok((
response.body(hyper::Body::wrap_stream(proxy_result.body.read()))?,
side_effects,
content_source_side_effects.clone(),
));
}
GetFromSourceResult::NotFound => {}
......@@ -227,7 +251,7 @@ pub async fn process_request_with_content_source(
Ok((
Response::builder().status(404).body(hyper::Body::empty())?,
side_effects,
content_source_side_effects.clone(),
))
}
......
......@@ -30,8 +30,8 @@ use socket2::{Domain, Protocol, Socket, Type};
use tokio::task::JoinHandle;
use tracing::{Instrument, Level, Span, event, info_span};
use turbo_tasks::{
Effects, NonLocalValue, OperationVc, PrettyPrintError, TurboTasksApi, Vc, get_effects,
run_once_with_reason, trace::TraceRawVcs, util::FormatDuration,
Effects, NonLocalValue, OperationVc, PrettyPrintError, TurboTasksApi, Vc, run_once_with_reason,
take_effects, trace::TraceRawVcs, util::FormatDuration,
};
use turbopack_core::issue::{IssueReporter, IssueSeverity, handle_issues};
......@@ -66,7 +66,7 @@ async fn get_source_with_issues_operation(
source_op: OperationVc<Box<dyn ContentSource>>,
) -> Result<Vc<ContentSourceWithIssues>> {
let _ = source_op.resolve().strongly_consistent().await?;
let effects = get_effects(source_op).await?;
let effects = take_effects(source_op).await?;
Ok(ContentSourceWithIssues { source_op, effects }.cell())
}
......
......@@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value as JsonValue;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ReadRef,
ResolvedVc, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects,
trace::TraceRawVcs,
Completion, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ReadRef, ResolvedVc,
TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, mark_session_dependent,
mark_top_level_task, take_effects, trace::TraceRawVcs,
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{File, FileContent, FileSystemPath, to_sys_path};
......@@ -187,19 +187,29 @@ async fn emit_evaluate_pool_assets_operation(
#[turbo_tasks::value(serialization = "none")]
struct EmittedEvaluatePoolAssetsWithEffects {
assets: ReadRef<EmittedEvaluatePoolAssets>,
effects: Arc<Effects>,
}
#[turbo_tasks::function(operation)]
async fn emit_evaluate_pool_assets_with_effects_operation(
async fn create_evaluate_pool_assets_operation(
entries: ResolvedVc<EvaluateEntries>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>,
) -> Result<Vc<EmittedEvaluatePoolAssetsWithEffects>> {
) -> Result<Vc<EmittedEvaluatePoolAssets>> {
mark_session_dependent();
let operation = emit_evaluate_pool_assets_operation(entries, chunking_context, module_graph);
let assets = operation.read_strongly_consistent().await?;
let effects = Arc::new(get_effects(operation).await?);
Ok(EmittedEvaluatePoolAssetsWithEffects { assets, effects }.cell())
let assets = operation.resolve().strongly_consistent().await?;
let effects = Arc::new(take_effects(operation).await?);
// HACK: `Effects::apply` normally panics if not called at the top-level. We want to apply most
// effects outside of turbo-task functions to avoid re-executing effects during invalidations.
//
// That's not possible here because we lazily create the pool, so instead, use
// `mark_top_level_task` to avoid the debug assertion. The consequence is that these effects
// might get evaluated more than once if this function is invalidated.
mark_top_level_task();
effects.apply().await?;
Ok(*assets)
}
#[derive(
......@@ -224,17 +234,14 @@ pub async fn get_evaluate_pool(
debug: bool,
env_var_tracking: EnvVarTracking,
) -> Result<Vc<EvaluatePool>> {
let operation =
emit_evaluate_pool_assets_with_effects_operation(entries, chunking_context, module_graph);
let EmittedEvaluatePoolAssetsWithEffects { assets, effects } =
&*operation.read_strongly_consistent().await?;
effects.apply().await?;
let assets_op = create_evaluate_pool_assets_operation(entries, chunking_context, module_graph);
let assets = assets_op.read_strongly_consistent().await?;
let EmittedEvaluatePoolAssets {
bootstrap,
output_root,
entrypoint,
} = &**assets;
} = &*assets;
let (Some(cwd), Some(entrypoint)) = (
to_sys_path(cwd.clone()).await?,
......
......@@ -14,8 +14,8 @@ use serde::Deserialize;
use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, NonLocalValue, OperationVc, ResolvedVc, TaskInput, TurboTasks, Vc, apply_effects,
debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs,
Completion, Effects, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TaskInput, TurboTasks,
Vc, debug::ValueDebugFormat, fxindexmap, take_effects, trace::TraceRawVcs,
};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_env::CommandLineProcessEnv;
......@@ -213,28 +213,51 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result<JsRe
noop_backing_storage(),
));
tt.run_once(async move {
let emit_op = run_inner_operation(resource.to_str().unwrap().into(), snapshot_mode);
let result = emit_op.read_strongly_consistent().owned().await?;
apply_effects(emit_op).await?;
#[turbo_tasks::value(serialization = "none")]
struct JsResultWithEffects {
result: ReadRef<JsResult>,
effects: Effects,
}
Ok(result)
})
.await
}
#[turbo_tasks::function(operation)]
async fn run_inner_operation(
resource: RcStr,
snapshot_mode: IssueSnapshotMode,
) -> Result<Vc<JsResult>> {
let prepared_test = prepare_test(resource).to_resolved().await?;
let run_result_op = run_test_operation(prepared_test);
if snapshot_mode == IssueSnapshotMode::Snapshots {
snapshot_issues(*prepared_test, run_result_op).await?;
}
#[turbo_tasks::function(operation)]
async fn run_inner_operation(
resource: RcStr,
snapshot_mode: IssueSnapshotMode,
) -> Result<Vc<JsResult>> {
let prepared_test = prepare_test(resource).to_resolved().await?;
let run_result_op = run_test_operation(prepared_test);
if snapshot_mode == IssueSnapshotMode::Snapshots {
snapshot_issues(*prepared_test, run_result_op).await?;
let result = (*run_result_op.connect().await?.js_result.await?).clone();
Ok(result.cell())
}
/// Wrapper operation that collects all effects (including snapshot issue file writes) from
/// [`run_inner_operation`].
#[turbo_tasks::function(operation)]
async fn run_inner_operation_with_effects(
resource: RcStr,
snapshot_mode: IssueSnapshotMode,
) -> Result<Vc<JsResultWithEffects>> {
let op = run_inner_operation(resource, snapshot_mode);
let result = op.connect().await?.clone();
let effects = take_effects(op).await?;
Ok(JsResultWithEffects { result, effects }.cell())
}
Ok(*run_result_op.connect().await?.js_result)
tt.run_once(async move {
let result_with_effects =
run_inner_operation_with_effects(resource.to_str().unwrap().into(), snapshot_mode)
.read_strongly_consistent()
.await?;
result_with_effects.effects.apply().await?;
Ok((*result_with_effects.result).clone())
})
.await
}
#[derive(
......
......@@ -11,7 +11,7 @@ use rustc_hash::FxHashSet;
use serde::Deserialize;
use serde_json::json;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Effects, OperationVc, ResolvedVc, TurboTasks, Vc, get_effects, turbofmt};
use turbo_tasks::{Effects, OperationVc, ResolvedVc, TurboTasks, Vc, take_effects, turbofmt};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_env::DotenvProcessEnv;
use turbo_tasks_fs::{
......@@ -245,7 +245,7 @@ async fn run(resource: PathBuf) -> Result<()> {
#[turbo_tasks::function(operation)]
async fn extract_effects(op: OperationVc<()>) -> Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(get_effects(op).await?.cell())
Ok(take_effects(op).await?.cell())
}
extract_effects(inner_operation(resource.to_str().unwrap().into()))
......
......@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf};
use criterion::{Bencher, BenchmarkId, Criterion};
use regex::Regex;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{TurboTasks, Vc, apply_effects};
use turbo_tasks::{Effects, OperationVc, TurboTasks, Vc, take_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem};
use turbopack::{
......@@ -22,6 +22,12 @@ use turbopack_core::{
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
// TODO this should move to the `node-file-trace` crate
pub fn benchmark(c: &mut Criterion) {
let bench_filter = Regex::new(r"(empty|simple|dynamic-in-package|react|whatwg-url|axios|azure-cosmos|cowsay|env-var|fast-glob)\.js$").unwrap();
......@@ -123,9 +129,11 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) {
.to_resolved()
.await?;
let emit_op = emit_assets_into_dir_operation(assets, output_dir);
emit_op.read_strongly_consistent().await?;
apply_effects(emit_op).await?;
extract_effects_operation(emit_assets_into_dir_operation(assets, output_dir))
.read_strongly_consistent()
.await?
.apply()
.await?;
Ok(())
})
......
......@@ -28,7 +28,8 @@ use serde::Serialize;
use tokio::{process::Command, time::timeout};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
ResolvedVc, TurboTasks, ValueToString, Vc, apply_effects, backend::Backend, trace::TraceRawVcs,
Effects, ResolvedVc, TurboTasks, ValueToString, Vc, backend::Backend, take_effects,
trace::TraceRawVcs,
};
use turbo_tasks_backend::TurboTasksBackend;
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath};
......@@ -345,12 +346,18 @@ fn bench_against_node_nft_inner(input: CaseInput) {
});
}
#[turbo_tasks::value(serialization = "none")]
struct NodeFileTraceResult {
rebased: ResolvedVc<RebasedAsset>,
effects: Effects,
}
#[turbo_tasks::function(operation)]
async fn node_file_trace_operation(
package_root: RcStr,
input: RcStr,
directory: RcStr,
) -> Result<Vc<RebasedAsset>> {
) -> Result<Vc<NodeFileTraceResult>> {
let workspace_fs: Vc<Box<dyn FileSystem>> = Vc::upcast(DiskFileSystem::new(
rcstr!("workspace"),
package_root.clone(),
......@@ -426,9 +433,9 @@ async fn node_file_trace_operation(
let emit_op = emit_assets_into_dir_operation(assets, output_dir.clone());
emit_op.read_strongly_consistent().await?;
apply_effects(emit_op).await?;
let effects = take_effects(emit_op).await?;
Ok(*rebased)
Ok(NodeFileTraceResult { rebased, effects }.cell())
}
fn node_file_trace<B: Backend + 'static>(
......@@ -480,14 +487,15 @@ fn node_file_trace<B: Backend + 'static>(
let directory = directory.clone();
let task = async move {
let before_start = Instant::now();
let rebased = node_file_trace_operation(
let trace_result = node_file_trace_operation(
package_root.clone(),
input.clone(),
directory.clone(),
)
.resolve()
.strongly_consistent()
.read_strongly_consistent()
.await?;
trace_result.effects.apply().await?;
let rebased = trace_result.rebased;
let duration = before_start.elapsed();
print_graph_operation(ResolvedVc::upcast(rebased))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment