Unverified Commit df6cd599 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Remove `turbo_tasks::apply_effects`, use `Effects::apply` instead (#91858)

Having both `apply_effects` and `Effects::apply` is confusing.

`apply_effect`'s API seems a bit bad/dangerous because it combines work that should be done inside of a strongly-consistent operation (collecting the effects and reading their `Vc`s) with work that should be done at the top-level outside of any operation or turbo-task function (applying the effects).

The first commit in this PR removes the `apply_effect` API and tries to improve documentation, including adding a doctest that compiles.

The second commit in this PR is an LLM-driven refactor to remove the `apply_effect` callsites.
parent b6ff360b
...@@ -3,7 +3,7 @@ use bincode::{Decode, Encode}; ...@@ -3,7 +3,7 @@ use bincode::{Decode, Encode};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
CollectiblesSource, FxIndexMap, NonLocalValue, OperationValue, OperationVc, ResolvedVc, CollectiblesSource, FxIndexMap, NonLocalValue, OperationValue, OperationVc, ResolvedVc,
TaskInput, Vc, debug::ValueDebugFormat, get_effects, trace::TraceRawVcs, TaskInput, Vc, debug::ValueDebugFormat, take_effects, trace::TraceRawVcs,
}; };
use turbopack_core::{diagnostics::Diagnostic, issue::CollectibleIssuesExt}; use turbopack_core::{diagnostics::Diagnostic, issue::CollectibleIssuesExt};
...@@ -40,7 +40,7 @@ async fn entrypoints_without_collectibles_operation( ...@@ -40,7 +40,7 @@ async fn entrypoints_without_collectibles_operation(
let _ = entrypoints.resolve().strongly_consistent().await?; let _ = entrypoints.resolve().strongly_consistent().await?;
entrypoints.drop_collectibles::<Box<dyn Diagnostic>>(); entrypoints.drop_collectibles::<Box<dyn Diagnostic>>();
entrypoints.drop_issues(); entrypoints.drop_issues();
let _ = get_effects(entrypoints).await?; let _ = take_effects(entrypoints).await?;
Ok(entrypoints.connect()) Ok(entrypoints.connect())
} }
......
...@@ -12,7 +12,9 @@ use next_api::{ ...@@ -12,7 +12,9 @@ use next_api::{
route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk}, route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
}; };
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ReadConsistency, ResolvedVc, TransientInstance, TurboTasks, Vc, get_effects}; use turbo_tasks::{
Effects, ReadConsistency, ReadRef, ResolvedVc, TransientInstance, TurboTasks, Vc, take_effects,
};
use turbo_tasks_backend::{NoopBackingStorage, TurboTasksBackend}; use turbo_tasks_backend::{NoopBackingStorage, TurboTasksBackend};
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
...@@ -186,21 +188,21 @@ pub async fn render_routes( ...@@ -186,21 +188,21 @@ pub async fn render_routes(
html_endpoint, html_endpoint,
data_endpoint: _, data_endpoint: _,
} => { } => {
endpoint_write_to_disk_with_effects(*html_endpoint).await?; endpoint_write_to_disk_with_apply(html_endpoint).await?;
} }
Route::PageApi { endpoint } => { Route::PageApi { endpoint } => {
endpoint_write_to_disk_with_effects(*endpoint).await?; endpoint_write_to_disk_with_apply(endpoint).await?;
} }
Route::AppPage(routes) => { Route::AppPage(routes) => {
for route in routes { for route in routes {
endpoint_write_to_disk_with_effects(*route.html_endpoint).await?; endpoint_write_to_disk_with_apply(route.html_endpoint).await?;
} }
} }
Route::AppRoute { Route::AppRoute {
original_name: _, original_name: _,
endpoint, endpoint,
} => { } => {
endpoint_write_to_disk_with_effects(*endpoint).await?; endpoint_write_to_disk_with_apply(endpoint).await?;
} }
Route::Conflict => { Route::Conflict => {
tracing::info!("WARN: conflict {}", name); tracing::info!("WARN: conflict {}", name);
...@@ -243,21 +245,43 @@ pub async fn render_routes( ...@@ -243,21 +245,43 @@ pub async fn render_routes(
Ok(stream.len()) Ok(stream.len())
} }
#[turbo_tasks::function] async fn endpoint_write_to_disk_with_apply(
async fn endpoint_write_to_disk_with_effects(
endpoint: ResolvedVc<Box<dyn Endpoint>>, endpoint: ResolvedVc<Box<dyn Endpoint>>,
) -> Result<Vc<EndpointOutputPaths>> { ) -> Result<ReadRef<EndpointOutputPaths>> {
let op = endpoint_write_to_disk_operation(endpoint); #[turbo_tasks::function(operation)]
let result = op.resolve().strongly_consistent().await?; fn inner_operation(endpoint: ResolvedVc<Box<dyn Endpoint>>) -> Vc<EndpointOutputPaths> {
get_effects(op).await?.apply().await?; // we must wrap this in an operation so we can get the Effects collectibles
Ok(*result) endpoint_write_to_disk(*endpoint)
} }
#[turbo_tasks::function(operation)] #[turbo_tasks::value(serialization = "none")]
pub fn endpoint_write_to_disk_operation( struct WithEffects {
endpoint: ResolvedVc<Box<dyn Endpoint>>, output_paths: ReadRef<EndpointOutputPaths>,
) -> Vc<EndpointOutputPaths> { effects: Effects,
endpoint_write_to_disk(*endpoint) }
#[turbo_tasks::function(operation)]
pub async fn inner_operation_with_effects(
endpoint: ResolvedVc<Box<dyn Endpoint>>,
) -> Result<Vc<WithEffects>> {
let op = inner_operation(endpoint);
let output_paths = op.read_strongly_consistent().await?;
let effects = take_effects(op).await?;
Ok(WithEffects {
output_paths,
effects,
}
.cell())
}
let op = inner_operation_with_effects(endpoint);
let WithEffects {
output_paths,
effects,
} = &*op.read_strongly_consistent().await?;
effects.apply().await?;
Ok(output_paths.clone())
} }
async fn hmr( async fn hmr(
......
...@@ -49,9 +49,11 @@ use turbo_rcstr::{RcStr, rcstr}; ...@@ -49,9 +49,11 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Effects, FxIndexSet, NonLocalValue, OperationValue, OperationVc, PrettyPrintError, ReadRef, Effects, FxIndexSet, NonLocalValue, OperationValue, OperationVc, PrettyPrintError, ReadRef,
ResolvedVc, TaskInput, TransientInstance, TryJoinIterExt, TurboTasksApi, TurboTasksCallApi, ResolvedVc, TaskInput, TransientInstance, TryJoinIterExt, TurboTasksApi, TurboTasksCallApi,
UpdateInfo, Vc, get_effects, UpdateInfo, Vc, mark_top_level_task,
message_queue::{CompilationEvent, Severity}, message_queue::{CompilationEvent, Severity},
take_effects,
trace::TraceRawVcs, trace::TraceRawVcs,
unmark_top_level_task_may_leak_eventually_consistent_state,
}; };
use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons}; use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
...@@ -1840,7 +1842,7 @@ async fn hmr_update_with_issues_operation( ...@@ -1840,7 +1842,7 @@ async fn hmr_update_with_issues_operation(
let filter = project.issue_filter(); let filter = project.issue_filter();
let issues = get_issues(update_op, filter).await?; let issues = get_issues(update_op, filter).await?;
let diagnostics = get_diagnostics(update_op).await?; let diagnostics = get_diagnostics(update_op).await?;
let effects = Arc::new(get_effects(update_op).await?); let effects = Arc::new(take_effects(update_op).await?);
Ok(HmrUpdateWithIssues { Ok(HmrUpdateWithIssues {
update, update,
issues, issues,
...@@ -1874,6 +1876,8 @@ pub fn project_hmr_events( ...@@ -1874,6 +1876,8 @@ pub fn project_hmr_events(
let chunk_name: RcStr = outer_chunk_name.clone(); let chunk_name: RcStr = outer_chunk_name.clone();
let session = session.clone(); let session = session.clone();
async move { async move {
// HACK(bgw): Remove this unmark call
unmark_top_level_task_may_leak_eventually_consistent_state();
let project = container.project().to_resolved().await?; let project = container.project().to_resolved().await?;
let state = project let state = project
.hmr_version_state(chunk_name.clone(), hmr_target, session) .hmr_version_state(chunk_name.clone(), hmr_target, session)
...@@ -1893,7 +1897,11 @@ pub fn project_hmr_events( ...@@ -1893,7 +1897,11 @@ pub fn project_hmr_events(
diagnostics, diagnostics,
effects, effects,
} = &*update; } = &*update;
// HACK(bgw): Remove this mark call
mark_top_level_task();
effects.apply().await?; effects.apply().await?;
// HACK(bgw): Remove this unmark call
unmark_top_level_task_may_leak_eventually_consistent_state();
match &**update { match &**update {
Update::Missing | Update::None => {} Update::Missing | Update::None => {}
Update::Total(TotalUpdate { to }) => { Update::Total(TotalUpdate { to }) => {
...@@ -1975,7 +1983,7 @@ async fn get_hmr_chunk_names_with_issues_operation( ...@@ -1975,7 +1983,7 @@ async fn get_hmr_chunk_names_with_issues_operation(
let filter = issue_filter_from_container(container); let filter = issue_filter_from_container(container);
let issues = get_issues(hmr_chunk_names_op, filter).await?; let issues = get_issues(hmr_chunk_names_op, filter).await?;
let diagnostics = get_diagnostics(hmr_chunk_names_op).await?; let diagnostics = get_diagnostics(hmr_chunk_names_op).await?;
let effects = Arc::new(get_effects(hmr_chunk_names_op).await?); let effects = Arc::new(take_effects(hmr_chunk_names_op).await?);
Ok(HmrChunkNamesWithIssues { Ok(HmrChunkNamesWithIssues {
chunk_names: hmr_chunk_names, chunk_names: hmr_chunk_names,
issues, issues,
......
...@@ -15,7 +15,7 @@ use rustc_hash::FxHashMap; ...@@ -15,7 +15,7 @@ use rustc_hash::FxHashMap;
use serde::Serialize; use serde::Serialize;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
Effects, OperationVc, ReadRef, TaskId, TryJoinIterExt, Vc, VcValueType, get_effects, Effects, OperationVc, ReadRef, TaskId, TryJoinIterExt, Vc, VcValueType, take_effects,
}; };
use turbo_tasks_fs::FileContent; use turbo_tasks_fs::FileContent;
use turbopack_core::{ use turbopack_core::{
...@@ -486,7 +486,7 @@ pub async fn strongly_consistent_catch_collectables<R: VcValueType + Send>( ...@@ -486,7 +486,7 @@ pub async fn strongly_consistent_catch_collectables<R: VcValueType + Send>(
let result = source_op.read_strongly_consistent().await; let result = source_op.read_strongly_consistent().await;
let issues = get_issues(source_op, filter).await?; let issues = get_issues(source_op, filter).await?;
let diagnostics = get_diagnostics(source_op).await?; let diagnostics = get_diagnostics(source_op).await?;
let effects = Arc::new(get_effects(source_op).await?); let effects = Arc::new(take_effects(source_op).await?);
let result = if result.is_err() && issues.iter().any(|i| i.severity <= IssueSeverity::Error) { let result = if result.is_err() && issues.iter().any(|i| i.severity <= IssueSeverity::Error) {
None None
......
This diff is collapsed.
...@@ -232,7 +232,7 @@ pub mod tests { ...@@ -232,7 +232,7 @@ pub mod tests {
}; };
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Completion, OperationVc, ReadRef, Vc, apply_effects}; use turbo_tasks::{Completion, Effects, OperationVc, ReadRef, Vc, take_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use crate::{ use crate::{
...@@ -486,10 +486,9 @@ pub mod tests { ...@@ -486,10 +486,9 @@ pub mod tests {
} }
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn apply_effects_operation(op: OperationVc<()>) -> anyhow::Result<()> { async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
op.read_strongly_consistent().await?; let _ = op.resolve().strongly_consistent().await?;
apply_effects(op).await?; Ok(take_effects(op).await?.cell())
Ok(())
} }
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
...@@ -559,8 +558,10 @@ pub mod tests { ...@@ -559,8 +558,10 @@ pub mod tests {
.await?; .await?;
// Delete a file that we shouldn't be tracking // Delete a file that we shouldn't be tracking
apply_effects_operation(delete(root.join("dir/sub/.vim/.gitignore")?)) extract_effects_operation(delete(root.join("dir/sub/.vim/.gitignore")?))
.read_strongly_consistent() .read_strongly_consistent()
.await?
.apply()
.await?; .await?;
let read_dir2 = track_star_star_glob(dir.clone()) let read_dir2 = track_star_star_glob(dir.clone())
...@@ -569,8 +570,10 @@ pub mod tests { ...@@ -569,8 +570,10 @@ pub mod tests {
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2)); assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
// Delete a file that we should be tracking // Delete a file that we should be tracking
apply_effects_operation(delete(root.join("dir/foo")?)) extract_effects_operation(delete(root.join("dir/foo")?))
.read_strongly_consistent() .read_strongly_consistent()
.await?
.apply()
.await?; .await?;
let read_dir2 = track_star_star_glob(dir.clone()) let read_dir2 = track_star_star_glob(dir.clone())
...@@ -580,8 +583,10 @@ pub mod tests { ...@@ -580,8 +583,10 @@ pub mod tests {
assert!(!ReadRef::ptr_eq(&read_dir, &read_dir2)); assert!(!ReadRef::ptr_eq(&read_dir, &read_dir2));
// Modify a symlink target file // Modify a symlink target file
apply_effects_operation(write(root.join("link_target.js")?, rcstr!("new_contents"))) extract_effects_operation(write(root.join("link_target.js")?, rcstr!("new_contents")))
.read_strongly_consistent() .read_strongly_consistent()
.await?
.apply()
.await?; .await?;
let read_dir3 = track_star_star_glob(dir.clone()) let read_dir3 = track_star_star_glob(dir.clone())
.read_strongly_consistent() .read_strongly_consistent()
......
...@@ -15,7 +15,8 @@ use rustc_hash::FxHashSet; ...@@ -15,7 +15,8 @@ use rustc_hash::FxHashSet;
use tokio::time::sleep; use tokio::time::sleep;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
NonLocalValue, ResolvedVc, TransientInstance, Vc, apply_effects, trace::TraceRawVcs, Effects, NonLocalValue, OperationVc, ResolvedVc, TransientInstance, Vc, take_effects,
trace::TraceRawVcs,
}; };
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
...@@ -88,6 +89,12 @@ impl SymlinkMode { ...@@ -88,6 +89,12 @@ impl SymlinkMode {
#[derive(Default, NonLocalValue, TraceRawVcs)] #[derive(Default, NonLocalValue, TraceRawVcs)]
struct PathInvalidations(#[turbo_tasks(trace_ignore)] Arc<Mutex<FxHashSet<RcStr>>>); struct PathInvalidations(#[turbo_tasks(trace_ignore)] Arc<Mutex<FxHashSet<RcStr>>>);
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
pub async fn run(args: FsWatcher) -> anyhow::Result<()> { pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
std::fs::create_dir(&args.fs_root)?; std::fs::create_dir(&args.fs_root)?;
let fs_root = args.fs_root.canonicalize()?; let fs_root = args.fs_root.canonicalize()?;
...@@ -135,7 +142,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> { ...@@ -135,7 +142,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
let symlink_is_directory = let symlink_is_directory =
symlink_mode.map(|m| m.to_link_type().contains(LinkType::DIRECTORY)); symlink_mode.map(|m| m.to_link_type().contains(LinkType::DIRECTORY));
let initial_op = read_or_write_all_paths_operation( let effects = extract_effects_operation(read_or_write_all_paths_operation(
invalidations.clone(), invalidations.clone(),
project_root.clone(), project_root.clone(),
args.depth, args.depth,
...@@ -143,10 +150,11 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> { ...@@ -143,10 +150,11 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
symlink_count, symlink_count,
symlink_is_directory, symlink_is_directory,
track_writes, track_writes,
); ))
initial_op.read_strongly_consistent().await?; .read_strongly_consistent()
.await?;
if track_writes { if track_writes {
apply_effects(initial_op).await?; effects.apply().await?;
let (total, mismatched) = verify_written_files( let (total, mismatched) = verify_written_files(
&fs_root, &fs_root,
args.depth, args.depth,
...@@ -220,7 +228,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> { ...@@ -220,7 +228,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
// there's no way to know when we've received all the pending events from the operating // there's no way to know when we've received all the pending events from the operating
// system, so just sleep and pray // system, so just sleep and pray
sleep(Duration::from_millis(args.notify_timeout_ms)).await; sleep(Duration::from_millis(args.notify_timeout_ms)).await;
let read_or_write_op = read_or_write_all_paths_operation( let effects = extract_effects_operation(read_or_write_all_paths_operation(
invalidations.clone(), invalidations.clone(),
project_root.clone(), project_root.clone(),
args.depth, args.depth,
...@@ -228,15 +236,16 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> { ...@@ -228,15 +236,16 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
symlink_count, symlink_count,
symlink_is_directory, symlink_is_directory,
track_writes, track_writes,
); ))
read_or_write_op.read_strongly_consistent().await?; .read_strongly_consistent()
.await?;
let symlink_info = if args.symlinks.is_some() { let symlink_info = if args.symlinks.is_some() {
" and symlinks" " and symlinks"
} else { } else {
"" ""
}; };
if track_writes { if track_writes {
apply_effects(read_or_write_op).await?; effects.apply().await?;
let (total, mismatched) = verify_written_files( let (total, mismatched) = verify_written_files(
&fs_root, &fs_root,
args.depth, args.depth,
......
...@@ -8,7 +8,7 @@ const PROGRESS_INTERVAL: Duration = Duration::from_secs(1); ...@@ -8,7 +8,7 @@ const PROGRESS_INTERVAL: Duration = Duration::from_secs(1);
use clap::Args; use clap::Args;
use rand::{RngExt, SeedableRng}; use rand::{RngExt, SeedableRng};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, apply_effects}; use turbo_tasks::{Effects, OperationVc, ResolvedVc, TryJoinIterExt, Vc, take_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, LinkContent, LinkType}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, LinkContent, LinkType};
...@@ -30,6 +30,12 @@ pub struct SymlinkStress { ...@@ -30,6 +30,12 @@ pub struct SymlinkStress {
duration_secs: u64, duration_secs: u64,
} }
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
pub async fn run(args: SymlinkStress) -> anyhow::Result<()> { pub async fn run(args: SymlinkStress) -> anyhow::Result<()> {
std::fs::create_dir(&args.fs_root)?; std::fs::create_dir(&args.fs_root)?;
let fs_root = args.fs_root.canonicalize()?; let fs_root = args.fs_root.canonicalize()?;
...@@ -76,10 +82,15 @@ pub async fn run(args: SymlinkStress) -> anyhow::Result<()> { ...@@ -76,10 +82,15 @@ pub async fn run(args: SymlinkStress) -> anyhow::Result<()> {
println!("creating {symlink_count} initial symlinks..."); println!("creating {symlink_count} initial symlinks...");
let initial_op = extract_effects_operation(create_initial_symlinks_operation(
create_initial_symlinks_operation(symlinks_path.clone(), symlink_count, initial_target); symlinks_path.clone(),
initial_op.read_strongly_consistent().await?; symlink_count,
apply_effects(initial_op).await?; initial_target,
))
.read_strongly_consistent()
.await?
.apply()
.await?;
println!( println!(
"starting stress test with parallelism={} for {}s...", "starting stress test with parallelism={} for {}s...",
...@@ -109,9 +120,14 @@ pub async fn run(args: SymlinkStress) -> anyhow::Result<()> { ...@@ -109,9 +120,14 @@ pub async fn run(args: SymlinkStress) -> anyhow::Result<()> {
.collect(); .collect();
// Execute writes in parallel via turbo-tasks // Execute writes in parallel via turbo-tasks
let write_op = write_symlinks_batch_operation(symlinks_path.clone(), updates); extract_effects_operation(write_symlinks_batch_operation(
write_op.read_strongly_consistent().await?; symlinks_path.clone(),
apply_effects(write_op).await?; updates,
))
.read_strongly_consistent()
.await?
.apply()
.await?;
total_writes += parallelism as u64; total_writes += parallelism as u64;
......
...@@ -3,13 +3,11 @@ use std::{ ...@@ -3,13 +3,11 @@ use std::{
error::Error as StdError, error::Error as StdError,
future::Future, future::Future,
mem::replace, mem::replace,
panic,
pin::Pin, pin::Pin,
sync::Arc, sync::Arc,
}; };
use anyhow::Result; use anyhow::Result;
use auto_hash_map::AutoSet;
use futures::{StreamExt, TryStreamExt}; use futures::{StreamExt, TryStreamExt};
use parking_lot::Mutex; use parking_lot::Mutex;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
...@@ -20,6 +18,7 @@ use crate::{ ...@@ -20,6 +18,7 @@ use crate::{
self as turbo_tasks, CollectiblesSource, NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt, self as turbo_tasks, CollectiblesSource, NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt,
emit, emit,
event::{Event, EventListener}, event::{Event, EventListener},
manager::{debug_assert_in_top_level_task, debug_assert_not_in_top_level_task},
spawn, spawn,
trace::TraceRawVcs, trace::TraceRawVcs,
}; };
...@@ -179,7 +178,8 @@ impl EffectInstance { ...@@ -179,7 +178,8 @@ impl EffectInstance {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl EffectCollectible for EffectInstance {} impl EffectCollectible for EffectInstance {}
/// Emits an effect to be applied. The effect is executed once `apply_effects` is called. /// Emits an effect to be applied. The effect is executed once [`Effects::apply`] is called (see
/// [`take_effects`]).
/// ///
/// The effect will only executed once. The effect is executed outside of the current task /// The effect will only executed once. The effect is executed outside of the current task
/// and can't read any Vcs. These need to be read before. ReadRefs can be passed into the effect. /// and can't read any Vcs. These need to be read before. ReadRefs can be passed into the effect.
...@@ -193,75 +193,67 @@ pub fn emit_effect(effect: impl Effect) { ...@@ -193,75 +193,67 @@ pub fn emit_effect(effect: impl Effect) {
)); ));
} }
/// Applies all effects that have been emitted by an operations. /// Capture effects. Call this from within a [turbo-tasks operation][crate::OperationVc].
/// ///
/// Usually it's important that effects are strongly consistent, so one want to use `apply_effects` /// Collectibles are read from `ResolvedVc`s, so this function, and the return value of this
/// only on operations that have been strongly consistently read before. /// function should be applied with [`Effect::apply`].
/// ///
/// The order of execution is not defined and effects are executed in parallel. /// It's important to wrap calls to this function in an [operation with a strongly consistent
/// /// read][crate::OperationVc::read_strongly_consistent] before applying the effects outside of the
/// `apply_effects` must only be used in a "once" task. When used in a "root" task, a /// operation at the top-level (e.g. in a `run_once` closure) with [`Effects::apply`].
/// combination of `get_effects` and `Effects::apply` must be used.
/// ///
/// # Example /// # Example
/// ///
/// ```rust /// ```rust
/// let operation = some_turbo_tasks_function(args); /// # #![feature(arbitrary_self_types_pointers)]
/// let result = operation.strongly_consistent().await?; /// #
/// apply_effects(operation).await?; /// # use anyhow::Result;
/// ``` /// # use turbo_tasks::{Effects, ReadRef, Vc, run_once, take_effects};
pub async fn apply_effects(source: impl CollectiblesSource) -> Result<()> { /// #
let effects: AutoSet<ResolvedVc<Box<dyn EffectCollectible>>> = source.take_collectibles(); /// # async fn _wrapper() -> Result<()> {
if effects.is_empty() { /// # type Example = ();
return Ok(()); /// # type Args = ();
} /// # let args = ();
let span = tracing::info_span!("apply effects", count = effects.len()); /// # #[turbo_tasks::function(operation)]
APPLY_EFFECTS_CONTEXT /// # fn some_turbo_tasks_operation(_args: Args) {}
.scope(Default::default(), async move { /// #
// Limit the concurrency of effects /// #[turbo_tasks::value(serialization = "none")]
futures::stream::iter(effects) /// struct OutputWithEffects {
.map(Ok) /// output: ReadRef<Example>,
.try_for_each_concurrent(APPLY_EFFECTS_CONCURRENCY_LIMIT, async |effect| { /// effects: Effects,
let Some(effect) = ResolvedVc::try_downcast_type::<EffectInstance>(effect) /// }
else {
panic!("Effect must only be implemented by EffectInstance");
};
effect.await?.apply().await
})
.await
})
.instrument(span)
.await
}
/// Capture effects from an turbo-tasks operation. Since this captures collectibles it might
/// invalidate the current task when effects are changing or even temporary change.
///
/// Therefore it's important to wrap this in a strongly consistent read before applying the effects
/// with `Effects::apply`.
///
/// # Example
/// ///
/// ```rust /// // ensure the return value and the collectibles match by using a single operation for both
/// async fn some_turbo_tasks_function_with_effects(args: Args) -> Result<ResultWithEffects> { /// #[turbo_tasks::function(operation)]
/// let operation = some_turbo_tasks_function(args); /// async fn some_turbo_tasks_operation_with_effects(args: Args) -> Result<Vc<OutputWithEffects>> {
/// let result = operation.strongly_consistent().await?; /// let operation = some_turbo_tasks_operation(args);
/// let effects = get_effects(operation).await?; /// // we must first read the operation to populate the collectibles
/// Ok(ResultWithEffects { result, effects }) /// let output = operation.connect().await?;
/// // read the effects from the collectibles
/// let effects = take_effects(operation).await?;
/// Ok(OutputWithEffects { output, effects }.cell())
/// } /// }
/// ///
/// let result_with_effects = some_turbo_tasks_function_with_effects(args).strongly_consistent().await?; /// // every operation must be read with strong consistency at the top-level
/// let result_with_effects = some_turbo_tasks_operation_with_effects(args)
/// .read_strongly_consistent()
/// .await?;
///
/// // apply the effects once outside of a turbo_tasks::function at the top-level (e.g. `run_once`)
/// result_with_effects.effects.apply().await?; /// result_with_effects.effects.apply().await?;
/// # Ok(())
/// # }
/// ``` /// ```
pub async fn get_effects(source: impl CollectiblesSource) -> Result<Effects> { pub async fn take_effects(source: impl CollectiblesSource) -> Result<Effects> {
let effects: AutoSet<ResolvedVc<Box<dyn EffectCollectible>>> = source.take_collectibles(); debug_assert_not_in_top_level_task("take_effects");
let effects = effects let effects = source
.take_collectibles::<Box<dyn EffectCollectible>>()
.into_iter() .into_iter()
.map(|effect| async move { .map(|effect| {
if let Some(effect) = ResolvedVc::try_downcast_type::<EffectInstance>(effect) { if let Some(effect) = ResolvedVc::try_downcast_type::<EffectInstance>(effect) {
Ok(effect.await?) effect
} else { } else {
panic!("Effect must only be implemented by EffectInstance"); unreachable!("EffectCollectible must only be implemented by EffectInstance");
} }
}) })
.try_join() .try_join()
...@@ -299,12 +291,27 @@ impl Eq for Effects {} ...@@ -299,12 +291,27 @@ impl Eq for Effects {}
impl Effects { impl Effects {
/// Applies all effects that have been captured by this struct. /// Applies all effects that have been captured by this struct.
///
/// The order of execution is not defined and effects are executed in parallel.
///
/// `apply` must only be used in a "top-level" task (e.g. [`run_once`][crate::run_once]), after
/// [`take_effects`] is called from an [operation read with strong
/// consistency][crate::OperationVc::read_strongly_consistent].
///
/// See [`take_effects`] for example usage.
pub async fn apply(&self) -> Result<()> { pub async fn apply(&self) -> Result<()> {
debug_assert_in_top_level_task(
"Effects::apply must be called from a top-level task to avoid unintended \
re-executions due to eventual consistency",
);
let span = tracing::info_span!("apply effects", count = self.effects.len()); let span = tracing::info_span!("apply effects", count = self.effects.len());
if self.effects.is_empty() {
return Ok(());
}
APPLY_EFFECTS_CONTEXT APPLY_EFFECTS_CONTEXT
.scope(Default::default(), async move { .scope(Default::default(), async move {
// Limit the concurrency of effects // Limit the concurrency of effects
futures::stream::iter(self.effects.iter()) futures::stream::iter(&self.effects)
.map(Ok) .map(Ok)
.try_for_each_concurrent(APPLY_EFFECTS_CONCURRENCY_LIMIT, async |effect| { .try_for_each_concurrent(APPLY_EFFECTS_CONCURRENCY_LIMIT, async |effect| {
effect.apply().await effect.apply().await
...@@ -381,17 +388,22 @@ impl ApplyEffectsContext { ...@@ -381,17 +388,22 @@ impl ApplyEffectsContext {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::{CollectiblesSource, apply_effects, get_effects}; use crate::{CollectiblesSource, Effects, take_effects};
#[test] #[test]
#[allow(dead_code)] #[allow(dead_code)]
fn is_sync_and_send() { fn is_sync_and_send() {
fn assert_sync<T: Sync + Send>(_: T) {} fn assert_sync<T: Sync + Send>(_: T) {}
fn check_apply_effects<T: CollectiblesSource + Send + Sync>(t: T) { fn check_effects_apply() {
assert_sync(apply_effects(t)); assert_sync(
Effects {
effects: Vec::new(),
}
.apply(),
);
} }
fn check_get_effects<T: CollectiblesSource + Send + Sync>(t: T) { fn check_take_effects<T: CollectiblesSource + Send + Sync>(t: T) {
assert_sync(get_effects(t)); assert_sync(take_effects(t));
} }
} }
} }
...@@ -76,9 +76,7 @@ pub use crate::{ ...@@ -76,9 +76,7 @@ pub use crate::{
collectibles::CollectiblesSource, collectibles::CollectiblesSource,
completion::{Completion, Completions}, completion::{Completion, Completions},
display::{ValueToString, ValueToStringRef}, display::{ValueToString, ValueToStringRef},
effect::{ effect::{ApplyEffectsContext, Effect, EffectError, Effects, emit_effect, take_effects},
ApplyEffectsContext, Effect, EffectError, Effects, apply_effects, emit_effect, get_effects,
},
error::PrettyPrintError, error::PrettyPrintError,
id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId}, id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId},
invalidation::{ invalidation::{
......
...@@ -684,6 +684,7 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -684,6 +684,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
TransientTaskType::Root(Box::new(move || { TransientTaskType::Root(Box::new(move || {
let functor = functor.clone(); let functor = functor.clone();
Box::pin(async move { Box::pin(async move {
mark_top_level_task();
let raw_vc = functor().await?.node; let raw_vc = functor().await?.node;
raw_vc.to_non_local().await raw_vc.to_non_local().await
}) })
...@@ -709,6 +710,7 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -709,6 +710,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
{ {
let id = self.backend.create_transient_task( let id = self.backend.create_transient_task(
TransientTaskType::Once(Box::pin(async move { TransientTaskType::Once(Box::pin(async move {
mark_top_level_task();
let raw_vc = future.await?.node; let raw_vc = future.await?.node;
raw_vc.to_non_local().await raw_vc.to_non_local().await
})), })),
...@@ -1730,8 +1732,24 @@ pub(crate) fn current_task(from: &str) -> TaskId { ...@@ -1730,8 +1732,24 @@ pub(crate) fn current_task(from: &str) -> TaskId {
} }
} }
/// Panics if we're not in a top-level task (e.g. [`run_once`]). Some function calls should only
/// happen in a top-level task (e.g. [`Effects::apply`][crate::Effects::apply]).
#[track_caller] #[track_caller]
fn debug_assert_not_in_top_level_task(operation: &str) { pub(crate) fn debug_assert_in_top_level_task(message: &str) {
if !cfg!(debug_assertions) {
return;
}
let in_top_level = CURRENT_TASK_STATE
.try_with(|ts| ts.read().unwrap().in_top_level_task)
.unwrap_or(true);
if !in_top_level {
panic!("{message}");
}
}
#[track_caller]
pub(crate) fn debug_assert_not_in_top_level_task(operation: &str) {
if !cfg!(debug_assertions) { if !cfg!(debug_assertions) {
return; return;
} }
......
...@@ -10,7 +10,10 @@ use either::Either; ...@@ -10,7 +10,10 @@ use either::Either;
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasks, Vc, apply_effects}; use turbo_tasks::{
Effects, OperationVc, ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasks, Vc,
take_effects,
};
use turbo_tasks_backend::{ use turbo_tasks_backend::{
BackendOptions, GitVersionInfo, NoopBackingStorage, StartupCacheState, StorageMode, BackendOptions, GitVersionInfo, NoopBackingStorage, StartupCacheState, StorageMode,
TurboBackingStorage, TurboTasksBackend, noop_backing_storage, turbo_backing_storage, TurboBackingStorage, TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
...@@ -144,7 +147,7 @@ impl TurbopackBuildBuilder { ...@@ -144,7 +147,7 @@ impl TurbopackBuildBuilder {
pub async fn build(self) -> Result<()> { pub async fn build(self) -> Result<()> {
self.turbo_tasks self.turbo_tasks
.run_once(async move { .run_once(async move {
let build_result_op = build_internal( let wrapper_op = extract_effects_operation(build_internal(
self.project_dir.clone(), self.project_dir.clone(),
self.root_dir, self.root_dir,
self.entry_requests.clone(), self.entry_requests.clone(),
...@@ -153,12 +156,12 @@ impl TurbopackBuildBuilder { ...@@ -153,12 +156,12 @@ impl TurbopackBuildBuilder {
self.minify_type, self.minify_type,
self.target, self.target,
self.scope_hoist, self.scope_hoist,
); ));
// Await the result to propagate any errors. // Await the result to propagate any errors and capture effects.
build_result_op.read_strongly_consistent().await?; let effects = wrapper_op.read_strongly_consistent().await?;
apply_effects(build_result_op).await?; effects.apply().await?;
let issue_reporter: Vc<Box<dyn IssueReporter>> = let issue_reporter: Vc<Box<dyn IssueReporter>> =
Vc::upcast(ConsoleUi::new(TransientInstance::new(LogOptions { Vc::upcast(ConsoleUi::new(TransientInstance::new(LogOptions {
...@@ -169,14 +172,7 @@ impl TurbopackBuildBuilder { ...@@ -169,14 +172,7 @@ impl TurbopackBuildBuilder {
log_level: self.log_level, log_level: self.log_level,
}))); })));
handle_issues( handle_issues(wrapper_op, issue_reporter, IssueSeverity::Error, None, None).await?;
build_result_op,
issue_reporter,
IssueSeverity::Error,
None,
None,
)
.await?;
Ok(()) Ok(())
}) })
...@@ -184,6 +180,12 @@ impl TurbopackBuildBuilder { ...@@ -184,6 +180,12 @@ impl TurbopackBuildBuilder {
} }
} }
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn build_internal( async fn build_internal(
project_dir: RcStr, project_dir: RcStr,
......
...@@ -11,8 +11,8 @@ use hyper::{ ...@@ -11,8 +11,8 @@ use hyper::{
}; };
use mime::Mime; use mime::Mime;
use turbo_tasks::{ use turbo_tasks::{
CollectiblesSource, OperationVc, ReadRef, ResolvedVc, TransientInstance, Vc, apply_effects, CollectiblesSource, Effects, OperationVc, ReadRef, ResolvedVc, TransientInstance, Vc,
util::SharedError, take_effects, util::SharedError,
}; };
use turbo_tasks_bytes::Bytes; use turbo_tasks_bytes::Bytes;
use turbo_tasks_fs::FileContent; use turbo_tasks_fs::FileContent;
...@@ -71,8 +71,29 @@ async fn get_from_source_operation( ...@@ -71,8 +71,29 @@ async fn get_from_source_operation(
) )
} }
/// Processes an HTTP request within a given content source and returns the #[turbo_tasks::value(serialization = "none")]
/// response. struct GetFromSourceResultWithCollectibles {
result: ReadRef<GetFromSourceResult>,
effects: Effects,
content_source_side_effects: AutoSet<ResolvedVc<Box<dyn ContentSourceSideEffect>>>,
}
#[turbo_tasks::function(operation)]
async fn get_from_source_with_collectibles_operation(
source_op: OperationVc<Box<dyn ContentSource>>,
request: TransientInstance<SourceRequest>,
) -> Result<Vc<GetFromSourceResultWithCollectibles>> {
let op = get_from_source_operation(source_op, request);
let result = op.read_strongly_consistent().await?;
Ok(GetFromSourceResultWithCollectibles {
result,
effects: take_effects(op).await?,
content_source_side_effects: op.peek_collectibles(),
}
.cell())
}
/// Processes an HTTP request within a given content source and returns the response.
pub async fn process_request_with_content_source( pub async fn process_request_with_content_source(
source: OperationVc<Box<dyn ContentSource>>, source: OperationVc<Box<dyn ContentSource>>,
request: Request<hyper::Body>, request: Request<hyper::Body>,
...@@ -83,20 +104,23 @@ pub async fn process_request_with_content_source( ...@@ -83,20 +104,23 @@ pub async fn process_request_with_content_source(
)> { )> {
let original_path = request.uri().path().to_string(); let original_path = request.uri().path().to_string();
let request = http_request_to_source_request(request).await?; let request = http_request_to_source_request(request).await?;
let result_op = get_from_source_operation(source, TransientInstance::new(request)); let wrapper_op =
let resolved_result = result_op.resolve().strongly_consistent().await?; get_from_source_with_collectibles_operation(source, TransientInstance::new(request));
apply_effects(result_op).await?; let GetFromSourceResultWithCollectibles {
let side_effects: AutoSet<ResolvedVc<Box<dyn ContentSourceSideEffect>>> = result,
result_op.peek_collectibles(); effects,
content_source_side_effects,
} = &*wrapper_op.read_strongly_consistent().await?;
effects.apply().await?;
handle_issues( handle_issues(
result_op, wrapper_op,
issue_reporter, issue_reporter,
IssueSeverity::Fatal, IssueSeverity::Fatal,
Some(&original_path), Some(&original_path),
Some("get_from_source_operation"), Some("get_from_source_operation"),
) )
.await?; .await?;
match &*resolved_result.await? { match &**result {
GetFromSourceResult::Static { GetFromSourceResult::Static {
content, content,
status_code, status_code,
...@@ -203,7 +227,7 @@ pub async fn process_request_with_content_source( ...@@ -203,7 +227,7 @@ pub async fn process_request_with_content_source(
response.body(hyper::Body::wrap_stream(stream::iter(owned_chunks)))? response.body(hyper::Body::wrap_stream(stream::iter(owned_chunks)))?
}; };
return Ok((response, side_effects)); return Ok((response, content_source_side_effects.clone()));
} }
} }
GetFromSourceResult::HttpProxy(proxy_result) => { GetFromSourceResult::HttpProxy(proxy_result) => {
...@@ -219,7 +243,7 @@ pub async fn process_request_with_content_source( ...@@ -219,7 +243,7 @@ pub async fn process_request_with_content_source(
return Ok(( return Ok((
response.body(hyper::Body::wrap_stream(proxy_result.body.read()))?, response.body(hyper::Body::wrap_stream(proxy_result.body.read()))?,
side_effects, content_source_side_effects.clone(),
)); ));
} }
GetFromSourceResult::NotFound => {} GetFromSourceResult::NotFound => {}
...@@ -227,7 +251,7 @@ pub async fn process_request_with_content_source( ...@@ -227,7 +251,7 @@ pub async fn process_request_with_content_source(
Ok(( Ok((
Response::builder().status(404).body(hyper::Body::empty())?, Response::builder().status(404).body(hyper::Body::empty())?,
side_effects, content_source_side_effects.clone(),
)) ))
} }
......
...@@ -30,8 +30,8 @@ use socket2::{Domain, Protocol, Socket, Type}; ...@@ -30,8 +30,8 @@ use socket2::{Domain, Protocol, Socket, Type};
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tracing::{Instrument, Level, Span, event, info_span}; use tracing::{Instrument, Level, Span, event, info_span};
use turbo_tasks::{ use turbo_tasks::{
Effects, NonLocalValue, OperationVc, PrettyPrintError, TurboTasksApi, Vc, get_effects, Effects, NonLocalValue, OperationVc, PrettyPrintError, TurboTasksApi, Vc, run_once_with_reason,
run_once_with_reason, trace::TraceRawVcs, util::FormatDuration, take_effects, trace::TraceRawVcs, util::FormatDuration,
}; };
use turbopack_core::issue::{IssueReporter, IssueSeverity, handle_issues}; use turbopack_core::issue::{IssueReporter, IssueSeverity, handle_issues};
...@@ -66,7 +66,7 @@ async fn get_source_with_issues_operation( ...@@ -66,7 +66,7 @@ async fn get_source_with_issues_operation(
source_op: OperationVc<Box<dyn ContentSource>>, source_op: OperationVc<Box<dyn ContentSource>>,
) -> Result<Vc<ContentSourceWithIssues>> { ) -> Result<Vc<ContentSourceWithIssues>> {
let _ = source_op.resolve().strongly_consistent().await?; let _ = source_op.resolve().strongly_consistent().await?;
let effects = get_effects(source_op).await?; let effects = take_effects(source_op).await?;
Ok(ContentSourceWithIssues { source_op, effects }.cell()) Ok(ContentSourceWithIssues { source_op, effects }.cell())
} }
......
...@@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; ...@@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ReadRef, Completion, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ReadRef, ResolvedVc,
ResolvedVc, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, mark_session_dependent,
trace::TraceRawVcs, mark_top_level_task, take_effects, trace::TraceRawVcs,
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{File, FileContent, FileSystemPath, to_sys_path}; use turbo_tasks_fs::{File, FileContent, FileSystemPath, to_sys_path};
...@@ -187,19 +187,29 @@ async fn emit_evaluate_pool_assets_operation( ...@@ -187,19 +187,29 @@ async fn emit_evaluate_pool_assets_operation(
#[turbo_tasks::value(serialization = "none")] #[turbo_tasks::value(serialization = "none")]
struct EmittedEvaluatePoolAssetsWithEffects { struct EmittedEvaluatePoolAssetsWithEffects {
assets: ReadRef<EmittedEvaluatePoolAssets>, assets: ReadRef<EmittedEvaluatePoolAssets>,
effects: Arc<Effects>,
} }
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn emit_evaluate_pool_assets_with_effects_operation( async fn create_evaluate_pool_assets_operation(
entries: ResolvedVc<EvaluateEntries>, entries: ResolvedVc<EvaluateEntries>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
) -> Result<Vc<EmittedEvaluatePoolAssetsWithEffects>> { ) -> Result<Vc<EmittedEvaluatePoolAssets>> {
mark_session_dependent();
let operation = emit_evaluate_pool_assets_operation(entries, chunking_context, module_graph); let operation = emit_evaluate_pool_assets_operation(entries, chunking_context, module_graph);
let assets = operation.read_strongly_consistent().await?; let assets = operation.resolve().strongly_consistent().await?;
let effects = Arc::new(get_effects(operation).await?); let effects = Arc::new(take_effects(operation).await?);
Ok(EmittedEvaluatePoolAssetsWithEffects { assets, effects }.cell())
// HACK: `Effects::apply` normally panics if not called at the top-level. We want to apply most
// effects outside of turbo-task functions to avoid re-executing effects during invalidations.
//
// That's not possible here because we lazily create the pool, so instead, use
// `mark_top_level_task` to avoid the debug assertion. The consequence is that these effects
// might get evaluated more than once if this function is invalidated.
mark_top_level_task();
effects.apply().await?;
Ok(*assets)
} }
#[derive( #[derive(
...@@ -224,17 +234,14 @@ pub async fn get_evaluate_pool( ...@@ -224,17 +234,14 @@ pub async fn get_evaluate_pool(
debug: bool, debug: bool,
env_var_tracking: EnvVarTracking, env_var_tracking: EnvVarTracking,
) -> Result<Vc<EvaluatePool>> { ) -> Result<Vc<EvaluatePool>> {
let operation = let assets_op = create_evaluate_pool_assets_operation(entries, chunking_context, module_graph);
emit_evaluate_pool_assets_with_effects_operation(entries, chunking_context, module_graph); let assets = assets_op.read_strongly_consistent().await?;
let EmittedEvaluatePoolAssetsWithEffects { assets, effects } =
&*operation.read_strongly_consistent().await?;
effects.apply().await?;
let EmittedEvaluatePoolAssets { let EmittedEvaluatePoolAssets {
bootstrap, bootstrap,
output_root, output_root,
entrypoint, entrypoint,
} = &**assets; } = &*assets;
let (Some(cwd), Some(entrypoint)) = ( let (Some(cwd), Some(entrypoint)) = (
to_sys_path(cwd.clone()).await?, to_sys_path(cwd.clone()).await?,
......
...@@ -14,8 +14,8 @@ use serde::Deserialize; ...@@ -14,8 +14,8 @@ use serde::Deserialize;
use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt}; use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Completion, NonLocalValue, OperationVc, ResolvedVc, TaskInput, TurboTasks, Vc, apply_effects, Completion, Effects, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TaskInput, TurboTasks,
debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs, Vc, debug::ValueDebugFormat, fxindexmap, take_effects, trace::TraceRawVcs,
}; };
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_env::CommandLineProcessEnv; use turbo_tasks_env::CommandLineProcessEnv;
...@@ -213,28 +213,51 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result<JsRe ...@@ -213,28 +213,51 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result<JsRe
noop_backing_storage(), noop_backing_storage(),
)); ));
tt.run_once(async move { #[turbo_tasks::value(serialization = "none")]
let emit_op = run_inner_operation(resource.to_str().unwrap().into(), snapshot_mode); struct JsResultWithEffects {
let result = emit_op.read_strongly_consistent().owned().await?; result: ReadRef<JsResult>,
apply_effects(emit_op).await?; effects: Effects,
}
Ok(result) #[turbo_tasks::function(operation)]
}) async fn run_inner_operation(
.await resource: RcStr,
} snapshot_mode: IssueSnapshotMode,
) -> Result<Vc<JsResult>> {
let prepared_test = prepare_test(resource).to_resolved().await?;
let run_result_op = run_test_operation(prepared_test);
if snapshot_mode == IssueSnapshotMode::Snapshots {
snapshot_issues(*prepared_test, run_result_op).await?;
}
#[turbo_tasks::function(operation)] let result = (*run_result_op.connect().await?.js_result.await?).clone();
async fn run_inner_operation(
resource: RcStr, Ok(result.cell())
snapshot_mode: IssueSnapshotMode, }
) -> Result<Vc<JsResult>> {
let prepared_test = prepare_test(resource).to_resolved().await?; /// Wrapper operation that collects all effects (including snapshot issue file writes) from
let run_result_op = run_test_operation(prepared_test); /// [`run_inner_operation`].
if snapshot_mode == IssueSnapshotMode::Snapshots { #[turbo_tasks::function(operation)]
snapshot_issues(*prepared_test, run_result_op).await?; async fn run_inner_operation_with_effects(
resource: RcStr,
snapshot_mode: IssueSnapshotMode,
) -> Result<Vc<JsResultWithEffects>> {
let op = run_inner_operation(resource, snapshot_mode);
let result = op.connect().await?.clone();
let effects = take_effects(op).await?;
Ok(JsResultWithEffects { result, effects }.cell())
} }
Ok(*run_result_op.connect().await?.js_result) tt.run_once(async move {
let result_with_effects =
run_inner_operation_with_effects(resource.to_str().unwrap().into(), snapshot_mode)
.read_strongly_consistent()
.await?;
result_with_effects.effects.apply().await?;
Ok((*result_with_effects.result).clone())
})
.await
} }
#[derive( #[derive(
......
...@@ -11,7 +11,7 @@ use rustc_hash::FxHashSet; ...@@ -11,7 +11,7 @@ use rustc_hash::FxHashSet;
use serde::Deserialize; use serde::Deserialize;
use serde_json::json; use serde_json::json;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Effects, OperationVc, ResolvedVc, TurboTasks, Vc, get_effects, turbofmt}; use turbo_tasks::{Effects, OperationVc, ResolvedVc, TurboTasks, Vc, take_effects, turbofmt};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_env::DotenvProcessEnv; use turbo_tasks_env::DotenvProcessEnv;
use turbo_tasks_fs::{ use turbo_tasks_fs::{
...@@ -245,7 +245,7 @@ async fn run(resource: PathBuf) -> Result<()> { ...@@ -245,7 +245,7 @@ async fn run(resource: PathBuf) -> Result<()> {
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn extract_effects(op: OperationVc<()>) -> Result<Vc<Effects>> { async fn extract_effects(op: OperationVc<()>) -> Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?; let _ = op.resolve().strongly_consistent().await?;
Ok(get_effects(op).await?.cell()) Ok(take_effects(op).await?.cell())
} }
extract_effects(inner_operation(resource.to_str().unwrap().into())) extract_effects(inner_operation(resource.to_str().unwrap().into()))
......
...@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf}; ...@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf};
use criterion::{Bencher, BenchmarkId, Criterion}; use criterion::{Bencher, BenchmarkId, Criterion};
use regex::Regex; use regex::Regex;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{TurboTasks, Vc, apply_effects}; use turbo_tasks::{Effects, OperationVc, TurboTasks, Vc, take_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem};
use turbopack::{ use turbopack::{
...@@ -22,6 +22,12 @@ use turbopack_core::{ ...@@ -22,6 +22,12 @@ use turbopack_core::{
}; };
use turbopack_resolve::resolve_options_context::ResolveOptionsContext; use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
#[turbo_tasks::function(operation)]
async fn extract_effects_operation(op: OperationVc<()>) -> anyhow::Result<Vc<Effects>> {
let _ = op.resolve().strongly_consistent().await?;
Ok(take_effects(op).await?.cell())
}
// TODO this should move to the `node-file-trace` crate // TODO this should move to the `node-file-trace` crate
pub fn benchmark(c: &mut Criterion) { pub fn benchmark(c: &mut Criterion) {
let bench_filter = Regex::new(r"(empty|simple|dynamic-in-package|react|whatwg-url|axios|azure-cosmos|cowsay|env-var|fast-glob)\.js$").unwrap(); let bench_filter = Regex::new(r"(empty|simple|dynamic-in-package|react|whatwg-url|axios|azure-cosmos|cowsay|env-var|fast-glob)\.js$").unwrap();
...@@ -123,9 +129,11 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) { ...@@ -123,9 +129,11 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) {
.to_resolved() .to_resolved()
.await?; .await?;
let emit_op = emit_assets_into_dir_operation(assets, output_dir); extract_effects_operation(emit_assets_into_dir_operation(assets, output_dir))
emit_op.read_strongly_consistent().await?; .read_strongly_consistent()
apply_effects(emit_op).await?; .await?
.apply()
.await?;
Ok(()) Ok(())
}) })
......
...@@ -28,7 +28,8 @@ use serde::Serialize; ...@@ -28,7 +28,8 @@ use serde::Serialize;
use tokio::{process::Command, time::timeout}; use tokio::{process::Command, time::timeout};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
ResolvedVc, TurboTasks, ValueToString, Vc, apply_effects, backend::Backend, trace::TraceRawVcs, Effects, ResolvedVc, TurboTasks, ValueToString, Vc, backend::Backend, take_effects,
trace::TraceRawVcs,
}; };
use turbo_tasks_backend::TurboTasksBackend; use turbo_tasks_backend::TurboTasksBackend;
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath};
...@@ -345,12 +346,18 @@ fn bench_against_node_nft_inner(input: CaseInput) { ...@@ -345,12 +346,18 @@ fn bench_against_node_nft_inner(input: CaseInput) {
}); });
} }
#[turbo_tasks::value(serialization = "none")]
struct NodeFileTraceResult {
rebased: ResolvedVc<RebasedAsset>,
effects: Effects,
}
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn node_file_trace_operation( async fn node_file_trace_operation(
package_root: RcStr, package_root: RcStr,
input: RcStr, input: RcStr,
directory: RcStr, directory: RcStr,
) -> Result<Vc<RebasedAsset>> { ) -> Result<Vc<NodeFileTraceResult>> {
let workspace_fs: Vc<Box<dyn FileSystem>> = Vc::upcast(DiskFileSystem::new( let workspace_fs: Vc<Box<dyn FileSystem>> = Vc::upcast(DiskFileSystem::new(
rcstr!("workspace"), rcstr!("workspace"),
package_root.clone(), package_root.clone(),
...@@ -426,9 +433,9 @@ async fn node_file_trace_operation( ...@@ -426,9 +433,9 @@ async fn node_file_trace_operation(
let emit_op = emit_assets_into_dir_operation(assets, output_dir.clone()); let emit_op = emit_assets_into_dir_operation(assets, output_dir.clone());
emit_op.read_strongly_consistent().await?; emit_op.read_strongly_consistent().await?;
apply_effects(emit_op).await?; let effects = take_effects(emit_op).await?;
Ok(*rebased) Ok(NodeFileTraceResult { rebased, effects }.cell())
} }
fn node_file_trace<B: Backend + 'static>( fn node_file_trace<B: Backend + 'static>(
...@@ -480,14 +487,15 @@ fn node_file_trace<B: Backend + 'static>( ...@@ -480,14 +487,15 @@ fn node_file_trace<B: Backend + 'static>(
let directory = directory.clone(); let directory = directory.clone();
let task = async move { let task = async move {
let before_start = Instant::now(); let before_start = Instant::now();
let rebased = node_file_trace_operation( let trace_result = node_file_trace_operation(
package_root.clone(), package_root.clone(),
input.clone(), input.clone(),
directory.clone(), directory.clone(),
) )
.resolve() .read_strongly_consistent()
.strongly_consistent()
.await?; .await?;
trace_result.effects.apply().await?;
let rebased = trace_result.rebased;
let duration = before_start.elapsed(); let duration = before_start.elapsed();
print_graph_operation(ResolvedVc::upcast(rebased)) print_graph_operation(ResolvedVc::upcast(rebased))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment