Unverified Commit 491e9c1a authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] Revert "refactor filesystem writes to an effect based system" (#73287)

Revert "Turbopack: apply effects in tests (#72952)"

This reverts commit b6b5ab88.

Revert "[Turbopack] fix effects tracing (#72928)"

This reverts commit 0714c457.

Revert "[Turbopack] refactor filesystem writes to an effect based system (#72847)"

This reverts commit be9b2ddb.

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the PR.
- Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to understand the PR)
- When linking to a Slack thread, you might want to share details of the conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->
parent fdaf35b9
...@@ -7,7 +7,7 @@ use next_api::{ ...@@ -7,7 +7,7 @@ use next_api::{
route::{Endpoint, WrittenEndpoint}, route::{Endpoint, WrittenEndpoint},
}; };
use tracing::Instrument; use tracing::Instrument;
use turbo_tasks::{get_effects, Completion, Effects, ReadRef, Vc, VcValueType}; use turbo_tasks::{Completion, ReadRef, Vc, VcValueType};
use turbopack_core::{ use turbopack_core::{
diagnostics::PlainDiagnostic, diagnostics::PlainDiagnostic,
error::PrettyPrintError, error::PrettyPrintError,
...@@ -104,12 +104,10 @@ async fn strongly_consistent_catch_collectables<R: VcValueType + Send>( ...@@ -104,12 +104,10 @@ async fn strongly_consistent_catch_collectables<R: VcValueType + Send>(
Option<ReadRef<R>>, Option<ReadRef<R>>,
Arc<Vec<ReadRef<PlainIssue>>>, Arc<Vec<ReadRef<PlainIssue>>>,
Arc<Vec<ReadRef<PlainDiagnostic>>>, Arc<Vec<ReadRef<PlainDiagnostic>>>,
Arc<Effects>,
)> { )> {
let result = source.strongly_consistent().await; let result = source.strongly_consistent().await;
let issues = get_issues(source).await?; let issues = get_issues(source).await?;
let diagnostics = get_diagnostics(source).await?; let diagnostics = get_diagnostics(source).await?;
let effects = Arc::new(get_effects(source).await?);
let result = if result.is_err() && issues.iter().any(|i| i.severity <= IssueSeverity::Error) { let result = if result.is_err() && issues.iter().any(|i| i.severity <= IssueSeverity::Error) {
None None
...@@ -117,7 +115,7 @@ async fn strongly_consistent_catch_collectables<R: VcValueType + Send>( ...@@ -117,7 +115,7 @@ async fn strongly_consistent_catch_collectables<R: VcValueType + Send>(
Some(result?) Some(result?)
}; };
Ok((result, issues, diagnostics, effects)) Ok((result, issues, diagnostics))
} }
#[turbo_tasks::value(serialization = "none")] #[turbo_tasks::value(serialization = "none")]
...@@ -125,7 +123,6 @@ struct WrittenEndpointWithIssues { ...@@ -125,7 +123,6 @@ struct WrittenEndpointWithIssues {
written: Option<ReadRef<WrittenEndpoint>>, written: Option<ReadRef<WrittenEndpoint>>,
issues: Arc<Vec<ReadRef<PlainIssue>>>, issues: Arc<Vec<ReadRef<PlainIssue>>>,
diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>, diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
effects: Arc<Effects>,
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -133,13 +130,12 @@ async fn get_written_endpoint_with_issues( ...@@ -133,13 +130,12 @@ async fn get_written_endpoint_with_issues(
endpoint: Vc<Box<dyn Endpoint>>, endpoint: Vc<Box<dyn Endpoint>>,
) -> Result<Vc<WrittenEndpointWithIssues>> { ) -> Result<Vc<WrittenEndpointWithIssues>> {
let write_to_disk = endpoint.write_to_disk(); let write_to_disk = endpoint.write_to_disk();
let (written, issues, diagnostics, effects) = let (written, issues, diagnostics) =
strongly_consistent_catch_collectables(write_to_disk).await?; strongly_consistent_catch_collectables(write_to_disk).await?;
Ok(WrittenEndpointWithIssues { Ok(WrittenEndpointWithIssues {
written, written,
issues, issues,
diagnostics, diagnostics,
effects,
} }
.cell()) .cell())
} }
...@@ -153,15 +149,13 @@ pub async fn endpoint_write_to_disk( ...@@ -153,15 +149,13 @@ pub async fn endpoint_write_to_disk(
let endpoint = ***endpoint; let endpoint = ***endpoint;
let (written, issues, diags) = turbo_tasks let (written, issues, diags) = turbo_tasks
.run_once(async move { .run_once(async move {
let operation = get_written_endpoint_with_issues(endpoint);
let WrittenEndpointWithIssues { let WrittenEndpointWithIssues {
written, written,
issues, issues,
diagnostics, diagnostics,
effects, } = &*get_written_endpoint_with_issues(endpoint)
} = &*operation.strongly_consistent().await?; .strongly_consistent()
effects.apply().await?; .await?;
Ok((written.clone(), issues.clone(), diagnostics.clone())) Ok((written.clone(), issues.clone(), diagnostics.clone()))
}) })
.await .await
...@@ -186,10 +180,9 @@ pub fn endpoint_server_changed_subscribe( ...@@ -186,10 +180,9 @@ pub fn endpoint_server_changed_subscribe(
func, func,
move || { move || {
async move { async move {
let operation = subscribe_issues_and_diags(endpoint, issues); subscribe_issues_and_diags(endpoint, issues)
let result = operation.strongly_consistent().await?; .strongly_consistent()
result.effects.apply().await?; .await
Ok(result)
} }
.instrument(tracing::info_span!("server changes subscription")) .instrument(tracing::info_span!("server changes subscription"))
}, },
...@@ -198,7 +191,6 @@ pub fn endpoint_server_changed_subscribe( ...@@ -198,7 +191,6 @@ pub fn endpoint_server_changed_subscribe(
changed: _, changed: _,
issues, issues,
diagnostics, diagnostics,
effects: _,
} = &*ctx.value; } = &*ctx.value;
Ok(vec![TurbopackResult { Ok(vec![TurbopackResult {
...@@ -218,7 +210,6 @@ struct EndpointIssuesAndDiags { ...@@ -218,7 +210,6 @@ struct EndpointIssuesAndDiags {
changed: Option<ReadRef<Completion>>, changed: Option<ReadRef<Completion>>,
issues: Arc<Vec<ReadRef<PlainIssue>>>, issues: Arc<Vec<ReadRef<PlainIssue>>>,
diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>, diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
effects: Arc<Effects>,
} }
impl PartialEq for EndpointIssuesAndDiags { impl PartialEq for EndpointIssuesAndDiags {
...@@ -242,13 +233,12 @@ async fn subscribe_issues_and_diags( ...@@ -242,13 +233,12 @@ async fn subscribe_issues_and_diags(
let changed = endpoint.server_changed(); let changed = endpoint.server_changed();
if should_include_issues { if should_include_issues {
let (changed_value, issues, diagnostics, effects) = let (changed_value, issues, diagnostics) =
strongly_consistent_catch_collectables(changed).await?; strongly_consistent_catch_collectables(changed).await?;
Ok(EndpointIssuesAndDiags { Ok(EndpointIssuesAndDiags {
changed: changed_value, changed: changed_value,
issues, issues,
diagnostics, diagnostics,
effects,
} }
.cell()) .cell())
} else { } else {
...@@ -257,7 +247,6 @@ async fn subscribe_issues_and_diags( ...@@ -257,7 +247,6 @@ async fn subscribe_issues_and_diags(
changed: Some(changed_value), changed: Some(changed_value),
issues: Arc::new(vec![]), issues: Arc::new(vec![]),
diagnostics: Arc::new(vec![]), diagnostics: Arc::new(vec![]),
effects: Arc::new(Effects::default()),
} }
.cell()) .cell())
} }
......
...@@ -24,7 +24,7 @@ use tokio::{io::AsyncWriteExt, time::Instant}; ...@@ -24,7 +24,7 @@ use tokio::{io::AsyncWriteExt, time::Instant};
use tracing::Instrument; use tracing::Instrument;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{get_effects, Completion, Effects, ReadRef, TransientInstance, UpdateInfo, Vc}; use turbo_tasks::{Completion, ReadRef, TransientInstance, UpdateInfo, Vc};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
util::uri_from_file, DiskFileSystem, FileContent, FileSystem, FileSystemPath, util::uri_from_file, DiskFileSystem, FileContent, FileSystem, FileSystemPath,
}; };
...@@ -623,7 +623,6 @@ struct EntrypointsWithIssues { ...@@ -623,7 +623,6 @@ struct EntrypointsWithIssues {
entrypoints: ReadRef<Entrypoints>, entrypoints: ReadRef<Entrypoints>,
issues: Arc<Vec<ReadRef<PlainIssue>>>, issues: Arc<Vec<ReadRef<PlainIssue>>>,
diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>, diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
effects: Arc<Effects>,
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -634,12 +633,10 @@ async fn get_entrypoints_with_issues( ...@@ -634,12 +633,10 @@ async fn get_entrypoints_with_issues(
let entrypoints = entrypoints_operation.strongly_consistent().await?; let entrypoints = entrypoints_operation.strongly_consistent().await?;
let issues = get_issues(entrypoints_operation).await?; let issues = get_issues(entrypoints_operation).await?;
let diagnostics = get_diagnostics(entrypoints_operation).await?; let diagnostics = get_diagnostics(entrypoints_operation).await?;
let effects = Arc::new(get_effects(entrypoints_operation).await?);
Ok(EntrypointsWithIssues { Ok(EntrypointsWithIssues {
entrypoints, entrypoints,
issues, issues,
diagnostics, diagnostics,
effects,
} }
.cell()) .cell())
} }
...@@ -656,14 +653,13 @@ pub fn project_entrypoints_subscribe( ...@@ -656,14 +653,13 @@ pub fn project_entrypoints_subscribe(
func, func,
move || { move || {
async move { async move {
let operation = get_entrypoints_with_issues(container);
let EntrypointsWithIssues { let EntrypointsWithIssues {
entrypoints, entrypoints,
issues, issues,
diagnostics, diagnostics,
effects, } = &*get_entrypoints_with_issues(container)
} = &*operation.strongly_consistent().await?; .strongly_consistent()
effects.apply().await?; .await?;
Ok((entrypoints.clone(), issues.clone(), diagnostics.clone())) Ok((entrypoints.clone(), issues.clone(), diagnostics.clone()))
} }
.instrument(tracing::info_span!("entrypoints subscription")) .instrument(tracing::info_span!("entrypoints subscription"))
...@@ -722,7 +718,6 @@ struct HmrUpdateWithIssues { ...@@ -722,7 +718,6 @@ struct HmrUpdateWithIssues {
update: ReadRef<Update>, update: ReadRef<Update>,
issues: Arc<Vec<ReadRef<PlainIssue>>>, issues: Arc<Vec<ReadRef<PlainIssue>>>,
diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>, diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
effects: Arc<Effects>,
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -735,12 +730,10 @@ async fn hmr_update( ...@@ -735,12 +730,10 @@ async fn hmr_update(
let update = update_operation.strongly_consistent().await?; let update = update_operation.strongly_consistent().await?;
let issues = get_issues(update_operation).await?; let issues = get_issues(update_operation).await?;
let diagnostics = get_diagnostics(update_operation).await?; let diagnostics = get_diagnostics(update_operation).await?;
let effects = Arc::new(get_effects(update_operation).await?);
Ok(HmrUpdateWithIssues { Ok(HmrUpdateWithIssues {
update, update,
issues, issues,
diagnostics, diagnostics,
effects,
} }
.cell()) .cell())
} }
...@@ -767,15 +760,14 @@ pub fn project_hmr_events( ...@@ -767,15 +760,14 @@ pub fn project_hmr_events(
let project = project.project().resolve().await?; let project = project.project().resolve().await?;
let state = project.hmr_version_state(identifier.clone(), session); let state = project.hmr_version_state(identifier.clone(), session);
let operation = hmr_update(project, identifier.clone(), state); let update = hmr_update(project, identifier.clone(), state)
let update = operation.strongly_consistent().await?; .strongly_consistent()
.await?;
let HmrUpdateWithIssues { let HmrUpdateWithIssues {
update, update,
issues, issues,
diagnostics, diagnostics,
effects,
} = &*update; } = &*update;
effects.apply().await?;
match &**update { match &**update {
Update::Missing | Update::None => {} Update::Missing | Update::None => {}
Update::Total(TotalUpdate { to }) => { Update::Total(TotalUpdate { to }) => {
...@@ -840,7 +832,6 @@ struct HmrIdentifiersWithIssues { ...@@ -840,7 +832,6 @@ struct HmrIdentifiersWithIssues {
identifiers: ReadRef<Vec<RcStr>>, identifiers: ReadRef<Vec<RcStr>>,
issues: Arc<Vec<ReadRef<PlainIssue>>>, issues: Arc<Vec<ReadRef<PlainIssue>>>,
diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>, diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
effects: Arc<Effects>,
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -851,12 +842,10 @@ async fn get_hmr_identifiers_with_issues( ...@@ -851,12 +842,10 @@ async fn get_hmr_identifiers_with_issues(
let hmr_identifiers = hmr_identifiers_operation.strongly_consistent().await?; let hmr_identifiers = hmr_identifiers_operation.strongly_consistent().await?;
let issues = get_issues(hmr_identifiers_operation).await?; let issues = get_issues(hmr_identifiers_operation).await?;
let diagnostics = get_diagnostics(hmr_identifiers_operation).await?; let diagnostics = get_diagnostics(hmr_identifiers_operation).await?;
let effects = Arc::new(get_effects(hmr_identifiers_operation).await?);
Ok(HmrIdentifiersWithIssues { Ok(HmrIdentifiersWithIssues {
identifiers: hmr_identifiers, identifiers: hmr_identifiers,
issues, issues,
diagnostics, diagnostics,
effects,
} }
.cell()) .cell())
} }
...@@ -872,14 +861,13 @@ pub fn project_hmr_identifiers_subscribe( ...@@ -872,14 +861,13 @@ pub fn project_hmr_identifiers_subscribe(
turbo_tasks.clone(), turbo_tasks.clone(),
func, func,
move || async move { move || async move {
let operation = get_hmr_identifiers_with_issues(container);
let HmrIdentifiersWithIssues { let HmrIdentifiersWithIssues {
identifiers, identifiers,
issues, issues,
diagnostics, diagnostics,
effects, } = &*get_hmr_identifiers_with_issues(container)
} = &*operation.strongly_consistent().await?; .strongly_consistent()
effects.apply().await?; .await?;
Ok((identifiers.clone(), issues.clone(), diagnostics.clone())) Ok((identifiers.clone(), issues.clone(), diagnostics.clone()))
}, },
......
...@@ -1633,10 +1633,10 @@ impl Endpoint for AppEndpoint { ...@@ -1633,10 +1633,10 @@ impl Endpoint for AppEndpoint {
let node_root_ref = &node_root.await?; let node_root_ref = &node_root.await?;
let _ = this this.app_project
.app_project
.project() .project()
.emit_all_output_assets(Vc::cell(output_assets)); .emit_all_output_assets(Vc::cell(output_assets))
.await?;
let (server_paths, client_paths) = if this let (server_paths, client_paths) = if this
.app_project .app_project
......
...@@ -252,7 +252,9 @@ impl Endpoint for InstrumentationEndpoint { ...@@ -252,7 +252,9 @@ impl Endpoint for InstrumentationEndpoint {
let this = self.await?; let this = self.await?;
let output_assets = self.output_assets(); let output_assets = self.output_assets();
let _ = output_assets.resolve().await?; let _ = output_assets.resolve().await?;
let _ = this.project.emit_all_output_assets(Vc::cell(output_assets)); this.project
.emit_all_output_assets(Vc::cell(output_assets))
.await?;
let server_paths = if this.project.next_mode().await?.is_development() { let server_paths = if this.project.next_mode().await?.is_development() {
let node_root = this.project.node_root(); let node_root = this.project.node_root();
......
...@@ -277,7 +277,9 @@ impl Endpoint for MiddlewareEndpoint { ...@@ -277,7 +277,9 @@ impl Endpoint for MiddlewareEndpoint {
let this = self.await?; let this = self.await?;
let output_assets = self.output_assets(); let output_assets = self.output_assets();
let _ = output_assets.resolve().await?; let _ = output_assets.resolve().await?;
let _ = this.project.emit_all_output_assets(Vc::cell(output_assets)); this.project
.emit_all_output_assets(Vc::cell(output_assets))
.await?;
let (server_paths, client_paths) = if this.project.next_mode().await?.is_development() { let (server_paths, client_paths) = if this.project.next_mode().await?.is_development() {
let node_root = this.project.node_root(); let node_root = this.project.node_root();
......
...@@ -1277,10 +1277,10 @@ impl Endpoint for PageEndpoint { ...@@ -1277,10 +1277,10 @@ impl Endpoint for PageEndpoint {
// single operation // single operation
let output_assets = self.output_assets(); let output_assets = self.output_assets();
let _ = this this.pages_project
.pages_project
.project() .project()
.emit_all_output_assets(Vc::cell(output_assets)); .emit_all_output_assets(Vc::cell(output_assets))
.await?;
let node_root = this.pages_project.project().node_root(); let node_root = this.pages_project.project().node_root();
......
...@@ -1170,7 +1170,7 @@ impl Project { ...@@ -1170,7 +1170,7 @@ impl Project {
pub async fn emit_all_output_assets( pub async fn emit_all_output_assets(
self: Vc<Self>, self: Vc<Self>,
output_assets: Vc<OutputAssetsOperation>, output_assets: Vc<OutputAssetsOperation>,
) -> Result<()> { ) -> Result<Vc<()>> {
let span = tracing::info_span!("emitting"); let span = tracing::info_span!("emitting");
async move { async move {
let all_output_assets = all_assets_from_entries_operation(output_assets); let all_output_assets = all_assets_from_entries_operation(output_assets);
...@@ -1179,23 +1179,27 @@ impl Project { ...@@ -1179,23 +1179,27 @@ impl Project {
let node_root = self.node_root(); let node_root = self.node_root();
if let Some(map) = self.await?.versioned_content_map { if let Some(map) = self.await?.versioned_content_map {
let _ = map.insert_output_assets( let _ = map
all_output_assets, .insert_output_assets(
node_root, all_output_assets,
client_relative_path, node_root,
node_root, client_relative_path,
); node_root,
)
.resolve()
.await?;
Ok(()) Ok(Vc::cell(()))
} else { } else {
let _ = emit_assets( let _ = emit_assets(
*all_output_assets.await?, *all_output_assets.await?,
node_root, node_root,
client_relative_path, client_relative_path,
node_root, node_root,
); )
.resolve()
Ok(()) .await?;
Ok(Vc::cell(()))
} }
} }
.instrument(span) .instrument(span)
......
...@@ -5,8 +5,8 @@ use next_core::emit_assets; ...@@ -5,8 +5,8 @@ use next_core::emit_assets;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
debug::ValueDebugFormat, trace::TraceRawVcs, FxIndexSet, ResolvedVc, State, TryFlatJoinIterExt, debug::ValueDebugFormat, trace::TraceRawVcs, Completion, FxIndexSet, ResolvedVc, State,
TryJoinIterExt, ValueDefault, ValueToString, Vc, TryFlatJoinIterExt, TryJoinIterExt, ValueDefault, ValueToString, Vc,
}; };
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
...@@ -24,8 +24,8 @@ pub struct OutputAssetsOperation(Vc<OutputAssets>); ...@@ -24,8 +24,8 @@ pub struct OutputAssetsOperation(Vc<OutputAssets>);
#[derive(Clone, TraceRawVcs, PartialEq, Eq, ValueDebugFormat, Serialize, Deserialize, Debug)] #[derive(Clone, TraceRawVcs, PartialEq, Eq, ValueDebugFormat, Serialize, Deserialize, Debug)]
struct MapEntry { struct MapEntry {
// must not be resolved
assets_operation: Vc<OutputAssets>, assets_operation: Vc<OutputAssets>,
side_effects: Vc<Completion>,
/// Precomputed map for quick access to output asset by filepath /// Precomputed map for quick access to output asset by filepath
path_to_asset: HashMap<ResolvedVc<FileSystemPath>, Vc<Box<dyn OutputAsset>>>, path_to_asset: HashMap<ResolvedVc<FileSystemPath>, Vc<Box<dyn OutputAsset>>>,
} }
...@@ -74,7 +74,7 @@ impl VersionedContentMap { ...@@ -74,7 +74,7 @@ impl VersionedContentMap {
node_root: Vc<FileSystemPath>, node_root: Vc<FileSystemPath>,
client_relative_path: Vc<FileSystemPath>, client_relative_path: Vc<FileSystemPath>,
client_output_path: Vc<FileSystemPath>, client_output_path: Vc<FileSystemPath>,
) -> Result<()> { ) -> Result<Vc<Completion>> {
let this = self.await?; let this = self.await?;
let compute_entry = self.compute_entry( let compute_entry = self.compute_entry(
assets_operation, assets_operation,
...@@ -85,7 +85,10 @@ impl VersionedContentMap { ...@@ -85,7 +85,10 @@ impl VersionedContentMap {
let assets = *assets_operation.await?; let assets = *assets_operation.await?;
this.map_op_to_compute_entry this.map_op_to_compute_entry
.update_conditionally(|map| map.insert(assets, compute_entry) != Some(compute_entry)); .update_conditionally(|map| map.insert(assets, compute_entry) != Some(compute_entry));
Ok(()) let Some(entry) = &*compute_entry.await? else {
unreachable!("compute_entry always returns Some(MapEntry)")
};
Ok(entry.side_effects)
} }
/// Creates a [`MapEntry`] (a pre-computed map for optimized lookup) for an output assets /// Creates a [`MapEntry`] (a pre-computed map for optimized lookup) for an output assets
...@@ -140,9 +143,10 @@ impl VersionedContentMap { ...@@ -140,9 +143,10 @@ impl VersionedContentMap {
}); });
// Make sure all written client assets are up-to-date // Make sure all written client assets are up-to-date
let _ = emit_assets(assets, node_root, client_relative_path, client_output_path); let side_effects = emit_assets(assets, node_root, client_relative_path, client_output_path);
let map_entry = Vc::cell(Some(MapEntry { let map_entry = Vc::cell(Some(MapEntry {
assets_operation: assets, assets_operation: assets,
side_effects,
path_to_asset: entries.into_iter().collect(), path_to_asset: entries.into_iter().collect(),
})); }));
Ok(map_entry) Ok(map_entry)
...@@ -191,9 +195,12 @@ impl VersionedContentMap { ...@@ -191,9 +195,12 @@ impl VersionedContentMap {
let result = self.raw_get(*path).await?; let result = self.raw_get(*path).await?;
if let Some(MapEntry { if let Some(MapEntry {
assets_operation: _, assets_operation: _,
side_effects,
path_to_asset, path_to_asset,
}) = &*result }) = &*result
{ {
side_effects.await?;
if let Some(asset) = path_to_asset.get(&path) { if let Some(asset) = path_to_asset.get(&path) {
return Ok(Vc::cell(Some(asset.to_resolved().await?))); return Ok(Vc::cell(Some(asset.to_resolved().await?)));
} else { } else {
......
...@@ -2,7 +2,7 @@ use anyhow::Result; ...@@ -2,7 +2,7 @@ use anyhow::Result;
use tracing::Instrument; use tracing::Instrument;
use turbo_tasks::{ use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal}, graph::{AdjacencyMap, GraphTraversal},
ResolvedVc, TryFlatJoinIterExt, ValueToString, Vc, Completion, Completions, ResolvedVc, TryFlatJoinIterExt, ValueToString, Vc,
}; };
use turbo_tasks_fs::{rebase, FileSystemPath}; use turbo_tasks_fs::{rebase, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
...@@ -21,13 +21,13 @@ pub fn emit_all_assets( ...@@ -21,13 +21,13 @@ pub fn emit_all_assets(
node_root: Vc<FileSystemPath>, node_root: Vc<FileSystemPath>,
client_relative_path: Vc<FileSystemPath>, client_relative_path: Vc<FileSystemPath>,
client_output_path: Vc<FileSystemPath>, client_output_path: Vc<FileSystemPath>,
) { ) -> Vc<Completion> {
let _ = emit_assets( emit_assets(
all_assets_from_entries(assets), all_assets_from_entries(assets),
node_root, node_root,
client_relative_path, client_relative_path,
client_output_path, client_output_path,
); )
} }
/// Emits all assets transitively reachable from the given chunks, that are /// Emits all assets transitively reachable from the given chunks, that are
...@@ -41,39 +41,42 @@ pub async fn emit_assets( ...@@ -41,39 +41,42 @@ pub async fn emit_assets(
node_root: Vc<FileSystemPath>, node_root: Vc<FileSystemPath>,
client_relative_path: Vc<FileSystemPath>, client_relative_path: Vc<FileSystemPath>,
client_output_path: Vc<FileSystemPath>, client_output_path: Vc<FileSystemPath>,
) -> Result<()> { ) -> Result<Vc<Completion>> {
let _: Vec<Vc<()>> = assets Ok(Vc::<Completions>::cell(
.await? assets
.iter() .await?
.copied() .iter()
.map(|asset| async move { .copied()
let asset = asset.resolve().await?; .map(|asset| async move {
let path = asset.ident().path(); let asset = asset.resolve().await?;
let span = tracing::info_span!("emit asset", name = %path.to_string().await?); let path = asset.ident().path();
async move { let span =
let path = path.await?; tracing::info_span!("emit asset", name = path.to_string().await?.as_str());
Ok(if path.is_inside_ref(&*node_root.await?) { async move {
Some(emit(asset)) let path = path.await?;
} else if path.is_inside_ref(&*client_relative_path.await?) { Ok(if path.is_inside_ref(&*node_root.await?) {
// Client assets are emitted to the client output path, which is prefixed Some(emit(asset))
// with _next. We need to rebase them to remove that } else if path.is_inside_ref(&*client_relative_path.await?) {
// prefix. // Client assets are emitted to the client output path, which is prefixed
Some(emit_rebase(asset, client_relative_path, client_output_path)) // with _next. We need to rebase them to remove that
} else { // prefix.
None Some(emit_rebase(asset, client_relative_path, client_output_path))
}) } else {
} None
.instrument(span) })
.await }
}) .instrument(span)
.try_flat_join() .await
.await?; })
Ok(()) .try_flat_join()
.await?,
)
.completed())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn emit(asset: Vc<Box<dyn OutputAsset>>) { fn emit(asset: Vc<Box<dyn OutputAsset>>) -> Vc<Completion> {
let _ = asset.content().write(asset.ident().path()); asset.content().write(asset.ident().path())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -81,11 +84,10 @@ async fn emit_rebase( ...@@ -81,11 +84,10 @@ async fn emit_rebase(
asset: Vc<Box<dyn OutputAsset>>, asset: Vc<Box<dyn OutputAsset>>,
from: Vc<FileSystemPath>, from: Vc<FileSystemPath>,
to: Vc<FileSystemPath>, to: Vc<FileSystemPath>,
) -> Result<()> { ) -> Result<Vc<Completion>> {
let path = rebase(asset.ident().path(), from, to); let path = rebase(asset.ident().path(), from, to);
let content = asset.content(); let content = asset.content();
let _ = content.resolve().await?.write(path.resolve().await?); Ok(content.resolve().await?.write(path.resolve().await?))
Ok(())
} }
/// Walks the asset graph from multiple assets and collect all referenced /// Walks the asset graph from multiple assets and collect all referenced
......
...@@ -164,12 +164,16 @@ impl FileSystem for AttachedFileSystem { ...@@ -164,12 +164,16 @@ impl FileSystem for AttachedFileSystem {
} }
#[turbo_tasks::function(fs)] #[turbo_tasks::function(fs)]
fn write(self: Vc<Self>, path: Vc<FileSystemPath>, content: Vc<FileContent>) -> Vc<()> { fn write(self: Vc<Self>, path: Vc<FileSystemPath>, content: Vc<FileContent>) -> Vc<Completion> {
self.get_inner_fs_path(path).write(content) self.get_inner_fs_path(path).write(content)
} }
#[turbo_tasks::function(fs)] #[turbo_tasks::function(fs)]
fn write_link(self: Vc<Self>, path: Vc<FileSystemPath>, target: Vc<LinkContent>) -> Vc<()> { fn write_link(
self: Vc<Self>,
path: Vc<FileSystemPath>,
target: Vc<LinkContent>,
) -> Vc<Completion> {
self.get_inner_fs_path(path).write_link(target) self.get_inner_fs_path(path).write_link(target)
} }
......
...@@ -76,12 +76,20 @@ impl FileSystem for EmbeddedFileSystem { ...@@ -76,12 +76,20 @@ impl FileSystem for EmbeddedFileSystem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn write(&self, _path: Vc<FileSystemPath>, _content: Vc<FileContent>) -> Result<Vc<()>> { fn write(
&self,
_path: Vc<FileSystemPath>,
_content: Vc<FileContent>,
) -> Result<Vc<Completion>> {
bail!("Writing is not possible to the embedded filesystem") bail!("Writing is not possible to the embedded filesystem")
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn write_link(&self, _path: Vc<FileSystemPath>, _target: Vc<LinkContent>) -> Result<Vc<()>> { fn write_link(
&self,
_path: Vc<FileSystemPath>,
_target: Vc<LinkContent>,
) -> Result<Vc<Completion>> {
bail!("Writing is not possible to the embedded filesystem") bail!("Writing is not possible to the embedded filesystem")
} }
......
This diff is collapsed.
...@@ -66,12 +66,20 @@ impl FileSystem for VirtualFileSystem { ...@@ -66,12 +66,20 @@ impl FileSystem for VirtualFileSystem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn write(&self, _fs_path: Vc<FileSystemPath>, _content: Vc<FileContent>) -> Result<Vc<()>> { fn write(
&self,
_fs_path: Vc<FileSystemPath>,
_content: Vc<FileContent>,
) -> Result<Vc<Completion>> {
bail!("Writing is not possible on the virtual file system") bail!("Writing is not possible on the virtual file system")
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn write_link(&self, _fs_path: Vc<FileSystemPath>, _target: Vc<LinkContent>) -> Result<Vc<()>> { fn write_link(
&self,
_fs_path: Vc<FileSystemPath>,
_target: Vc<LinkContent>,
) -> Result<Vc<Completion>> {
bail!("Writing is not possible on the virtual file system") bail!("Writing is not possible on the virtual file system")
} }
......
...@@ -16,13 +16,16 @@ use notify::{ ...@@ -16,13 +16,16 @@ use notify::{
}; };
use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use tracing::instrument; use tracing::instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{spawn_thread, Invalidator}; use turbo_tasks::{spawn_thread, Invalidator};
use crate::{ use crate::{
format_absolute_fs_path, format_absolute_fs_path,
invalidation::{WatchChange, WatchStart}, invalidation::{WatchChange, WatchStart},
path_to_key, DiskFileSystemInner, invalidator_map::InvalidatorMap,
path_to_key,
}; };
enum DiskWatcherInternal { enum DiskWatcherInternal {
...@@ -134,9 +137,13 @@ impl DiskWatcher { ...@@ -134,9 +137,13 @@ impl DiskWatcher {
/// - Doesn't emit duplicate create events /// - Doesn't emit duplicate create events
/// - Doesn't emit Modify events after a Create event /// - Doesn't emit Modify events after a Create event
pub(crate) fn start_watching( pub(crate) fn start_watching(
&self, self: Arc<Self>,
inner: Arc<DiskFileSystemInner>, name: RcStr,
report_invalidation_reason: bool, root_path: PathBuf,
report_invalidation_reason: Option<(RcStr, PathBuf)>,
invalidation_lock: Arc<RwLock<()>>,
invalidator_map: Arc<InvalidatorMap>,
dir_invalidator_map: Arc<InvalidatorMap>,
poll_interval: Option<Duration>, poll_interval: Option<Duration>,
) -> Result<()> { ) -> Result<()> {
let mut watcher_guard = self.watcher.lock().unwrap(); let mut watcher_guard = self.watcher.lock().unwrap();
...@@ -162,7 +169,7 @@ impl DiskWatcher { ...@@ -162,7 +169,7 @@ impl DiskWatcher {
// below will be monitored for changes. // below will be monitored for changes.
#[cfg(any(target_os = "macos", target_os = "windows"))] #[cfg(any(target_os = "macos", target_os = "windows"))]
{ {
watcher.watch(inner.root_path(), RecursiveMode::Recursive)?; watcher.watch(&root_path, RecursiveMode::Recursive)?;
} }
#[cfg(not(any(target_os = "macos", target_os = "windows")))] #[cfg(not(any(target_os = "macos", target_os = "windows")))]
...@@ -175,17 +182,17 @@ impl DiskWatcher { ...@@ -175,17 +182,17 @@ impl DiskWatcher {
{ {
let _span = tracing::info_span!("invalidate filesystem").entered(); let _span = tracing::info_span!("invalidate filesystem").entered();
let span = tracing::Span::current(); let span = tracing::Span::current();
let invalidator_map = take(&mut *inner.invalidator_map.lock().unwrap()); let invalidator_map = take(&mut *invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *inner.dir_invalidator_map.lock().unwrap()); let dir_invalidator_map = take(&mut *dir_invalidator_map.lock().unwrap());
let iter = invalidator_map let iter = invalidator_map
.into_par_iter() .into_par_iter()
.chain(dir_invalidator_map.into_par_iter()); .chain(dir_invalidator_map.into_par_iter());
let handle = tokio::runtime::Handle::current(); let handle = tokio::runtime::Handle::current();
if report_invalidation_reason { if report_invalidation_reason.is_some() {
iter.flat_map(|(path, invalidators)| { iter.flat_map(|(path, invalidators)| {
let _span = span.clone().entered(); let _span = span.clone().entered();
let reason = WatchStart { let reason = WatchStart {
name: inner.name.clone(), name: name.clone(),
path: path.into(), path: path.into(),
}; };
invalidators invalidators
...@@ -214,10 +221,14 @@ impl DiskWatcher { ...@@ -214,10 +221,14 @@ impl DiskWatcher {
drop(watcher_guard); drop(watcher_guard);
spawn_thread(move || { spawn_thread(move || {
inner self.watch_thread(
.clone() rx,
.watcher root_path,
.watch_thread(rx, inner, report_invalidation_reason) report_invalidation_reason,
invalidation_lock,
invalidator_map,
dir_invalidator_map,
)
}); });
Ok(()) Ok(())
...@@ -237,8 +248,11 @@ impl DiskWatcher { ...@@ -237,8 +248,11 @@ impl DiskWatcher {
fn watch_thread( fn watch_thread(
&self, &self,
rx: Receiver<notify::Result<notify::Event>>, rx: Receiver<notify::Result<notify::Event>>,
inner: Arc<DiskFileSystemInner>, root_path: PathBuf,
report_invalidation_reason: bool, report_invalidation_reason: Option<(RcStr, PathBuf)>,
invalidation_lock: Arc<RwLock<()>>,
invalidator_map: Arc<InvalidatorMap>,
dir_invalidator_map: Arc<InvalidatorMap>,
) { ) {
let mut batched_invalidate_path = HashSet::new(); let mut batched_invalidate_path = HashSet::new();
let mut batched_invalidate_path_dir = HashSet::new(); let mut batched_invalidate_path_dir = HashSet::new();
...@@ -363,10 +377,8 @@ impl DiskWatcher { ...@@ -363,10 +377,8 @@ impl DiskWatcher {
println!("watch error ({:?}): {:?} ", paths, kind); println!("watch error ({:?}): {:?} ", paths, kind);
if paths.is_empty() { if paths.is_empty() {
batched_invalidate_path_and_children batched_invalidate_path_and_children.insert(root_path.clone());
.insert(inner.root_path().to_path_buf()); batched_invalidate_path_and_children_dir.insert(root_path.clone());
batched_invalidate_path_and_children_dir
.insert(inner.root_path().to_path_buf());
} else { } else {
batched_invalidate_path_and_children.extend(paths.clone()); batched_invalidate_path_and_children.extend(paths.clone());
batched_invalidate_path_and_children_dir.extend(paths.clone()); batched_invalidate_path_and_children_dir.extend(paths.clone());
...@@ -401,37 +413,33 @@ impl DiskWatcher { ...@@ -401,37 +413,33 @@ impl DiskWatcher {
#[cfg(not(any(target_os = "macos", target_os = "windows")))] #[cfg(not(any(target_os = "macos", target_os = "windows")))]
{ {
for path in batched_new_paths.drain() { for path in batched_new_paths.drain() {
let _ = self.restore_if_watching(&path, inner.root_path()); let _ = self.restore_if_watching(&path, &root_path);
} }
} }
let _lock = inner.invalidation_lock.blocking_write(); let _lock = invalidation_lock.blocking_write();
{ {
let mut invalidator_map = inner.invalidator_map.lock().unwrap(); let mut invalidator_map = invalidator_map.lock().unwrap();
invalidate_path( invalidate_path(
&inner, &report_invalidation_reason,
report_invalidation_reason,
&mut invalidator_map, &mut invalidator_map,
batched_invalidate_path.drain(), batched_invalidate_path.drain(),
); );
invalidate_path_and_children_execute( invalidate_path_and_children_execute(
&inner, &report_invalidation_reason,
report_invalidation_reason,
&mut invalidator_map, &mut invalidator_map,
batched_invalidate_path_and_children.drain(), batched_invalidate_path_and_children.drain(),
); );
} }
{ {
let mut dir_invalidator_map = inner.dir_invalidator_map.lock().unwrap(); let mut dir_invalidator_map = dir_invalidator_map.lock().unwrap();
invalidate_path( invalidate_path(
&inner, &report_invalidation_reason,
report_invalidation_reason,
&mut dir_invalidator_map, &mut dir_invalidator_map,
batched_invalidate_path_dir.drain(), batched_invalidate_path_dir.drain(),
); );
invalidate_path_and_children_execute( invalidate_path_and_children_execute(
&inner, &report_invalidation_reason,
report_invalidation_reason,
&mut dir_invalidator_map, &mut dir_invalidator_map,
batched_invalidate_path_and_children_dir.drain(), batched_invalidate_path_and_children_dir.drain(),
); );
...@@ -442,13 +450,12 @@ impl DiskWatcher { ...@@ -442,13 +450,12 @@ impl DiskWatcher {
#[instrument(parent = None, level = "info", name = "DiskFileSystem file change", skip_all, fields(name = display(path.display())))] #[instrument(parent = None, level = "info", name = "DiskFileSystem file change", skip_all, fields(name = display(path.display())))]
fn invalidate( fn invalidate(
inner: &DiskFileSystemInner, report_invalidation_reason: &Option<(RcStr, PathBuf)>,
report_invalidation_reason: bool,
path: &Path, path: &Path,
invalidator: Invalidator, invalidator: Invalidator,
) { ) {
if report_invalidation_reason { if let Some((name, root_path)) = report_invalidation_reason {
if let Some(path) = format_absolute_fs_path(path, &inner.name, inner.root_path()) { if let Some(path) = format_absolute_fs_path(path, name, root_path) {
invalidator.invalidate_with_reason(WatchChange { path }); invalidator.invalidate_with_reason(WatchChange { path });
return; return;
} }
...@@ -457,8 +464,7 @@ fn invalidate( ...@@ -457,8 +464,7 @@ fn invalidate(
} }
fn invalidate_path( fn invalidate_path(
inner: &DiskFileSystemInner, report_invalidation_reason: &Option<(RcStr, PathBuf)>,
report_invalidation_reason: bool,
invalidator_map: &mut HashMap<String, HashSet<Invalidator>>, invalidator_map: &mut HashMap<String, HashSet<Invalidator>>,
paths: impl Iterator<Item = PathBuf>, paths: impl Iterator<Item = PathBuf>,
) { ) {
...@@ -467,14 +473,13 @@ fn invalidate_path( ...@@ -467,14 +473,13 @@ fn invalidate_path(
if let Some(invalidators) = invalidator_map.remove(&key) { if let Some(invalidators) = invalidator_map.remove(&key) {
invalidators invalidators
.into_iter() .into_iter()
.for_each(|i| invalidate(inner, report_invalidation_reason, &path, i)); .for_each(|i| invalidate(report_invalidation_reason, &path, i));
} }
} }
} }
fn invalidate_path_and_children_execute( fn invalidate_path_and_children_execute(
inner: &DiskFileSystemInner, report_invalidation_reason: &Option<(RcStr, PathBuf)>,
report_invalidation_reason: bool,
invalidator_map: &mut HashMap<String, HashSet<Invalidator>>, invalidator_map: &mut HashMap<String, HashSet<Invalidator>>,
paths: impl Iterator<Item = PathBuf>, paths: impl Iterator<Item = PathBuf>,
) { ) {
...@@ -483,7 +488,7 @@ fn invalidate_path_and_children_execute( ...@@ -483,7 +488,7 @@ fn invalidate_path_and_children_execute(
for (_, invalidators) in invalidator_map.extract_if(|key, _| key.starts_with(&path_key)) { for (_, invalidators) in invalidator_map.extract_if(|key, _| key.starts_with(&path_key)) {
invalidators invalidators
.into_iter() .into_iter()
.for_each(|i| invalidate(inner, report_invalidation_reason, &path, i)); .for_each(|i| invalidate(report_invalidation_reason, &path, i));
} }
} }
} }
use std::{borrow::Cow, collections::HashSet, future::Future, panic, pin::Pin};
use anyhow::{anyhow, Result};
use auto_hash_map::AutoSet;
use parking_lot::Mutex;
use tokio::task::JoinHandle;
use tracing::{Instrument, Span};
use turbo_tasks_macros::{TraceRawVcs, ValueDebugFormat};
use crate::{
self as turbo_tasks, emit, manager::turbo_tasks_future_scope, CollectiblesSource, ReadRef,
TryJoinIterExt, Vc,
};
/// A trait to emit a task effect as collectible. This trait only has one
/// implementation, `EffectInstance` and no other implementation is allowed.
/// The trait is private to this module so that no other implementation can be
/// added.
#[turbo_tasks::value_trait]
trait Effect {}
/// A future that represents the effect of a task. The future is executed when
/// the effect is applied.
type EffectFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + Sync + 'static>>;
/// The inner state of an effect instance if it has not been applied yet.
struct EffectInner {
future: EffectFuture,
span: Span,
}
/// The Effect instance collectible that is emitted for effects.
#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")]
struct EffectInstance {
#[turbo_tasks(trace_ignore, debug_ignore)]
inner: Mutex<Option<EffectInner>>,
}
impl EffectInstance {
fn new(future: impl Future<Output = Result<()>> + Send + Sync + 'static) -> Self {
Self {
inner: Mutex::new(Some(EffectInner {
future: Box::pin(future),
span: Span::current(),
})),
}
}
fn apply(&self) -> Option<JoinHandle<Result<()>>> {
let future = self.inner.lock().take();
future.map(|EffectInner { future, span }| {
tokio::spawn(
turbo_tasks_future_scope(turbo_tasks::turbo_tasks(), future).instrument(span),
)
})
}
}
#[turbo_tasks::value_impl]
impl Effect for EffectInstance {}
/// Schedules an effect to be applied. The passed future is executed once `apply_effects` is called.
///
/// The effect will only executed once. The passed future is executed outside of the current task
/// and can't read any Vcs. These need to be read before. ReadRefs can be passed into the future.
///
/// Effects are executed in parallel, so they might need to use async locking to avoid problems.
/// Order of execution of multiple effects is not defined. You must not use mutliple conflicting
/// effects to avoid non-deterministic behavior.
pub fn effect(future: impl Future<Output = Result<()>> + Send + Sync + 'static) {
emit::<Box<dyn Effect>>(Vc::upcast(EffectInstance::new(future).cell()));
}
/// Applies all effects that have been emitted by an operations.
///
/// Usually it's important that effects are strongly consistent, so one want to use `apply_effects`
/// only on operations that have been strongly consistently read before.
///
/// The order of execution is not defined and effects are executed in parallel.
///
/// `apply_effects` must only be used in a "once" task. When used in a "root" task, a
/// combination of `get_effects` and `Effects::apply` must be used.
///
/// # Example
///
/// ```rust
/// let operation = some_turbo_tasks_function(args);
/// let result = operation.strongly_consistent().await?;
/// apply_effects(operation).await?;
/// ```
pub async fn apply_effects(source: impl CollectiblesSource) -> Result<()> {
let effects: AutoSet<Vc<Box<dyn Effect>>> = source.take_collectibles();
if effects.is_empty() {
return Ok(());
}
let span = tracing::info_span!("apply effects", count = effects.len());
async move {
let mut first_error = anyhow::Ok(());
for effect in effects {
let Some(effect) = Vc::try_resolve_downcast_type::<EffectInstance>(effect).await?
else {
panic!("Effect must only be implemented by EffectInstance");
};
apply_effect(&effect.await?, &mut first_error).await;
}
first_error
}
.instrument(span)
.await
}
/// Capture effects from an turbo-tasks operation. Since this captures collectibles it might
/// invalidate the current task when effects are changing or even temporary change.
///
/// Therefore it's important to wrap this in a strongly consistent read before applying the effects
/// with `Effects::apply`.
///
/// # Example
///
/// ```rust
/// async fn some_turbo_tasks_function_with_effects(args: Args) -> Result<ResultWithEffects> {
/// let operation = some_turbo_tasks_function(args);
/// let result = operation.strongly_consistent().await?;
/// let effects = get_effects(operation).await?;
/// Ok(ResultWithEffects { result, effects })
/// }
///
/// let result_with_effects = some_turbo_tasks_function_with_effects(args).strongly_consistent().await?;
/// result_with_effects.effects.apply().await?;
/// ```
pub async fn get_effects(source: impl CollectiblesSource) -> Result<Effects> {
let effects: AutoSet<Vc<Box<dyn Effect>>> = source.take_collectibles();
let effects = effects
.into_iter()
.map(|effect| async move {
if let Some(effect) = Vc::try_resolve_downcast_type::<EffectInstance>(effect).await? {
Ok(effect.await?)
} else {
panic!("Effect must only be implemented by EffectInstance");
}
})
.try_join()
.await?;
Ok(Effects { effects })
}
/// Captured effects from an operation. This struct can be used to return Effects from a turbo-tasks
/// function and apply them later.
#[derive(TraceRawVcs, Default, ValueDebugFormat)]
pub struct Effects {
#[turbo_tasks(trace_ignore, debug_ignore)]
effects: Vec<ReadRef<EffectInstance>>,
}
impl PartialEq for Effects {
fn eq(&self, other: &Self) -> bool {
if self.effects.len() != other.effects.len() {
return false;
}
let effect_ptrs = self
.effects
.iter()
.map(ReadRef::ptr)
.collect::<HashSet<_>>();
other
.effects
.iter()
.all(|e| effect_ptrs.contains(&ReadRef::ptr(e)))
}
}
impl Eq for Effects {}
impl Effects {
/// Applies all effects that have been captured by this struct.
pub async fn apply(&self) -> Result<()> {
let span = tracing::info_span!("apply effects", count = self.effects.len());
async move {
let mut first_error = anyhow::Ok(());
for effect in self.effects.iter() {
apply_effect(effect, &mut first_error).await;
}
first_error
}
.instrument(span)
.await
}
}
async fn apply_effect(
effect: &ReadRef<EffectInstance>,
first_error: &mut std::result::Result<(), anyhow::Error>,
) {
if let Some(join_handle) = effect.apply() {
match join_handle.await {
Ok(Err(err)) if first_error.is_ok() => {
*first_error = Err(err);
}
Err(err) if first_error.is_ok() => {
let any = err.into_panic();
let panic = match any.downcast::<String>() {
Ok(owned) => Some(Cow::Owned(*owned)),
Err(any) => match any.downcast::<&'static str>() {
Ok(str) => Some(Cow::Borrowed(*str)),
Err(_) => None,
},
};
*first_error = Err(if let Some(panic) = panic {
anyhow!("Task effect panicked: {panic}")
} else {
anyhow!("Task effect panicked")
});
}
_ => {}
}
}
}
#[cfg(test)]
mod tests {
use crate::{apply_effects, get_effects, CollectiblesSource};
#[test]
#[allow(dead_code)]
fn is_sync_and_send() {
fn assert_sync<T: Sync + Send>(_: T) {}
fn check_apply_effects<T: CollectiblesSource + Send + Sync>(t: T) {
assert_sync(apply_effects(t));
}
fn check_get_effects<T: CollectiblesSource + Send + Sync>(t: T) {
assert_sync(get_effects(t));
}
}
}
...@@ -43,7 +43,6 @@ mod completion; ...@@ -43,7 +43,6 @@ mod completion;
pub mod debug; pub mod debug;
mod display; mod display;
pub mod duration_span; pub mod duration_span;
mod effect;
pub mod event; pub mod event;
pub mod graph; pub mod graph;
mod id; mod id;
...@@ -86,7 +85,6 @@ use auto_hash_map::AutoSet; ...@@ -86,7 +85,6 @@ use auto_hash_map::AutoSet;
pub use collectibles::CollectiblesSource; pub use collectibles::CollectiblesSource;
pub use completion::{Completion, Completions}; pub use completion::{Completion, Completions};
pub use display::ValueToString; pub use display::ValueToString;
pub use effect::{apply_effects, effect, get_effects, Effects};
pub use id::{ pub use id::{
ExecutionId, FunctionId, LocalTaskId, SessionId, TaskId, TraitTypeId, ValueTypeId, ExecutionId, FunctionId, LocalTaskId, SessionId, TaskId, TraitTypeId, ValueTypeId,
TRANSIENT_TASK_BIT, TRANSIENT_TASK_BIT,
......
...@@ -1672,13 +1672,6 @@ pub fn turbo_tasks_scope<T>(tt: Arc<dyn TurboTasksApi>, f: impl FnOnce() -> T) - ...@@ -1672,13 +1672,6 @@ pub fn turbo_tasks_scope<T>(tt: Arc<dyn TurboTasksApi>, f: impl FnOnce() -> T) -
TURBO_TASKS.sync_scope(tt, f) TURBO_TASKS.sync_scope(tt, f)
} }
pub fn turbo_tasks_future_scope<T>(
tt: Arc<dyn TurboTasksApi>,
f: impl Future<Output = T>,
) -> impl Future<Output = T> {
TURBO_TASKS.scope(tt, f)
}
pub fn with_turbo_tasks_for_testing<T>( pub fn with_turbo_tasks_for_testing<T>(
tt: Arc<dyn TurboTasksApi>, tt: Arc<dyn TurboTasksApi>,
current_task: TaskId, current_task: TaskId,
......
...@@ -234,10 +234,6 @@ impl<T> ReadRef<T> { ...@@ -234,10 +234,6 @@ impl<T> ReadRef<T> {
pub fn ptr_eq(&self, other: &ReadRef<T>) -> bool { pub fn ptr_eq(&self, other: &ReadRef<T>) -> bool {
triomphe::Arc::ptr_eq(&self.0, &other.0) triomphe::Arc::ptr_eq(&self.0, &other.0)
} }
pub fn ptr(&self) -> *const T {
&*self.0 as *const T
}
} }
impl<T> ReadRef<T> impl<T> ReadRef<T>
......
...@@ -8,8 +8,7 @@ use std::{ ...@@ -8,8 +8,7 @@ use std::{
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
apply_effects, ReadConsistency, ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasks, ReadConsistency, ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasks, Value, Vc,
Value, Vc,
}; };
use turbo_tasks_fs::FileSystem; use turbo_tasks_fs::FileSystem;
use turbo_tasks_memory::MemoryBackend; use turbo_tasks_memory::MemoryBackend;
...@@ -129,9 +128,7 @@ impl TurbopackBuildBuilder { ...@@ -129,9 +128,7 @@ impl TurbopackBuildBuilder {
); );
// Await the result to propagate any errors. // Await the result to propagate any errors.
build_result.strongly_consistent().await?; build_result.await?;
apply_effects(build_result).await?;
let issue_reporter: Vc<Box<dyn IssueReporter>> = let issue_reporter: Vc<Box<dyn IssueReporter>> =
Vc::upcast(ConsoleUi::new(TransientInstance::new(LogOptions { Vc::upcast(ConsoleUi::new(TransientInstance::new(LogOptions {
......
use anyhow::Result; use anyhow::Result;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{Completion, ResolvedVc, Vc};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
FileContent, FileJsonContent, FileLinesContent, FileSystemPath, LinkContent, LinkType, FileContent, FileJsonContent, FileLinesContent, FileSystemPath, LinkContent, LinkType,
}; };
...@@ -86,22 +86,17 @@ impl AssetContent { ...@@ -86,22 +86,17 @@ impl AssetContent {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn write(self: Vc<Self>, path: Vc<FileSystemPath>) -> Result<()> { pub async fn write(self: Vc<Self>, path: Vc<FileSystemPath>) -> Result<Vc<Completion>> {
let this = self.await?; let this = self.await?;
match &*this { Ok(match &*this {
AssetContent::File(file) => { AssetContent::File(file) => path.write(**file),
let _ = path.write(**file); AssetContent::Redirect { target, link_type } => path.write_link(
} LinkContent::Link {
AssetContent::Redirect { target, link_type } => { target: target.clone(),
let _ = path.write_link( link_type: *link_type,
LinkContent::Link { }
target: target.clone(), .cell(),
link_type: *link_type, ),
} })
.cell(),
);
}
}
Ok(())
} }
} }
...@@ -251,7 +251,7 @@ pub async fn fileify_source_map( ...@@ -251,7 +251,7 @@ pub async fn fileify_source_map(
.await? .await?
.context("Expected the chunking context to have a DiskFileSystem")? .context("Expected the chunking context to have a DiskFileSystem")?
.await?; .await?;
let prefix = format!("{}[{}]/", SOURCE_MAP_PREFIX, context_fs.name()); let prefix = format!("{}[{}]/", SOURCE_MAP_PREFIX, context_fs.name);
let mut transformed = flattened.into_owned(); let mut transformed = flattened.into_owned();
let mut updates = IndexMap::new(); let mut updates = IndexMap::new();
......
...@@ -39,12 +39,20 @@ impl FileSystem for ServerFileSystem { ...@@ -39,12 +39,20 @@ impl FileSystem for ServerFileSystem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn write(&self, _fs_path: Vc<FileSystemPath>, _content: Vc<FileContent>) -> Result<Vc<()>> { fn write(
&self,
_fs_path: Vc<FileSystemPath>,
_content: Vc<FileContent>,
) -> Result<Vc<Completion>> {
bail!("Writing is not possible to the marker filesystem for the server") bail!("Writing is not possible to the marker filesystem for the server")
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn write_link(&self, _fs_path: Vc<FileSystemPath>, _target: Vc<LinkContent>) -> Result<Vc<()>> { fn write_link(
&self,
_fs_path: Vc<FileSystemPath>,
_target: Vc<LinkContent>,
) -> Result<Vc<Completion>> {
bail!("Writing is not possible to the marker filesystem for the server") bail!("Writing is not possible to the marker filesystem for the server")
} }
......
...@@ -10,9 +10,7 @@ use hyper::{ ...@@ -10,9 +10,7 @@ use hyper::{
}; };
use mime::Mime; use mime::Mime;
use tokio_util::io::{ReaderStream, StreamReader}; use tokio_util::io::{ReaderStream, StreamReader};
use turbo_tasks::{ use turbo_tasks::{util::SharedError, CollectiblesSource, ReadRef, TransientInstance, Vc};
apply_effects, util::SharedError, CollectiblesSource, ReadRef, TransientInstance, Vc,
};
use turbo_tasks_bytes::Bytes; use turbo_tasks_bytes::Bytes;
use turbo_tasks_fs::FileContent; use turbo_tasks_fs::FileContent;
use turbopack_core::{ use turbopack_core::{
...@@ -82,7 +80,6 @@ pub async fn process_request_with_content_source( ...@@ -82,7 +80,6 @@ pub async fn process_request_with_content_source(
let request = http_request_to_source_request(request).await?; let request = http_request_to_source_request(request).await?;
let result = get_from_source(source, TransientInstance::new(request)); let result = get_from_source(source, TransientInstance::new(request));
let resolved_result = result.resolve_strongly_consistent().await?; let resolved_result = result.resolve_strongly_consistent().await?;
apply_effects(result).await?;
let side_effects: AutoSet<Vc<Box<dyn ContentSourceSideEffect>>> = result.peek_collectibles(); let side_effects: AutoSet<Vc<Box<dyn ContentSourceSideEffect>>> = result.peek_collectibles();
handle_issues( handle_issues(
result, result,
......
...@@ -33,8 +33,7 @@ use socket2::{Domain, Protocol, Socket, Type}; ...@@ -33,8 +33,7 @@ use socket2::{Domain, Protocol, Socket, Type};
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tracing::{event, info_span, Instrument, Level, Span}; use tracing::{event, info_span, Instrument, Level, Span};
use turbo_tasks::{ use turbo_tasks::{
apply_effects, run_once_with_reason, trace::TraceRawVcs, util::FormatDuration, TurboTasksApi, run_once_with_reason, trace::TraceRawVcs, util::FormatDuration, TurboTasksApi, Vc,
Vc,
}; };
use turbopack_core::{ use turbopack_core::{
error::PrettyPrintError, error::PrettyPrintError,
...@@ -212,7 +211,6 @@ impl DevServerBuilder { ...@@ -212,7 +211,6 @@ impl DevServerBuilder {
let path = uri.path().to_string(); let path = uri.path().to_string();
let source = source_provider.get_source(); let source = source_provider.get_source();
let resolved_source = source.resolve_strongly_consistent().await?; let resolved_source = source.resolve_strongly_consistent().await?;
apply_effects(source).await?;
handle_issues( handle_issues(
source, source,
issue_reporter, issue_reporter,
......
...@@ -12,8 +12,8 @@ use parking_lot::Mutex; ...@@ -12,8 +12,8 @@ use parking_lot::Mutex;
use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
use turbo_tasks::{ use turbo_tasks::{
apply_effects, duration_span, fxindexmap, mark_finished, prevent_gc, util::SharedError, duration_span, fxindexmap, mark_finished, prevent_gc, util::SharedError, Completion,
Completion, FxIndexMap, RawVc, ResolvedVc, TaskInput, TryJoinIterExt, Value, Vc, FxIndexMap, RawVc, ResolvedVc, TaskInput, TryJoinIterExt, Value, Vc,
}; };
use turbo_tasks_bytes::{Bytes, Stream}; use turbo_tasks_bytes::{Bytes, Stream};
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
...@@ -28,7 +28,7 @@ use turbopack_core::{ ...@@ -28,7 +28,7 @@ use turbopack_core::{
ident::AssetIdent, ident::AssetIdent,
issue::{Issue, IssueExt, IssueStage, OptionStyledString, StyledString}, issue::{Issue, IssueExt, IssueStage, OptionStyledString, StyledString},
module::Module, module::Module,
output::{OutputAsset, OutputAssets}, output::OutputAssets,
reference_type::{InnerAssets, ReferenceType}, reference_type::{InnerAssets, ReferenceType},
virtual_source::VirtualSource, virtual_source::VirtualSource,
}; };
...@@ -77,20 +77,26 @@ pub struct JavaScriptStreamSender { ...@@ -77,20 +77,26 @@ pub struct JavaScriptStreamSender {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct JavaScriptEvaluation(#[turbo_tasks(trace_ignore)] JavaScriptStream); pub struct JavaScriptEvaluation(#[turbo_tasks(trace_ignore)] JavaScriptStream);
#[turbo_tasks::value] #[derive(Clone, Copy, Hash, Debug, PartialEq, Eq, Serialize, Deserialize, TaskInput)]
struct EmittedEvaluatePoolAssets { pub enum EnvVarTracking {
bootstrap: ResolvedVc<Box<dyn OutputAsset>>, WholeEnvTracked,
output_root: ResolvedVc<FileSystemPath>, Untracked,
entrypoint: ResolvedVc<FileSystemPath>,
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn emit_evaluate_pool_assets( /// Pass the file you cared as `runtime_entries` to invalidate and reload the
/// evaluated result automatically.
pub async fn get_evaluate_pool(
module_asset: ResolvedVc<Box<dyn Module>>, module_asset: ResolvedVc<Box<dyn Module>>,
cwd: Vc<FileSystemPath>,
env: Vc<Box<dyn ProcessEnv>>,
asset_context: Vc<Box<dyn AssetContext>>, asset_context: Vc<Box<dyn AssetContext>>,
chunking_context: Vc<Box<dyn ChunkingContext>>, chunking_context: Vc<Box<dyn ChunkingContext>>,
runtime_entries: Option<Vc<EvaluatableAssets>>, runtime_entries: Option<Vc<EvaluatableAssets>>,
) -> Result<Vc<EmittedEvaluatePoolAssets>> { additional_invalidation: Vc<Completion>,
debug: bool,
env_var_tracking: EnvVarTracking,
) -> Result<Vc<NodeJsPool>> {
let runtime_asset = asset_context let runtime_asset = asset_context
.process( .process(
Vc::upcast(FileSource::new(embed_file_path("ipc/evaluate.ts".into()))), Vc::upcast(FileSource::new(embed_file_path("ipc/evaluate.ts".into()))),
...@@ -111,7 +117,7 @@ async fn emit_evaluate_pool_assets( ...@@ -111,7 +117,7 @@ async fn emit_evaluate_pool_assets(
} else { } else {
Cow::Owned(format!("{file_name}.js")) Cow::Owned(format!("{file_name}.js"))
}; };
let entrypoint = chunking_context.output_root().join(file_name.into()); let path = chunking_context.output_root().join(file_name.into());
let entry_module = asset_context let entry_module = asset_context
.process( .process(
Vc::upcast(VirtualSource::new( Vc::upcast(VirtualSource::new(
...@@ -127,6 +133,10 @@ async fn emit_evaluate_pool_assets( ...@@ -127,6 +133,10 @@ async fn emit_evaluate_pool_assets(
) )
.module(); .module();
let (Some(cwd), Some(entrypoint)) = (to_sys_path(cwd).await?, to_sys_path(path).await?) else {
panic!("can only evaluate from a disk filesystem");
};
let runtime_entries = { let runtime_entries = {
let globals_module = asset_context let globals_module = asset_context
.process( .process(
...@@ -154,7 +164,7 @@ async fn emit_evaluate_pool_assets( ...@@ -154,7 +164,7 @@ async fn emit_evaluate_pool_assets(
}; };
let bootstrap = chunking_context.root_entry_chunk_group_asset( let bootstrap = chunking_context.root_entry_chunk_group_asset(
entrypoint, path,
entry_module, entry_module,
OutputAssets::empty(), OutputAssets::empty(),
runtime_entries, runtime_entries,
...@@ -162,75 +172,10 @@ async fn emit_evaluate_pool_assets( ...@@ -162,75 +172,10 @@ async fn emit_evaluate_pool_assets(
let output_root = chunking_context.output_root().to_resolved().await?; let output_root = chunking_context.output_root().to_resolved().await?;
let _ = emit_package_json(*output_root); let _ = emit_package_json(*output_root);
let _ = emit(bootstrap, *output_root);
Ok(EmittedEvaluatePoolAssets {
bootstrap: bootstrap.to_resolved().await?,
output_root,
entrypoint: entrypoint.to_resolved().await?,
}
.cell())
}
#[turbo_tasks::function]
async fn emit_evaluate_pool_assets_with_effects(
module_asset: Vc<Box<dyn Module>>,
asset_context: Vc<Box<dyn AssetContext>>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
runtime_entries: Option<Vc<EvaluatableAssets>>,
) -> Result<Vc<EmittedEvaluatePoolAssets>> {
let operation = emit_evaluate_pool_assets(
module_asset,
asset_context,
chunking_context,
runtime_entries,
);
let result = operation.resolve_strongly_consistent().await?;
apply_effects(operation).await?;
Ok(result)
}
#[derive(Clone, Copy, Hash, Debug, PartialEq, Eq, Serialize, Deserialize, TaskInput)]
pub enum EnvVarTracking {
WholeEnvTracked,
Untracked,
}
#[turbo_tasks::function]
/// Pass the file you cared as `runtime_entries` to invalidate and reload the
/// evaluated result automatically.
pub async fn get_evaluate_pool(
module_asset: Vc<Box<dyn Module>>,
cwd: Vc<FileSystemPath>,
env: Vc<Box<dyn ProcessEnv>>,
asset_context: Vc<Box<dyn AssetContext>>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
runtime_entries: Option<Vc<EvaluatableAssets>>,
additional_invalidation: Vc<Completion>,
debug: bool,
env_var_tracking: EnvVarTracking,
) -> Result<Vc<NodeJsPool>> {
let EmittedEvaluatePoolAssets {
bootstrap,
output_root,
entrypoint,
} = *emit_evaluate_pool_assets_with_effects(
module_asset,
asset_context,
chunking_context,
runtime_entries,
)
.strongly_consistent()
.await?;
let (Some(cwd), Some(entrypoint)) = (to_sys_path(cwd).await?, to_sys_path(*entrypoint).await?)
else {
panic!("can only evaluate from a disk filesystem");
};
// Invalidate pool when code content changes // Invalidate pool when code content changes
content_changed(Vc::upcast(*bootstrap)).await?; content_changed(Vc::upcast(bootstrap)).await?;
let assets_for_source_mapping = internal_assets_for_source_mapping(*bootstrap, *output_root) let _ = emit(bootstrap, *output_root);
let assets_for_source_mapping = internal_assets_for_source_mapping(bootstrap, *output_root)
.to_resolved() .to_resolved()
.await?; .await?;
let env = match env_var_tracking { let env = match env_var_tracking {
...@@ -412,12 +357,12 @@ pub async fn compute( ...@@ -412,12 +357,12 @@ pub async fn compute(
}; };
let stream = generator! { let stream = generator! {
let pool_operation = evaluate_context.pool(); let pool = evaluate_context.pool();
let mut state = Default::default(); let mut state = Default::default();
// Read this strongly consistent, since we don't want to run inconsistent // Read this strongly consistent, since we don't want to run inconsistent
// node.js code. // node.js code.
let pool = pool_operation.strongly_consistent().await?; let pool = pool.strongly_consistent().await?;
let args = evaluate_context.args().iter().try_join().await?; let args = evaluate_context.args().iter().try_join().await?;
// Assume this is a one-off operation, so we can kill the process // Assume this is a one-off operation, so we can kill the process
......
...@@ -11,13 +11,12 @@ pub use node_entry::{NodeEntry, NodeRenderingEntries, NodeRenderingEntry}; ...@@ -11,13 +11,12 @@ pub use node_entry::{NodeEntry, NodeRenderingEntries, NodeRenderingEntry};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal}, graph::{AdjacencyMap, GraphTraversal},
FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc, Completion, Completions, FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc,
}; };
use turbo_tasks_env::ProcessEnv; use turbo_tasks_env::ProcessEnv;
use turbo_tasks_fs::{to_sys_path, File, FileSystemPath}; use turbo_tasks_fs::{to_sys_path, File, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
changed::content_changed,
chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAssets}, chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAssets},
module::Module, module::Module,
output::{OutputAsset, OutputAssets, OutputAssetsSet}, output::{OutputAsset, OutputAssets, OutputAssetsSet},
...@@ -42,11 +41,16 @@ pub mod transforms; ...@@ -42,11 +41,16 @@ pub mod transforms;
async fn emit( async fn emit(
intermediate_asset: Vc<Box<dyn OutputAsset>>, intermediate_asset: Vc<Box<dyn OutputAsset>>,
intermediate_output_path: Vc<FileSystemPath>, intermediate_output_path: Vc<FileSystemPath>,
) -> Result<()> { ) -> Result<Vc<Completion>> {
for asset in internal_assets(intermediate_asset, intermediate_output_path).await? { Ok(Vc::<Completions>::cell(
let _ = asset.content().write(asset.ident().path()); internal_assets(intermediate_asset, intermediate_output_path)
} .strongly_consistent()
Ok(()) .await?
.iter()
.map(|a| a.content().write(a.ident().path()))
.collect(),
)
.completed())
} }
/// List of the all assets of the "internal" subgraph and a list of boundary /// List of the all assets of the "internal" subgraph and a list of boundary
...@@ -192,7 +196,7 @@ async fn separate_assets( ...@@ -192,7 +196,7 @@ async fn separate_assets(
/// Emit a basic package.json that sets the type of the package to commonjs. /// Emit a basic package.json that sets the type of the package to commonjs.
/// Currently code generated for Node is CommonJS, while authored code may be /// Currently code generated for Node is CommonJS, while authored code may be
/// ESM, for example. /// ESM, for example.
fn emit_package_json(dir: Vc<FileSystemPath>) -> Vc<()> { fn emit_package_json(dir: Vc<FileSystemPath>) -> Vc<Completion> {
emit( emit(
Vc::upcast(VirtualOutputAsset::new( Vc::upcast(VirtualOutputAsset::new(
dir.join("package.json".into()), dir.join("package.json".into()),
...@@ -215,7 +219,7 @@ pub async fn get_renderer_pool( ...@@ -215,7 +219,7 @@ pub async fn get_renderer_pool(
) -> Result<Vc<NodeJsPool>> { ) -> Result<Vc<NodeJsPool>> {
emit_package_json(intermediate_output_path).await?; emit_package_json(intermediate_output_path).await?;
let _ = emit(intermediate_asset, *output_root); let emit = emit(intermediate_asset, *output_root);
let assets_for_source_mapping = let assets_for_source_mapping =
internal_assets_for_source_mapping(intermediate_asset, *output_root); internal_assets_for_source_mapping(intermediate_asset, *output_root);
...@@ -233,9 +237,8 @@ pub async fn get_renderer_pool( ...@@ -233,9 +237,8 @@ pub async fn get_renderer_pool(
entrypoint.to_string().await? entrypoint.to_string().await?
); );
}; };
// Invalidate pool when code content changes
content_changed(Vc::upcast(intermediate_asset)).await?;
emit.await?;
Ok(NodeJsPool::new( Ok(NodeJsPool::new(
cwd, cwd,
entrypoint, entrypoint,
......
...@@ -12,7 +12,7 @@ use dunce::canonicalize; ...@@ -12,7 +12,7 @@ use dunce::canonicalize;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
apply_effects, debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs, Completion, ResolvedVc, debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs, Completion, ResolvedVc,
TryJoinIterExt, TurboTasks, Value, Vc, TryJoinIterExt, TurboTasks, Value, Vc,
}; };
use turbo_tasks_bytes::stream::SingleValue; use turbo_tasks_bytes::stream::SingleValue;
...@@ -66,8 +66,6 @@ struct JsResult { ...@@ -66,8 +66,6 @@ struct JsResult {
jest_result: JestRunResult, jest_result: JestRunResult,
} }
#[turbo_tasks::value]
#[derive(Copy, Clone, Debug, Hash)]
enum IssueSnapshotMode { enum IssueSnapshotMode {
Snapshots, Snapshots,
NoSnapshots, NoSnapshots,
...@@ -170,29 +168,18 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result<JsRe ...@@ -170,29 +168,18 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result<JsRe
let tt = TurboTasks::new(MemoryBackend::default()); let tt = TurboTasks::new(MemoryBackend::default());
tt.run_once(async move { tt.run_once(async move {
let emit = run_inner(resource.to_str().unwrap().into(), Value::new(snapshot_mode)); let resource_str = resource.to_str().unwrap();
let result = emit.strongly_consistent().await?; let prepared_test = prepare_test(resource_str.into());
apply_effects(emit).await?; let run_result = run_test(prepared_test);
if matches!(snapshot_mode, IssueSnapshotMode::Snapshots) {
snapshot_issues(prepared_test, run_result).await?;
}
Ok(result.clone_value()) Ok((*run_result.await.unwrap().js_result.await.unwrap()).clone())
}) })
.await .await
} }
#[turbo_tasks::function]
async fn run_inner(
resource: RcStr,
snapshot_mode: Value<IssueSnapshotMode>,
) -> Result<Vc<JsResult>> {
let prepared_test = prepare_test(resource);
let run_result = run_test(prepared_test);
if *snapshot_mode == IssueSnapshotMode::Snapshots {
snapshot_issues(prepared_test, run_result).await?;
}
Ok(*run_result.await?.js_result)
}
#[derive(PartialEq, Eq, Debug, Default, Serialize, Deserialize, TraceRawVcs, ValueDebugFormat)] #[derive(PartialEq, Eq, Debug, Default, Serialize, Deserialize, TraceRawVcs, ValueDebugFormat)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase", deny_unknown_fields)]
struct TestOptions { struct TestOptions {
......
...@@ -15,8 +15,7 @@ use serde::Deserialize; ...@@ -15,8 +15,7 @@ use serde::Deserialize;
use serde_json::json; use serde_json::json;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
apply_effects, ReadConsistency, ReadRef, ResolvedVc, TryJoinIterExt, TurboTasks, Value, ReadConsistency, ReadRef, ResolvedVc, TryJoinIterExt, TurboTasks, Value, ValueToString, Vc,
ValueToString, Vc,
}; };
use turbo_tasks_env::DotenvProcessEnv; use turbo_tasks_env::DotenvProcessEnv;
use turbo_tasks_fs::{ use turbo_tasks_fs::{
...@@ -157,10 +156,19 @@ async fn run(resource: PathBuf) -> Result<()> { ...@@ -157,10 +156,19 @@ async fn run(resource: PathBuf) -> Result<()> {
let tt = TurboTasks::new(MemoryBackend::default()); let tt = TurboTasks::new(MemoryBackend::default());
let task = tt.spawn_once_task(async move { let task = tt.spawn_once_task(async move {
let emit = run_inner(resource.to_str().unwrap().into()); let out = run_test(resource.to_str().unwrap().into());
emit.strongly_consistent().await?; let _ = out.resolve_strongly_consistent().await?;
apply_effects(emit).await?; let captured_issues = out.peek_issues_with_path().await?;
let plain_issues = captured_issues
.iter_with_shortest_path()
.map(|(issue_vc, path)| async move { issue_vc.into_plain(path).await })
.try_join()
.await?;
snapshot_issues(plain_issues, out.join("issues".into()), &REPO_ROOT)
.await
.context("Unable to handle issues")?;
Ok(Vc::<()>::default()) Ok(Vc::<()>::default())
}); });
tt.wait_task_completion(task, ReadConsistency::Strong) tt.wait_task_completion(task, ReadConsistency::Strong)
...@@ -169,25 +177,6 @@ async fn run(resource: PathBuf) -> Result<()> { ...@@ -169,25 +177,6 @@ async fn run(resource: PathBuf) -> Result<()> {
Ok(()) Ok(())
} }
#[turbo_tasks::function]
async fn run_inner(resource: RcStr) -> Result<()> {
let out = run_test(resource);
let _ = out.resolve_strongly_consistent().await?;
let captured_issues = out.peek_issues_with_path().await?;
let plain_issues = captured_issues
.iter_with_shortest_path()
.map(|(issue_vc, path)| async move { issue_vc.into_plain(path).await })
.try_join()
.await?;
snapshot_issues(plain_issues, out.join("issues".into()), &REPO_ROOT)
.await
.context("Unable to handle issues")?;
Ok(())
}
#[turbo_tasks::function] #[turbo_tasks::function]
async fn run_test(resource: RcStr) -> Result<Vc<FileSystemPath>> { async fn run_test(resource: RcStr) -> Result<Vc<FileSystemPath>> {
let test_path = canonicalize(&resource)?; let test_path = canonicalize(&resource)?;
......
...@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf}; ...@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf};
use criterion::{Bencher, BenchmarkId, Criterion}; use criterion::{Bencher, BenchmarkId, Criterion};
use regex::Regex; use regex::Regex;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{apply_effects, ReadConsistency, TurboTasks, Value, Vc}; use turbo_tasks::{ReadConsistency, TurboTasks, Value, Vc};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem};
use turbo_tasks_memory::MemoryBackend; use turbo_tasks_memory::MemoryBackend;
use turbopack::{ use turbopack::{
...@@ -117,9 +117,7 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) { ...@@ -117,9 +117,7 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) {
.module(); .module();
let rebased = RebasedAsset::new(Vc::upcast(module), input_dir, output_dir); let rebased = RebasedAsset::new(Vc::upcast(module), input_dir, output_dir);
let emit = emit_with_completion(Vc::upcast(rebased), output_dir); emit_with_completion(Vc::upcast(rebased), output_dir).await?;
emit.strongly_consistent().await?;
apply_effects(emit).await?;
Ok::<Vc<()>, _>(Default::default()) Ok::<Vc<()>, _>(Default::default())
}); });
......
...@@ -30,7 +30,7 @@ use graph::{aggregate, AggregatedGraph, AggregatedGraphNodeContent}; ...@@ -30,7 +30,7 @@ use graph::{aggregate, AggregatedGraph, AggregatedGraphNodeContent};
use module_options::{ModuleOptions, ModuleOptionsContext, ModuleRuleEffect, ModuleType}; use module_options::{ModuleOptions, ModuleOptionsContext, ModuleRuleEffect, ModuleType};
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Value, ValueToString, Vc}; use turbo_tasks::{Completion, ResolvedVc, Value, ValueToString, Vc};
use turbo_tasks_fs::{glob::Glob, FileSystemPath}; use turbo_tasks_fs::{glob::Glob, FileSystemPath};
pub use turbopack_core::condition; pub use turbopack_core::condition;
use turbopack_core::{ use turbopack_core::{
...@@ -899,49 +899,54 @@ impl AssetContext for ModuleAssetContext { ...@@ -899,49 +899,54 @@ impl AssetContext for ModuleAssetContext {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn emit_with_completion(asset: Vc<Box<dyn OutputAsset>>, output_dir: Vc<FileSystemPath>) { pub fn emit_with_completion(
let _ = emit_assets_aggregated(asset, output_dir); asset: Vc<Box<dyn OutputAsset>>,
output_dir: Vc<FileSystemPath>,
) -> Vc<Completion> {
emit_assets_aggregated(asset, output_dir)
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn emit_assets_aggregated(asset: Vc<Box<dyn OutputAsset>>, output_dir: Vc<FileSystemPath>) { fn emit_assets_aggregated(
asset: Vc<Box<dyn OutputAsset>>,
output_dir: Vc<FileSystemPath>,
) -> Vc<Completion> {
let aggregated = aggregate(asset); let aggregated = aggregate(asset);
let _ = emit_aggregated_assets(aggregated, output_dir); emit_aggregated_assets(aggregated, output_dir)
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn emit_aggregated_assets( async fn emit_aggregated_assets(
aggregated: Vc<AggregatedGraph>, aggregated: Vc<AggregatedGraph>,
output_dir: Vc<FileSystemPath>, output_dir: Vc<FileSystemPath>,
) -> Result<()> { ) -> Result<Vc<Completion>> {
match &*aggregated.content().await? { Ok(match &*aggregated.content().await? {
AggregatedGraphNodeContent::Asset(asset) => { AggregatedGraphNodeContent::Asset(asset) => emit_asset_into_dir(**asset, output_dir),
let _ = emit_asset_into_dir(**asset, output_dir);
}
AggregatedGraphNodeContent::Children(children) => { AggregatedGraphNodeContent::Children(children) => {
for aggregated in children { for aggregated in children {
let _ = emit_aggregated_assets(**aggregated, output_dir); emit_aggregated_assets(**aggregated, output_dir).await?;
} }
Completion::new()
} }
} })
Ok(())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn emit_asset(asset: Vc<Box<dyn OutputAsset>>) { pub fn emit_asset(asset: Vc<Box<dyn OutputAsset>>) -> Vc<Completion> {
let _ = asset.content().write(asset.ident().path()); asset.content().write(asset.ident().path())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn emit_asset_into_dir( pub async fn emit_asset_into_dir(
asset: Vc<Box<dyn OutputAsset>>, asset: Vc<Box<dyn OutputAsset>>,
output_dir: Vc<FileSystemPath>, output_dir: Vc<FileSystemPath>,
) -> Result<()> { ) -> Result<Vc<Completion>> {
let dir = &*output_dir.await?; let dir = &*output_dir.await?;
if asset.ident().path().await?.is_inside_ref(dir) { Ok(if asset.ident().path().await?.is_inside_ref(dir) {
let _ = emit_asset(asset); emit_asset(asset)
} } else {
Ok(()) Completion::new()
})
} }
type OutputAssetSet = HashSet<Vc<Box<dyn OutputAsset>>>; type OutputAssetSet = HashSet<Vc<Box<dyn OutputAsset>>>;
......
...@@ -30,9 +30,7 @@ use rstest_reuse::{ ...@@ -30,9 +30,7 @@ use rstest_reuse::{
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::{process::Command, time::timeout}; use tokio::{process::Command, time::timeout};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{backend::Backend, ReadRef, ResolvedVc, TurboTasks, Value, ValueToString, Vc};
apply_effects, backend::Backend, ReadRef, ResolvedVc, TurboTasks, Value, ValueToString, Vc,
};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath};
use turbo_tasks_memory::MemoryBackend; use turbo_tasks_memory::MemoryBackend;
use turbopack::{ use turbopack::{
...@@ -466,9 +464,7 @@ fn node_file_trace<B: Backend + 'static>( ...@@ -466,9 +464,7 @@ fn node_file_trace<B: Backend + 'static>(
print_graph(ResolvedVc::upcast(rebased)).await?; print_graph(ResolvedVc::upcast(rebased)).await?;
let emit = emit_with_completion(*ResolvedVc::upcast(rebased), output_dir); emit_with_completion(*ResolvedVc::upcast(rebased), output_dir).await?;
emit.strongly_consistent().await?;
apply_effects(emit).await?;
#[cfg(not(feature = "bench_against_node_nft"))] #[cfg(not(feature = "bench_against_node_nft"))]
{ {
...@@ -663,12 +659,10 @@ fn clean_stderr(str: &str) -> String { ...@@ -663,12 +659,10 @@ fn clean_stderr(str: &str) -> String {
lazy_static! { lazy_static! {
static ref EXPERIMENTAL_WARNING: Regex = static ref EXPERIMENTAL_WARNING: Regex =
Regex::new(r"\(node:\d+\) ExperimentalWarning:").unwrap(); Regex::new(r"\(node:\d+\) ExperimentalWarning:").unwrap();
static ref DEPRECATION_WARNING: Regex =
Regex::new(r"\(node:\d+\) \[DEP\d+] DeprecationWarning:").unwrap();
} }
let str = EXPERIMENTAL_WARNING.replace_all(str, "(node:XXXX) ExperimentalWarning:"); EXPERIMENTAL_WARNING
let str = DEPRECATION_WARNING.replace_all(&str, "(node:XXXX) [DEPXXXX] DeprecationWarning:"); .replace_all(str, "(node:XXXX) ExperimentalWarning:")
str.to_string() .to_string()
} }
fn diff(expected: &str, actual: &str) -> String { fn diff(expected: &str, actual: &str) -> String {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment