Unverified Commit 5ee70052 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: use module graph for NFT (#92901)

- [x] skip traced edges properly in all graph traversals
- [x] iter_nodes and enumerate_nodes include the traced modules
    - [x] Don't propagate is_self_async inside of traced references as a perf optimization
- [x] fix performance
- [x] validate that NFT JSON files are identical
- [x] ~~validate/adjust bundle analyzer output~~ will be in followup
- [x] audit `fn chunking_type()` changes

Benchmark:
```
* ced85b3108 - (2 weeks ago) refactor - Niklas Mischkulnig (HEAD -> mischnic/nft-module-graph, origin/mischnic/nft-module-graph)

TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   334.96s user 44.57s system 788% cpu 48.152 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   343.39s user 42.61s system 809% cpu 47.659 total

* 5edf4823 - (77 minutes ago) v16.3.0-canary.32 - next-js-bot[bot] (HEAD, tag: v16.3.0-canary.32)

TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   333.42s user 43.57s system 712% cpu 52.877 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   332.95s user 43.15s system 791% cpu 47.515 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   337.59s user 41.28s system 771% cpu 49.117 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   335.25s user 45.53s system 718% cpu 52.975 total
```

<details>
<summary>Previous benchmarks</summary>

```
52212418699f362904d64c08e544d4994e8d27fa mischnic/nft-module-graph

pnpm next build   371.41s user 66.39s system 737% cpu 59.353 total
pnpm next build   380.13s user 59.62s system 751% cpu 58.548 total
pnpm next build   374.25s user 60.55s system 761% cpu 57.102 total

8e4cfc50 canary

pnpm next build   370.77s user 53.89s system 762% cpu 55.725 total
pnpm next build   383.16s user 55.61s system 744% cpu 58.921 total
pnpm next build   382.34s user 53.51s system 757% cpu 57.512 total
pnpm next build   381.15s user 54.50s system 751% cpu 57.943 total
```


```
b0d5411a79 mischnic/nft-module-graph
pnpm next build --experimental-build-mode=compil  368.75s user 62.05s system 749% cpu 57.445 total
pnpm next build --experimental-build-mode=compil  368.95s user 58.42s system 763% cpu 56.004 total

0e3ad2768bd3b036e96f66e4b770a16bb0b38705 mischnic/nft-module-graph
pnpm next build --experimental-build-mode=compil  369.48s user 60.62s system 748% cpu 57.494 total
pnpm next build --experimental-build-mode=compil  374.89s user 52.27s system 730% cpu 58.449 total
pnpm next build --experimental-build-mode=compil  375.02s user 52.89s system 747% cpu 57.274 total

f37073e1d96f6d846cd333b93fccd1dd7bdc253e mischnic/speed-next-dynamic-map
pnpm next build --experimental-build-mode=compil  389.58s user 55.98s system 704% cpu 1:03.21 total

e599b11e2d6b09a92600cfca6d9b338ed1f7537d mischnic/fix-side-effect-edge-removal
pnpm next build --experimental-build-mode=compil  386.60s user 53.34s system 720% cpu 1:01.09 total

f68f70b0 canary
pnpm next build --experimental-build-mode=compil  387.46s user 53.72s system 738% cpu 59.716 total
pnpm next build --experimental-build-mode=compil  390.33s user 53.54s system 734% cpu 1:00.43 total
pnpm next build --experimental-build-mode=compil  385.33s user 56.55s system 722% cpu 1:01.18 total
pnpm next build --experimental-build-mode=compil  386.49s user 56.44s system 726% cpu 1:00.96 total
```



```
b1ed4553e6 origin/mischnic/nft-module-graph
pnpm next build --experimental-build-mode=compile  467.28s user 75.24s system 791% cpu 1:08.56 total
pnpm next build --experimental-build-mode=compile  465.00s user 76.80s system 831% cpu 1:05.20 total
pnpm next build --experimental-build-mode=compile  473.86s user 70.73s system 790% cpu 1:08.90 total

3cb77c84 canary
pnpm next build --experimental-build-mode=compile  367.91s user 58.06s system 726% cpu 58.672 total
pnpm next build --experimental-build-mode=compile  362.97s user 57.68s system 746% cpu 56.379 total
pnpm next build --experimental-build-mode=compile  364.59s user 57.01s system 724% cpu 58.167 total
```

</details>
parent ee0d02ad
...@@ -10773,6 +10773,7 @@ dependencies = [ ...@@ -10773,6 +10773,7 @@ dependencies = [
"regex", "regex",
"rstest", "rstest",
"rstest_reuse", "rstest_reuse",
"rustc-hash 2.1.1",
"serde", "serde",
"serde_json", "serde_json",
"similar", "similar",
......
...@@ -1747,6 +1747,8 @@ impl AppEndpoint { ...@@ -1747,6 +1747,8 @@ impl AppEndpoint {
.chain(loadable_manifest_output.iter().flat_map(|m| &**m).copied()) .chain(loadable_manifest_output.iter().flat_map(|m| &**m).copied())
.map(|m| *m) .map(|m| *m)
.collect(), .collect(),
*module_graphs.full,
vec![*rsc_entry],
) )
.to_resolved() .to_resolved()
.await?, .await?,
......
...@@ -184,10 +184,18 @@ impl InstrumentationEndpoint { ...@@ -184,10 +184,18 @@ impl InstrumentationEndpoint {
let chunk = self.node_chunk().to_resolved().await?; let chunk = self.node_chunk().to_resolved().await?;
let mut output_assets = vec![chunk]; let mut output_assets = vec![chunk];
if this.project.next_mode().await?.is_production() { if this.project.next_mode().await?.is_production() {
let userland_module = self.entry_module();
output_assets.push(ResolvedVc::upcast( output_assets.push(ResolvedVc::upcast(
NftJsonAsset::new(*this.project, None, *chunk, vec![]) NftJsonAsset::new(
.to_resolved() *this.project,
.await?, None,
*chunk,
vec![],
this.project.module_graph(userland_module),
vec![userland_module],
)
.to_resolved()
.await?,
)); ));
} }
Ok(Vc::cell(output_assets)) Ok(Vc::cell(output_assets))
......
...@@ -226,10 +226,18 @@ impl MiddlewareEndpoint { ...@@ -226,10 +226,18 @@ impl MiddlewareEndpoint {
let chunk = self.node_chunk().to_resolved().await?; let chunk = self.node_chunk().to_resolved().await?;
let mut output_assets = vec![chunk]; let mut output_assets = vec![chunk];
if this.project.next_mode().await?.is_production() { if this.project.next_mode().await?.is_production() {
let userland_module = self.entry_module();
output_assets.push(ResolvedVc::upcast( output_assets.push(ResolvedVc::upcast(
NftJsonAsset::new(*this.project, None, *chunk, vec![]) NftJsonAsset::new(
.to_resolved() *this.project,
.await?, None,
*chunk,
vec![],
this.project.module_graph(userland_module),
vec![userland_module],
)
.to_resolved()
.await?,
)); ));
} }
let middleware_manifest_v2 = MiddlewaresManifestV2 { let middleware_manifest_v2 = MiddlewaresManifestV2 {
......
...@@ -226,6 +226,7 @@ impl NextDynamicGraph { ...@@ -226,6 +226,7 @@ impl NextDynamicGraph {
}) })
}, },
|_, _, _| Ok(()), |_, _, _| Ok(()),
false,
)?; )?;
Ok(Vc::cell(result)) Ok(Vc::cell(result))
} }
...@@ -834,6 +835,7 @@ async fn validate_pages_css_imports_individual( ...@@ -834,6 +835,7 @@ async fn validate_pages_css_imports_individual(
Ok(GraphTraversalAction::Continue) Ok(GraphTraversalAction::Continue)
}, },
|_, _, _| Ok(()), |_, _, _| Ok(()),
false,
)?; )?;
candidates candidates
......
...@@ -19,15 +19,16 @@ use turbopack_core::{ ...@@ -19,15 +19,16 @@ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
context::AssetContext, context::AssetContext,
file_source::FileSource, file_source::FileSource,
module::{Module, Modules},
module_graph::{ModuleGraph, SingleModuleGraph, chunk_group_info::ChunkGroupEntry},
output::{OutputAsset, OutputAssets, OutputAssetsReference}, output::{OutputAsset, OutputAssets, OutputAssetsReference},
reference_type::{CommonJsReferenceSubType, ReferenceType}, reference_type::{CommonJsReferenceSubType, ReferenceType},
resolve::{ResolveErrorMode, origin::PlainResolveOrigin, parse::Request}, resolve::{ResolveErrorMode, origin::PlainResolveOrigin, parse::Request},
traced_asset::TracedAsset,
}; };
use turbopack_resolve::ecmascript::cjs_resolve; use turbopack_resolve::ecmascript::cjs_resolve;
use crate::{ use crate::{
nft_json::{all_assets_from_entries_filtered, relativize_glob}, nft_json::{relativize_glob, traced_modules_for_entries},
project::Project, project::Project,
}; };
...@@ -106,6 +107,7 @@ impl Asset for ServerNftJsonAsset { ...@@ -106,6 +107,7 @@ impl Asset for ServerNftJsonAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> { async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
let this = self.await?; let this = self.await?;
// Example: [project]/apps/my-website/.next/ // Example: [project]/apps/my-website/.next/
let base_dir = this let base_dir = this
.project .project
...@@ -113,16 +115,31 @@ impl Asset for ServerNftJsonAsset { ...@@ -113,16 +115,31 @@ impl Asset for ServerNftJsonAsset {
.await? .await?
.join(&this.project.node_root().await?.path)?; .join(&this.project.node_root().await?.path)?;
let module_graph = ModuleGraph::from_graphs(
vec![SingleModuleGraph::new_with_traced_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(self.entries().owned().await?)]),
true,
false,
)],
None,
)
.connect();
let mut server_output_assets = let mut server_output_assets =
all_assets_from_entries_filtered(self.entries(), None, Some(self.ignores())) traced_modules_for_entries(module_graph, self.entries(), Some(self.ignores()), true)
.await? .await?
.iter() .iter()
.map(async |m| { .map(async |m| {
Ok(( Ok((
base_dir base_dir
.get_relative_path_to(&*m.path().await?) .get_relative_path_to(&m.ident().await?.path)
.context("failed to compute relative path for server NFT JSON")?, .context("failed to compute relative path for server NFT JSON")?,
m.content().hash(HashAlgorithm::Xxh3Hash128Hex).await?, m.source()
.await?
.context("NFT module has no content")?
.content()
.hash(HashAlgorithm::Xxh3Hash128Hex)
.await?,
)) ))
}) })
.try_join() .try_join()
...@@ -186,7 +203,7 @@ impl Asset for ServerNftJsonAsset { ...@@ -186,7 +203,7 @@ impl Asset for ServerNftJsonAsset {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ServerNftJsonAsset { impl ServerNftJsonAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn entries(&self) -> Result<Vc<OutputAssets>> { async fn entries(&self) -> Result<Vc<Modules>> {
let is_standalone = *self.project.next_config().is_standalone().await?; let is_standalone = *self.project.next_config().is_standalone().await?;
let asset_context = Vc::upcast(externals_tracing_module_context( let asset_context = Vc::upcast(externals_tracing_module_context(
...@@ -215,14 +232,20 @@ impl ServerNftJsonAsset { ...@@ -215,14 +232,20 @@ impl ServerNftJsonAsset {
// These are used by packages/next/src/server/require-hook.ts // These are used by packages/next/src/server/require-hook.ts
let shared_entries = ["styled-jsx", "styled-jsx/style", "styled-jsx/style.js"]; let shared_entries = ["styled-jsx", "styled-jsx/style", "styled-jsx/style.js"];
let cache_handler_entries = cache_handler.iter().chain(cache_handlers.iter()).map(|f| { let cache_handler_entries = cache_handler
asset_context .iter()
.process( .chain(cache_handlers.iter())
Vc::upcast(FileSource::new(f.clone())), .map(|f| {
ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined), asset_context
) .process(
.module() Vc::upcast(FileSource::new(f.clone())),
}); ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined),
)
.module()
})
.map(|m| m.to_resolved())
.try_join()
.await?;
let entries = match self.ty { let entries = match self.ty {
ServerNftType::Full => Either::Left( ServerNftType::Full => Either::Left(
...@@ -247,6 +270,7 @@ impl ServerNftJsonAsset { ...@@ -247,6 +270,7 @@ impl ServerNftJsonAsset {
Ok(Vc::cell( Ok(Vc::cell(
cache_handler_entries cache_handler_entries
.into_iter()
.chain( .chain(
shared_entries shared_entries
.into_iter() .into_iter()
...@@ -262,15 +286,12 @@ impl ServerNftJsonAsset { ...@@ -262,15 +286,12 @@ impl ServerNftJsonAsset {
.await? .await?
.primary_modules() .primary_modules()
.await? .await?
.into_iter() .into_iter())
.map(|m| *m))
}) })
.try_flat_join() .try_flat_join()
.await?, .await?,
) )
.map(|m| Vc::upcast::<Box<dyn OutputAsset>>(TracedAsset::new(m)).to_resolved()) .collect(),
.try_join()
.await?,
)) ))
} }
......
This diff is collapsed.
...@@ -1087,6 +1087,8 @@ impl PageEndpoint { ...@@ -1087,6 +1087,8 @@ impl PageEndpoint {
Some(pages_function_name(&this.original_name).into()), Some(pages_function_name(&this.original_name).into()),
*ssr_entry_chunk, *ssr_entry_chunk,
additional_assets, additional_assets,
ssr_module_graph,
vec![*ssr_module],
) )
.to_resolved() .to_resolved()
.await?, .await?,
......
...@@ -189,7 +189,7 @@ impl Visit<FindServerEntriesNode> for FindServerEntries { ...@@ -189,7 +189,7 @@ impl Visit<FindServerEntriesNode> for FindServerEntries {
let referenced_modules = referenced_modules let referenced_modules = referenced_modules
.iter() .iter()
.flat_map(|(_, resolved)| match resolved.chunking_type { .flat_map(|(_, resolved)| match resolved.chunking_type {
ChunkingType::Traced => None, ChunkingType::Traced { .. } => None,
_ => Some(resolved.modules.iter()), _ => Some(resolved.modules.iter()),
}) })
.flatten() .flatten()
......
...@@ -1095,7 +1095,6 @@ pub async fn get_server_chunking_context_with_client_assets( ...@@ -1095,7 +1095,6 @@ pub async fn get_server_chunking_context_with_client_assets(
.module_id_strategy(module_id_strategy.to_resolved().await?) .module_id_strategy(module_id_strategy.to_resolved().await?)
.export_usage(*export_usage.await?) .export_usage(*export_usage.await?)
.unused_references(unused_references.to_resolved().await?) .unused_references(unused_references.to_resolved().await?)
.file_tracing(next_mode.is_production())
.debug_ids(*debug_ids.await?) .debug_ids(*debug_ids.await?)
.hash_salt(hash_salt) .hash_salt(hash_salt)
.nested_async_availability(*nested_async_chunking.await?) .nested_async_availability(*nested_async_chunking.await?)
...@@ -1203,7 +1202,6 @@ pub async fn get_server_chunking_context( ...@@ -1203,7 +1202,6 @@ pub async fn get_server_chunking_context(
.module_id_strategy(module_id_strategy.to_resolved().await?) .module_id_strategy(module_id_strategy.to_resolved().await?)
.export_usage(*export_usage.await?) .export_usage(*export_usage.await?)
.unused_references(unused_references.to_resolved().await?) .unused_references(unused_references.to_resolved().await?)
.file_tracing(next_mode.is_production())
.debug_ids(*debug_ids.await?) .debug_ids(*debug_ids.await?)
.hash_salt(hash_salt) .hash_salt(hash_salt)
.nested_async_availability(*nested_async_chunking.await?) .nested_async_availability(*nested_async_chunking.await?)
......
...@@ -110,8 +110,6 @@ async fn get_analyze_data_operation( ...@@ -110,8 +110,6 @@ async fn get_analyze_data_operation(
.try_join() .try_join()
.await?; .await?;
whole_app_module_graphs.as_side_effect().await?;
let modules_data = ResolvedVc::upcast( let modules_data = ResolvedVc::upcast(
ModulesDataOutputAsset::new( ModulesDataOutputAsset::new(
analyze_output_root.join("modules.data")?, analyze_output_root.join("modules.data")?,
......
...@@ -1690,13 +1690,14 @@ async fn output_assets_operation( ...@@ -1690,13 +1690,14 @@ async fn output_assets_operation(
} }
let whole_app_module_graphs = project.whole_app_module_graphs(); let whole_app_module_graphs = project.whole_app_module_graphs();
// This makes the trace file nicer to look at
whole_app_module_graphs.as_side_effect().await?;
let nft = next_server_nft_assets(project).await?; let nft = next_server_nft_assets(project).await?;
let routes_hashes_manifest = routes_hashes_manifest_asset_if_enabled(project).await?; let routes_hashes_manifest = routes_hashes_manifest_asset_if_enabled(project).await?;
let immutable_hashes_manifest_asset = let immutable_hashes_manifest_asset =
immutable_hashes_manifest_asset_if_enabled(project).await?; immutable_hashes_manifest_asset_if_enabled(project).await?;
whole_app_module_graphs.as_side_effect().await?;
Ok(Vc::cell( Ok(Vc::cell(
output_assets output_assets
.into_iter() .into_iter()
......
...@@ -69,11 +69,6 @@ impl BrowserChunkingContextBuilder { ...@@ -69,11 +69,6 @@ impl BrowserChunkingContextBuilder {
self self
} }
pub fn tracing(mut self, enable_tracing: bool) -> Self {
self.chunking_context.enable_tracing = enable_tracing;
self
}
pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self { pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self {
self.chunking_context.enable_nested_async_availability = enable_nested_async_availability; self.chunking_context.enable_nested_async_availability = enable_nested_async_availability;
self self
...@@ -297,8 +292,6 @@ pub struct BrowserChunkingContext { ...@@ -297,8 +292,6 @@ pub struct BrowserChunkingContext {
default_url_behavior: Option<UrlBehavior>, default_url_behavior: Option<UrlBehavior>,
/// Enable HMR for this chunking /// Enable HMR for this chunking
enable_hot_module_replacement: bool, enable_hot_module_replacement: bool,
/// Enable tracing for this chunking
enable_tracing: bool,
/// Enable nested async availability for this chunking /// Enable nested async availability for this chunking
enable_nested_async_availability: bool, enable_nested_async_availability: bool,
/// Enable module merging /// Enable module merging
...@@ -375,7 +368,6 @@ impl BrowserChunkingContext { ...@@ -375,7 +368,6 @@ impl BrowserChunkingContext {
url_behaviors: Default::default(), url_behaviors: Default::default(),
default_url_behavior: None, default_url_behavior: None,
enable_hot_module_replacement: false, enable_hot_module_replacement: false,
enable_tracing: false,
enable_nested_async_availability: false, enable_nested_async_availability: false,
enable_module_merging: false, enable_module_merging: false,
enable_dynamic_chunk_content_loading: false, enable_dynamic_chunk_content_loading: false,
...@@ -728,11 +720,6 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -728,11 +720,6 @@ impl ChunkingContext for BrowserChunkingContext {
self.source_map_source_type.cell() self.source_map_source_type.cell()
} }
#[turbo_tasks::function]
fn is_tracing_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_tracing)
}
#[turbo_tasks::function] #[turbo_tasks::function]
fn is_nested_async_availability_enabled(&self) -> Vc<bool> { fn is_nested_async_availability_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_nested_async_availability) Vc::cell(self.enable_nested_async_availability)
...@@ -772,7 +759,6 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -772,7 +759,6 @@ impl ChunkingContext for BrowserChunkingContext {
let input_availability_info = availability_info; let input_availability_info = availability_info;
let MakeChunkGroupResult { let MakeChunkGroupResult {
chunks, chunks,
referenced_output_assets,
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
...@@ -818,7 +804,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -818,7 +804,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult { Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets), assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets), referenced_assets: OutputAssets::empty_resolved(),
references: ResolvedVc::cell(references), references: ResolvedVc::cell(references),
availability_info, availability_info,
} }
...@@ -845,7 +831,6 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -845,7 +831,6 @@ impl ChunkingContext for BrowserChunkingContext {
let this = self.await?; let this = self.await?;
let MakeChunkGroupResult { let MakeChunkGroupResult {
chunks, chunks,
referenced_output_assets,
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
...@@ -908,7 +893,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -908,7 +893,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult { Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets), assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets), referenced_assets: OutputAssets::empty_resolved(),
references: ResolvedVc::cell(references), references: ResolvedVc::cell(references),
availability_info, availability_info,
} }
......
...@@ -21,7 +21,6 @@ use crate::{ ...@@ -21,7 +21,6 @@ use crate::{
chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo}, chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
}, },
environment::ChunkLoading, environment::ChunkLoading,
module::Module,
module_graph::{ module_graph::{
GraphTraversalAction, ModuleGraph, GraphTraversalAction, ModuleGraph,
chunk_group_info::ChunkGroup, chunk_group_info::ChunkGroup,
...@@ -32,13 +31,11 @@ use crate::{ ...@@ -32,13 +31,11 @@ use crate::{
}, },
module_batches::{BatchingConfig, ModuleBatchesGraphEdge}, module_batches::{BatchingConfig, ModuleBatchesGraphEdge},
}, },
output::{OutputAsset, OutputAssetsReference}, output::OutputAssetsReference,
traced_asset::TracedAsset,
}; };
pub struct MakeChunkGroupResult { pub struct MakeChunkGroupResult {
pub chunks: ResolvedVc<Chunks>, pub chunks: ResolvedVc<Chunks>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>, pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>,
pub availability_info: AvailabilityInfo, pub availability_info: AvailabilityInfo,
} }
...@@ -57,7 +54,6 @@ pub async fn make_chunk_group( ...@@ -57,7 +54,6 @@ pub async fn make_chunk_group(
let is_nested_async_availability_enabled = *chunking_context let is_nested_async_availability_enabled = *chunking_context
.is_nested_async_availability_enabled() .is_nested_async_availability_enabled()
.await?; .await?;
let should_trace = *chunking_context.is_tracing_enabled().await?;
let should_merge_modules = *chunking_context.is_module_merging_enabled().await?; let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
let batching_config = chunking_context.batching_config().to_resolved().await?; let batching_config = chunking_context.batching_config().to_resolved().await?;
...@@ -70,7 +66,6 @@ pub async fn make_chunk_group( ...@@ -70,7 +66,6 @@ pub async fn make_chunk_group(
ChunkGroupContentOptions { ChunkGroupContentOptions {
availability_info, availability_info,
can_split_async, can_split_async,
should_trace,
should_merge_modules, should_merge_modules,
batching_config, batching_config,
}, },
...@@ -80,7 +75,6 @@ pub async fn make_chunk_group( ...@@ -80,7 +75,6 @@ pub async fn make_chunk_group(
chunkable_items, chunkable_items,
batch_groups, batch_groups,
async_modules, async_modules,
traced_modules,
available_modules: _, available_modules: _,
} = &*inner; } = &*inner;
...@@ -156,17 +150,6 @@ pub async fn make_chunk_group( ...@@ -156,17 +150,6 @@ pub async fn make_chunk_group(
.try_join() .try_join()
.await?; .await?;
let referenced_output_assets = traced_modules
.iter()
.copied()
.map(|module| async move {
Ok(ResolvedVc::upcast(
TracedAsset::new(*module).to_resolved().await?,
))
})
.try_join()
.await?;
chunk_items.extend(async_loader_chunk_items); chunk_items.extend(async_loader_chunk_items);
// Pass chunk items to chunking algorithm // Pass chunk items to chunking algorithm
...@@ -182,7 +165,6 @@ pub async fn make_chunk_group( ...@@ -182,7 +165,6 @@ pub async fn make_chunk_group(
Ok(MakeChunkGroupResult { Ok(MakeChunkGroupResult {
chunks, chunks,
referenced_output_assets,
references: ResolvedVc::upcast_vec(async_loaders), references: ResolvedVc::upcast_vec(async_loaders),
availability_info: new_availability_info, availability_info: new_availability_info,
}) })
...@@ -196,8 +178,6 @@ pub struct ChunkGroupContentOptions { ...@@ -196,8 +178,6 @@ pub struct ChunkGroupContentOptions {
pub availability_info: AvailabilityInfo, pub availability_info: AvailabilityInfo,
/// Whether async modules can be split into separate chunks /// Whether async modules can be split into separate chunks
pub can_split_async: bool, pub can_split_async: bool,
/// Whether traced modules should be collected
pub should_trace: bool,
/// Whether module merging is enabled /// Whether module merging is enabled
pub should_merge_modules: bool, pub should_merge_modules: bool,
/// The batching config to use /// The batching config to use
...@@ -235,7 +215,6 @@ async fn chunk_group_content_operation( ...@@ -235,7 +215,6 @@ async fn chunk_group_content_operation(
ChunkGroupContentOptions { ChunkGroupContentOptions {
availability_info, availability_info,
can_split_async, can_split_async,
should_trace,
should_merge_modules, should_merge_modules,
batching_config, batching_config,
}: ChunkGroupContentOptions, }: ChunkGroupContentOptions,
...@@ -248,14 +227,12 @@ async fn chunk_group_content_operation( ...@@ -248,14 +227,12 @@ async fn chunk_group_content_operation(
unsorted_items: ModuleToChunkableMap, unsorted_items: ModuleToChunkableMap,
chunkable_items: FxIndexSet<ChunkableModuleOrBatch>, chunkable_items: FxIndexSet<ChunkableModuleOrBatch>,
async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>, async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
} }
let mut state = TraverseState { let mut state = TraverseState {
unsorted_items: FxHashMap::default(), unsorted_items: FxHashMap::default(),
chunkable_items: FxIndexSet::default(), chunkable_items: FxIndexSet::default(),
async_modules: FxIndexSet::default(), async_modules: FxIndexSet::default(),
traced_modules: FxIndexSet::default(),
}; };
let available_modules = match availability_info.available_modules() { let available_modules = match availability_info.available_modules() {
...@@ -277,21 +254,15 @@ async fn chunk_group_content_operation( ...@@ -277,21 +254,15 @@ async fn chunk_group_content_operation(
if matches!(node, ModuleOrBatch::None(_)) { if matches!(node, ModuleOrBatch::None(_)) {
return Ok(GraphTraversalAction::Continue); return Ok(GraphTraversalAction::Continue);
} }
// Traced modules need to have a special handling // Traced modules are completely ignored during chunking
if let Some(( if let Some((
_, _,
ModuleBatchesGraphEdge { ModuleBatchesGraphEdge {
ty: ChunkingType::Traced, ty: ChunkingType::Traced { .. },
.. ..
}, },
)) = parent_info )) = parent_info
{ {
if should_trace {
let ModuleOrBatch::Module(module) = node else {
unreachable!();
};
state.traced_modules.insert(module);
}
return Ok(GraphTraversalAction::Exclude); return Ok(GraphTraversalAction::Exclude);
} }
...@@ -359,7 +330,7 @@ async fn chunk_group_content_operation( ...@@ -359,7 +330,7 @@ async fn chunk_group_content_operation(
GraphTraversalAction::Exclude GraphTraversalAction::Exclude
} }
} }
ChunkingType::Traced => { ChunkingType::Traced { .. } => {
// handled above before the sidecast // handled above before the sidecast
unreachable!(); unreachable!();
} }
...@@ -458,7 +429,6 @@ async fn chunk_group_content_operation( ...@@ -458,7 +429,6 @@ async fn chunk_group_content_operation(
chunkable_items, chunkable_items,
batch_groups, batch_groups,
async_modules: state.async_modules, async_modules: state.async_modules,
traced_modules: state.traced_modules,
available_modules, available_modules,
} }
.cell()) .cell())
......
...@@ -381,13 +381,6 @@ pub trait ChunkingContext { ...@@ -381,13 +381,6 @@ pub trait ChunkingContext {
}) })
} }
/// Whether `ChunkingType::Traced` are used to create corresponding output assets for each
/// traced module.
#[turbo_tasks::function]
fn is_tracing_enabled(self: Vc<Self>) -> Vc<bool> {
Vc::cell(false)
}
/// Whether async modules should create an new availability boundary and therefore nested async /// Whether async modules should create an new availability boundary and therefore nested async
/// modules include less modules. Enabling this will lead to better optimized async chunks, /// modules include less modules. Enabling this will lead to better optimized async chunks,
/// but it will require to compute all possible paths in the application, which might lead to /// but it will require to compute all possible paths in the application, which might lead to
......
...@@ -281,6 +281,40 @@ pub trait OutputChunk: Asset { ...@@ -281,6 +281,40 @@ pub trait OutputChunk: Asset {
fn runtime_info(self: Vc<Self>) -> Vc<OutputChunkRuntimeInfo>; fn runtime_info(self: Vc<Self>) -> Vc<OutputChunkRuntimeInfo>;
} }
/// Whether this reference is an entry point for a traced subgraph.
#[derive(
Debug,
Clone,
Copy,
Hash,
TraceRawVcs,
Serialize,
Deserialize,
Eq,
PartialEq,
ValueDebugFormat,
NonLocalValue,
Encode,
Decode,
TaskInput,
)]
pub enum TracedMode {
/// Going from bundled to unbundled code, i.e. an external dependency or readFile static assets.
Entry,
/// This reference should only be respected from unbundled code (e.g. for package.json needed by
/// externals (sort of affecting_sources)
Transitive,
}
impl Display for TracedMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TracedMode::Entry => write!(f, "Entry"),
TracedMode::Transitive => write!(f, "Transitive"),
}
}
}
/// Specifies how a chunk interacts with other chunks when building a chunk /// Specifies how a chunk interacts with other chunks when building a chunk
/// group /// group
#[derive( #[derive(
...@@ -324,8 +358,13 @@ pub enum ChunkingType { ...@@ -324,8 +358,13 @@ pub enum ChunkingType {
inherit_async: bool, inherit_async: bool,
merge_tag: Option<RcStr>, merge_tag: Option<RcStr>,
}, },
// Module not placed in chunk group, but its references are still followed. /// The module not placed in chunk group, but its references are still followed. This is used
Traced, /// for NFT, to list all unbundled files that are still needed at runtime (some static assets,
/// or externals and their transitive dependencies).
Traced {
/// Whether this reference is an entry point for a traced subgraph.
mode: TracedMode,
},
} }
impl Display for ChunkingType { impl Display for ChunkingType {
...@@ -368,7 +407,7 @@ impl Display for ChunkingType { ...@@ -368,7 +407,7 @@ impl Display for ChunkingType {
} => { } => {
write!(f, "Shared(inherit_async: {inherit_async})") write!(f, "Shared(inherit_async: {inherit_async})")
} }
ChunkingType::Traced => write!(f, "Traced"), ChunkingType::Traced { mode } => write!(f, "Traced(mode: {mode})"),
} }
} }
} }
...@@ -391,6 +430,10 @@ impl ChunkingType { ...@@ -391,6 +430,10 @@ impl ChunkingType {
matches!(self, ChunkingType::Parallel { .. }) matches!(self, ChunkingType::Parallel { .. })
} }
pub fn is_traced(&self) -> bool {
matches!(self, ChunkingType::Traced { .. })
}
pub fn is_merged(&self) -> bool { pub fn is_merged(&self) -> bool {
matches!( matches!(
self, self,
...@@ -422,7 +465,7 @@ impl ChunkingType { ...@@ -422,7 +465,7 @@ impl ChunkingType {
inherit_async: false, inherit_async: false,
merge_tag: merge_tag.clone(), merge_tag: merge_tag.clone(),
}, },
ChunkingType::Traced => ChunkingType::Traced, ChunkingType::Traced { mode } => ChunkingType::Traced { mode: *mode },
} }
} }
} }
...@@ -433,8 +476,6 @@ pub struct ChunkGroupContentInner { ...@@ -433,8 +476,6 @@ pub struct ChunkGroupContentInner {
pub batch_groups: Vec<ResolvedVc<ModuleBatchGroup>>, pub batch_groups: Vec<ResolvedVc<ModuleBatchGroup>>,
#[bincode(with = "turbo_bincode::indexset")] #[bincode(with = "turbo_bincode::indexset")]
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>, pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
#[bincode(with = "turbo_bincode::indexset")]
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
pub available_modules: ResolvedVc<AvailableModulesSet>, pub available_modules: ResolvedVc<AvailableModulesSet>,
} }
......
...@@ -81,7 +81,7 @@ pub async fn children_from_module_references( ...@@ -81,7 +81,7 @@ pub async fn children_from_module_references(
Some(ChunkingType::Async) => async_reference_ty(), Some(ChunkingType::Async) => async_reference_ty(),
Some(ChunkingType::Isolated { .. }) => isolated_reference_ty(), Some(ChunkingType::Isolated { .. }) => isolated_reference_ty(),
Some(ChunkingType::Shared { .. }) => shared_reference_ty(), Some(ChunkingType::Shared { .. }) => shared_reference_ty(),
Some(ChunkingType::Traced) => traced_reference_ty(), Some(ChunkingType::Traced { .. }) => traced_reference_ty(),
}; };
for &module in reference for &module in reference
......
...@@ -41,7 +41,6 @@ pub mod source_map; ...@@ -41,7 +41,6 @@ pub mod source_map;
pub mod source_pos; pub mod source_pos;
pub mod source_transform; pub mod source_transform;
pub mod target; pub mod target;
pub mod traced_asset;
mod utils; mod utils;
pub mod version; pub mod version;
pub mod virtual_output; pub mod virtual_output;
......
...@@ -600,7 +600,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro ...@@ -600,7 +600,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
chunk_groups, chunk_groups,
))) )))
} }
ChunkingType::Traced => { ChunkingType::Traced { .. } => {
// Traced modules are not placed in chunk groups // Traced modules are not placed in chunk groups
return Ok(GraphTraversalAction::Skip); return Ok(GraphTraversalAction::Skip);
} }
......
...@@ -423,6 +423,7 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc< ...@@ -423,6 +423,7 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
} }
Ok(()) Ok(())
}, },
false,
)?; )?;
list_lists.push(lists); list_lists.push(lists);
...@@ -543,6 +544,7 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc< ...@@ -543,6 +544,7 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
} }
Ok(()) Ok(())
}, },
false,
)?; )?;
drop(inner_span); drop(inner_span);
......
...@@ -335,6 +335,7 @@ impl PreBatches { ...@@ -335,6 +335,7 @@ impl PreBatches {
state.items.push(item); state.items.push(item);
Ok(()) Ok(())
}, },
false,
)?; )?;
Ok(state.items) Ok(state.items)
} }
......
...@@ -7,6 +7,7 @@ use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Vc}; ...@@ -7,6 +7,7 @@ use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks_fs::{FileSystemEntryType, FileSystemPath}; use turbo_tasks_fs::{FileSystemEntryType, FileSystemPath};
use crate::{ use crate::{
chunk::TracedMode,
file_source::FileSource, file_source::FileSource,
ident::AssetIdent, ident::AssetIdent,
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
...@@ -146,9 +147,10 @@ async fn dir_references(package_dir: FileSystemPath) -> Result<Vc<ModuleReferenc ...@@ -146,9 +147,10 @@ async fn dir_references(package_dir: FileSystemPath) -> Result<Vc<ModuleReferenc
.into_iter() .into_iter()
.map(async |p| { .map(async |p| {
Ok(ResolvedVc::upcast( Ok(ResolvedVc::upcast(
TracedModuleReference::new(Vc::upcast(RawModule::new(Vc::upcast( TracedModuleReference::new(
FileSource::new(p), Vc::upcast(RawModule::new(Vc::upcast(FileSource::new(p)))),
)))) TracedMode::Entry,
)
.to_resolved() .to_resolved()
.await?, .await?,
)) ))
......
...@@ -41,10 +41,11 @@ impl RebasedAsset { ...@@ -41,10 +41,11 @@ impl RebasedAsset {
impl OutputAssetsReference for RebasedAsset { impl OutputAssetsReference for RebasedAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> { async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let references = referenced_modules_and_affecting_sources(*self.module) let references = referenced_modules_and_affecting_sources(*self.module, false)
.await? .await?
.iter() .iter()
.map(|module| async move { .flat_map(|(_, ref_data)| ref_data.modules.iter())
.map(async |module| {
Ok(ResolvedVc::upcast( Ok(ResolvedVc::upcast(
RebasedAsset::new(**module, self.input_dir.clone(), self.output_dir.clone()) RebasedAsset::new(**module, self.input_dir.clone(), self.output_dir.clone())
.to_resolved() .to_resolved()
......
...@@ -4,12 +4,12 @@ use anyhow::Result; ...@@ -4,12 +4,12 @@ use anyhow::Result;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
FxIndexSet, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs, debug::ValueDebugFormat, trace::TraceRawVcs,
}; };
use crate::{ use crate::{
chunk::ChunkingType, chunk::{ChunkingType, TracedMode},
module::{Module, Modules}, module::{Module, Modules},
output::{ output::{
ExpandOutputAssetsInput, ExpandedOutputAssets, OutputAsset, OutputAssets, ExpandOutputAssetsInput, ExpandedOutputAssets, OutputAsset, OutputAssets,
...@@ -34,9 +34,7 @@ pub trait ModuleReference: ValueToString { ...@@ -34,9 +34,7 @@ pub trait ModuleReference: ValueToString {
// TODO think about different types // TODO think about different types
// fn kind(&self) -> Vc<AssetReferenceType>; // fn kind(&self) -> Vc<AssetReferenceType>;
fn chunking_type(&self) -> Option<ChunkingType> { fn chunking_type(&self) -> Option<ChunkingType>;
None
}
fn binding_usage(&self) -> BindingUsage { fn binding_usage(&self) -> BindingUsage {
BindingUsage::default() BindingUsage::default()
...@@ -111,33 +109,51 @@ impl ModuleReference for SingleChunkableModuleReference { ...@@ -111,33 +109,51 @@ impl ModuleReference for SingleChunkableModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn referenced_modules_and_affecting_sources( pub async fn referenced_modules_and_affecting_sources(
module: Vc<Box<dyn Module>>, module: Vc<Box<dyn Module>>,
) -> Result<Vc<Modules>> { include_binding_usage: bool,
let references = module.references().await?; ) -> Result<Vc<ModulesWithRefData>> {
let modules = module
let resolved_references = references .references()
.await?
.iter() .iter()
.map(|r| r.resolve_reference()) .map(|reference| async {
.try_join() let trait_ref = reference.into_trait_ref().await?;
.await?; let resolve_result = reference.resolve_reference().await?;
let mut modules = Vec::new(); if let Some(chunking_type) = &trait_ref.chunking_type() {
for resolve_result in resolved_references { let mut modules = resolve_result
modules.extend(resolve_result.primary_modules_raw_iter()); .primary_modules_raw_iter()
modules.extend( .collect::<Vec<_>>();
resolve_result modules.extend(
.affecting_sources_iter() resolve_result
.map(|source| async move { .affecting_sources_iter()
Ok(ResolvedVc::upcast( .map(|source| async move {
RawModule::new(*source).to_resolved().await?, Ok(ResolvedVc::upcast(
)) RawModule::new(*source).to_resolved().await?,
}) ))
.try_join() })
.await?, .try_join()
); .await?,
} );
let resolved_modules: FxIndexSet<_> = modules.into_iter().collect(); let binding_usage = if include_binding_usage {
trait_ref.binding_usage()
} else {
BindingUsage::default()
};
Ok(Vc::cell(resolved_modules.into_iter().collect())) return Ok(Some((
*reference,
ResolvedReference {
chunking_type: chunking_type.clone(),
binding_usage,
modules,
},
)));
}
Ok(None)
})
.try_flat_join()
.await?;
Ok(Vc::cell(modules))
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -145,6 +161,7 @@ pub async fn referenced_modules_and_affecting_sources( ...@@ -145,6 +161,7 @@ pub async fn referenced_modules_and_affecting_sources(
#[value_to_string("traced {}", self.module.ident())] #[value_to_string("traced {}", self.module.ident())]
pub struct TracedModuleReference { pub struct TracedModuleReference {
module: ResolvedVc<Box<dyn Module>>, module: ResolvedVc<Box<dyn Module>>,
mode: TracedMode,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -155,15 +172,15 @@ impl ModuleReference for TracedModuleReference { ...@@ -155,15 +172,15 @@ impl ModuleReference for TracedModuleReference {
} }
fn chunking_type(&self) -> Option<ChunkingType> { fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced) Some(ChunkingType::Traced { mode: self.mode })
} }
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl TracedModuleReference { impl TracedModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> { pub fn new(module: ResolvedVc<Box<dyn Module>>, mode: TracedMode) -> Vc<Self> {
Self::cell(TracedModuleReference { module }) Self::cell(TracedModuleReference { module, mode })
} }
} }
...@@ -216,7 +233,7 @@ pub async fn primary_chunkable_referenced_modules( ...@@ -216,7 +233,7 @@ pub async fn primary_chunkable_referenced_modules(
.map(|reference| async { .map(|reference| async {
let trait_ref = reference.into_trait_ref().await?; let trait_ref = reference.into_trait_ref().await?;
if let Some(chunking_type) = &trait_ref.chunking_type() { if let Some(chunking_type) = &trait_ref.chunking_type() {
if !include_traced && matches!(chunking_type, ChunkingType::Traced) { if !include_traced && chunking_type.is_traced() {
return Ok(None); return Ok(None);
} }
......
...@@ -4,6 +4,7 @@ use turbo_tasks_fs::{File, FileContent, FileSystemEntryType, FileSystemPath}; ...@@ -4,6 +4,7 @@ use turbo_tasks_fs::{File, FileContent, FileSystemEntryType, FileSystemPath};
use super::ModuleReference; use super::ModuleReference;
use crate::{ use crate::{
chunk::{ChunkingType, TracedMode},
file_source::FileSource, file_source::FileSource,
raw_module::RawModule, raw_module::RawModule,
resolve::ModuleResolveResult, resolve::ModuleResolveResult,
...@@ -51,6 +52,12 @@ impl ModuleReference for SourceMapReference { ...@@ -51,6 +52,12 @@ impl ModuleReference for SourceMapReference {
} }
Ok(*ModuleResolveResult::unresolvable()) Ok(*ModuleResolveResult::unresolvable())
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
......
use anyhow::Result;
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
module::Module,
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
reference::referenced_modules_and_affecting_sources,
};
/// Converts a traced external [Module] graph into a graph consisting of [TracedAsset]s.
#[turbo_tasks::value]
pub struct TracedAsset {
module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl]
impl TracedAsset {
#[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(TracedAsset { module })
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for TracedAsset {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let references = referenced_modules_and_affecting_sources(*self.module)
.await?
.iter()
.map(async |module| {
Ok(ResolvedVc::upcast(
TracedAsset::new(**module).to_resolved().await?,
))
})
.try_join()
.await?;
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for TracedAsset {
#[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> {
Ok(self.module.ident().await?.path.clone().cell())
}
}
#[turbo_tasks::value_impl]
impl Asset for TracedAsset {
#[turbo_tasks::function]
async fn content(&self) -> Result<Vc<AssetContent>> {
if let Some(source) = *self.module.source().await? {
Ok(source.content())
} else {
turbobail!("Module {} has no source", self.module.ident());
}
}
}
...@@ -10,7 +10,7 @@ use turbo_tasks_fs::{FileSystem, FileSystemPath, LinkType, VirtualFileSystem, ro ...@@ -10,7 +10,7 @@ use turbo_tasks_fs::{FileSystem, FileSystemPath, LinkType, VirtualFileSystem, ro
use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64}; use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext}, chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext, TracedMode},
ident::{AssetIdent, Layer}, ident::{AssetIdent, Layer},
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
module_graph::ModuleGraph, module_graph::ModuleGraph,
...@@ -334,22 +334,13 @@ impl Module for CachedExternalModule { ...@@ -334,22 +334,13 @@ impl Module for CachedExternalModule {
rcstr!("affecting source"), rcstr!("affecting source"),
)) ))
}) })
.chain( .chain(external_result.primary_modules_raw_iter().map(|m| *m))
external_result
.primary_modules_raw_iter()
// These modules aren't bundled but still need to be part of the module
// graph for chunking. `compute_async_module_info` computes
// `is_self_async` for every module, but at least for traced modules,
// that value is never used as `ChunkingType::Traced.is_inherit_async()
// == false`. Optimize this case by using
// `SideEffectfulModuleWithoutSelfAsync` to
// short circuit that computation and thus defer parsing traced modules
// to emitting to not block all of chunking on this.
.map(|m| Vc::upcast(SideEffectfulModuleWithoutSelfAsync::new(*m))),
)
.map(|s| { .map(|s| {
Vc::upcast::<Box<dyn ModuleReference>>(TracedModuleReference::new(s)) Vc::upcast::<Box<dyn ModuleReference>>(TracedModuleReference::new(
.to_resolved() s,
TracedMode::Entry,
))
.to_resolved()
}) })
.try_join() .try_join()
.await?; .await?;
...@@ -458,45 +449,6 @@ impl EcmascriptChunkPlaceable for CachedExternalModule { ...@@ -458,45 +449,6 @@ impl EcmascriptChunkPlaceable for CachedExternalModule {
} }
} }
/// A wrapper "passthrough" module type that always returns `false` for `is_self_async` and
/// `SideEffects` for `side_effects`.Be careful when using it, as it may hide async dependencies.
#[turbo_tasks::value]
struct SideEffectfulModuleWithoutSelfAsync {
module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl]
impl SideEffectfulModuleWithoutSelfAsync {
#[turbo_tasks::function]
fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(SideEffectfulModuleWithoutSelfAsync { module })
}
}
#[turbo_tasks::value_impl]
impl Module for SideEffectfulModuleWithoutSelfAsync {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
fn source(&self) -> Vc<turbopack_core::source::OptionSource> {
self.module.source()
}
#[turbo_tasks::function]
fn references(&self) -> Vc<ModuleReferences> {
self.module.references()
}
#[turbo_tasks::function]
fn side_effects(&self) -> Vc<ModuleSideEffects> {
ModuleSideEffects::SideEffectful.cell()
}
// Don't override and use default is_self_async that always returns false
}
#[derive(Debug)] #[derive(Debug)]
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct ExternalsSymlinkAsset { pub struct ExternalsSymlinkAsset {
......
...@@ -2,7 +2,10 @@ use anyhow::Result; ...@@ -2,7 +2,10 @@ use anyhow::Result;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
file_source::FileSource, raw_module::RawModule, reference::ModuleReference, chunk::{ChunkingType, TracedMode},
file_source::FileSource,
raw_module::RawModule,
reference::ModuleReference,
resolve::ModuleResolveResult, resolve::ModuleResolveResult,
}; };
...@@ -31,4 +34,10 @@ impl ModuleReference for PackageJsonReference { ...@@ -31,4 +34,10 @@ impl ModuleReference for PackageJsonReference {
.await?, .await?,
))) )))
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
...@@ -4,7 +4,7 @@ use turbo_rcstr::rcstr; ...@@ -4,7 +4,7 @@ use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
chunk::ChunkingType, chunk::{ChunkingType, TracedMode},
file_source::FileSource, file_source::FileSource,
issue::IssueSource, issue::IssueSource,
raw_module::RawModule, raw_module::RawModule,
...@@ -79,7 +79,9 @@ impl ModuleReference for FileSourceReference { ...@@ -79,7 +79,9 @@ impl ModuleReference for FileSourceReference {
} }
fn chunking_type(&self) -> Option<ChunkingType> { fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced) Some(ChunkingType::Traced {
mode: TracedMode::Entry,
})
} }
} }
...@@ -207,6 +209,8 @@ impl ModuleReference for DirAssetReference { ...@@ -207,6 +209,8 @@ impl ModuleReference for DirAssetReference {
} }
fn chunking_type(&self) -> Option<ChunkingType> { fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced) Some(ChunkingType::Traced {
mode: TracedMode::Entry,
})
} }
} }
...@@ -3,6 +3,7 @@ use turbo_rcstr::RcStr; ...@@ -3,6 +3,7 @@ use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkingType, TracedMode},
context::AssetContext, context::AssetContext,
file_source::FileSource, file_source::FileSource,
reference::ModuleReference, reference::ModuleReference,
...@@ -42,6 +43,12 @@ impl ModuleReference for TsConfigReference { ...@@ -42,6 +43,12 @@ impl ModuleReference for TsConfigReference {
.await?, .await?,
))) )))
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -88,6 +95,12 @@ impl ModuleReference for TsReferencePathAssetReference { ...@@ -88,6 +95,12 @@ impl ModuleReference for TsReferencePathAssetReference {
}, },
) )
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -120,4 +133,10 @@ impl ModuleReference for TsReferenceTypeAssetReference { ...@@ -120,4 +133,10 @@ impl ModuleReference for TsReferenceTypeAssetReference {
), ),
) )
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
...@@ -5,6 +5,7 @@ use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc}; ...@@ -5,6 +5,7 @@ use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks_fs::DirectoryContent; use turbo_tasks_fs::DirectoryContent;
use turbopack_core::{ use turbopack_core::{
asset::Asset, asset::Asset,
chunk::{ChunkingType, TracedMode},
ident::AssetIdent, ident::AssetIdent,
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
raw_module::RawModule, raw_module::RawModule,
...@@ -208,6 +209,12 @@ impl ModuleReference for CompilerReference { ...@@ -208,6 +209,12 @@ impl ModuleReference for CompilerReference {
ResolveErrorMode::Error, ResolveErrorMode::Error,
) )
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -233,6 +240,12 @@ impl ModuleReference for TsExtendsReference { ...@@ -233,6 +240,12 @@ impl ModuleReference for TsExtendsReference {
RawModule::new(*self.config).to_resolved().await?, RawModule::new(*self.config).to_resolved().await?,
))) )))
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -266,6 +279,12 @@ impl ModuleReference for TsNodeRequireReference { ...@@ -266,6 +279,12 @@ impl ModuleReference for TsNodeRequireReference {
ResolveErrorMode::Error, ResolveErrorMode::Error,
) )
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -293,4 +312,10 @@ impl ModuleReference for TsConfigTypesReference { ...@@ -293,4 +312,10 @@ impl ModuleReference for TsConfigTypesReference {
fn resolve_reference(&self) -> Vc<ModuleResolveResult> { fn resolve_reference(&self) -> Vc<ModuleResolveResult> {
type_resolve(*self.origin, *self.request) type_resolve(*self.origin, *self.request)
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
...@@ -3,6 +3,7 @@ use swc_core::ecma::ast::Lit; ...@@ -3,6 +3,7 @@ use swc_core::ecma::ast::Lit;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkingType, TracedMode},
compile_time_info::CompileTimeInfo, compile_time_info::CompileTimeInfo,
file_source::FileSource, file_source::FileSource,
ident::AssetIdent, ident::AssetIdent,
...@@ -138,6 +139,12 @@ impl ModuleReference for WebpackChunkAssetReference { ...@@ -138,6 +139,12 @@ impl ModuleReference for WebpackChunkAssetReference {
WebpackRuntime::None => *ModuleResolveResult::unresolvable(), WebpackRuntime::None => *ModuleResolveResult::unresolvable(),
}) })
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
...@@ -165,6 +172,12 @@ impl ModuleReference for WebpackEntryAssetReference { ...@@ -165,6 +172,12 @@ impl ModuleReference for WebpackEntryAssetReference {
.await?, .await?,
))) )))
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
...@@ -210,4 +223,10 @@ impl ModuleReference for WebpackRuntimeAssetReference { ...@@ -210,4 +223,10 @@ impl ModuleReference for WebpackRuntimeAssetReference {
.await? .await?
.cell()) .cell())
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
use std::{collections::HashSet, env::current_dir, path::PathBuf}; use std::{env::current_dir, path::PathBuf};
use anyhow::Result; use anyhow::Result;
use rustc_hash::FxHashSet;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TransientInstance, TryJoinIterExt, Vc, turbofmt}; use turbo_tasks::{TransientInstance, Vc, turbofmt};
use turbo_tasks_fs::{DiskFileSystem, FileSystem}; use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{ use turbopack::{
ModuleAssetContext, ModuleAssetContext,
...@@ -19,11 +20,10 @@ use turbopack_core::{ ...@@ -19,11 +20,10 @@ use turbopack_core::{
file_source::FileSource, file_source::FileSource,
ident::Layer, ident::Layer,
issue::{IssueReporter, IssueSeverity, handle_issues}, issue::{IssueReporter, IssueSeverity, handle_issues},
output::{OutputAsset, OutputAssetsReference}, module::Module,
reference::all_assets_from_entries, reference::referenced_modules_and_affecting_sources,
reference_type::ReferenceType, reference_type::ReferenceType,
resolve::options::ConditionValue, resolve::options::ConditionValue,
traced_asset::TracedAsset,
}; };
use turbopack_ecmascript::AnalyzeMode; use turbopack_ecmascript::AnalyzeMode;
use turbopack_resolve::resolve_options_context::ResolveOptionsContext; use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
...@@ -123,52 +123,65 @@ async fn node_file_trace_operation( ...@@ -123,52 +123,65 @@ async fn node_file_trace_operation(
.process(Vc::upcast(source), ReferenceType::Undefined) .process(Vc::upcast(source), ReferenceType::Undefined)
.module(); .module();
let asset = TracedAsset::new(module).to_resolved().await?;
Ok(Vc::cell(if graph { Ok(Vc::cell(if graph {
to_graph(ResolvedVc::upcast(asset), max_depth.unwrap_or(usize::MAX)).await? to_graph(module, max_depth.unwrap_or(usize::MAX)).await?
} else { } else {
to_list(ResolvedVc::upcast(asset)).await? to_list(module).await?
})) }))
} }
async fn to_list(asset: ResolvedVc<Box<dyn OutputAsset>>) -> Result<Vec<RcStr>> { async fn to_list(asset: Vc<Box<dyn Module>>) -> Result<Vec<RcStr>> {
let mut assets = all_assets_from_entries(Vc::cell(vec![asset])) let mut assets = vec![];
.await?
.iter() let mut visited = FxHashSet::default();
.map(async |a| Ok(a.path().await?.path.clone())) let mut queue = Vec::new();
.try_join() queue.push(asset);
.await?;
while let Some(asset) = queue.pop() {
let references = referenced_modules_and_affecting_sources(asset, false).await?;
let path = &asset.ident().await?.path;
if visited.insert(asset) {
for (_, references) in references.iter().rev() {
for asset in references.modules.iter() {
queue.push(**asset);
}
}
}
assets.push(path.path.clone());
}
assets.sort(); assets.sort();
assets.dedup(); assets.dedup();
Ok(assets) Ok(assets)
} }
async fn to_graph(asset: ResolvedVc<Box<dyn OutputAsset>>, max_depth: usize) -> Result<Vec<RcStr>> { async fn to_graph(asset: Vc<Box<dyn Module>>, max_depth: usize) -> Result<Vec<RcStr>> {
let mut visited = HashSet::new(); let mut visited = FxHashSet::default();
let mut queue = Vec::new(); let mut queue = Vec::new();
queue.push((0, asset)); queue.push((0, asset));
let mut result = vec![]; let mut result = vec![];
while let Some((depth, asset)) = queue.pop() { while let Some((depth, asset)) = queue.pop() {
let references = asset.references().all_assets().await?; let references = referenced_modules_and_affecting_sources(asset, false).await?;
let mut indent = String::new(); let mut indent = String::new();
for _ in 0..depth { for _ in 0..depth {
indent.push_str(" "); indent.push_str(" ");
} }
let path = asset.path(); let path = &asset.ident().await?.path;
if visited.insert(asset) { if visited.insert(asset) {
if depth < max_depth { if depth < max_depth {
for &asset in references.iter().rev() { for (_, references) in references.iter().rev() {
queue.push((depth + 1, asset)); for asset in references.modules.iter() {
queue.push((depth + 1, **asset));
}
} }
} }
result.push(turbofmt!("{indent}{path}").await?); result.push(turbofmt!("{indent}{}", path.path).await?);
} else if references.is_empty() { } else if references.is_empty() {
result.push(turbofmt!("{indent}{path} *").await?); result.push(turbofmt!("{indent}{} *", path.path).await?);
} else { } else {
result.push(turbofmt!("{indent}{path} *...").await?); result.push(turbofmt!("{indent}{} *...", path.path).await?);
} }
} }
result.push("".into()); result.push("".into());
......
...@@ -84,11 +84,6 @@ impl NodeJsChunkingContextBuilder { ...@@ -84,11 +84,6 @@ impl NodeJsChunkingContextBuilder {
self self
} }
pub fn file_tracing(mut self, enable_tracing: bool) -> Self {
self.chunking_context.enable_file_tracing = enable_tracing;
self
}
pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self { pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self {
self.chunking_context.enable_nested_async_availability = enable_nested_async_availability; self.chunking_context.enable_nested_async_availability = enable_nested_async_availability;
self self
...@@ -212,8 +207,6 @@ pub struct NodeJsChunkingContext { ...@@ -212,8 +207,6 @@ pub struct NodeJsChunkingContext {
environment: ResolvedVc<Environment>, environment: ResolvedVc<Environment>,
/// The kind of runtime to include in the output. /// The kind of runtime to include in the output.
runtime_type: RuntimeType, runtime_type: RuntimeType,
/// Enable tracing for this chunking
enable_file_tracing: bool,
/// Enable nested async availability for this chunking /// Enable nested async availability for this chunking
enable_nested_async_availability: bool, enable_nested_async_availability: bool,
/// Enable module merging /// Enable module merging
...@@ -272,7 +265,6 @@ impl NodeJsChunkingContext { ...@@ -272,7 +265,6 @@ impl NodeJsChunkingContext {
asset_prefixes: Default::default(), asset_prefixes: Default::default(),
url_behaviors: Default::default(), url_behaviors: Default::default(),
default_url_behavior: None, default_url_behavior: None,
enable_file_tracing: false,
enable_nested_async_availability: false, enable_nested_async_availability: false,
enable_module_merging: false, enable_module_merging: false,
enable_dynamic_chunk_content_loading: false, enable_dynamic_chunk_content_loading: false,
...@@ -374,11 +366,6 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -374,11 +366,6 @@ impl ChunkingContext for NodeJsChunkingContext {
*self.environment *self.environment
} }
#[turbo_tasks::function]
fn is_tracing_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_file_tracing)
}
#[turbo_tasks::function] #[turbo_tasks::function]
fn is_nested_async_availability_enabled(&self) -> Vc<bool> { fn is_nested_async_availability_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_nested_async_availability) Vc::cell(self.enable_nested_async_availability)
...@@ -535,7 +522,6 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -535,7 +522,6 @@ impl ChunkingContext for NodeJsChunkingContext {
async move { async move {
let MakeChunkGroupResult { let MakeChunkGroupResult {
chunks, chunks,
referenced_output_assets,
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
...@@ -556,7 +542,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -556,7 +542,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Ok(ChunkGroupResult { Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets), assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets), referenced_assets: OutputAssets::empty_resolved(),
references: ResolvedVc::cell(references), references: ResolvedVc::cell(references),
availability_info, availability_info,
} }
...@@ -584,7 +570,6 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -584,7 +570,6 @@ impl ChunkingContext for NodeJsChunkingContext {
async move { async move {
let MakeChunkGroupResult { let MakeChunkGroupResult {
chunks, chunks,
mut referenced_output_assets,
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
...@@ -605,8 +590,6 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -605,8 +590,6 @@ impl ChunkingContext for NodeJsChunkingContext {
.await?; .await?;
other_chunks.extend(extra_chunks.iter().copied()); other_chunks.extend(extra_chunks.iter().copied());
referenced_output_assets.extend(extra_referenced_assets.await?.iter().copied());
let Some(module) = ResolvedVc::try_sidecast(chunk_group.entries().last().unwrap()) let Some(module) = ResolvedVc::try_sidecast(chunk_group.entries().last().unwrap())
else { else {
bail!("module must be placeable in an ecmascript chunk"); bail!("module must be placeable in an ecmascript chunk");
...@@ -626,7 +609,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -626,7 +609,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Vc::cell(other_chunks), Vc::cell(other_chunks),
Vc::cell(evaluatable_assets), Vc::cell(evaluatable_assets),
*module, *module,
Vc::cell(referenced_output_assets), extra_referenced_assets,
Vc::cell(references), Vc::cell(references),
*module_graph, *module_graph,
*self, *self,
......
...@@ -11,6 +11,7 @@ use turbo_tasks_fs::{ ...@@ -11,6 +11,7 @@ use turbo_tasks_fs::{
}; };
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{ChunkingType, TracedMode},
file_source::FileSource, file_source::FileSource,
raw_module::RawModule, raw_module::RawModule,
reference::ModuleReference, reference::ModuleReference,
...@@ -71,6 +72,12 @@ impl ModuleReference for NodePreGypConfigReference { ...@@ -71,6 +72,12 @@ impl ModuleReference for NodePreGypConfigReference {
) )
.await .await
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
async fn resolve_node_pre_gyp_files( async fn resolve_node_pre_gyp_files(
...@@ -253,6 +260,12 @@ impl ModuleReference for NodeGypBuildReference { ...@@ -253,6 +260,12 @@ impl ModuleReference for NodeGypBuildReference {
) )
.await .await
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
async fn resolve_node_gyp_build_files( async fn resolve_node_gyp_build_files(
...@@ -376,6 +389,12 @@ impl ModuleReference for NodeBindingsReference { ...@@ -376,6 +389,12 @@ impl ModuleReference for NodeBindingsReference {
) )
.await .await
} }
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
} }
async fn resolve_node_bindings_files( async fn resolve_node_bindings_files(
......
...@@ -21,6 +21,7 @@ turbopack = { workspace = true } ...@@ -21,6 +21,7 @@ turbopack = { workspace = true }
[dev-dependencies] [dev-dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
rustc-hash = { workspace = true }
regex = { workspace = true } regex = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
......
...@@ -7,9 +7,10 @@ use std::{path::PathBuf, sync::LazyLock}; ...@@ -7,9 +7,10 @@ use std::{path::PathBuf, sync::LazyLock};
use anyhow::Result; use anyhow::Result;
use regex::Regex; use regex::Regex;
use rstest::*; use rstest::*;
use rustc_hash::FxHashSet;
use similar::TextDiff; use similar::TextDiff;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, TurboTasks, Vc}; use turbo_tasks::{FxIndexSet, ResolvedVc, TurboTasks, Vc};
use turbo_tasks_backend::TurboTasksBackend; use turbo_tasks_backend::TurboTasksBackend;
use turbo_tasks_fs::{DiskFileSystem, FileSystem}; use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{ use turbopack::{
...@@ -27,11 +28,9 @@ use turbopack_core::{ ...@@ -27,11 +28,9 @@ use turbopack_core::{
file_source::FileSource, file_source::FileSource,
ident::Layer, ident::Layer,
module::Module, module::Module,
output::OutputAsset, reference::referenced_modules_and_affecting_sources,
reference::all_assets_from_entries,
reference_type::ReferenceType, reference_type::ReferenceType,
resolve::options::ConditionValue, resolve::options::ConditionValue,
traced_asset::TracedAsset,
}; };
use turbopack_ecmascript::AnalyzeMode; use turbopack_ecmascript::AnalyzeMode;
use turbopack_resolve::resolve_options_context::ResolveOptionsContext; use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
...@@ -265,22 +264,32 @@ async fn node_file_trace_operation(package_root: RcStr, input: RcStr) -> Result< ...@@ -265,22 +264,32 @@ async fn node_file_trace_operation(package_root: RcStr, input: RcStr) -> Result<
.module(); .module();
// We treat the entry as an external // We treat the entry as an external
let mut paths = to_list(vec![ResolvedVc::upcast( let mut paths = to_list(module).await?;
TracedAsset::new(module).to_resolved().await?,
)])
.await?;
paths.push(module.ident().await?.path.path.clone()); paths.push(module.ident().await?.path.path.clone());
Ok(Vc::cell(paths)) Ok(Vc::cell(paths))
} }
async fn to_list(assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>) -> Result<Vec<RcStr>> { async fn to_list(asset: Vc<Box<dyn Module>>) -> Result<Vec<RcStr>> {
let mut assets = all_assets_from_entries(Vc::cell(assets)) let mut assets = vec![];
.await?
.iter() let mut visited = FxHashSet::default();
.map(async |a| Ok(a.path().await?.path.clone())) let mut queue = Vec::new();
.try_join() queue.push(asset);
.await?;
while let Some(asset) = queue.pop() {
let references = referenced_modules_and_affecting_sources(asset, false).await?;
let path = &asset.ident().await?.path;
if visited.insert(asset) {
for (_, references) in references.iter().rev() {
for asset in references.modules.iter() {
queue.push(**asset);
}
}
}
assets.push(path.path.clone());
}
assets.sort(); assets.sort();
assets.dedup(); assets.dedup();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment