Unverified Commit 5ee70052 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: use module graph for NFT (#92901)

- [x] skip traced edges properly in all graph traversals
- [x] iter_nodes and enumerate_nodes include the traced modules
    - [x] Don't propagate is_self_async inside of traced references as a perf optimization
- [x] fix performance
- [x] validate that NFT JSON files are identical
- [x] ~~validate/adjust bundle analyzer output~~ will be in followup
- [x] audit `fn chunking_type()` changes

Benchmark:
```
* ced85b3108 - (2 weeks ago) refactor - Niklas Mischkulnig (HEAD -> mischnic/nft-module-graph, origin/mischnic/nft-module-graph)

TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   334.96s user 44.57s system 788% cpu 48.152 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   343.39s user 42.61s system 809% cpu 47.659 total

* 5edf4823 - (77 minutes ago) v16.3.0-canary.32 - next-js-bot[bot] (HEAD, tag: v16.3.0-canary.32)

TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   333.42s user 43.57s system 712% cpu 52.877 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   332.95s user 43.15s system 791% cpu 47.515 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   337.59s user 41.28s system 771% cpu 49.117 total
TURBOPACK_PERSISTENT_CACHE=0 NEXT_TURBOPACK_TRACING= pnpm next build   335.25s user 45.53s system 718% cpu 52.975 total
```

<details>
<summary>Previous benchmarks</summary>

```
52212418699f362904d64c08e544d4994e8d27fa mischnic/nft-module-graph

pnpm next build   371.41s user 66.39s system 737% cpu 59.353 total
pnpm next build   380.13s user 59.62s system 751% cpu 58.548 total
pnpm next build   374.25s user 60.55s system 761% cpu 57.102 total

8e4cfc50 canary

pnpm next build   370.77s user 53.89s system 762% cpu 55.725 total
pnpm next build   383.16s user 55.61s system 744% cpu 58.921 total
pnpm next build   382.34s user 53.51s system 757% cpu 57.512 total
pnpm next build   381.15s user 54.50s system 751% cpu 57.943 total
```


```
b0d5411a79 mischnic/nft-module-graph
pnpm next build --experimental-build-mode=compil  368.75s user 62.05s system 749% cpu 57.445 total
pnpm next build --experimental-build-mode=compil  368.95s user 58.42s system 763% cpu 56.004 total

0e3ad2768bd3b036e96f66e4b770a16bb0b38705 mischnic/nft-module-graph
pnpm next build --experimental-build-mode=compil  369.48s user 60.62s system 748% cpu 57.494 total
pnpm next build --experimental-build-mode=compil  374.89s user 52.27s system 730% cpu 58.449 total
pnpm next build --experimental-build-mode=compil  375.02s user 52.89s system 747% cpu 57.274 total

f37073e1d96f6d846cd333b93fccd1dd7bdc253e mischnic/speed-next-dynamic-map
pnpm next build --experimental-build-mode=compil  389.58s user 55.98s system 704% cpu 1:03.21 total

e599b11e2d6b09a92600cfca6d9b338ed1f7537d mischnic/fix-side-effect-edge-removal
pnpm next build --experimental-build-mode=compil  386.60s user 53.34s system 720% cpu 1:01.09 total

f68f70b0 canary
pnpm next build --experimental-build-mode=compil  387.46s user 53.72s system 738% cpu 59.716 total
pnpm next build --experimental-build-mode=compil  390.33s user 53.54s system 734% cpu 1:00.43 total
pnpm next build --experimental-build-mode=compil  385.33s user 56.55s system 722% cpu 1:01.18 total
pnpm next build --experimental-build-mode=compil  386.49s user 56.44s system 726% cpu 1:00.96 total
```



```
b1ed4553e6 origin/mischnic/nft-module-graph
pnpm next build --experimental-build-mode=compile  467.28s user 75.24s system 791% cpu 1:08.56 total
pnpm next build --experimental-build-mode=compile  465.00s user 76.80s system 831% cpu 1:05.20 total
pnpm next build --experimental-build-mode=compile  473.86s user 70.73s system 790% cpu 1:08.90 total

3cb77c84 canary
pnpm next build --experimental-build-mode=compile  367.91s user 58.06s system 726% cpu 58.672 total
pnpm next build --experimental-build-mode=compile  362.97s user 57.68s system 746% cpu 56.379 total
pnpm next build --experimental-build-mode=compile  364.59s user 57.01s system 724% cpu 58.167 total
```

</details>
parent ee0d02ad
......@@ -10773,6 +10773,7 @@ dependencies = [
"regex",
"rstest",
"rstest_reuse",
"rustc-hash 2.1.1",
"serde",
"serde_json",
"similar",
......
......@@ -1747,6 +1747,8 @@ impl AppEndpoint {
.chain(loadable_manifest_output.iter().flat_map(|m| &**m).copied())
.map(|m| *m)
.collect(),
*module_graphs.full,
vec![*rsc_entry],
)
.to_resolved()
.await?,
......
......@@ -184,10 +184,18 @@ impl InstrumentationEndpoint {
let chunk = self.node_chunk().to_resolved().await?;
let mut output_assets = vec![chunk];
if this.project.next_mode().await?.is_production() {
let userland_module = self.entry_module();
output_assets.push(ResolvedVc::upcast(
NftJsonAsset::new(*this.project, None, *chunk, vec![])
.to_resolved()
.await?,
NftJsonAsset::new(
*this.project,
None,
*chunk,
vec![],
this.project.module_graph(userland_module),
vec![userland_module],
)
.to_resolved()
.await?,
));
}
Ok(Vc::cell(output_assets))
......
......@@ -226,10 +226,18 @@ impl MiddlewareEndpoint {
let chunk = self.node_chunk().to_resolved().await?;
let mut output_assets = vec![chunk];
if this.project.next_mode().await?.is_production() {
let userland_module = self.entry_module();
output_assets.push(ResolvedVc::upcast(
NftJsonAsset::new(*this.project, None, *chunk, vec![])
.to_resolved()
.await?,
NftJsonAsset::new(
*this.project,
None,
*chunk,
vec![],
this.project.module_graph(userland_module),
vec![userland_module],
)
.to_resolved()
.await?,
));
}
let middleware_manifest_v2 = MiddlewaresManifestV2 {
......
......@@ -226,6 +226,7 @@ impl NextDynamicGraph {
})
},
|_, _, _| Ok(()),
false,
)?;
Ok(Vc::cell(result))
}
......@@ -834,6 +835,7 @@ async fn validate_pages_css_imports_individual(
Ok(GraphTraversalAction::Continue)
},
|_, _, _| Ok(()),
false,
)?;
candidates
......
......@@ -19,15 +19,16 @@ use turbopack_core::{
asset::{Asset, AssetContent},
context::AssetContext,
file_source::FileSource,
module::{Module, Modules},
module_graph::{ModuleGraph, SingleModuleGraph, chunk_group_info::ChunkGroupEntry},
output::{OutputAsset, OutputAssets, OutputAssetsReference},
reference_type::{CommonJsReferenceSubType, ReferenceType},
resolve::{ResolveErrorMode, origin::PlainResolveOrigin, parse::Request},
traced_asset::TracedAsset,
};
use turbopack_resolve::ecmascript::cjs_resolve;
use crate::{
nft_json::{all_assets_from_entries_filtered, relativize_glob},
nft_json::{relativize_glob, traced_modules_for_entries},
project::Project,
};
......@@ -106,6 +107,7 @@ impl Asset for ServerNftJsonAsset {
#[turbo_tasks::function]
async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
let this = self.await?;
// Example: [project]/apps/my-website/.next/
let base_dir = this
.project
......@@ -113,16 +115,31 @@ impl Asset for ServerNftJsonAsset {
.await?
.join(&this.project.node_root().await?.path)?;
let module_graph = ModuleGraph::from_graphs(
vec![SingleModuleGraph::new_with_traced_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(self.entries().owned().await?)]),
true,
false,
)],
None,
)
.connect();
let mut server_output_assets =
all_assets_from_entries_filtered(self.entries(), None, Some(self.ignores()))
traced_modules_for_entries(module_graph, self.entries(), Some(self.ignores()), true)
.await?
.iter()
.map(async |m| {
Ok((
base_dir
.get_relative_path_to(&*m.path().await?)
.get_relative_path_to(&m.ident().await?.path)
.context("failed to compute relative path for server NFT JSON")?,
m.content().hash(HashAlgorithm::Xxh3Hash128Hex).await?,
m.source()
.await?
.context("NFT module has no content")?
.content()
.hash(HashAlgorithm::Xxh3Hash128Hex)
.await?,
))
})
.try_join()
......@@ -186,7 +203,7 @@ impl Asset for ServerNftJsonAsset {
#[turbo_tasks::value_impl]
impl ServerNftJsonAsset {
#[turbo_tasks::function]
async fn entries(&self) -> Result<Vc<OutputAssets>> {
async fn entries(&self) -> Result<Vc<Modules>> {
let is_standalone = *self.project.next_config().is_standalone().await?;
let asset_context = Vc::upcast(externals_tracing_module_context(
......@@ -215,14 +232,20 @@ impl ServerNftJsonAsset {
// These are used by packages/next/src/server/require-hook.ts
let shared_entries = ["styled-jsx", "styled-jsx/style", "styled-jsx/style.js"];
let cache_handler_entries = cache_handler.iter().chain(cache_handlers.iter()).map(|f| {
asset_context
.process(
Vc::upcast(FileSource::new(f.clone())),
ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined),
)
.module()
});
let cache_handler_entries = cache_handler
.iter()
.chain(cache_handlers.iter())
.map(|f| {
asset_context
.process(
Vc::upcast(FileSource::new(f.clone())),
ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined),
)
.module()
})
.map(|m| m.to_resolved())
.try_join()
.await?;
let entries = match self.ty {
ServerNftType::Full => Either::Left(
......@@ -247,6 +270,7 @@ impl ServerNftJsonAsset {
Ok(Vc::cell(
cache_handler_entries
.into_iter()
.chain(
shared_entries
.into_iter()
......@@ -262,15 +286,12 @@ impl ServerNftJsonAsset {
.await?
.primary_modules()
.await?
.into_iter()
.map(|m| *m))
.into_iter())
})
.try_flat_join()
.await?,
)
.map(|m| Vc::upcast::<Box<dyn OutputAsset>>(TracedAsset::new(m)).to_resolved())
.try_join()
.await?,
.collect(),
))
}
......
This diff is collapsed.
......@@ -1087,6 +1087,8 @@ impl PageEndpoint {
Some(pages_function_name(&this.original_name).into()),
*ssr_entry_chunk,
additional_assets,
ssr_module_graph,
vec![*ssr_module],
)
.to_resolved()
.await?,
......
......@@ -189,7 +189,7 @@ impl Visit<FindServerEntriesNode> for FindServerEntries {
let referenced_modules = referenced_modules
.iter()
.flat_map(|(_, resolved)| match resolved.chunking_type {
ChunkingType::Traced => None,
ChunkingType::Traced { .. } => None,
_ => Some(resolved.modules.iter()),
})
.flatten()
......
......@@ -1095,7 +1095,6 @@ pub async fn get_server_chunking_context_with_client_assets(
.module_id_strategy(module_id_strategy.to_resolved().await?)
.export_usage(*export_usage.await?)
.unused_references(unused_references.to_resolved().await?)
.file_tracing(next_mode.is_production())
.debug_ids(*debug_ids.await?)
.hash_salt(hash_salt)
.nested_async_availability(*nested_async_chunking.await?)
......@@ -1203,7 +1202,6 @@ pub async fn get_server_chunking_context(
.module_id_strategy(module_id_strategy.to_resolved().await?)
.export_usage(*export_usage.await?)
.unused_references(unused_references.to_resolved().await?)
.file_tracing(next_mode.is_production())
.debug_ids(*debug_ids.await?)
.hash_salt(hash_salt)
.nested_async_availability(*nested_async_chunking.await?)
......
......@@ -110,8 +110,6 @@ async fn get_analyze_data_operation(
.try_join()
.await?;
whole_app_module_graphs.as_side_effect().await?;
let modules_data = ResolvedVc::upcast(
ModulesDataOutputAsset::new(
analyze_output_root.join("modules.data")?,
......
......@@ -1690,13 +1690,14 @@ async fn output_assets_operation(
}
let whole_app_module_graphs = project.whole_app_module_graphs();
// This makes the trace file nicer to look at
whole_app_module_graphs.as_side_effect().await?;
let nft = next_server_nft_assets(project).await?;
let routes_hashes_manifest = routes_hashes_manifest_asset_if_enabled(project).await?;
let immutable_hashes_manifest_asset =
immutable_hashes_manifest_asset_if_enabled(project).await?;
whole_app_module_graphs.as_side_effect().await?;
Ok(Vc::cell(
output_assets
.into_iter()
......
......@@ -69,11 +69,6 @@ impl BrowserChunkingContextBuilder {
self
}
pub fn tracing(mut self, enable_tracing: bool) -> Self {
self.chunking_context.enable_tracing = enable_tracing;
self
}
pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self {
self.chunking_context.enable_nested_async_availability = enable_nested_async_availability;
self
......@@ -297,8 +292,6 @@ pub struct BrowserChunkingContext {
default_url_behavior: Option<UrlBehavior>,
/// Enable HMR for this chunking
enable_hot_module_replacement: bool,
/// Enable tracing for this chunking
enable_tracing: bool,
/// Enable nested async availability for this chunking
enable_nested_async_availability: bool,
/// Enable module merging
......@@ -375,7 +368,6 @@ impl BrowserChunkingContext {
url_behaviors: Default::default(),
default_url_behavior: None,
enable_hot_module_replacement: false,
enable_tracing: false,
enable_nested_async_availability: false,
enable_module_merging: false,
enable_dynamic_chunk_content_loading: false,
......@@ -728,11 +720,6 @@ impl ChunkingContext for BrowserChunkingContext {
self.source_map_source_type.cell()
}
#[turbo_tasks::function]
fn is_tracing_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_tracing)
}
#[turbo_tasks::function]
fn is_nested_async_availability_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_nested_async_availability)
......@@ -772,7 +759,6 @@ impl ChunkingContext for BrowserChunkingContext {
let input_availability_info = availability_info;
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
......@@ -818,7 +804,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
referenced_assets: OutputAssets::empty_resolved(),
references: ResolvedVc::cell(references),
availability_info,
}
......@@ -845,7 +831,6 @@ impl ChunkingContext for BrowserChunkingContext {
let this = self.await?;
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
......@@ -908,7 +893,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
referenced_assets: OutputAssets::empty_resolved(),
references: ResolvedVc::cell(references),
availability_info,
}
......
......@@ -21,7 +21,6 @@ use crate::{
chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
},
environment::ChunkLoading,
module::Module,
module_graph::{
GraphTraversalAction, ModuleGraph,
chunk_group_info::ChunkGroup,
......@@ -32,13 +31,11 @@ use crate::{
},
module_batches::{BatchingConfig, ModuleBatchesGraphEdge},
},
output::{OutputAsset, OutputAssetsReference},
traced_asset::TracedAsset,
output::OutputAssetsReference,
};
pub struct MakeChunkGroupResult {
pub chunks: ResolvedVc<Chunks>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>,
pub availability_info: AvailabilityInfo,
}
......@@ -57,7 +54,6 @@ pub async fn make_chunk_group(
let is_nested_async_availability_enabled = *chunking_context
.is_nested_async_availability_enabled()
.await?;
let should_trace = *chunking_context.is_tracing_enabled().await?;
let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
let batching_config = chunking_context.batching_config().to_resolved().await?;
......@@ -70,7 +66,6 @@ pub async fn make_chunk_group(
ChunkGroupContentOptions {
availability_info,
can_split_async,
should_trace,
should_merge_modules,
batching_config,
},
......@@ -80,7 +75,6 @@ pub async fn make_chunk_group(
chunkable_items,
batch_groups,
async_modules,
traced_modules,
available_modules: _,
} = &*inner;
......@@ -156,17 +150,6 @@ pub async fn make_chunk_group(
.try_join()
.await?;
let referenced_output_assets = traced_modules
.iter()
.copied()
.map(|module| async move {
Ok(ResolvedVc::upcast(
TracedAsset::new(*module).to_resolved().await?,
))
})
.try_join()
.await?;
chunk_items.extend(async_loader_chunk_items);
// Pass chunk items to chunking algorithm
......@@ -182,7 +165,6 @@ pub async fn make_chunk_group(
Ok(MakeChunkGroupResult {
chunks,
referenced_output_assets,
references: ResolvedVc::upcast_vec(async_loaders),
availability_info: new_availability_info,
})
......@@ -196,8 +178,6 @@ pub struct ChunkGroupContentOptions {
pub availability_info: AvailabilityInfo,
/// Whether async modules can be split into separate chunks
pub can_split_async: bool,
/// Whether traced modules should be collected
pub should_trace: bool,
/// Whether module merging is enabled
pub should_merge_modules: bool,
/// The batching config to use
......@@ -235,7 +215,6 @@ async fn chunk_group_content_operation(
ChunkGroupContentOptions {
availability_info,
can_split_async,
should_trace,
should_merge_modules,
batching_config,
}: ChunkGroupContentOptions,
......@@ -248,14 +227,12 @@ async fn chunk_group_content_operation(
unsorted_items: ModuleToChunkableMap,
chunkable_items: FxIndexSet<ChunkableModuleOrBatch>,
async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
}
let mut state = TraverseState {
unsorted_items: FxHashMap::default(),
chunkable_items: FxIndexSet::default(),
async_modules: FxIndexSet::default(),
traced_modules: FxIndexSet::default(),
};
let available_modules = match availability_info.available_modules() {
......@@ -277,21 +254,15 @@ async fn chunk_group_content_operation(
if matches!(node, ModuleOrBatch::None(_)) {
return Ok(GraphTraversalAction::Continue);
}
// Traced modules need to have a special handling
// Traced modules are completely ignored during chunking
if let Some((
_,
ModuleBatchesGraphEdge {
ty: ChunkingType::Traced,
ty: ChunkingType::Traced { .. },
..
},
)) = parent_info
{
if should_trace {
let ModuleOrBatch::Module(module) = node else {
unreachable!();
};
state.traced_modules.insert(module);
}
return Ok(GraphTraversalAction::Exclude);
}
......@@ -359,7 +330,7 @@ async fn chunk_group_content_operation(
GraphTraversalAction::Exclude
}
}
ChunkingType::Traced => {
ChunkingType::Traced { .. } => {
// handled above before the sidecast
unreachable!();
}
......@@ -458,7 +429,6 @@ async fn chunk_group_content_operation(
chunkable_items,
batch_groups,
async_modules: state.async_modules,
traced_modules: state.traced_modules,
available_modules,
}
.cell())
......
......@@ -381,13 +381,6 @@ pub trait ChunkingContext {
})
}
/// Whether `ChunkingType::Traced` are used to create corresponding output assets for each
/// traced module.
#[turbo_tasks::function]
fn is_tracing_enabled(self: Vc<Self>) -> Vc<bool> {
Vc::cell(false)
}
/// Whether async modules should create an new availability boundary and therefore nested async
/// modules include less modules. Enabling this will lead to better optimized async chunks,
/// but it will require to compute all possible paths in the application, which might lead to
......
......@@ -281,6 +281,40 @@ pub trait OutputChunk: Asset {
fn runtime_info(self: Vc<Self>) -> Vc<OutputChunkRuntimeInfo>;
}
/// Whether this reference is an entry point for a traced subgraph.
#[derive(
Debug,
Clone,
Copy,
Hash,
TraceRawVcs,
Serialize,
Deserialize,
Eq,
PartialEq,
ValueDebugFormat,
NonLocalValue,
Encode,
Decode,
TaskInput,
)]
pub enum TracedMode {
/// Going from bundled to unbundled code, i.e. an external dependency or readFile static assets.
Entry,
/// This reference should only be respected from unbundled code (e.g. for package.json needed by
/// externals (sort of affecting_sources)
Transitive,
}
impl Display for TracedMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TracedMode::Entry => write!(f, "Entry"),
TracedMode::Transitive => write!(f, "Transitive"),
}
}
}
/// Specifies how a chunk interacts with other chunks when building a chunk
/// group
#[derive(
......@@ -324,8 +358,13 @@ pub enum ChunkingType {
inherit_async: bool,
merge_tag: Option<RcStr>,
},
// Module not placed in chunk group, but its references are still followed.
Traced,
/// The module not placed in chunk group, but its references are still followed. This is used
/// for NFT, to list all unbundled files that are still needed at runtime (some static assets,
/// or externals and their transitive dependencies).
Traced {
/// Whether this reference is an entry point for a traced subgraph.
mode: TracedMode,
},
}
impl Display for ChunkingType {
......@@ -368,7 +407,7 @@ impl Display for ChunkingType {
} => {
write!(f, "Shared(inherit_async: {inherit_async})")
}
ChunkingType::Traced => write!(f, "Traced"),
ChunkingType::Traced { mode } => write!(f, "Traced(mode: {mode})"),
}
}
}
......@@ -391,6 +430,10 @@ impl ChunkingType {
matches!(self, ChunkingType::Parallel { .. })
}
pub fn is_traced(&self) -> bool {
matches!(self, ChunkingType::Traced { .. })
}
pub fn is_merged(&self) -> bool {
matches!(
self,
......@@ -422,7 +465,7 @@ impl ChunkingType {
inherit_async: false,
merge_tag: merge_tag.clone(),
},
ChunkingType::Traced => ChunkingType::Traced,
ChunkingType::Traced { mode } => ChunkingType::Traced { mode: *mode },
}
}
}
......@@ -433,8 +476,6 @@ pub struct ChunkGroupContentInner {
pub batch_groups: Vec<ResolvedVc<ModuleBatchGroup>>,
#[bincode(with = "turbo_bincode::indexset")]
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
#[bincode(with = "turbo_bincode::indexset")]
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
pub available_modules: ResolvedVc<AvailableModulesSet>,
}
......
......@@ -81,7 +81,7 @@ pub async fn children_from_module_references(
Some(ChunkingType::Async) => async_reference_ty(),
Some(ChunkingType::Isolated { .. }) => isolated_reference_ty(),
Some(ChunkingType::Shared { .. }) => shared_reference_ty(),
Some(ChunkingType::Traced) => traced_reference_ty(),
Some(ChunkingType::Traced { .. }) => traced_reference_ty(),
};
for &module in reference
......
......@@ -41,7 +41,6 @@ pub mod source_map;
pub mod source_pos;
pub mod source_transform;
pub mod target;
pub mod traced_asset;
mod utils;
pub mod version;
pub mod virtual_output;
......
......@@ -600,7 +600,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
chunk_groups,
)))
}
ChunkingType::Traced => {
ChunkingType::Traced { .. } => {
// Traced modules are not placed in chunk groups
return Ok(GraphTraversalAction::Skip);
}
......
......@@ -423,6 +423,7 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
}
Ok(())
},
false,
)?;
list_lists.push(lists);
......@@ -543,6 +544,7 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
}
Ok(())
},
false,
)?;
drop(inner_span);
......
......@@ -335,6 +335,7 @@ impl PreBatches {
state.items.push(item);
Ok(())
},
false,
)?;
Ok(state.items)
}
......
......@@ -7,6 +7,7 @@ use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks_fs::{FileSystemEntryType, FileSystemPath};
use crate::{
chunk::TracedMode,
file_source::FileSource,
ident::AssetIdent,
module::{Module, ModuleSideEffects},
......@@ -146,9 +147,10 @@ async fn dir_references(package_dir: FileSystemPath) -> Result<Vc<ModuleReferenc
.into_iter()
.map(async |p| {
Ok(ResolvedVc::upcast(
TracedModuleReference::new(Vc::upcast(RawModule::new(Vc::upcast(
FileSource::new(p),
))))
TracedModuleReference::new(
Vc::upcast(RawModule::new(Vc::upcast(FileSource::new(p)))),
TracedMode::Entry,
)
.to_resolved()
.await?,
))
......
......@@ -41,10 +41,11 @@ impl RebasedAsset {
impl OutputAssetsReference for RebasedAsset {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let references = referenced_modules_and_affecting_sources(*self.module)
let references = referenced_modules_and_affecting_sources(*self.module, false)
.await?
.iter()
.map(|module| async move {
.flat_map(|(_, ref_data)| ref_data.modules.iter())
.map(async |module| {
Ok(ResolvedVc::upcast(
RebasedAsset::new(**module, self.input_dir.clone(), self.output_dir.clone())
.to_resolved()
......
......@@ -4,12 +4,12 @@ use anyhow::Result;
use bincode::{Decode, Encode};
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexSet, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs,
};
use crate::{
chunk::ChunkingType,
chunk::{ChunkingType, TracedMode},
module::{Module, Modules},
output::{
ExpandOutputAssetsInput, ExpandedOutputAssets, OutputAsset, OutputAssets,
......@@ -34,9 +34,7 @@ pub trait ModuleReference: ValueToString {
// TODO think about different types
// fn kind(&self) -> Vc<AssetReferenceType>;
fn chunking_type(&self) -> Option<ChunkingType> {
None
}
fn chunking_type(&self) -> Option<ChunkingType>;
fn binding_usage(&self) -> BindingUsage {
BindingUsage::default()
......@@ -111,33 +109,51 @@ impl ModuleReference for SingleChunkableModuleReference {
#[turbo_tasks::function]
pub async fn referenced_modules_and_affecting_sources(
module: Vc<Box<dyn Module>>,
) -> Result<Vc<Modules>> {
let references = module.references().await?;
let resolved_references = references
include_binding_usage: bool,
) -> Result<Vc<ModulesWithRefData>> {
let modules = module
.references()
.await?
.iter()
.map(|r| r.resolve_reference())
.try_join()
.await?;
let mut modules = Vec::new();
for resolve_result in resolved_references {
modules.extend(resolve_result.primary_modules_raw_iter());
modules.extend(
resolve_result
.affecting_sources_iter()
.map(|source| async move {
Ok(ResolvedVc::upcast(
RawModule::new(*source).to_resolved().await?,
))
})
.try_join()
.await?,
);
}
.map(|reference| async {
let trait_ref = reference.into_trait_ref().await?;
let resolve_result = reference.resolve_reference().await?;
if let Some(chunking_type) = &trait_ref.chunking_type() {
let mut modules = resolve_result
.primary_modules_raw_iter()
.collect::<Vec<_>>();
modules.extend(
resolve_result
.affecting_sources_iter()
.map(|source| async move {
Ok(ResolvedVc::upcast(
RawModule::new(*source).to_resolved().await?,
))
})
.try_join()
.await?,
);
let resolved_modules: FxIndexSet<_> = modules.into_iter().collect();
let binding_usage = if include_binding_usage {
trait_ref.binding_usage()
} else {
BindingUsage::default()
};
Ok(Vc::cell(resolved_modules.into_iter().collect()))
return Ok(Some((
*reference,
ResolvedReference {
chunking_type: chunking_type.clone(),
binding_usage,
modules,
},
)));
}
Ok(None)
})
.try_flat_join()
.await?;
Ok(Vc::cell(modules))
}
#[turbo_tasks::value]
......@@ -145,6 +161,7 @@ pub async fn referenced_modules_and_affecting_sources(
#[value_to_string("traced {}", self.module.ident())]
pub struct TracedModuleReference {
module: ResolvedVc<Box<dyn Module>>,
mode: TracedMode,
}
#[turbo_tasks::value_impl]
......@@ -155,15 +172,15 @@ impl ModuleReference for TracedModuleReference {
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced)
Some(ChunkingType::Traced { mode: self.mode })
}
}
#[turbo_tasks::value_impl]
impl TracedModuleReference {
#[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(TracedModuleReference { module })
pub fn new(module: ResolvedVc<Box<dyn Module>>, mode: TracedMode) -> Vc<Self> {
Self::cell(TracedModuleReference { module, mode })
}
}
......@@ -216,7 +233,7 @@ pub async fn primary_chunkable_referenced_modules(
.map(|reference| async {
let trait_ref = reference.into_trait_ref().await?;
if let Some(chunking_type) = &trait_ref.chunking_type() {
if !include_traced && matches!(chunking_type, ChunkingType::Traced) {
if !include_traced && chunking_type.is_traced() {
return Ok(None);
}
......
......@@ -4,6 +4,7 @@ use turbo_tasks_fs::{File, FileContent, FileSystemEntryType, FileSystemPath};
use super::ModuleReference;
use crate::{
chunk::{ChunkingType, TracedMode},
file_source::FileSource,
raw_module::RawModule,
resolve::ModuleResolveResult,
......@@ -51,6 +52,12 @@ impl ModuleReference for SourceMapReference {
}
Ok(*ModuleResolveResult::unresolvable())
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value_impl]
......
use anyhow::Result;
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
module::Module,
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
reference::referenced_modules_and_affecting_sources,
};
/// Converts a traced external [Module] graph into a graph consisting of [TracedAsset]s.
#[turbo_tasks::value]
pub struct TracedAsset {
module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl]
impl TracedAsset {
#[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(TracedAsset { module })
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for TracedAsset {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let references = referenced_modules_and_affecting_sources(*self.module)
.await?
.iter()
.map(async |module| {
Ok(ResolvedVc::upcast(
TracedAsset::new(**module).to_resolved().await?,
))
})
.try_join()
.await?;
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for TracedAsset {
#[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> {
Ok(self.module.ident().await?.path.clone().cell())
}
}
#[turbo_tasks::value_impl]
impl Asset for TracedAsset {
#[turbo_tasks::function]
async fn content(&self) -> Result<Vc<AssetContent>> {
if let Some(source) = *self.module.source().await? {
Ok(source.content())
} else {
turbobail!("Module {} has no source", self.module.ident());
}
}
}
......@@ -10,7 +10,7 @@ use turbo_tasks_fs::{FileSystem, FileSystemPath, LinkType, VirtualFileSystem, ro
use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext},
chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext, TracedMode},
ident::{AssetIdent, Layer},
module::{Module, ModuleSideEffects},
module_graph::ModuleGraph,
......@@ -334,22 +334,13 @@ impl Module for CachedExternalModule {
rcstr!("affecting source"),
))
})
.chain(
external_result
.primary_modules_raw_iter()
// These modules aren't bundled but still need to be part of the module
// graph for chunking. `compute_async_module_info` computes
// `is_self_async` for every module, but at least for traced modules,
// that value is never used as `ChunkingType::Traced.is_inherit_async()
// == false`. Optimize this case by using
// `SideEffectfulModuleWithoutSelfAsync` to
// short circuit that computation and thus defer parsing traced modules
// to emitting to not block all of chunking on this.
.map(|m| Vc::upcast(SideEffectfulModuleWithoutSelfAsync::new(*m))),
)
.chain(external_result.primary_modules_raw_iter().map(|m| *m))
.map(|s| {
Vc::upcast::<Box<dyn ModuleReference>>(TracedModuleReference::new(s))
.to_resolved()
Vc::upcast::<Box<dyn ModuleReference>>(TracedModuleReference::new(
s,
TracedMode::Entry,
))
.to_resolved()
})
.try_join()
.await?;
......@@ -458,45 +449,6 @@ impl EcmascriptChunkPlaceable for CachedExternalModule {
}
}
/// A wrapper "passthrough" module type that always returns `false` for `is_self_async` and
/// `SideEffects` for `side_effects`.Be careful when using it, as it may hide async dependencies.
#[turbo_tasks::value]
struct SideEffectfulModuleWithoutSelfAsync {
module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl]
impl SideEffectfulModuleWithoutSelfAsync {
#[turbo_tasks::function]
fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(SideEffectfulModuleWithoutSelfAsync { module })
}
}
#[turbo_tasks::value_impl]
impl Module for SideEffectfulModuleWithoutSelfAsync {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
fn source(&self) -> Vc<turbopack_core::source::OptionSource> {
self.module.source()
}
#[turbo_tasks::function]
fn references(&self) -> Vc<ModuleReferences> {
self.module.references()
}
#[turbo_tasks::function]
fn side_effects(&self) -> Vc<ModuleSideEffects> {
ModuleSideEffects::SideEffectful.cell()
}
// Don't override and use default is_self_async that always returns false
}
#[derive(Debug)]
#[turbo_tasks::value(shared)]
pub struct ExternalsSymlinkAsset {
......
......@@ -2,7 +2,10 @@ use anyhow::Result;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
file_source::FileSource, raw_module::RawModule, reference::ModuleReference,
chunk::{ChunkingType, TracedMode},
file_source::FileSource,
raw_module::RawModule,
reference::ModuleReference,
resolve::ModuleResolveResult,
};
......@@ -31,4 +34,10 @@ impl ModuleReference for PackageJsonReference {
.await?,
)))
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
......@@ -4,7 +4,7 @@ use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
chunk::ChunkingType,
chunk::{ChunkingType, TracedMode},
file_source::FileSource,
issue::IssueSource,
raw_module::RawModule,
......@@ -79,7 +79,9 @@ impl ModuleReference for FileSourceReference {
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced)
Some(ChunkingType::Traced {
mode: TracedMode::Entry,
})
}
}
......@@ -207,6 +209,8 @@ impl ModuleReference for DirAssetReference {
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced)
Some(ChunkingType::Traced {
mode: TracedMode::Entry,
})
}
}
......@@ -3,6 +3,7 @@ use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
chunk::{ChunkingType, TracedMode},
context::AssetContext,
file_source::FileSource,
reference::ModuleReference,
......@@ -42,6 +43,12 @@ impl ModuleReference for TsConfigReference {
.await?,
)))
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value]
......@@ -88,6 +95,12 @@ impl ModuleReference for TsReferencePathAssetReference {
},
)
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value]
......@@ -120,4 +133,10 @@ impl ModuleReference for TsReferenceTypeAssetReference {
),
)
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
......@@ -5,6 +5,7 @@ use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks_fs::DirectoryContent;
use turbopack_core::{
asset::Asset,
chunk::{ChunkingType, TracedMode},
ident::AssetIdent,
module::{Module, ModuleSideEffects},
raw_module::RawModule,
......@@ -208,6 +209,12 @@ impl ModuleReference for CompilerReference {
ResolveErrorMode::Error,
)
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value]
......@@ -233,6 +240,12 @@ impl ModuleReference for TsExtendsReference {
RawModule::new(*self.config).to_resolved().await?,
)))
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value]
......@@ -266,6 +279,12 @@ impl ModuleReference for TsNodeRequireReference {
ResolveErrorMode::Error,
)
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value]
......@@ -293,4 +312,10 @@ impl ModuleReference for TsConfigTypesReference {
fn resolve_reference(&self) -> Vc<ModuleResolveResult> {
type_resolve(*self.origin, *self.request)
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
......@@ -3,6 +3,7 @@ use swc_core::ecma::ast::Lit;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbopack_core::{
chunk::{ChunkingType, TracedMode},
compile_time_info::CompileTimeInfo,
file_source::FileSource,
ident::AssetIdent,
......@@ -138,6 +139,12 @@ impl ModuleReference for WebpackChunkAssetReference {
WebpackRuntime::None => *ModuleResolveResult::unresolvable(),
})
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value(shared)]
......@@ -165,6 +172,12 @@ impl ModuleReference for WebpackEntryAssetReference {
.await?,
)))
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
#[turbo_tasks::value(shared)]
......@@ -210,4 +223,10 @@ impl ModuleReference for WebpackRuntimeAssetReference {
.await?
.cell())
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
use std::{collections::HashSet, env::current_dir, path::PathBuf};
use std::{env::current_dir, path::PathBuf};
use anyhow::Result;
use rustc_hash::FxHashSet;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TransientInstance, TryJoinIterExt, Vc, turbofmt};
use turbo_tasks::{TransientInstance, Vc, turbofmt};
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{
ModuleAssetContext,
......@@ -19,11 +20,10 @@ use turbopack_core::{
file_source::FileSource,
ident::Layer,
issue::{IssueReporter, IssueSeverity, handle_issues},
output::{OutputAsset, OutputAssetsReference},
reference::all_assets_from_entries,
module::Module,
reference::referenced_modules_and_affecting_sources,
reference_type::ReferenceType,
resolve::options::ConditionValue,
traced_asset::TracedAsset,
};
use turbopack_ecmascript::AnalyzeMode;
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......@@ -123,52 +123,65 @@ async fn node_file_trace_operation(
.process(Vc::upcast(source), ReferenceType::Undefined)
.module();
let asset = TracedAsset::new(module).to_resolved().await?;
Ok(Vc::cell(if graph {
to_graph(ResolvedVc::upcast(asset), max_depth.unwrap_or(usize::MAX)).await?
to_graph(module, max_depth.unwrap_or(usize::MAX)).await?
} else {
to_list(ResolvedVc::upcast(asset)).await?
to_list(module).await?
}))
}
async fn to_list(asset: ResolvedVc<Box<dyn OutputAsset>>) -> Result<Vec<RcStr>> {
let mut assets = all_assets_from_entries(Vc::cell(vec![asset]))
.await?
.iter()
.map(async |a| Ok(a.path().await?.path.clone()))
.try_join()
.await?;
async fn to_list(asset: Vc<Box<dyn Module>>) -> Result<Vec<RcStr>> {
let mut assets = vec![];
let mut visited = FxHashSet::default();
let mut queue = Vec::new();
queue.push(asset);
while let Some(asset) = queue.pop() {
let references = referenced_modules_and_affecting_sources(asset, false).await?;
let path = &asset.ident().await?.path;
if visited.insert(asset) {
for (_, references) in references.iter().rev() {
for asset in references.modules.iter() {
queue.push(**asset);
}
}
}
assets.push(path.path.clone());
}
assets.sort();
assets.dedup();
Ok(assets)
}
async fn to_graph(asset: ResolvedVc<Box<dyn OutputAsset>>, max_depth: usize) -> Result<Vec<RcStr>> {
let mut visited = HashSet::new();
async fn to_graph(asset: Vc<Box<dyn Module>>, max_depth: usize) -> Result<Vec<RcStr>> {
let mut visited = FxHashSet::default();
let mut queue = Vec::new();
queue.push((0, asset));
let mut result = vec![];
while let Some((depth, asset)) = queue.pop() {
let references = asset.references().all_assets().await?;
let references = referenced_modules_and_affecting_sources(asset, false).await?;
let mut indent = String::new();
for _ in 0..depth {
indent.push_str(" ");
}
let path = asset.path();
let path = &asset.ident().await?.path;
if visited.insert(asset) {
if depth < max_depth {
for &asset in references.iter().rev() {
queue.push((depth + 1, asset));
for (_, references) in references.iter().rev() {
for asset in references.modules.iter() {
queue.push((depth + 1, **asset));
}
}
}
result.push(turbofmt!("{indent}{path}").await?);
result.push(turbofmt!("{indent}{}", path.path).await?);
} else if references.is_empty() {
result.push(turbofmt!("{indent}{path} *").await?);
result.push(turbofmt!("{indent}{} *", path.path).await?);
} else {
result.push(turbofmt!("{indent}{path} *...").await?);
result.push(turbofmt!("{indent}{} *...", path.path).await?);
}
}
result.push("".into());
......
......@@ -84,11 +84,6 @@ impl NodeJsChunkingContextBuilder {
self
}
pub fn file_tracing(mut self, enable_tracing: bool) -> Self {
self.chunking_context.enable_file_tracing = enable_tracing;
self
}
pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self {
self.chunking_context.enable_nested_async_availability = enable_nested_async_availability;
self
......@@ -212,8 +207,6 @@ pub struct NodeJsChunkingContext {
environment: ResolvedVc<Environment>,
/// The kind of runtime to include in the output.
runtime_type: RuntimeType,
/// Enable tracing for this chunking
enable_file_tracing: bool,
/// Enable nested async availability for this chunking
enable_nested_async_availability: bool,
/// Enable module merging
......@@ -272,7 +265,6 @@ impl NodeJsChunkingContext {
asset_prefixes: Default::default(),
url_behaviors: Default::default(),
default_url_behavior: None,
enable_file_tracing: false,
enable_nested_async_availability: false,
enable_module_merging: false,
enable_dynamic_chunk_content_loading: false,
......@@ -374,11 +366,6 @@ impl ChunkingContext for NodeJsChunkingContext {
*self.environment
}
#[turbo_tasks::function]
fn is_tracing_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_file_tracing)
}
#[turbo_tasks::function]
fn is_nested_async_availability_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_nested_async_availability)
......@@ -535,7 +522,6 @@ impl ChunkingContext for NodeJsChunkingContext {
async move {
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
......@@ -556,7 +542,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
referenced_assets: OutputAssets::empty_resolved(),
references: ResolvedVc::cell(references),
availability_info,
}
......@@ -584,7 +570,6 @@ impl ChunkingContext for NodeJsChunkingContext {
async move {
let MakeChunkGroupResult {
chunks,
mut referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
......@@ -605,8 +590,6 @@ impl ChunkingContext for NodeJsChunkingContext {
.await?;
other_chunks.extend(extra_chunks.iter().copied());
referenced_output_assets.extend(extra_referenced_assets.await?.iter().copied());
let Some(module) = ResolvedVc::try_sidecast(chunk_group.entries().last().unwrap())
else {
bail!("module must be placeable in an ecmascript chunk");
......@@ -626,7 +609,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Vc::cell(other_chunks),
Vc::cell(evaluatable_assets),
*module,
Vc::cell(referenced_output_assets),
extra_referenced_assets,
Vc::cell(references),
*module_graph,
*self,
......
......@@ -11,6 +11,7 @@ use turbo_tasks_fs::{
};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{ChunkingType, TracedMode},
file_source::FileSource,
raw_module::RawModule,
reference::ModuleReference,
......@@ -71,6 +72,12 @@ impl ModuleReference for NodePreGypConfigReference {
)
.await
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
async fn resolve_node_pre_gyp_files(
......@@ -253,6 +260,12 @@ impl ModuleReference for NodeGypBuildReference {
)
.await
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
async fn resolve_node_gyp_build_files(
......@@ -376,6 +389,12 @@ impl ModuleReference for NodeBindingsReference {
)
.await
}
fn chunking_type(&self) -> Option<ChunkingType> {
Some(ChunkingType::Traced {
mode: TracedMode::Transitive,
})
}
}
async fn resolve_node_bindings_files(
......
......@@ -21,6 +21,7 @@ turbopack = { workspace = true }
[dev-dependencies]
anyhow = { workspace = true }
rustc-hash = { workspace = true }
regex = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
......
......@@ -7,9 +7,10 @@ use std::{path::PathBuf, sync::LazyLock};
use anyhow::Result;
use regex::Regex;
use rstest::*;
use rustc_hash::FxHashSet;
use similar::TextDiff;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, TurboTasks, Vc};
use turbo_tasks::{FxIndexSet, ResolvedVc, TurboTasks, Vc};
use turbo_tasks_backend::TurboTasksBackend;
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{
......@@ -27,11 +28,9 @@ use turbopack_core::{
file_source::FileSource,
ident::Layer,
module::Module,
output::OutputAsset,
reference::all_assets_from_entries,
reference::referenced_modules_and_affecting_sources,
reference_type::ReferenceType,
resolve::options::ConditionValue,
traced_asset::TracedAsset,
};
use turbopack_ecmascript::AnalyzeMode;
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......@@ -265,22 +264,32 @@ async fn node_file_trace_operation(package_root: RcStr, input: RcStr) -> Result<
.module();
// We treat the entry as an external
let mut paths = to_list(vec![ResolvedVc::upcast(
TracedAsset::new(module).to_resolved().await?,
)])
.await?;
let mut paths = to_list(module).await?;
paths.push(module.ident().await?.path.path.clone());
Ok(Vc::cell(paths))
}
async fn to_list(assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>) -> Result<Vec<RcStr>> {
let mut assets = all_assets_from_entries(Vc::cell(assets))
.await?
.iter()
.map(async |a| Ok(a.path().await?.path.clone()))
.try_join()
.await?;
async fn to_list(asset: Vc<Box<dyn Module>>) -> Result<Vec<RcStr>> {
let mut assets = vec![];
let mut visited = FxHashSet::default();
let mut queue = Vec::new();
queue.push(asset);
while let Some(asset) = queue.pop() {
let references = referenced_modules_and_affecting_sources(asset, false).await?;
let path = &asset.ident().await?.path;
if visited.insert(asset) {
for (_, references) in references.iter().rev() {
for asset in references.modules.iter() {
queue.push(**asset);
}
}
}
assets.push(path.path.clone());
}
assets.sort();
assets.dedup();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment