Unverified Commit fa57e47f authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] avoid attaching referenced output assets to chunks (#75933)

### What?

`referenced_output_assets` are different for every chunk group. Attaching them as references to a random chunk in the group causes a different Chunk instance which not only causes duplicate work, but also multiple writes to the same file.

This fixes that by refactoring that and attaching the `referenced_output_assets`  higher. Actually it simplifies it.

Closes PACK-4322
Closes PACK-4392
parent 7ff17dd1
This diff is collapsed.
......@@ -68,7 +68,7 @@ pub(crate) async fn collect_next_dynamic_chunks(
.client_component_client_chunks
.get(&parent_client_reference.unwrap())
.unwrap()
.1
.availability_info
}
NextDynamicChunkAvailability::AvailabilityInfo(availability_info) => {
*availability_info
......
......@@ -20,7 +20,7 @@ use turbopack_core::{
GraphEntries,
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
reference_type::{EntryReferenceSubType, ReferenceType},
source::Source,
virtual_output::VirtualOutputAsset,
......@@ -97,21 +97,19 @@ impl InstrumentationEndpoint {
}
#[turbo_tasks::function]
async fn edge_files(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
async fn edge_chunk_group(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let module = self.core_modules().await?.edge_entry_module;
let module_graph = this.project.module_graph(*module);
let edge_chunking_context = this.project.edge_chunking_context(false);
let edge_files: Vc<OutputAssets> = edge_chunking_context.evaluated_chunk_group_assets(
Ok(edge_chunking_context.evaluated_chunk_group_assets(
module.ident(),
ChunkGroup::Entry(vec![module]),
module_graph,
AvailabilityInfo::Root,
);
Ok(edge_files)
))
}
#[turbo_tasks::function]
......@@ -136,6 +134,7 @@ impl InstrumentationEndpoint {
Vc::cell(vec![module]),
module_graph,
OutputAssets::empty(),
OutputAssets::empty(),
AvailabilityInfo::Root,
)
.await?;
......@@ -147,19 +146,20 @@ impl InstrumentationEndpoint {
let this = self.await?;
if this.is_edge {
let edge_files = self.edge_files();
let mut output_assets = edge_files.owned().await?;
let edge_chunk_group = self.edge_chunk_group();
let edge_all_assets = edge_chunk_group.all_assets();
let node_root = this.project.node_root().owned().await?;
let node_root_value = node_root.clone();
let file_paths_from_root =
get_js_paths_from_root(&node_root_value, &output_assets).await?;
get_js_paths_from_root(&node_root_value, &edge_chunk_group.await?.assets.await?)
.await?;
let all_output_assets = all_assets_from_entries(edge_files).await?;
let mut output_assets = all_assets_from_entries(edge_all_assets).owned().await?;
let wasm_paths_from_root =
get_wasm_paths_from_root(&node_root_value, &all_output_assets).await?;
get_wasm_paths_from_root(&node_root_value, &output_assets).await?;
let instrumentation_definition = InstrumentationDefinition {
files: file_paths_from_root,
......
......@@ -26,7 +26,7 @@ use turbopack_core::{
GraphEntries,
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
reference_type::{EntryReferenceSubType, ReferenceType},
source::Source,
virtual_output::VirtualOutputAsset,
......@@ -104,20 +104,20 @@ impl MiddlewareEndpoint {
}
#[turbo_tasks::function]
async fn edge_files(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
async fn edge_chunk_group(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let module = self.entry_module().to_resolved().await?;
let module_graph = this.project.module_graph(*module);
let edge_chunking_context = this.project.edge_chunking_context(false);
let edge_files = edge_chunking_context.evaluated_chunk_group_assets(
let edge_chunk_grou = edge_chunking_context.evaluated_chunk_group_assets(
module.ident(),
ChunkGroup::Entry(vec![module]),
module_graph,
AvailabilityInfo::Root,
);
Ok(edge_files)
Ok(edge_chunk_grou)
}
#[turbo_tasks::function]
......@@ -142,6 +142,7 @@ impl MiddlewareEndpoint {
Vc::cell(vec![module]),
module_graph,
OutputAssets::empty(),
OutputAssets::empty(),
AvailabilityInfo::Root,
)
.await?;
......@@ -254,22 +255,22 @@ impl MiddlewareEndpoint {
Ok(Vc::cell(output_assets))
} else {
let edge_files = self.edge_files();
let mut output_assets = edge_files.owned().await?;
let edge_chunk_group = self.edge_chunk_group();
let edge_all_assets = edge_chunk_group.all_assets();
let node_root = this.project.node_root().owned().await?;
let node_root_value = node_root.clone();
let file_paths_from_root =
get_js_paths_from_root(&node_root_value, &output_assets).await?;
get_js_paths_from_root(&node_root_value, &edge_chunk_group.await?.assets.await?)
.await?;
let all_output_assets = all_assets_from_entries(edge_files).await?;
let mut output_assets = all_assets_from_entries(edge_all_assets).owned().await?;
let wasm_paths_from_root =
get_wasm_paths_from_root(&node_root_value, &all_output_assets).await?;
get_wasm_paths_from_root(&node_root_value, &output_assets).await?;
let all_assets =
get_asset_paths_from_root(&node_root_value, &all_output_assets).await?;
let all_assets = get_asset_paths_from_root(&node_root_value, &output_assets).await?;
let regions = if let Some(regions) = config.preferred_region.as_ref() {
if regions.len() == 1 {
......
......@@ -56,7 +56,7 @@ use turbopack_core::{
GraphEntries, ModuleGraph, SingleModuleGraph, VisitedModules,
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
},
output::{OptionOutputAsset, OutputAsset, OutputAssets},
output::{OptionOutputAsset, OutputAsset, OutputAssets, OutputAssetsWithReferenced},
reference_type::{EcmaScriptModulesReferenceSubType, EntryReferenceSubType, ReferenceType},
resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern},
source::Source,
......@@ -756,7 +756,7 @@ impl PageEndpoint {
}
#[turbo_tasks::function]
async fn client_chunks(self: Vc<Self>) -> Result<Vc<ChunkGroupResult>> {
async fn client_chunk_group(self: Vc<Self>) -> Result<Vc<ChunkGroupResult>> {
async move {
let this = self.await?;
......@@ -933,7 +933,7 @@ impl PageEndpoint {
let ssr_module_graph = self.ssr_module_graph();
let next_dynamic_imports = if let PageEndpointType::Html = this.ty {
let client_availability_info = self.client_chunks().await?.availability_info;
let client_availability_info = self.client_chunk_group().await?.availability_info;
let client_module_graph = self.client_module_graph();
......@@ -998,6 +998,7 @@ impl PageEndpoint {
};
let mut current_chunks = OutputAssets::empty();
let mut current_referenced_assets = OutputAssets::empty();
let mut current_availability_info = AvailabilityInfo::Root;
for layout in [document_module, app_module].iter().flatten().copied() {
let span = tracing::trace_span!(
......@@ -1005,7 +1006,11 @@ impl PageEndpoint {
name = display(layout.ident().to_string().await?)
);
async {
let chunk_group = chunking_context
let ChunkGroupResult {
assets,
referenced_assets,
availability_info,
} = *chunking_context
.chunk_group(
layout.ident(),
ChunkGroup::Shared(layout),
......@@ -1014,11 +1019,12 @@ impl PageEndpoint {
)
.await?;
current_chunks = current_chunks
.concatenate(*chunk_group.assets)
current_chunks = current_chunks.concatenate(*assets).resolve().await?;
current_referenced_assets = current_referenced_assets
.concatenate(*referenced_assets)
.resolve()
.await?;
current_availability_info = chunk_group.availability_info;
current_availability_info = availability_info;
anyhow::Ok(())
}
......@@ -1030,15 +1036,27 @@ impl PageEndpoint {
.context("could not process page loader entry module")?;
let is_edge = matches!(runtime, NextRuntime::Edge);
if is_edge {
let edge_files = edge_chunking_context.evaluated_chunk_group_assets(
ssr_module.ident(),
ChunkGroup::Entry(vec![ResolvedVc::upcast(ssr_module_evaluatable)]),
ssr_module_graph,
current_availability_info,
);
let OutputAssetsWithReferenced {
assets: edge_assets,
referenced_assets: edge_referenced_assets,
} = *edge_chunking_context
.evaluated_chunk_group_assets(
ssr_module.ident(),
ChunkGroup::Entry(vec![ResolvedVc::upcast(ssr_module_evaluatable)]),
ssr_module_graph,
current_availability_info,
)
.await?;
Ok(SsrChunk::Edge {
files: current_chunks.concatenate(edge_files).to_resolved().await?,
assets: current_chunks
.concatenate(*edge_assets)
.to_resolved()
.await?,
referenced_assets: current_referenced_assets
.concatenate(*edge_referenced_assets)
.to_resolved()
.await?,
dynamic_import_entries,
regions: regions.clone(),
}
......@@ -1056,6 +1074,7 @@ impl PageEndpoint {
EvaluatableAssets::empty().with_entry(*ssr_module_evaluatable),
ssr_module_graph,
current_chunks,
current_referenced_assets,
current_availability_info,
)
.to_resolved()
......@@ -1296,8 +1315,16 @@ impl PageEndpoint {
let ssr_chunk = match this.ty {
PageEndpointType::Html => {
let client_chunks = *self.client_chunks().await?.assets;
let client_chunk_group = self.client_chunk_group().await?;
let client_chunks = *client_chunk_group.assets;
client_assets.extend(client_chunks.await?.iter().map(|asset| **asset));
client_assets.extend(
client_chunk_group
.referenced_assets
.await?
.iter()
.map(|asset| **asset),
);
let build_manifest = self.build_manifest(client_chunks).to_resolved().await?;
let page_loader = self.page_loader(client_chunks);
......@@ -1394,7 +1421,8 @@ impl PageEndpoint {
}
}
SsrChunk::Edge {
files,
assets,
referenced_assets,
dynamic_import_entries,
ref regions,
} => {
......@@ -1414,9 +1442,14 @@ impl PageEndpoint {
fxindexset![]
};
let files_value = files.await?;
let all_assets = assets.concatenate(*referenced_assets);
let assets_ref = assets.await?;
server_assets.extend(referenced_assets.await?.iter().copied());
if let Some(&file) = files_value.first() {
// TODO(sokra): accessing the 1st asset is a bit hacky, we should find a better
// way to get the main entry asset
if let Some(&file) = assets_ref.first() {
let pages_manifest = self.pages_manifest(*file).to_resolved().await?;
server_assets.push(pages_manifest);
}
......@@ -1424,9 +1457,9 @@ impl PageEndpoint {
// Only include the actual edge files if pages should be created
let pages_structure = this.pages_structure.await?;
if pages_structure.should_create_pages_entries {
server_assets.extend(files_value.iter().copied());
server_assets.extend(assets_ref.iter().copied());
file_paths_from_root
.extend(get_js_paths_from_root(&node_root, &files_value).await?);
.extend(get_js_paths_from_root(&node_root, &assets_ref).await?);
}
if emit_manifests == EmitManifests::Full {
......@@ -1444,7 +1477,7 @@ impl PageEndpoint {
let (wasm_paths_from_root, all_assets) =
if pages_structure.should_create_pages_entries {
let all_output_assets = all_assets_from_entries(*files).await?;
let all_output_assets = all_assets_from_entries(all_assets).await?;
let mut wasm_paths_from_root = fxindexset![];
wasm_paths_from_root.extend(
......@@ -1512,7 +1545,7 @@ impl PageEndpoint {
}
PageEndpointOutput::Edge {
files,
files: assets,
server_assets: ResolvedVc::cell(server_assets),
client_assets,
}
......@@ -1745,7 +1778,8 @@ pub enum SsrChunk {
server_asset_trace_file: ResolvedVc<OptionOutputAsset>,
},
Edge {
files: ResolvedVc<OutputAssets>,
assets: ResolvedVc<OutputAssets>,
referenced_assets: ResolvedVc<OutputAssets>,
dynamic_import_entries: ResolvedVc<DynamicImportedChunks>,
regions: Option<Vec<RcStr>>,
},
......
......@@ -3,10 +3,10 @@ use tracing::Instrument;
use turbo_rcstr::rcstr;
use turbo_tasks::{FxIndexMap, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
use turbopack_core::{
chunk::{ChunkingContext, availability_info::AvailabilityInfo},
chunk::{ChunkGroupResult, ChunkingContext, availability_info::AvailabilityInfo},
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::OutputAssets,
output::{OutputAssets, OutputAssetsWithReferenced},
};
use crate::{
......@@ -22,12 +22,10 @@ use crate::{
#[turbo_tasks::value]
pub struct ClientReferencesChunks {
pub client_component_client_chunks:
FxIndexMap<ClientReferenceType, (ResolvedVc<OutputAssets>, AvailabilityInfo)>,
pub client_component_ssr_chunks:
FxIndexMap<ClientReferenceType, (ResolvedVc<OutputAssets>, AvailabilityInfo)>,
pub client_component_client_chunks: FxIndexMap<ClientReferenceType, ChunkGroupResult>,
pub client_component_ssr_chunks: FxIndexMap<ClientReferenceType, ChunkGroupResult>,
pub layout_segment_client_chunks:
FxIndexMap<ResolvedVc<NextServerComponentModule>, ResolvedVc<OutputAssets>>,
FxIndexMap<ResolvedVc<NextServerComponentModule>, OutputAssetsWithReferenced>,
}
/// Computes all client references chunks.
......@@ -161,8 +159,10 @@ pub async fn get_app_client_references_chunks(
let mut current_client_availability_info = client_availability_info;
let mut current_client_chunks = OutputAssets::empty().to_resolved().await?;
let mut current_client_referenced_assets = OutputAssets::empty().to_resolved().await?;
let mut current_ssr_availability_info = AvailabilityInfo::Root;
let mut current_ssr_chunks = OutputAssets::empty().to_resolved().await?;
let mut current_ssr_referenced_assets = OutputAssets::empty().to_resolved().await?;
let mut layout_segment_client_chunks = FxIndexMap::default();
let mut client_component_ssr_chunks = FxIndexMap::default();
......@@ -264,18 +264,34 @@ pub async fn get_app_client_references_chunks(
};
if let Some(client_chunk_group) = client_chunk_group {
let client_chunk_group = client_chunk_group.await?;
let ChunkGroupResult {
assets,
referenced_assets,
availability_info,
} = *client_chunk_group.await?;
let client_chunks =
current_client_chunks.concatenate(*client_chunk_group.assets);
let client_chunks = client_chunks.to_resolved().await?;
let client_chunks = current_client_chunks
.concatenate(*assets)
.to_resolved()
.await?;
let client_referenced_assets = current_client_referenced_assets
.concatenate(*referenced_assets)
.to_resolved()
.await?;
if is_layout {
current_client_availability_info = client_chunk_group.availability_info;
current_client_availability_info = availability_info;
current_client_chunks = client_chunks;
current_client_referenced_assets = client_referenced_assets;
}
layout_segment_client_chunks.insert(server_component, client_chunks);
layout_segment_client_chunks.insert(
server_component,
OutputAssetsWithReferenced {
assets: client_chunks,
referenced_assets: client_referenced_assets,
},
);
for &client_reference_ty in client_reference_types.iter() {
if let ClientReferenceType::EcmascriptClientReference(_) =
......@@ -283,7 +299,11 @@ pub async fn get_app_client_references_chunks(
{
client_component_client_chunks.insert(
client_reference_ty,
(client_chunks, client_chunk_group.availability_info),
ChunkGroupResult {
assets: client_chunks,
referenced_assets: client_referenced_assets,
availability_info,
},
);
}
}
......@@ -292,12 +312,19 @@ pub async fn get_app_client_references_chunks(
if let Some(ssr_chunk_group) = ssr_chunk_group {
let ssr_chunk_group = ssr_chunk_group.await?;
let ssr_chunks = current_ssr_chunks.concatenate(*ssr_chunk_group.assets);
let ssr_chunks = ssr_chunks.to_resolved().await?;
let ssr_chunks = current_ssr_chunks
.concatenate(*ssr_chunk_group.assets)
.to_resolved()
.await?;
let ssr_referenced_assets = current_ssr_referenced_assets
.concatenate(*ssr_chunk_group.referenced_assets)
.to_resolved()
.await?;
if is_layout {
current_ssr_availability_info = ssr_chunk_group.availability_info;
current_ssr_chunks = ssr_chunks;
current_ssr_referenced_assets = ssr_referenced_assets;
}
for &client_reference_ty in client_reference_types.iter() {
......@@ -306,7 +333,11 @@ pub async fn get_app_client_references_chunks(
{
client_component_ssr_chunks.insert(
client_reference_ty,
(ssr_chunks, ssr_chunk_group.availability_info),
ChunkGroupResult {
assets: ssr_chunks,
referenced_assets: ssr_referenced_assets,
availability_info: ssr_chunk_group.availability_info,
},
);
}
}
......
......@@ -20,6 +20,7 @@ pub async fn get_app_client_shared_chunk_group(
if app_client_runtime_entries.await?.is_empty() {
return Ok(ChunkGroupResult {
assets: OutputAssets::empty().to_resolved().await?,
referenced_assets: OutputAssets::empty().to_resolved().await?,
availability_info: AvailabilityInfo::Root,
}
.cell());
......
......@@ -13,9 +13,11 @@ use turbo_tasks::{
use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{ChunkingContext, ModuleChunkItemIdExt, ModuleId as TurbopackModuleId},
chunk::{
ChunkGroupResult, ChunkingContext, ModuleChunkItemIdExt, ModuleId as TurbopackModuleId,
},
module_graph::async_module_info::AsyncModulesInfo,
output::OutputAsset,
output::{OutputAsset, OutputAssetsWithReferenced},
virtual_output::VirtualOutputAsset,
};
use turbopack_ecmascript::utils::StringifyJs;
......@@ -172,44 +174,50 @@ impl ClientReferenceManifest {
.chunk_item_id(*client_chunking_context)
.await?;
let (client_chunks_paths, client_is_async) =
if let Some((client_chunks, _client_availability_info)) =
client_component_client_chunks.get(&app_client_reference_ty)
{
let client_chunks = client_chunks.await?;
references.extend(client_chunks.iter());
let client_chunks_paths = cached_chunk_paths(
&mut client_chunk_path_cache,
client_chunks.iter().copied(),
)
.await?;
let (client_chunks_paths, client_is_async) = if let Some(ChunkGroupResult {
assets: client_chunks,
referenced_assets: client_referenced_assets,
availability_info: _,
}) =
client_component_client_chunks.get(&app_client_reference_ty)
{
let client_chunks = client_chunks.await?;
let client_referenced_assets = client_referenced_assets.await?;
references.extend(client_chunks.iter());
references.extend(client_referenced_assets.iter());
let client_chunks_paths = cached_chunk_paths(
&mut client_chunk_path_cache,
client_chunks.iter().copied(),
)
.await?;
let chunk_paths = client_chunks_paths
.filter_map(|(_, chunk_path)| {
client_relative_path
.get_path_to(&chunk_path)
.map(ToString::to_string)
})
// It's possible that a chunk also emits CSS files, that will
// be handled separately.
.filter(|path| path.ends_with(".js"))
.map(|path| {
format!(
"{}{}{}",
prefix_path,
path.split('/').map(encode_uri_component).format("/"),
suffix_path
)
})
.map(RcStr::from)
.collect::<Vec<_>>();
let chunk_paths = client_chunks_paths
.filter_map(|(_, chunk_path)| {
client_relative_path
.get_path_to(&chunk_path)
.map(ToString::to_string)
})
// It's possible that a chunk also emits CSS files, that will
// be handled separately.
.filter(|path| path.ends_with(".js"))
.map(|path| {
format!(
"{}{}{}",
prefix_path,
path.split('/').map(encode_uri_component).format("/"),
suffix_path
)
})
.map(RcStr::from)
.collect::<Vec<_>>();
let is_async = async_modules.contains(&ResolvedVc::upcast(client_module));
let is_async = async_modules.contains(&ResolvedVc::upcast(client_module));
(chunk_paths, is_async)
} else {
(Vec::new(), false)
};
(chunk_paths, is_async)
} else {
(Vec::new(), false)
};
if let Some(ssr_chunking_context) = ssr_chunking_context {
let ssr_module = client_reference_module_ref.ssr_module;
......@@ -225,11 +233,16 @@ impl ClientReferenceManifest {
// edge runtime doesn't support dynamically
// loading chunks.
(Vec::new(), false)
} else if let Some((ssr_chunks, _ssr_availability_info)) =
client_component_ssr_chunks.get(&app_client_reference_ty)
} else if let Some(ChunkGroupResult {
assets: ssr_chunks,
referenced_assets: ssr_referenced_assets,
availability_info: _,
}) = client_component_ssr_chunks.get(&app_client_reference_ty)
{
let ssr_chunks = ssr_chunks.await?;
let ssr_referenced_assets = ssr_referenced_assets.await?;
references.extend(ssr_chunks.iter());
references.extend(ssr_referenced_assets.iter());
let ssr_chunks_paths = cached_chunk_paths(
&mut ssr_chunk_path_cache,
......@@ -317,7 +330,14 @@ impl ClientReferenceManifest {
}
// per layout segment chunks need to be emitted into the manifest too
for (server_component, client_chunks) in layout_segment_client_chunks.iter() {
for (
server_component,
OutputAssetsWithReferenced {
assets: client_chunks,
referenced_assets: _,
},
) in layout_segment_client_chunks.iter()
{
let server_component_name = server_component
.server_path()
.await?
......
......@@ -12,7 +12,7 @@
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"jsx": "react-jsx",
"plugins": [
{
"name": "next"
......@@ -21,6 +21,12 @@
"strictNullChecks": true,
"target": "ES2017"
},
"include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"],
"exclude": ["node_modules"]
"include": [
"**/*.ts",
"**/*.tsx",
".next/types/**/*.ts",
"next-env.d.ts",
"**/*.mts"
],
"exclude": ["node_modules", "**/*.test.ts", "**/*.test.tsx"]
}
import { nextTestSetup } from 'e2e-utils'
import { check } from 'next-test-utils'
import { retry } from 'next-test-utils'
import { join } from 'node:path'
describe('deployment-id-handling disabled', () => {
......@@ -62,10 +62,7 @@ describe('deployment-id-handling disabled', () => {
await browser.elementByCss('#dynamic-import').click()
await check(
() => (requests.length > 0 ? 'success' : JSON.stringify(requests)),
'success'
)
await retry(() => expect(requests).not.toBeEmpty())
try {
expect(
......
import { nextTestSetup } from 'e2e-utils'
import { check, retry } from 'next-test-utils'
import { retry } from 'next-test-utils'
import { join } from 'node:path'
describe.each(['NEXT_DEPLOYMENT_ID', 'CUSTOM_DEPLOYMENT_ID'])(
......@@ -58,10 +58,7 @@ describe.each(['NEXT_DEPLOYMENT_ID', 'CUSTOM_DEPLOYMENT_ID'])(
await browser.elementByCss('#dynamic-import').click()
await check(
() => (requests.length > 0 ? 'success' : JSON.stringify(requests)),
'success'
)
await retry(() => expect(requests).not.toBeEmpty())
try {
expect(
......@@ -190,10 +187,7 @@ describe('deployment-id-handling disabled', () => {
await browser.elementByCss('#dynamic-import').click()
await check(
() => (requests.length > 0 ? 'success' : JSON.stringify(requests)),
'success'
)
await retry(() => expect(requests).not.toBeEmpty())
try {
expect(
......
......@@ -577,6 +577,7 @@ impl ChunkingContext for BrowserChunkingContext {
let input_availability_info = availability_info;
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
availability_info,
} = make_chunk_group(
entries,
......@@ -619,6 +620,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
availability_info,
}
.cell())
......@@ -644,6 +646,7 @@ impl ChunkingContext for BrowserChunkingContext {
let entries = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
availability_info,
} = make_chunk_group(
entries,
......@@ -703,6 +706,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
availability_info,
}
.cell())
......@@ -718,6 +722,7 @@ impl ChunkingContext for BrowserChunkingContext {
_evaluatable_assets: Vc<EvaluatableAssets>,
_module_graph: Vc<ModuleGraph>,
_extra_chunks: Vc<OutputAssets>,
_extra_referenced_assets: Vc<OutputAssets>,
_availability_info: AvailabilityInfo,
) -> Result<Vc<EntryChunkGroupResult>> {
bail!("Browser chunking context does not support entry chunk groups")
......
......@@ -23,8 +23,8 @@ use turbopack_cli_utils::issue::{ConsoleUi, LogOptions};
use turbopack_core::{
asset::Asset,
chunk::{
ChunkingConfig, ChunkingContext, EvaluatableAsset, EvaluatableAssets, MangleType,
MinifyType, SourceMapsType, availability_info::AvailabilityInfo,
ChunkingConfig, ChunkingContext, ChunkingContextExt, EvaluatableAsset, EvaluatableAssets,
MangleType, MinifyType, SourceMapsType, availability_info::AvailabilityInfo,
},
environment::{BrowserEnvironment, Environment, ExecutionEnvironment, NodeJsEnvironment},
ident::AssetIdent,
......@@ -35,7 +35,7 @@ use turbopack_core::{
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
export_usage::compute_export_usage_info,
},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
reference::all_assets_from_entries,
reference_type::{EntryReferenceSubType, ReferenceType},
resolve::{
......@@ -428,8 +428,8 @@ async fn build_internal(
{
match target {
Target::Browser => {
chunking_context
.evaluated_chunk_group(
*chunking_context
.evaluated_chunk_group_assets(
AssetIdent::from_path(
build_output_root
.join(
......@@ -449,29 +449,32 @@ async fn build_internal(
AvailabilityInfo::Root,
)
.await?
.assets
}
Target::Node => ResolvedVc::cell(vec![
chunking_context
.entry_chunk_group(
build_output_root
.join(
ecmascript
.ident()
.path()
.await?
.file_stem()
.unwrap(),
)?
.with_extension("entry.js"),
EvaluatableAssets::one(*ecmascript),
module_graph,
OutputAssets::empty(),
AvailabilityInfo::Root,
)
.await?
.asset,
]),
Target::Node => OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![
chunking_context
.entry_chunk_group(
build_output_root
.join(
ecmascript
.ident()
.path()
.await?
.file_stem()
.unwrap(),
)?
.with_extension("entry.js"),
EvaluatableAssets::one(*ecmascript),
module_graph,
OutputAssets::empty(),
OutputAssets::empty(),
AvailabilityInfo::Root,
)
.await?
.asset,
]),
referenced_assets: ResolvedVc::cell(vec![]),
},
}
} else {
bail!(
......@@ -485,16 +488,27 @@ async fn build_internal(
.try_join()
.await?;
let mut chunks: FxHashSet<ResolvedVc<Box<dyn OutputAsset>>> = FxHashSet::default();
for chunk_group in entry_chunk_groups {
chunks.extend(
&*async move { all_assets_from_entries(*chunk_group).await }
.instrument(tracing::info_span!("list chunks"))
.await?,
);
let all_assets = async move {
let mut all_assets: FxHashSet<ResolvedVc<Box<dyn OutputAsset>>> = FxHashSet::default();
for OutputAssetsWithReferenced {
assets,
referenced_assets,
} in entry_chunk_groups
{
all_assets.extend(all_assets_from_entries(*assets).await?.into_iter().copied());
all_assets.extend(
all_assets_from_entries(*referenced_assets)
.await?
.into_iter()
.copied(),
);
}
anyhow::Ok(all_assets)
}
.instrument(tracing::info_span!("list chunks"))
.await?;
chunks
all_assets
.iter()
.map(|c| async move { c.content().write(c.path().owned().await?).await })
.try_join()
......
......@@ -25,13 +25,14 @@ use crate::{
},
module_batches::{BatchingConfig, ModuleBatchesGraphEdge},
},
output::OutputAssets,
output::{OutputAsset, OutputAssets},
reference::ModuleReference,
traced_asset::TracedAsset,
};
pub struct MakeChunkGroupResult {
pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
pub availability_info: AvailabilityInfo,
}
......@@ -155,12 +156,12 @@ pub async fn make_chunk_group(
chunk_items,
chunk_item_batch_groups,
rcstr!(""),
ResolvedVc::cell(referenced_output_assets),
)
.await?;
Ok(MakeChunkGroupResult {
chunks,
referenced_output_assets,
availability_info,
})
}
......
use std::{future::IntoFuture, mem::replace};
use std::future::IntoFuture;
use anyhow::Result;
use rustc_hash::FxHashMap;
......@@ -26,7 +26,6 @@ use crate::{
},
},
module_graph::ModuleGraph,
output::OutputAssets,
};
mod dev;
......@@ -283,15 +282,13 @@ async fn batch_chunk_items_with_info_with_type(
Ok(Vc::cell(map))
}
/// Creates chunks based on heuristics for the passed `chunk_items`. Also
/// attaches `referenced_output_assets` to the first chunk.
/// Creates chunks based on heuristics for the passed `chunk_items`.
pub async fn make_chunks(
module_graph: Vc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
chunk_items_or_batches: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
key_prefix: RcStr,
mut referenced_output_assets: ResolvedVc<OutputAssets>,
) -> Result<Vec<ResolvedVc<Box<dyn Chunk>>>> {
let chunking_configs = &*chunking_context.chunking_configs().await?;
......@@ -330,8 +327,6 @@ pub async fn make_chunks(
ty,
chunking_context,
chunks: &mut chunks,
referenced_output_assets: &mut referenced_output_assets,
empty_referenced_output_assets: OutputAssets::empty().to_resolved().await?,
};
if let Some(chunking_config) = chunking_configs.get(&ty) {
......@@ -400,8 +395,6 @@ struct SplitContext<'a> {
// resolution of `chunks` is deferred so it can be done with `try_join` at the end, letting as
// much work happen in parallel as possible.
chunks: &'a mut Vec<Vc<Box<dyn Chunk>>>,
referenced_output_assets: &'a mut ResolvedVc<OutputAssets>,
empty_referenced_output_assets: ResolvedVc<OutputAssets>,
}
/// Creates a chunk with the given `chunk_items. `key` should be unique.
......@@ -427,10 +420,6 @@ async fn make_chunk<'l>(
})
.collect(),
ResolvedVc::deref_vec(batch_groups),
*replace(
split_context.referenced_output_assets,
split_context.empty_referenced_output_assets,
),
),
);
Ok(())
......
......@@ -17,7 +17,7 @@ use crate::{
ModuleGraph, chunk_group_info::ChunkGroup, export_usage::ModuleExportUsage,
module_batches::BatchingConfig,
},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
};
#[derive(
......@@ -87,8 +87,10 @@ pub enum ChunkGroupType {
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Copy)]
pub struct ChunkGroupResult {
pub assets: ResolvedVc<OutputAssets>,
pub referenced_assets: ResolvedVc<OutputAssets>,
pub availability_info: AvailabilityInfo,
}
......@@ -273,6 +275,7 @@ pub trait ChunkingContext {
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
) -> Result<Vc<EntryChunkGroupResult>>;
......@@ -313,7 +316,7 @@ pub trait ChunkingContextExt {
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
) -> Vc<OutputAssets>
) -> Vc<OutputAssetsWithReferenced>
where
Self: Send;
......@@ -323,7 +326,7 @@ pub trait ChunkingContextExt {
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Vc<OutputAssets>
) -> Vc<OutputAssetsWithReferenced>
where
Self: Send;
......@@ -333,6 +336,7 @@ pub trait ChunkingContextExt {
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
) -> Vc<Box<dyn OutputAsset>>
where
......@@ -344,6 +348,7 @@ pub trait ChunkingContextExt {
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
) -> Vc<EntryChunkGroupResult>
where
Self: Send;
......@@ -354,6 +359,7 @@ pub trait ChunkingContextExt {
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
) -> Vc<Box<dyn OutputAsset>>
where
Self: Send;
......@@ -364,7 +370,7 @@ pub trait ChunkingContextExt {
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Vc<OutputAssets>
) -> Vc<OutputAssetsWithReferenced>
where
Self: Send;
}
......@@ -384,7 +390,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
) -> Vc<OutputAssets> {
) -> Vc<OutputAssetsWithReferenced> {
root_chunk_group_assets(
Vc::upcast_non_strict(self),
ident,
......@@ -399,7 +405,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Vc<OutputAssets> {
) -> Vc<OutputAssetsWithReferenced> {
evaluated_chunk_group_assets(
Vc::upcast_non_strict(self),
ident,
......@@ -415,6 +421,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
) -> Vc<Box<dyn OutputAsset>> {
entry_chunk_group_asset(
......@@ -423,6 +430,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
evaluatable_assets,
module_graph,
extra_chunks,
extra_referenced_assets,
availability_info,
)
}
......@@ -433,12 +441,14 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
) -> Vc<EntryChunkGroupResult> {
self.entry_chunk_group(
path,
evaluatable_assets,
module_graph,
extra_chunks,
extra_referenced_assets,
AvailabilityInfo::Root,
)
}
......@@ -449,6 +459,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
) -> Vc<Box<dyn OutputAsset>> {
entry_chunk_group_asset(
Vc::upcast_non_strict(self),
......@@ -456,6 +467,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
evaluatable_assets,
module_graph,
extra_chunks,
extra_referenced_assets,
AvailabilityInfo::Root,
)
}
......@@ -466,7 +478,7 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Vc<OutputAssets> {
) -> Vc<OutputAssetsWithReferenced> {
chunk_group_assets(
Vc::upcast_non_strict(self),
ident,
......@@ -483,11 +495,15 @@ async fn root_chunk_group_assets(
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
) -> Result<Vc<OutputAssets>> {
Ok(*chunking_context
) -> Result<Vc<OutputAssetsWithReferenced>> {
let root_chunk_group = chunking_context
.root_chunk_group(ident, chunk_group, module_graph)
.await?
.assets)
.await?;
Ok(OutputAssetsWithReferenced {
assets: root_chunk_group.assets,
referenced_assets: root_chunk_group.referenced_assets,
}
.cell())
}
#[turbo_tasks::function]
......@@ -497,11 +513,15 @@ async fn evaluated_chunk_group_assets(
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<OutputAssets>> {
Ok(*chunking_context
) -> Result<Vc<OutputAssetsWithReferenced>> {
let evaluated_chunk_group = chunking_context
.evaluated_chunk_group(ident, chunk_group, module_graph, availability_info)
.await?
.assets)
.await?;
Ok(OutputAssetsWithReferenced {
assets: evaluated_chunk_group.assets,
referenced_assets: evaluated_chunk_group.referenced_assets,
}
.cell())
}
#[turbo_tasks::function]
......@@ -511,6 +531,7 @@ async fn entry_chunk_group_asset(
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
) -> Result<Vc<Box<dyn OutputAsset>>> {
Ok(*chunking_context
......@@ -519,6 +540,7 @@ async fn entry_chunk_group_asset(
evaluatable_assets,
module_graph,
extra_chunks,
extra_referenced_assets,
availability_info,
)
.await?
......@@ -532,9 +554,13 @@ async fn chunk_group_assets(
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<OutputAssets>> {
Ok(*chunking_context
) -> Result<Vc<OutputAssetsWithReferenced>> {
let chunk_group = chunking_context
.chunk_group(ident, chunk_group, module_graph, availability_info)
.await?
.assets)
.await?;
Ok(OutputAssetsWithReferenced {
assets: chunk_group.assets,
referenced_assets: chunk_group.referenced_assets,
}
.cell())
}
......@@ -481,7 +481,6 @@ pub trait ChunkType: ValueToString {
chunking_context: Vc<Box<dyn ChunkingContext>>,
chunk_items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
referenced_output_assets: Vc<OutputAssets>,
) -> Vc<Box<dyn Chunk>>;
#[turbo_tasks::function]
......
......@@ -59,3 +59,18 @@ impl OutputAssets {
/// A set of [OutputAsset]s
#[turbo_tasks::value(transparent)]
pub struct OutputAssetsSet(FxIndexSet<ResolvedVc<Box<dyn OutputAsset>>>);
#[turbo_tasks::value(shared)]
#[derive(Clone, Copy)]
pub struct OutputAssetsWithReferenced {
pub assets: ResolvedVc<OutputAssets>,
pub referenced_assets: ResolvedVc<OutputAssets>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsWithReferenced {
#[turbo_tasks::function]
pub fn all_assets(&self) -> Vc<OutputAssets> {
self.assets.concatenate(*self.referenced_assets)
}
}
......@@ -236,7 +236,6 @@ pub async fn write_import_context(
#[turbo_tasks::value]
pub struct CssChunkContent {
pub chunk_items: Vec<ResolvedVc<Box<dyn CssChunkItem>>>,
pub referenced_output_assets: ResolvedVc<OutputAssets>,
}
#[turbo_tasks::value_impl]
......@@ -337,33 +336,30 @@ impl OutputAsset for CssChunk {
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = self.await?;
let content = this.content.await?;
let mut references = content.referenced_output_assets.owned().await?;
let should_generate_single_item_chunks = content.chunk_items.len() > 1
&& *this
.chunking_context
.is_dynamic_chunk_content_loading_enabled()
.await?;
references.extend(
content
.chunk_items
.iter()
.map(|item| async {
let references = item.references().await?.into_iter().copied();
Ok(if should_generate_single_item_chunks {
Either::Left(
references.chain(std::iter::once(ResolvedVc::upcast(
SingleItemCssChunk::new(*this.chunking_context, **item)
.to_resolved()
.await?,
))),
)
} else {
Either::Right(references)
})
let mut references = content
.chunk_items
.iter()
.map(|item| async {
let references = item.references().await?.into_iter().copied();
Ok(if should_generate_single_item_chunks {
Either::Left(
references.chain(std::iter::once(ResolvedVc::upcast(
SingleItemCssChunk::new(*this.chunking_context, **item)
.to_resolved()
.await?,
))),
)
} else {
Either::Right(references)
})
.try_flat_join()
.await?,
);
})
.try_flat_join()
.await?;
if *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
......@@ -501,7 +497,6 @@ impl ChunkType for CssChunkType {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
chunk_items_or_batches: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
_batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
referenced_output_assets: ResolvedVc<OutputAssets>,
) -> Result<Vc<Box<dyn Chunk>>> {
let mut chunk_items = Vec::new();
// TODO operate with batches
......@@ -530,7 +525,6 @@ impl ChunkType for CssChunkType {
})
.try_join()
.await?,
referenced_output_assets,
}
.cell();
Ok(Vc::upcast(CssChunk::new(*chunking_context, content)))
......
......@@ -15,7 +15,7 @@ use turbopack_core::{
},
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
version::{Version, VersionedContent},
};
......@@ -49,7 +49,7 @@ impl OutputAsset for DevHtmlAsset {
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
self.chunks()
self.chunk_group().all_assets()
}
}
......@@ -116,7 +116,7 @@ impl DevHtmlAsset {
let this = self.await?;
let context_path = this.path.parent();
let mut chunk_paths = vec![];
for chunk in &*self.chunks().await? {
for chunk in &*self.chunk_group().await?.assets.await? {
let chunk_path = &*chunk.path().await?;
if let Some(relative_path) = context_path.get_path_to(chunk_path) {
chunk_paths.push(format!("/{relative_path}").into());
......@@ -127,8 +127,8 @@ impl DevHtmlAsset {
}
#[turbo_tasks::function]
async fn chunks(&self) -> Result<Vc<OutputAssets>> {
let all_assets = self
async fn chunk_group(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let all_chunk_groups = self
.entries
.iter()
.map(|entry| async move {
......@@ -139,7 +139,7 @@ impl DevHtmlAsset {
runtime_entries,
} = entry;
let assets = if let Some(runtime_entries) = runtime_entries {
let asset_with_referenced = if let Some(runtime_entries) = runtime_entries {
let runtime_entries =
if let Some(evaluatable) = ResolvedVc::try_downcast(chunkable_module) {
runtime_entries
......@@ -149,36 +149,50 @@ impl DevHtmlAsset {
} else {
runtime_entries
};
chunking_context.evaluated_chunk_group_assets(
chunkable_module.ident(),
ChunkGroup::Entry(
runtime_entries
.await?
.iter()
.map(|v| ResolvedVc::upcast(*v))
.collect(),
),
*module_graph,
AvailabilityInfo::Root,
)
chunking_context
.evaluated_chunk_group_assets(
chunkable_module.ident(),
ChunkGroup::Entry(
runtime_entries
.await?
.iter()
.map(|v| ResolvedVc::upcast(*v))
.collect(),
),
*module_graph,
AvailabilityInfo::Root,
)
.await?
} else {
chunking_context.root_chunk_group_assets(
chunkable_module.ident(),
ChunkGroup::Entry(vec![ResolvedVc::upcast(chunkable_module)]),
*module_graph,
)
chunking_context
.root_chunk_group_assets(
chunkable_module.ident(),
ChunkGroup::Entry(vec![ResolvedVc::upcast(chunkable_module)]),
*module_graph,
)
.await?
};
assets.await
Ok((
asset_with_referenced.assets.await?,
asset_with_referenced.referenced_assets.await?,
))
})
.try_join()
.await?
.iter()
.flatten()
.copied()
.collect();
.await?;
Ok(Vc::cell(all_assets))
let mut all_assets = Vec::new();
let mut all_referenced_assets = Vec::new();
for (asset, referenced_asset) in all_chunk_groups {
all_assets.extend(asset);
all_referenced_assets.extend(referenced_asset);
}
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(all_assets),
referenced_assets: ResolvedVc::cell(all_referenced_assets),
}
.cell())
}
}
......
......@@ -11,7 +11,7 @@ use turbopack_core::{
module_graph::{
ModuleGraph, chunk_group_info::ChunkGroup, module_batch::ChunkableModuleOrBatch,
},
output::OutputAssets,
output::{OutputAssets, OutputAssetsWithReferenced},
};
use crate::{
......@@ -34,7 +34,7 @@ pub struct AsyncLoaderChunkItem {
#[turbo_tasks::value_impl]
impl AsyncLoaderChunkItem {
#[turbo_tasks::function]
pub(super) async fn chunks(&self) -> Result<Vc<OutputAssets>> {
pub(super) async fn chunk_group(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let module = self.module.await?;
if let Some(chunk_items) = module.availability_info.available_modules() {
let inner_module = ResolvedVc::upcast(module.inner);
......@@ -47,7 +47,11 @@ impl AsyncLoaderChunkItem {
ChunkableModuleOrBatch::from_module_or_batch(module_or_batch)
&& *chunk_items.get(chunkable_module_or_batch).await?
{
return Ok(Vc::cell(vec![]));
return Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
}
.cell());
}
}
Ok(self.chunking_context.chunk_group_assets(
......@@ -63,7 +67,7 @@ impl AsyncLoaderChunkItem {
let this = self.await?;
Ok(ChunkData::from_assets(
this.chunking_context.output_root().owned().await?,
self.chunks(),
*self.chunk_group().await?.assets,
))
}
}
......@@ -162,7 +166,7 @@ impl ChunkItem for AsyncLoaderChunkItem {
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
self.chunks()
self.chunk_group().all_assets()
}
#[turbo_tasks::function]
......
use anyhow::{Result, bail};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueDefault, ValueToString, Vc};
use turbopack_core::{
chunk::{
AsyncModuleInfo, Chunk, ChunkItem, ChunkItemBatchGroup,
ChunkItemOrBatchWithAsyncModuleInfo, ChunkType, ChunkingContext, round_chunk_item_size,
},
output::OutputAssets,
use turbopack_core::chunk::{
AsyncModuleInfo, Chunk, ChunkItem, ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo,
ChunkType, ChunkingContext, round_chunk_item_size,
};
use super::{EcmascriptChunk, EcmascriptChunkContent, EcmascriptChunkItem};
......@@ -37,7 +34,6 @@ impl ChunkType for EcmascriptChunkType {
chunking_context: Vc<Box<dyn ChunkingContext>>,
chunk_items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
referenced_output_assets: Vc<OutputAssets>,
) -> Result<Vc<Box<dyn Chunk>>> {
let content = EcmascriptChunkContent {
chunk_items: chunk_items
......@@ -53,7 +49,6 @@ impl ChunkType for EcmascriptChunkType {
})
.try_join()
.await?,
referenced_output_assets: referenced_output_assets.owned().await?,
}
.cell();
Ok(Vc::upcast(EcmascriptChunk::new(chunking_context, content)))
......
......@@ -3,10 +3,7 @@ use std::future::IntoFuture;
use anyhow::Result;
use either::Either;
use turbo_tasks::{ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
chunk::{ChunkItem, ChunkItems, batch_info},
output::OutputAsset,
};
use turbopack_core::chunk::{ChunkItem, ChunkItems, batch_info};
use crate::chunk::{
CodeAndIds,
......@@ -18,7 +15,6 @@ use crate::chunk::{
pub struct EcmascriptChunkContent {
pub chunk_items: Vec<EcmascriptChunkItemOrBatchWithAsyncInfo>,
pub batch_groups: Vec<ResolvedVc<EcmascriptChunkItemBatchGroup>>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
}
#[turbo_tasks::value_impl]
......
......@@ -19,7 +19,7 @@ use turbopack_core::{
Introspectable, IntrospectableChildren, module::IntrospectableModule,
utils::children_from_output_assets,
},
output::{OutputAsset, OutputAssets},
output::OutputAssets,
server_fs::ServerFileSystem,
};
......@@ -130,13 +130,12 @@ impl Chunk for EcmascriptChunk {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
let content = self.content.await?;
let mut referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = content
let referenced_output_assets = content
.chunk_items
.iter()
.map(async |with_info| Ok(with_info.references().await?.into_iter().copied()))
.try_flat_join()
.await?;
referenced_output_assets.extend(content.referenced_output_assets.iter().copied());
Ok(Vc::cell(referenced_output_assets))
}
......
......@@ -11,7 +11,7 @@ use turbopack_core::{
module_graph::{
ModuleGraph, chunk_group_info::ChunkGroup, module_batch::ChunkableModuleOrBatch,
},
output::OutputAssets,
output::OutputAssetsWithReferenced,
reference::{ModuleReferences, SingleOutputAssetReference},
};
......@@ -55,7 +55,7 @@ impl ManifestAsyncModule {
}
#[turbo_tasks::function]
pub(super) fn chunks(&self) -> Vc<OutputAssets> {
pub(super) fn chunk_group(&self) -> Vc<OutputAssetsWithReferenced> {
self.chunking_context.chunk_group_assets(
self.inner.ident(),
ChunkGroup::Async(ResolvedVc::upcast(self.inner)),
......@@ -65,7 +65,9 @@ impl ManifestAsyncModule {
}
#[turbo_tasks::function]
pub async fn manifest_chunks(self: ResolvedVc<Self>) -> Result<Vc<OutputAssets>> {
pub async fn manifest_chunk_group(
self: ResolvedVc<Self>,
) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
if let Some(chunk_items) = this.availability_info.available_modules() {
let inner_module = ResolvedVc::upcast(this.inner);
......@@ -78,7 +80,11 @@ impl ManifestAsyncModule {
ChunkableModuleOrBatch::from_module_or_batch(module_or_batch)
&& *chunk_items.get(chunkable_module_or_batch).await?
{
return Ok(Vc::cell(vec![]));
return Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
}
.cell());
}
}
Ok(this.chunking_context.chunk_group_assets(
......@@ -119,12 +125,11 @@ impl Module for ManifestAsyncModule {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> {
let chunks = self.chunks();
let assets = self.chunk_group().all_assets().await?;
Ok(Vc::cell(
chunks
.await?
.iter()
assets
.into_iter()
.copied()
.map(|chunk| async move {
Ok(ResolvedVc::upcast(
......
......@@ -33,7 +33,7 @@ impl ManifestChunkItem {
async fn chunks_data(&self) -> Result<Vc<ChunksData>> {
Ok(ChunkData::from_assets(
self.chunking_context.output_root().owned().await?,
self.manifest.chunks(),
*self.manifest.chunk_group().await?.assets,
))
}
}
......@@ -77,8 +77,8 @@ impl ChunkItem for ManifestChunkItem {
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
Ok(self.manifest.chunks())
fn references(&self) -> Vc<OutputAssets> {
self.manifest.chunk_group().all_assets()
}
#[turbo_tasks::function]
......
......@@ -67,10 +67,10 @@ impl ManifestLoaderChunkItem {
#[turbo_tasks::function]
pub async fn chunks_data(&self) -> Result<Vc<ChunksData>> {
let chunks = self.manifest.manifest_chunks();
let chunks = self.manifest.manifest_chunk_group().await?.assets;
Ok(ChunkData::from_assets(
self.chunking_context.output_root().owned().await?,
chunks,
*chunks,
))
}
......@@ -93,8 +93,8 @@ impl ChunkItem for ManifestLoaderChunkItem {
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
Ok(self.manifest.manifest_chunks())
fn references(&self) -> Vc<OutputAssets> {
self.manifest.manifest_chunk_group().all_assets()
}
#[turbo_tasks::function]
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::OutputAssets,
output::{OutputAssets, OutputAssetsWithReferenced},
};
use super::module::WorkerLoaderModule;
......@@ -33,7 +33,7 @@ pub struct WorkerLoaderChunkItem {
#[turbo_tasks::value_impl]
impl WorkerLoaderChunkItem {
#[turbo_tasks::function]
async fn chunks(&self) -> Result<Vc<OutputAssets>> {
async fn chunk_group(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let module = self.module.await?;
Ok(self.chunking_context.evaluated_chunk_group_assets(
......@@ -49,7 +49,7 @@ impl WorkerLoaderChunkItem {
let this = self.await?;
Ok(ChunkData::from_assets(
this.chunking_context.output_root().owned().await?,
self.chunks(),
*self.chunk_group().await?.assets,
))
}
}
......@@ -94,7 +94,7 @@ impl ChunkItem for WorkerLoaderChunkItem {
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
self.chunks()
self.chunk_group().all_assets()
}
#[turbo_tasks::function]
......
......@@ -177,6 +177,7 @@ async fn emit_evaluate_pool_assets_operation(
.with_entry(*ResolvedVc::try_downcast(entry_module).unwrap()),
module_graph,
OutputAssets::empty(),
OutputAssets::empty(),
);
let output_root = chunking_context.output_root().owned().await?;
......
......@@ -280,6 +280,7 @@ pub async fn get_intermediate_asset(
false,
),
OutputAssets::empty(),
OutputAssets::empty(),
))
}
......
......@@ -388,6 +388,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let modules = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
availability_info,
} = make_chunk_group(
modules,
......@@ -405,6 +406,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
availability_info,
}
.cell())
......@@ -420,6 +422,7 @@ impl ChunkingContext for NodeJsChunkingContext {
evaluatable_assets: Vc<EvaluatableAssets>,
module_graph: Vc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
) -> Result<Vc<EntryChunkGroupResult>> {
let evaluatable_assets_ref = evaluatable_assets.await?;
......@@ -429,6 +432,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let MakeChunkGroupResult {
chunks,
mut referenced_output_assets,
availability_info,
} = make_chunk_group(
entries,
......@@ -439,17 +443,14 @@ impl ChunkingContext for NodeJsChunkingContext {
.await?;
let extra_chunks = extra_chunks.await?;
let other_chunks: Vec<_> = extra_chunks
let mut other_chunks = chunks
.iter()
.copied()
.chain(
chunks
.iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved())
.try_join()
.await?,
)
.collect();
.map(|chunk| self.generate_chunk(**chunk).to_resolved())
.try_join()
.await?;
other_chunks.extend(extra_chunks.iter().copied());
referenced_output_assets.extend(extra_referenced_assets.await?.iter().copied());
let Some(module) = ResolvedVc::try_sidecast(*evaluatable_assets_ref.last().unwrap()) else {
bail!("module must be placeable in an ecmascript chunk");
......@@ -461,6 +462,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Vc::cell(other_chunks),
evaluatable_assets,
*module,
Vc::cell(referenced_output_assets),
module_graph,
*self,
)
......
......@@ -26,6 +26,7 @@ pub(crate) struct EcmascriptBuildNodeEntryChunk {
other_chunks: ResolvedVc<OutputAssets>,
evaluatable_assets: ResolvedVc<EvaluatableAssets>,
exported_module: ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>,
referenced_output_assets: ResolvedVc<OutputAssets>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<NodeJsChunkingContext>,
}
......@@ -39,6 +40,7 @@ impl EcmascriptBuildNodeEntryChunk {
other_chunks: ResolvedVc<OutputAssets>,
evaluatable_assets: ResolvedVc<EvaluatableAssets>,
exported_module: ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>,
referenced_output_assets: ResolvedVc<OutputAssets>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<NodeJsChunkingContext>,
) -> Vc<Self> {
......@@ -47,6 +49,7 @@ impl EcmascriptBuildNodeEntryChunk {
other_chunks,
evaluatable_assets,
exported_module,
referenced_output_assets,
module_graph,
chunking_context,
}
......@@ -191,6 +194,11 @@ impl OutputAsset for EcmascriptBuildNodeEntryChunk {
let other_chunks = this.other_chunks.await?;
references.extend(other_chunks.iter().copied());
let referenced_output_assets = this.referenced_output_assets.await?;
for &referenced_output_asset in &*referenced_output_assets {
references.push(referenced_output_asset);
}
Ok(Vc::cell(references))
}
}
......
......@@ -50,7 +50,7 @@ use turbopack_core::{
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
export_usage::compute_export_usage_info,
},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
reference_type::{EntryReferenceSubType, ReferenceType},
source::Source,
};
......@@ -511,28 +511,33 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
AvailabilityInfo::Root,
),
Runtime::NodeJs => {
Vc::cell(vec![
Vc::try_resolve_downcast_type::<NodeJsChunkingContext>(chunking_context)
.await?
.unwrap()
.entry_chunk_group(
// `expected` expects a completely flat output directory.
chunk_root_path
.join(entry_module.ident().path().await?.file_stem().unwrap())?
.with_extension("entry.js"),
evaluatable_assets,
module_graph,
OutputAssets::empty(),
AvailabilityInfo::Root,
)
.await?
.asset,
])
OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![
Vc::try_resolve_downcast_type::<NodeJsChunkingContext>(chunking_context)
.await?
.unwrap()
.entry_chunk_group(
// `expected` expects a completely flat output directory.
chunk_root_path
.join(entry_module.ident().path().await?.file_stem().unwrap())?
.with_extension("entry.js"),
evaluatable_assets,
module_graph,
OutputAssets::empty(),
OutputAssets::empty(),
AvailabilityInfo::Root,
)
.await?
.asset,
]),
referenced_assets: ResolvedVc::cell(vec![]),
}
.cell()
}
};
let mut seen = FxHashSet::default();
let mut queue: VecDeque<_> = chunks.await?.iter().copied().collect();
let mut queue: VecDeque<_> = chunks.all_assets().await?.iter().copied().collect();
let output_path = project_path.clone();
while let Some(asset) = queue.pop_front() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment