Unverified Commit a8d73c25 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: Refactor output assets to allow to lazy compute output assets (#85753)

### What?

Refactor the OutputAssets of chunk groups to not eagerly return all referenced output assets from async loaders, but instead return a list of references that can be followed to get the full list.
That solves a problem that will happen when the async loaders create cycles in the output graph.
parent f6c2d491
......@@ -19,7 +19,7 @@ use turbopack_core::{
asset::{Asset, AssetContent},
chunk::ChunkingType,
module::Module,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference},
reference::all_assets_from_entries,
};
......@@ -566,6 +566,9 @@ impl Asset for AnalyzeDataOutputAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for AnalyzeDataOutputAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for AnalyzeDataOutputAsset {
#[turbo_tasks::function]
......@@ -601,6 +604,9 @@ impl Asset for ModulesDataOutputAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ModulesDataOutputAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for ModulesDataOutputAsset {
#[turbo_tasks::function]
......
This diff is collapsed.
......@@ -31,12 +31,12 @@ use turbo_tasks::{
};
use turbopack_core::{
chunk::{
ChunkItem, ChunkableModule, ChunkingContext, ModuleChunkItemIdExt, ModuleId,
ChunkableModule, ChunkingContext, ModuleChunkItemIdExt, ModuleId,
availability_info::AvailabilityInfo,
},
module::Module,
module_graph::{ModuleGraph, SingleModuleGraph},
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
};
use crate::module_graph::DynamicImportEntriesWithImporter;
......@@ -72,6 +72,7 @@ pub(crate) async fn collect_next_dynamic_chunks(
)?,
)
.context("Client reference chunk group not found for next/dynamic import")?
.await?
.availability_info
}
NextDynamicChunkAvailability::AvailabilityInfo(availability_info) => {
......@@ -103,7 +104,7 @@ pub(crate) async fn collect_next_dynamic_chunks(
pub struct DynamicImportedChunks(
pub FxIndexMap<
ResolvedVc<NextDynamicEntryModule>,
(ResolvedVc<ModuleId>, ResolvedVc<OutputAssets>),
(ResolvedVc<ModuleId>, ResolvedVc<OutputAssetsWithReferenced>),
>,
);
......
use anyhow::Result;
use next_core::{all_assets_from_entries, next_manifests::NextFontManifest};
use next_core::next_manifests::NextFontManifest;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference},
reference::all_assets_from_entries,
};
use crate::paths::get_font_paths_from_root;
......@@ -22,6 +23,9 @@ pub struct FontManifest {
pub app_dir: bool,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for FontManifest {}
#[turbo_tasks::value_impl]
impl OutputAsset for FontManifest {
#[turbo_tasks::function]
......
use anyhow::{Result, bail};
use next_core::{
all_assets_from_entries,
next_edge::entry::wrap_edge_entry,
next_manifests::{InstrumentationDefinition, MiddlewaresManifestV2},
};
......@@ -147,7 +146,7 @@ impl InstrumentationEndpoint {
if this.is_edge {
let edge_chunk_group = self.edge_chunk_group();
let edge_all_assets = edge_chunk_group.all_assets();
let edge_all_assets = edge_chunk_group.expand_all_assets();
let node_root = this.project.node_root().owned().await?;
let node_root_value = node_root.clone();
......@@ -156,10 +155,10 @@ impl InstrumentationEndpoint {
get_js_paths_from_root(&node_root_value, &edge_chunk_group.await?.assets.await?)
.await?;
let mut output_assets = all_assets_from_entries(edge_all_assets).owned().await?;
let mut output_assets = edge_chunk_group.all_assets().owned().await?;
let wasm_paths_from_root =
get_wasm_paths_from_root(&node_root_value, &output_assets).await?;
get_wasm_paths_from_root(&node_root_value, edge_all_assets.await?).await?;
let instrumentation_definition = InstrumentationDefinition {
files: file_paths_from_root,
......
......@@ -23,7 +23,7 @@ pub async fn create_react_loadable_manifest(
let mut loadable_manifest: FxIndexMap<String, LoadableManifest> = FxIndexMap::default();
for (_, (module_id, chunk_output)) in dynamic_import_entries.into_iter() {
let chunk_output = chunk_output.await?;
let chunk_output = chunk_output.primary_assets().await?;
let id = &*module_id.await?;
......
......@@ -2,7 +2,6 @@ use std::future::IntoFuture;
use anyhow::{Result, bail};
use next_core::{
all_assets_from_entries,
middleware::get_middleware_module,
next_edge::entry::wrap_edge_entry,
next_manifests::{EdgeFunctionDefinition, MiddlewaresManifestV2, ProxyMatcher, Regions},
......@@ -258,7 +257,7 @@ impl MiddlewareEndpoint {
Ok(Vc::cell(output_assets))
} else {
let edge_chunk_group = self.edge_chunk_group();
let edge_all_assets = edge_chunk_group.all_assets();
let edge_all_assets = edge_chunk_group.expand_all_assets();
let node_root = this.project.node_root().owned().await?;
let node_root_value = node_root.clone();
......@@ -267,12 +266,13 @@ impl MiddlewareEndpoint {
get_js_paths_from_root(&node_root_value, &edge_chunk_group.await?.assets.await?)
.await?;
let mut output_assets = all_assets_from_entries(edge_all_assets).owned().await?;
let mut output_assets = edge_chunk_group.all_assets().owned().await?;
let wasm_paths_from_root =
get_wasm_paths_from_root(&node_root_value, &output_assets).await?;
get_wasm_paths_from_root(&node_root_value, edge_all_assets.await?).await?;
let all_assets = get_asset_paths_from_root(&node_root_value, &output_assets).await?;
let all_assets =
get_asset_paths_from_root(&node_root_value, &edge_all_assets.await?).await?;
let regions = if let Some(regions) = config.preferred_region.as_ref() {
if regions.len() == 1 {
......
......@@ -16,7 +16,7 @@ use turbopack_core::{
asset::{Asset, AssetContent},
context::AssetContext,
file_source::FileSource,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference},
reference_type::{CommonJsReferenceSubType, ReferenceType},
resolve::{origin::PlainResolveOrigin, parse::Request},
traced_asset::TracedAsset,
......@@ -76,6 +76,9 @@ impl ServerNftJsonAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ServerNftJsonAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for ServerNftJsonAsset {
#[turbo_tasks::function]
......
......@@ -14,7 +14,7 @@ use turbo_tasks_fs::{
};
use turbopack_core::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference},
};
use crate::project::Project;
......@@ -59,6 +59,9 @@ impl NftJsonAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for NftJsonAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for NftJsonAsset {
#[turbo_tasks::function]
......@@ -454,7 +457,7 @@ impl Visit<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>
node: &(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
) -> Self::EdgesFuture {
let client_root = self.client_root.clone();
let exclude_glob = self.exclude_glob.clone();
let exclude_glob: Option<ReadRef<Glob>> = self.exclude_glob.clone();
get_referenced_server_assets(self.emit_spans, node.0, client_root, exclude_glob)
}
......@@ -478,7 +481,7 @@ async fn get_referenced_server_assets(
client_root: Option<FileSystemPath>,
exclude_glob: Option<ReadRef<Glob>>,
) -> Result<Vec<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>> {
let refs = asset.references().await?;
let refs = asset.references().all_assets().await?;
refs.iter()
.map(async |asset| {
......
use anyhow::{Context, Result, bail};
use futures::future::BoxFuture;
use next_core::{
PageLoaderAsset, all_assets_from_entries, create_page_loader_entry_module,
get_asset_path_from_pathname, get_edge_resolve_options_context,
PageLoaderAsset, create_page_loader_entry_module, get_asset_path_from_pathname,
get_edge_resolve_options_context,
hmr_entry::HmrEntryModule,
mode::NextMode,
next_client::{
......@@ -56,7 +56,8 @@ use turbopack_core::{
GraphEntries, ModuleGraph, SingleModuleGraph, VisitedModules,
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
},
output::{OptionOutputAsset, OutputAsset, OutputAssets, OutputAssetsWithReferenced},
output::{OptionOutputAsset, OutputAsset, OutputAssets},
reference::all_assets_from_entries,
reference_type::{EcmaScriptModulesReferenceSubType, EntryReferenceSubType, ReferenceType},
resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern},
source::Source,
......@@ -1003,34 +1004,24 @@ impl PageEndpoint {
NextRuntime::Edge => edge_chunking_context,
};
let mut current_chunks = OutputAssets::empty();
let mut current_referenced_assets = OutputAssets::empty();
let mut current_availability_info = AvailabilityInfo::Root;
let mut current_chunk_group = ChunkGroupResult::empty_resolved();
for layout in [document_module, app_module].iter().flatten().copied() {
let span = tracing::trace_span!(
"layout segment",
name = display(layout.ident().to_string().await?)
);
async {
let ChunkGroupResult {
assets,
referenced_assets,
availability_info,
} = *chunking_context
.chunk_group(
layout.ident(),
ChunkGroup::Shared(layout),
ssr_module_graph,
current_availability_info,
)
.await?;
let chunk_group = chunking_context.chunk_group(
layout.ident(),
ChunkGroup::Shared(layout),
ssr_module_graph,
current_chunk_group.await?.availability_info,
);
current_chunks = current_chunks.concatenate(*assets).resolve().await?;
current_referenced_assets = current_referenced_assets
.concatenate(*referenced_assets)
.resolve()
current_chunk_group = current_chunk_group
.concatenate(chunk_group)
.to_resolved()
.await?;
current_availability_info = availability_info;
anyhow::Ok(())
}
......@@ -1042,27 +1033,22 @@ impl PageEndpoint {
.context("could not process page loader entry module")?;
let is_edge = matches!(runtime, NextRuntime::Edge);
if is_edge {
let OutputAssetsWithReferenced {
assets: edge_assets,
referenced_assets: edge_referenced_assets,
} = *edge_chunking_context
.evaluated_chunk_group_assets(
ssr_module.ident(),
ChunkGroup::Entry(vec![ResolvedVc::upcast(ssr_module_evaluatable)]),
ssr_module_graph,
current_availability_info,
)
let chunk_assets = edge_chunking_context.evaluated_chunk_group_assets(
ssr_module.ident(),
ChunkGroup::Entry(vec![ResolvedVc::upcast(ssr_module_evaluatable)]),
ssr_module_graph,
current_chunk_group.await?.availability_info,
);
let chunk_assets = current_chunk_group
.output_assets_with_referenced()
.concatenate(chunk_assets)
.to_resolved()
.await?;
Ok(SsrChunk::Edge {
assets: current_chunks
.concatenate(*edge_assets)
.to_resolved()
.await?,
referenced_assets: current_referenced_assets
.concatenate(*edge_referenced_assets)
.to_resolved()
.await?,
assets: chunk_assets.primary_assets().to_resolved().await?,
referenced_assets: chunk_assets.referenced_assets().to_resolved().await?,
dynamic_import_entries,
regions: regions.clone(),
}
......@@ -1079,9 +1065,9 @@ impl PageEndpoint {
ssr_entry_chunk_path,
EvaluatableAssets::empty().with_entry(*ssr_module_evaluatable),
ssr_module_graph,
current_chunks,
current_referenced_assets,
current_availability_info,
current_chunk_group.primary_assets(),
current_chunk_group.referenced_assets(),
current_chunk_group.await?.availability_info,
)
.to_resolved()
.await?;
......@@ -1321,21 +1307,14 @@ impl PageEndpoint {
let ssr_chunk = match this.ty {
PageEndpointType::Html => {
let client_chunk_group = self.client_chunk_group().await?;
let client_chunks = *client_chunk_group.assets;
client_assets.extend(client_chunks.await?.iter().map(|asset| **asset));
client_assets.extend(
client_chunk_group
.referenced_assets
.await?
.iter()
.map(|asset| **asset),
);
let client_chunk_group = self.client_chunk_group();
client_assets.extend(client_chunk_group.all_assets().await?.iter().copied());
let client_chunks = *client_chunk_group.await?.assets;
let build_manifest = self.build_manifest(client_chunks).to_resolved().await?;
let page_loader = self.page_loader(client_chunks);
let page_loader = self.page_loader(client_chunks).to_resolved().await?;
let client_build_manifest = self
.client_build_manifest(page_loader)
.client_build_manifest(*page_loader)
.to_resolved()
.await?;
client_assets.push(page_loader);
......@@ -1350,7 +1329,7 @@ impl PageEndpoint {
PageEndpointType::SsrOnly => self.ssr_chunk(emit_manifests),
};
let client_assets = OutputAssets::new(client_assets).to_resolved().await?;
let client_assets: ResolvedVc<OutputAssets> = ResolvedVc::cell(client_assets);
let manifest_path_prefix = get_asset_prefix_from_pathname(&this.pathname);
let node_root = this.pages_project.project().node_root().owned().await?;
......@@ -1381,7 +1360,7 @@ impl PageEndpoint {
let webpack_stats = generate_webpack_stats(
self.client_module_graph(),
this.original_name.clone(),
client_assets.await?.iter().copied(),
client_assets.await?.into_iter().copied(),
)
.await?;
let stats_output = VirtualOutputAsset::new(
......
use anyhow::Result;
use next_core::{all_assets_from_entries, next_manifests::AssetBinding};
use next_core::next_manifests::AssetBinding;
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
......@@ -7,6 +7,7 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::Asset,
output::{OutputAsset, OutputAssets},
reference::all_assets_from_entries,
};
use turbopack_wasm::wasm_edge_var_name;
......
......@@ -3,7 +3,6 @@ use std::time::Duration;
use anyhow::{Context, Result, bail};
use indexmap::map::Entry;
use next_core::{
all_assets_from_entries,
app_structure::find_app_dir,
emit_assets, get_edge_chunking_context, get_edge_chunking_context_with_client_assets,
get_edge_compile_time_info, get_edge_resolve_options_context,
......@@ -31,11 +30,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, Completions, FxIndexMap, IntoTraitRef, NonLocalValue, OperationValue, OperationVc,
ReadRef, ResolvedVc, State, TaskInput, TransientInstance, TryFlatJoinIterExt, Vc,
debug::ValueDebugFormat,
fxindexmap,
graph::{AdjacencyMap, GraphTraversal},
mark_root,
trace::TraceRawVcs,
debug::ValueDebugFormat, fxindexmap, mark_root, trace::TraceRawVcs,
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, invalidation};
......@@ -67,7 +62,11 @@ use turbopack_core::{
chunk_group_info::ChunkGroupEntry,
export_usage::{OptionExportUsageInfo, compute_export_usage_info},
},
output::{OutputAsset, OutputAssets},
output::{
ExpandOutputAssetsInput, ExpandedOutputAssets, OutputAsset, OutputAssets,
expand_output_assets,
},
reference::all_assets_from_entries,
resolve::{FindContextFileResult, find_context_file},
source_map::OptionStringifiedSourceMap,
version::{
......@@ -1939,13 +1938,16 @@ async fn any_output_changed(
path: FileSystemPath,
server: bool,
) -> Result<Vc<Completion>> {
let completions = AdjacencyMap::new()
.skip_duplicates()
.visit(roots.await?.iter().copied(), get_referenced_output_assets)
.await
.completed()?
.into_inner()
.into_postorder_topological()
let all_assets = expand_output_assets(
roots
.await?
.into_iter()
.map(|&a| ExpandOutputAssetsInput::Asset(a)),
true,
)
.await?;
let completions = all_assets
.into_iter()
.map(|m| {
let path = path.clone();
......@@ -1971,16 +1973,10 @@ async fn any_output_changed(
Ok(Vc::<Completions>::cell(completions).completed())
}
async fn get_referenced_output_assets(
parent: ResolvedVc<Box<dyn OutputAsset>>,
) -> Result<impl Iterator<Item = ResolvedVc<Box<dyn OutputAsset>>> + Send> {
Ok(parent.references().owned().await?.into_iter())
}
#[turbo_tasks::function(operation)]
fn all_assets_from_entries_operation(
operation: OperationVc<OutputAssets>,
) -> Result<Vc<OutputAssets>> {
) -> Result<Vc<ExpandedOutputAssets>> {
let assets = operation.connect();
Ok(all_assets_from_entries(assets))
}
......
......@@ -10,7 +10,7 @@ use turbo_tasks::{
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::Asset,
output::{OptionOutputAsset, OutputAsset, OutputAssets},
output::{ExpandedOutputAssets, OptionOutputAsset, OutputAsset},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap},
version::OptionVersionedContent,
};
......@@ -27,7 +27,7 @@ use turbopack_core::{
NonLocalValue,
)]
struct MapEntry {
assets_operation: OperationVc<OutputAssets>,
assets_operation: OperationVc<ExpandedOutputAssets>,
/// Precomputed map for quick access to output asset by filepath
path_to_asset: FxHashMap<FileSystemPath, ResolvedVc<Box<dyn OutputAsset>>>,
}
......@@ -47,7 +47,7 @@ pub struct PathToOutputOperation(
/// It may not be 100% correct for the key (`FileSystemPath`) to be in a `ResolvedVc` here, but
/// it's impractical to make it an `OperationVc`/`OperationValue`, and it's unlikely to
/// change/break?
FxHashMap<FileSystemPath, FxIndexSet<OperationVc<OutputAssets>>>,
FxHashMap<FileSystemPath, FxIndexSet<OperationVc<ExpandedOutputAssets>>>,
);
// HACK: This is technically incorrect because the map's key is a `ResolvedVc`...
......@@ -55,11 +55,12 @@ unsafe impl OperationValue for PathToOutputOperation {}
// A precomputed map for quick access to output asset by filepath
type OutputOperationToComputeEntry =
FxHashMap<OperationVc<OutputAssets>, OperationVc<OptionMapEntry>>;
FxHashMap<OperationVc<ExpandedOutputAssets>, OperationVc<OptionMapEntry>>;
#[turbo_tasks::value]
pub struct VersionedContentMap {
// TODO: turn into a bi-directional multimap, OutputAssets -> FxIndexSet<FileSystemPath>
// TODO: turn into a bi-directional multimap, ExpandedOutputAssets ->
// FxIndexSet<FileSystemPath>
map_path_to_op: State<PathToOutputOperation>,
map_op_to_compute_entry: State<OutputOperationToComputeEntry>,
}
......@@ -90,7 +91,7 @@ impl VersionedContentMap {
pub async fn insert_output_assets(
self: ResolvedVc<Self>,
// Output assets to emit
assets_operation: OperationVc<OutputAssets>,
assets_operation: OperationVc<ExpandedOutputAssets>,
node_root: FileSystemPath,
client_relative_path: FileSystemPath,
client_output_path: FileSystemPath,
......@@ -114,7 +115,7 @@ impl VersionedContentMap {
#[turbo_tasks::function]
async fn compute_entry(
&self,
assets_operation: OperationVc<OutputAssets>,
assets_operation: OperationVc<ExpandedOutputAssets>,
node_root: FileSystemPath,
client_relative_path: FileSystemPath,
client_output_path: FileSystemPath,
......@@ -259,7 +260,7 @@ type GetEntriesResultT = Vec<(FileSystemPath, ResolvedVc<Box<dyn OutputAsset>>)>
struct GetEntriesResult(GetEntriesResultT);
#[turbo_tasks::function(operation)]
async fn get_entries(assets: OperationVc<OutputAssets>) -> Result<Vc<GetEntriesResult>> {
async fn get_entries(assets: OperationVc<ExpandedOutputAssets>) -> Result<Vc<GetEntriesResult>> {
let assets_ref = assets.connect().await?;
let entries = assets_ref
.iter()
......@@ -275,7 +276,7 @@ async fn get_entries(assets: OperationVc<OutputAssets>) -> Result<Vc<GetEntriesR
#[turbo_tasks::function(operation)]
fn compute_entry_operation(
map: ResolvedVc<VersionedContentMap>,
assets_operation: OperationVc<OutputAssets>,
assets_operation: OperationVc<ExpandedOutputAssets>,
node_root: FileSystemPath,
client_relative_path: FileSystemPath,
client_output_path: FileSystemPath,
......
......@@ -11,7 +11,7 @@ use turbopack_core::{
chunk::{Chunk, ChunkItem, ChunkItemExt, ModuleId},
module::Module,
module_graph::ModuleGraph,
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
};
#[instrument(level = "info", name = "generate webpack stats", skip_all)]
......@@ -38,7 +38,7 @@ where
let mut queue = entry_assets.clone();
while let Some(asset) = queue.pop() {
if visited.insert(asset) {
let references = asset.references().await?;
let references = asset.references().all_assets().await?;
asset_children.insert(asset, references.clone());
queue.extend(references);
}
......
use anyhow::Result;
use tracing::{Instrument, Level, Span};
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexSet, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc,
graph::{AdjacencyMap, GraphTraversal, Visit, VisitControlFlow},
};
use tracing::Instrument;
use turbo_tasks::{TryFlatJoinIterExt, Vc};
use turbo_tasks_fs::{FileSystemPath, rebase};
use turbopack_core::{
asset::Asset,
output::{OutputAsset, OutputAssets},
output::{ExpandedOutputAssets, OutputAsset, OutputAssets},
reference::all_assets_from_entries,
};
/// Emits all assets transitively reachable from the given chunks, that are
......@@ -41,7 +38,7 @@ pub async fn emit_all_assets(
/// path.
#[turbo_tasks::function]
pub async fn emit_assets(
assets: Vc<OutputAssets>,
assets: Vc<ExpandedOutputAssets>,
node_root: FileSystemPath,
client_relative_path: FileSystemPath,
client_output_path: FileSystemPath,
......@@ -113,98 +110,3 @@ async fn emit_rebase(
.await?;
Ok(())
}
struct OutputAssetVisit {
emit_spans: bool,
}
impl Visit<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)> for OutputAssetVisit {
type Edge = (ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>);
type EdgesIntoIter = Vec<Self::Edge>;
type EdgesFuture = impl Future<Output = Result<Self::EdgesIntoIter>>;
fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow<Self::Edge> {
VisitControlFlow::Continue(edge)
}
fn edges(
&mut self,
node: &(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
) -> Self::EdgesFuture {
get_referenced_assets(self.emit_spans, node.0)
}
fn span(
&mut self,
node: &(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
) -> tracing::Span {
if let Some(ident) = &node.1 {
tracing::trace_span!("asset", name = display(ident))
} else {
Span::current()
}
}
}
/// Walks the asset graph from multiple assets and collect all referenced
/// assets.
#[turbo_tasks::function]
pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<OutputAssets>> {
let emit_spans = tracing::enabled!(Level::INFO);
Ok(Vc::cell(
AdjacencyMap::new()
.skip_duplicates()
.visit(
entries
.await?
.iter()
.map(async |asset| {
Ok((
*asset,
if emit_spans {
// INVALIDATION: we don't need to invalidate when the span name
// changes
Some(asset.path_string().untracked().await?)
} else {
None
},
))
})
.try_join()
.await?,
OutputAssetVisit { emit_spans },
)
.await
.completed()?
.into_inner()
.into_postorder_topological()
.map(|(asset, _)| asset)
.collect::<FxIndexSet<_>>()
.into_iter()
.collect(),
))
}
/// Computes the list of all chunk children of a given chunk.
async fn get_referenced_assets(
emit_spans: bool,
asset: ResolvedVc<Box<dyn OutputAsset>>,
) -> Result<Vec<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>> {
asset
.references()
.await?
.iter()
.map(async |asset| {
Ok((
*asset,
if emit_spans {
// INVALIDATION: we don't need to invalidate the list of assets when the span
// name changes
Some(asset.path_string().untracked().await?)
} else {
None
},
))
})
.try_join()
.await
}
......@@ -13,6 +13,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::{ModuleReference, ModuleReferences},
resolve::ModuleResolveResult,
};
......@@ -154,6 +155,9 @@ struct HmrEntryChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for HmrEntryChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for HmrEntryChunkItem {
#[turbo_tasks::function]
......
......@@ -41,7 +41,7 @@ mod transform_options;
pub mod url_node;
pub mod util;
pub use emit::{all_assets_from_entries, emit_all_assets, emit_assets};
pub use emit::{emit_all_assets, emit_assets};
pub use next_edge::context::{
get_edge_chunking_context, get_edge_chunking_context_with_client_assets,
get_edge_compile_time_info, get_edge_resolve_options_context,
......
......@@ -6,7 +6,7 @@ use turbopack_core::{
chunk::{ChunkGroupResult, ChunkingContext, availability_info::AvailabilityInfo},
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::{OutputAssets, OutputAssetsWithReferenced},
output::OutputAssetsWithReferenced,
};
use crate::{
......@@ -22,10 +22,12 @@ use crate::{
#[turbo_tasks::value]
pub struct ClientReferencesChunks {
pub client_component_client_chunks: FxIndexMap<ClientReferenceType, ChunkGroupResult>,
pub client_component_ssr_chunks: FxIndexMap<ClientReferenceType, ChunkGroupResult>,
pub client_component_client_chunks:
FxIndexMap<ClientReferenceType, ResolvedVc<ChunkGroupResult>>,
pub client_component_ssr_chunks:
FxIndexMap<ClientReferenceType, ResolvedVc<OutputAssetsWithReferenced>>,
pub layout_segment_client_chunks:
FxIndexMap<ResolvedVc<NextServerComponentModule>, OutputAssetsWithReferenced>,
FxIndexMap<ResolvedVc<NextServerComponentModule>, ResolvedVc<OutputAssetsWithReferenced>>,
}
/// Computes all client references chunks.
......@@ -156,12 +158,14 @@ pub async fn get_app_client_references_chunks(
let chunk_group_info = module_graph.chunk_group_info();
let mut current_client_availability_info = client_availability_info;
let mut current_client_chunks = OutputAssets::empty().to_resolved().await?;
let mut current_client_referenced_assets = OutputAssets::empty().to_resolved().await?;
let mut current_ssr_availability_info = AvailabilityInfo::Root;
let mut current_ssr_chunks = OutputAssets::empty().to_resolved().await?;
let mut current_ssr_referenced_assets = OutputAssets::empty().to_resolved().await?;
let mut current_client_chunk_group = ChunkGroupResult {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
availability_info: client_availability_info,
}
.resolved_cell();
let mut current_ssr_chunk_group = ChunkGroupResult::empty_resolved();
let mut layout_segment_client_chunks = FxIndexMap::default();
let mut client_component_ssr_chunks = FxIndexMap::default();
......@@ -202,25 +206,26 @@ pub async fn get_app_client_references_chunks(
.try_flat_join()
.await?;
let ssr_chunk_group = if !ssr_modules.is_empty() {
ssr_chunking_context.map(|ssr_chunking_context| {
let _span = tracing::info_span!(
"server side rendering",
layout_segment = display(&server_component_path),
)
.entered();
let ssr_chunk_group = if !ssr_modules.is_empty()
&& let Some(ssr_chunking_context) = ssr_chunking_context
{
let availability_info = current_ssr_chunk_group.await?.availability_info;
let _span = tracing::info_span!(
"server side rendering",
layout_segment = display(&server_component_path),
)
.entered();
ssr_chunking_context.chunk_group(
base_ident.with_modifier(rcstr!("ssr modules")),
ChunkGroup::IsolatedMerged {
parent: parent_chunk_group,
merge_tag: ecmascript_client_reference_merge_tag_ssr(),
entries: ssr_modules,
},
module_graph,
current_ssr_availability_info,
)
})
Some(ssr_chunking_context.chunk_group(
base_ident.with_modifier(rcstr!("ssr modules")),
ChunkGroup::IsolatedMerged {
parent: parent_chunk_group,
merge_tag: ecmascript_client_reference_merge_tag_ssr(),
entries: ssr_modules,
},
module_graph,
availability_info,
))
} else {
None
};
......@@ -242,6 +247,7 @@ pub async fn get_app_client_references_chunks(
.try_join()
.await?;
let client_chunk_group = if !client_modules.is_empty() {
let availability_info = current_client_chunk_group.await?.availability_info;
let _span = tracing::info_span!(
"client side rendering",
layout_segment = display(&server_component_path),
......@@ -256,88 +262,57 @@ pub async fn get_app_client_references_chunks(
entries: client_modules,
},
module_graph,
current_client_availability_info,
availability_info,
))
} else {
None
};
if let Some(client_chunk_group) = client_chunk_group {
let ChunkGroupResult {
assets,
referenced_assets,
availability_info,
} = *client_chunk_group.await?;
let client_chunks = current_client_chunks
.concatenate(*assets)
.to_resolved()
.await?;
let client_referenced_assets = current_client_referenced_assets
.concatenate(*referenced_assets)
let client_chunk_group = current_client_chunk_group
.concatenate(client_chunk_group)
.to_resolved()
.await?;
if is_layout {
current_client_availability_info = availability_info;
current_client_chunks = client_chunks;
current_client_referenced_assets = client_referenced_assets;
current_client_chunk_group = client_chunk_group;
}
layout_segment_client_chunks.insert(
server_component,
OutputAssetsWithReferenced {
assets: client_chunks,
referenced_assets: client_referenced_assets,
},
);
let assets = client_chunk_group
.output_assets_with_referenced()
.to_resolved()
.await?;
layout_segment_client_chunks.insert(server_component, assets);
for &client_reference_ty in client_reference_types.iter() {
if let ClientReferenceType::EcmascriptClientReference(_) =
client_reference_ty
{
client_component_client_chunks.insert(
client_reference_ty,
ChunkGroupResult {
assets: client_chunks,
referenced_assets: client_referenced_assets,
availability_info,
},
);
client_component_client_chunks
.insert(client_reference_ty, client_chunk_group);
}
}
}
if let Some(ssr_chunk_group) = ssr_chunk_group {
let ssr_chunk_group = ssr_chunk_group.await?;
let ssr_chunks = current_ssr_chunks
.concatenate(*ssr_chunk_group.assets)
.to_resolved()
.await?;
let ssr_referenced_assets = current_ssr_referenced_assets
.concatenate(*ssr_chunk_group.referenced_assets)
let ssr_chunk_group = current_ssr_chunk_group
.concatenate(ssr_chunk_group)
.to_resolved()
.await?;
if is_layout {
current_ssr_availability_info = ssr_chunk_group.availability_info;
current_ssr_chunks = ssr_chunks;
current_ssr_referenced_assets = ssr_referenced_assets;
current_ssr_chunk_group = ssr_chunk_group;
}
let assets = ssr_chunk_group
.output_assets_with_referenced()
.to_resolved()
.await?;
for &client_reference_ty in client_reference_types.iter() {
if let ClientReferenceType::EcmascriptClientReference(_) =
client_reference_ty
{
client_component_ssr_chunks.insert(
client_reference_ty,
ChunkGroupResult {
assets: ssr_chunks,
referenced_assets: ssr_referenced_assets,
availability_info: ssr_chunk_group.availability_info,
},
);
client_component_ssr_chunks.insert(client_reference_ty, assets);
}
}
}
......
......@@ -7,7 +7,6 @@ use turbopack_core::{
},
ident::AssetIdent,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::OutputAssets,
};
#[turbo_tasks::function]
......@@ -18,12 +17,7 @@ pub async fn get_app_client_shared_chunk_group(
client_chunking_context: Vc<Box<dyn ChunkingContext>>,
) -> Result<Vc<ChunkGroupResult>> {
if app_client_runtime_entries.await?.is_empty() {
return Ok(ChunkGroupResult {
assets: OutputAssets::empty().to_resolved().await?,
referenced_assets: OutputAssets::empty().to_resolved().await?,
availability_info: AvailabilityInfo::Root,
}
.cell());
return Ok(ChunkGroupResult::empty());
}
let span = tracing::trace_span!("app client shared");
......
......@@ -16,6 +16,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::{ModuleGraph, export_usage::ModuleExportUsageInfo},
output::OutputAssetsReference,
reference::{ModuleReference, ModuleReferences},
reference_type::ReferenceType,
resolve::ModuleResolveResult,
......@@ -291,6 +292,9 @@ struct EcmascriptClientReferenceProxyChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptClientReferenceProxyChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for EcmascriptClientReferenceProxyChunkItem {
#[turbo_tasks::function]
......
......@@ -10,6 +10,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::{ModuleReferences, SingleChunkableModuleReference},
resolve::ExportUsage,
};
......@@ -131,6 +132,9 @@ struct NextDynamicEntryChunkItem {
inner: ResolvedVc<NextDynamicEntryModule>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for NextDynamicEntryChunkItem {}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for NextDynamicEntryChunkItem {
#[turbo_tasks::function]
......
......@@ -12,11 +12,9 @@ use turbo_tasks::{
use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{
ChunkGroupResult, ChunkingContext, ModuleChunkItemIdExt, ModuleId as TurbopackModuleId,
},
chunk::{ChunkingContext, ModuleChunkItemIdExt, ModuleId as TurbopackModuleId},
module_graph::async_module_info::AsyncModulesInfo,
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
};
use turbopack_ecmascript::utils::StringifyJs;
......@@ -107,6 +105,16 @@ pub struct ClientReferenceManifest {
pub mode: NextMode,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ClientReferenceManifest {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
Ok(OutputAssetsWithReferenced::from_assets(
*build_manifest(self).await?.references,
))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for ClientReferenceManifest {
#[turbo_tasks::function]
......@@ -119,11 +127,6 @@ impl OutputAsset for ClientReferenceManifest {
))?
.cell())
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
Ok(*build_manifest(self).await?.references)
}
}
#[turbo_tasks::value_impl]
......@@ -253,15 +256,11 @@ async fn build_manifest(
.chunk_item_id(**client_chunking_context)
.await?;
let (client_chunks_paths, client_is_async) = if let Some(ChunkGroupResult {
assets: client_chunks,
referenced_assets: client_referenced_assets,
availability_info: _,
}) =
let (client_chunks_paths, client_is_async) = if let Some(client_assets) =
client_component_client_chunks.get(&app_client_reference_ty)
{
let client_chunks = client_chunks.await?;
let client_referenced_assets = client_referenced_assets.await?;
let client_chunks = client_assets.primary_assets().await?;
let client_referenced_assets = client_assets.referenced_assets().await?;
references.extend(client_chunks.iter());
references.extend(client_referenced_assets.iter());
......@@ -310,14 +309,11 @@ async fn build_manifest(
// edge runtime doesn't support dynamically
// loading chunks.
(Vec::new(), false)
} else if let Some(ChunkGroupResult {
assets: ssr_chunks,
referenced_assets: ssr_referenced_assets,
availability_info: _,
}) = client_component_ssr_chunks.get(&app_client_reference_ty)
} else if let Some(ssr_assets) =
client_component_ssr_chunks.get(&app_client_reference_ty)
{
let ssr_chunks = ssr_chunks.await?;
let ssr_referenced_assets = ssr_referenced_assets.await?;
let ssr_chunks = ssr_assets.primary_assets().await?;
let ssr_referenced_assets = ssr_assets.referenced_assets().await?;
references.extend(ssr_chunks.iter());
references.extend(ssr_referenced_assets.iter());
......@@ -405,14 +401,7 @@ async fn build_manifest(
}
// per layout segment chunks need to be emitted into the manifest too
for (
server_component,
OutputAssetsWithReferenced {
assets: client_chunks,
referenced_assets: _,
},
) in layout_segment_client_chunks.iter()
{
for (server_component, client_assets) in layout_segment_client_chunks.iter() {
let server_component_name = server_component
.server_path()
.await?
......@@ -429,7 +418,7 @@ async fn build_manifest(
.entry(server_component_name)
.or_default();
let client_chunks = &client_chunks.await?;
let client_chunks = client_assets.primary_assets().await?;
let client_chunks_with_path =
cached_chunk_paths(&mut client_chunk_path_cache, client_chunks.iter().copied())
.await?;
......
......@@ -13,7 +13,7 @@ use turbo_tasks::{
use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
};
use crate::next_config::RouteHas;
......@@ -36,14 +36,9 @@ pub struct BuildManifest {
}
#[turbo_tasks::value_impl]
impl OutputAsset for BuildManifest {
#[turbo_tasks::function]
async fn path(&self) -> Vc<FileSystemPath> {
self.output_path.clone().cell()
}
impl OutputAssetsReference for BuildManifest {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let chunks: Vec<ReadRef<OutputAssets>> = self.pages.values().try_join().await?;
let root_main_files = self
......@@ -61,7 +56,17 @@ impl OutputAsset for BuildManifest {
.chain(self.polyfill_files.iter().copied())
.collect();
Ok(Vc::cell(references))
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for BuildManifest {
#[turbo_tasks::function]
async fn path(&self) -> Vc<FileSystemPath> {
self.output_path.clone().cell()
}
}
......@@ -163,16 +168,19 @@ pub struct ClientBuildManifest {
}
#[turbo_tasks::value_impl]
impl OutputAsset for ClientBuildManifest {
impl OutputAssetsReference for ClientBuildManifest {
#[turbo_tasks::function]
async fn path(&self) -> Vc<FileSystemPath> {
self.output_path.clone().cell()
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let chunks: Vec<ResolvedVc<Box<dyn OutputAsset>>> = self.pages.values().copied().collect();
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(chunks)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for ClientBuildManifest {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
let chunks: Vec<ResolvedVc<Box<dyn OutputAsset>>> = self.pages.values().copied().collect();
Ok(Vc::cell(chunks))
async fn path(&self) -> Vc<FileSystemPath> {
self.output_path.clone().cell()
}
}
......
......@@ -11,6 +11,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::ModuleReferences,
};
use turbopack_ecmascript::{
......@@ -124,6 +125,9 @@ struct NextServerComponentChunkItem {
inner: ResolvedVc<NextServerComponentModule>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for NextServerComponentChunkItem {}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for NextServerComponentChunkItem {
#[turbo_tasks::function]
......
......@@ -11,6 +11,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::ModuleReferences,
};
use turbopack_ecmascript::{
......@@ -124,6 +125,9 @@ struct NextServerComponentChunkItem {
inner: ResolvedVc<NextServerUtilityModule>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for NextServerComponentChunkItem {}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for NextServerComponentChunkItem {
#[turbo_tasks::function]
......
......@@ -12,7 +12,7 @@ use turbopack_core::{
context::AssetContext,
ident::AssetIdent,
module::Module,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
proxied_asset::ProxiedAsset,
reference_type::{EntryReferenceSubType, ReferenceType},
source::Source,
......@@ -153,6 +153,16 @@ impl PageLoaderAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for PageLoaderAsset {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
Ok(OutputAssetsWithReferenced::from_assets(
*self.await?.page_chunks,
))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for PageLoaderAsset {
#[turbo_tasks::function]
......@@ -171,11 +181,6 @@ impl OutputAsset for PageLoaderAsset {
.chunk_path(Some(Vc::upcast(self)), ident, None, rcstr!(".js")))
}
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
Ok(*self.await?.page_chunks)
}
}
#[turbo_tasks::value_impl]
......
......@@ -20,6 +20,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
resolve::ModulePart,
source::Source,
source_map::GenerateSourceMap,
......@@ -138,6 +139,9 @@ struct RawEcmascriptChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for RawEcmascriptChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for RawEcmascriptChunkItem {
#[turbo_tasks::function]
......
......@@ -221,6 +221,22 @@ where
}
}
/// Upcasts the given `Vec<ResolvedVc<T>>` to a `Vec<ResolvedVc<K>>`.
///
/// See also: [`Vc::upcast`].
#[inline(always)]
pub fn upcast_vec<K>(vec: Vec<Self>) -> Vec<ResolvedVc<K>>
where
T: UpcastStrict<K>,
K: VcValueTrait + ?Sized,
{
debug_assert!(size_of::<ResolvedVc<T>>() == size_of::<ResolvedVc<K>>());
debug_assert!(size_of::<Vec<ResolvedVc<T>>>() == size_of::<Vec<ResolvedVc<K>>>());
let (ptr, len, capacity) = vec.into_raw_parts();
// Safety: The memory layout of `ResolvedVc<T>` and `ResolvedVc<K>` is the same.
unsafe { Vec::from_raw_parts(ptr as *mut ResolvedVc<K>, len, capacity) }
}
/// Cheaply converts a Vec of resolved Vcs to a Vec of Vcs.
pub fn deref_vec(vec: Vec<ResolvedVc<T>>) -> Vec<Vc<T>> {
debug_assert!(size_of::<ResolvedVc<T>>() == size_of::<Vc<T>>());
......
......@@ -12,7 +12,7 @@ use turbopack_analyze::split_chunk::{ChunkPart, ChunkPartRange, split_output_ass
use turbopack_core::{
asset::{Asset, AssetContent},
code_builder::{Code, CodeBuilder},
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap},
};
......@@ -102,6 +102,9 @@ struct TestAsset {
code: ResolvedVc<Code>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for TestAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for TestAsset {
#[turbo_tasks::function]
......
......@@ -662,6 +662,7 @@ impl ChunkingContext for BrowserChunkingContext {
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
......@@ -705,6 +706,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
references: ResolvedVc::cell(references),
availability_info,
}
.cell())
......@@ -732,6 +734,7 @@ impl ChunkingContext for BrowserChunkingContext {
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
......@@ -792,6 +795,7 @@ impl ChunkingContext for BrowserChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
references: ResolvedVc::cell(references),
availability_info,
}
.cell())
......
......@@ -7,7 +7,7 @@ use turbopack_core::{
chunk::{Chunk, ChunkingContext, OutputChunk, OutputChunkRuntimeInfo},
ident::AssetIdent,
introspect::{Introspectable, IntrospectableChildren},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, SourceMapAsset},
version::VersionedContent,
};
......@@ -95,6 +95,35 @@ impl EcmascriptBrowserChunk {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptBrowserChunk {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let chunk_references = this.chunk.references().await?;
let include_source_map = *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?;
let ref_assets = chunk_references.assets.await?;
let mut assets =
Vec::with_capacity(ref_assets.len() + if include_source_map { 1 } else { 0 });
assets.extend(ref_assets.iter().copied());
if include_source_map {
assets.push(ResolvedVc::upcast(self.source_map().to_resolved().await?));
}
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(assets),
referenced_assets: chunk_references.referenced_assets,
references: chunk_references.references,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBrowserChunk {
#[turbo_tasks::function]
......@@ -110,26 +139,6 @@ impl OutputAsset for EcmascriptBrowserChunk {
fn size_bytes(self: Vc<Self>) -> Vc<Option<u64>> {
self.own_content().content().len()
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = self.await?;
let chunk_references = this.chunk.references().await?;
let include_source_map = *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?;
let mut references =
Vec::with_capacity(chunk_references.len() + if include_source_map { 1 } else { 0 });
references.extend(chunk_references.iter().copied());
if include_source_map {
references.push(ResolvedVc::upcast(self.source_map().to_resolved().await?));
}
Ok(Vc::cell(references))
}
}
#[turbo_tasks::value_impl]
......
......@@ -17,7 +17,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, SourceMapAsset},
};
use turbopack_ecmascript::{
......@@ -243,21 +243,9 @@ impl ValueToString for EcmascriptBrowserEvaluateChunk {
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBrowserEvaluateChunk {
#[turbo_tasks::function]
async fn path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let ident = self.ident_for_path();
Ok(this.chunking_context.chunk_path(
Some(Vc::upcast(self)),
ident,
Some(rcstr!("turbopack")),
rcstr!(".js"),
))
}
impl OutputAssetsReference for EcmascriptBrowserEvaluateChunk {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let mut references = Vec::new();
......@@ -272,7 +260,24 @@ impl OutputAsset for EcmascriptBrowserEvaluateChunk {
references.extend(this.other_chunks.await?.iter().copied());
Ok(Vc::cell(references))
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBrowserEvaluateChunk {
#[turbo_tasks::function]
async fn path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let ident = self.ident_for_path();
Ok(this.chunking_context.chunk_path(
Some(Vc::upcast(self)),
ident,
Some(rcstr!("turbopack")),
rcstr!(".js"),
))
}
}
......
......@@ -7,7 +7,7 @@ use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{ChunkingContext, EvaluatableAssets},
ident::AssetIdent,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
version::VersionedContent,
};
......@@ -69,6 +69,14 @@ impl ValueToString for EcmascriptDevChunkList {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptDevChunkList {
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
OutputAssetsWithReferenced::from_assets(*self.chunks)
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptDevChunkList {
#[turbo_tasks::function]
......@@ -93,11 +101,6 @@ impl OutputAsset for EcmascriptDevChunkList {
.chunking_context
.chunk_path(Some(Vc::upcast(self)), ident, None, rcstr!(".js")))
}
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
*self.chunks
}
}
#[turbo_tasks::value_impl]
......
......@@ -37,7 +37,6 @@ use turbopack_core::{
export_usage::compute_export_usage_info,
},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
reference::all_assets_from_entries,
reference_type::{EntryReferenceSubType, ReferenceType},
resolve::{
origin::{PlainResolveOrigin, ResolveOriginExt},
......@@ -433,29 +432,20 @@ async fn build_internal(
ResolvedVc::try_sidecast::<Box<dyn EvaluatableAsset>>(entry_module)
{
match target {
Target::Browser => {
*chunking_context
.evaluated_chunk_group_assets(
AssetIdent::from_path(
build_output_root
.join(
ecmascript
.ident()
.path()
.await?
.file_stem()
.unwrap(),
)?
.with_extension("entry.js"),
),
ChunkGroup::Entry(
[ResolvedVc::upcast(ecmascript)].into_iter().collect(),
),
module_graph,
AvailabilityInfo::Root,
)
.await?
}
Target::Browser => chunking_context.evaluated_chunk_group_assets(
AssetIdent::from_path(
build_output_root
.join(
ecmascript.ident().path().await?.file_stem().unwrap(),
)?
.with_extension("entry.js"),
),
ChunkGroup::Entry(
[ResolvedVc::upcast(ecmascript)].into_iter().collect(),
),
module_graph,
AvailabilityInfo::Root,
),
Target::Node => OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![
chunking_context
......@@ -480,7 +470,9 @@ async fn build_internal(
.asset,
]),
referenced_assets: ResolvedVc::cell(vec![]),
},
references: ResolvedVc::cell(vec![]),
}
.cell(),
}
} else {
bail!(
......@@ -496,18 +488,8 @@ async fn build_internal(
let all_assets = async move {
let mut all_assets: FxHashSet<ResolvedVc<Box<dyn OutputAsset>>> = FxHashSet::default();
for OutputAssetsWithReferenced {
assets,
referenced_assets,
} in entry_chunk_groups
{
all_assets.extend(all_assets_from_entries(*assets).await?.into_iter().copied());
all_assets.extend(
all_assets_from_entries(*referenced_assets)
.await?
.into_iter()
.copied(),
);
for group in entry_chunk_groups {
all_assets.extend(group.expand_all_assets().await?);
}
anyhow::Ok(all_assets)
}
......
use anyhow::Result;
use turbo_tasks::{
Completion, Completions, ResolvedVc, TryJoinIterExt, Vc,
graph::{AdjacencyMap, GraphTraversal, NonDeterministic},
graph::{AdjacencyMap, GraphTraversal},
};
use crate::{
asset::Asset,
module::Module,
output::{OutputAsset, OutputAssets},
output::{ExpandOutputAssetsInput, OutputAsset, OutputAssets, expand_output_assets},
reference::primary_referenced_modules,
};
async fn get_referenced_output_assets(
parent: ResolvedVc<Box<dyn OutputAsset>>,
) -> Result<impl Iterator<Item = ResolvedVc<Box<dyn OutputAsset>>> + Send> {
Ok(parent.references().owned().await?.into_iter())
}
pub async fn get_referenced_modules(
parent: ResolvedVc<Box<dyn Module>>,
) -> Result<impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send> {
......@@ -53,17 +47,14 @@ pub async fn any_content_changed_of_module(
pub async fn any_content_changed_of_output_asset(
root: ResolvedVc<Box<dyn OutputAsset>>,
) -> Result<Vc<Completion>> {
let completions = NonDeterministic::new()
.skip_duplicates()
.visit([root], get_referenced_output_assets)
.await
.completed()?
.into_inner()
.into_iter()
.map(|m| content_changed(*ResolvedVc::upcast(m)))
.map(|v| v.to_resolved())
.try_join()
.await?;
let completions =
expand_output_assets(std::iter::once(ExpandOutputAssetsInput::Asset(root)), true)
.await?
.into_iter()
.map(|m| content_changed(*ResolvedVc::upcast(m)))
.map(|v| v.to_resolved())
.try_join()
.await?;
Ok(Vc::<Completions>::cell(completions).completed())
}
......
......@@ -6,7 +6,7 @@ use turbo_rcstr::rcstr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
use super::{
Chunk, ChunkGroupContent, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext,
Chunk, ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext,
availability_info::AvailabilityInfo, chunking::make_chunks,
};
use crate::{
......@@ -25,7 +25,10 @@ use crate::{
},
module_batches::{BatchingConfig, ModuleBatchesGraphEdge},
},
output::{OutputAsset, OutputAssets},
output::{
OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsReferences,
OutputAssetsWithReferenced,
},
reference::ModuleReference,
traced_asset::TracedAsset,
};
......@@ -33,6 +36,7 @@ use crate::{
pub struct MakeChunkGroupResult {
pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>,
pub availability_info: AvailabilityInfo,
}
......@@ -123,14 +127,7 @@ pub async fn make_chunk_group(
})
});
// And also add output assets referenced by async chunk loaders
let async_loader_references = async_loaders
.iter()
.map(|&loader| loader.references())
.try_join()
.await?;
let mut referenced_output_assets = traced_modules
let referenced_output_assets = traced_modules
.into_iter()
.map(|module| async move {
Ok(ResolvedVc::upcast(
......@@ -141,13 +138,6 @@ pub async fn make_chunk_group(
.await?;
chunk_items.extend(async_loader_chunk_items);
referenced_output_assets.reserve(
async_loader_references
.iter()
.map(|r| r.len())
.sum::<usize>(),
);
referenced_output_assets.extend(async_loader_references.into_iter().flatten());
// Pass chunk items to chunking algorithm
let chunks = make_chunks(
......@@ -162,13 +152,14 @@ pub async fn make_chunk_group(
Ok(MakeChunkGroupResult {
chunks,
referenced_output_assets,
references: ResolvedVc::upcast_vec(async_loaders),
availability_info,
})
}
pub async fn references_to_output_assets(
references: impl IntoIterator<Item = &ResolvedVc<Box<dyn ModuleReference>>>,
) -> Result<Vc<OutputAssets>> {
) -> Result<Vc<OutputAssetsWithReferenced>> {
let output_assets = references
.into_iter()
.map(|reference| reference.resolve_reference().primary_output_assets())
......@@ -180,9 +171,13 @@ pub async fn references_to_output_assets(
.flatten()
.copied()
.filter(|&asset| set.insert(asset))
.map(|asset| *asset)
.collect::<Vec<_>>();
Ok(OutputAssets::new(output_assets))
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(output_assets),
referenced_assets: OutputAssets::empty_resolved(),
references: OutputAssetsReferences::empty_resolved(),
}
.cell())
}
pub async fn chunk_group_content(
......
......@@ -17,7 +17,10 @@ use crate::{
ModuleGraph, chunk_group_info::ChunkGroup, export_usage::ModuleExportUsage,
module_batches::BatchingConfig,
},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
output::{
ExpandOutputAssetsInput, OutputAsset, OutputAssets, OutputAssetsReferences,
OutputAssetsWithReferenced, expand_output_assets,
},
};
#[derive(
......@@ -87,13 +90,121 @@ pub enum ChunkGroupType {
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Copy)]
#[derive(Clone)]
pub struct ChunkGroupResult {
pub assets: ResolvedVc<OutputAssets>,
pub referenced_assets: ResolvedVc<OutputAssets>,
pub references: ResolvedVc<OutputAssetsReferences>,
pub availability_info: AvailabilityInfo,
}
impl ChunkGroupResult {
pub fn empty() -> Vc<Self> {
ChunkGroupResult {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
availability_info: AvailabilityInfo::Root,
}
.cell()
}
pub fn empty_resolved() -> ResolvedVc<Self> {
ChunkGroupResult {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
availability_info: AvailabilityInfo::Root,
}
.resolved_cell()
}
}
#[turbo_tasks::value_impl]
impl ChunkGroupResult {
#[turbo_tasks::function]
pub async fn output_assets_with_referenced(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
Ok(OutputAssetsWithReferenced {
assets: self.assets,
referenced_assets: self.referenced_assets,
references: self.references,
}
.cell())
}
#[turbo_tasks::function]
pub async fn concatenate(&self, next: Vc<Self>) -> Result<Vc<Self>> {
let next = next.await?;
Ok(ChunkGroupResult {
assets: self.assets.concatenate(*next.assets).to_resolved().await?,
referenced_assets: self
.referenced_assets
.concatenate(*next.referenced_assets)
.to_resolved()
.await?,
references: self
.references
.concatenate(*next.references)
.to_resolved()
.await?,
availability_info: next.availability_info,
}
.cell())
}
#[turbo_tasks::function]
pub async fn all_assets(&self) -> Result<Vc<OutputAssets>> {
Ok(Vc::cell(
expand_output_assets(
self.assets
.await?
.into_iter()
.chain(self.referenced_assets.await?.into_iter())
.copied()
.map(ExpandOutputAssetsInput::Asset)
.chain(
self.references
.await?
.into_iter()
.copied()
.map(ExpandOutputAssetsInput::Reference),
),
false,
)
.await?,
))
}
/// Returns only primary asset entries. Doesn't expand OutputAssets. Doesn't return referenced
/// assets.
#[turbo_tasks::function]
pub fn primary_assets(&self) -> Vc<OutputAssets> {
*self.assets
}
#[turbo_tasks::function]
pub async fn referenced_assets(&self) -> Result<Vc<OutputAssets>> {
Ok(Vc::cell(
expand_output_assets(
self.referenced_assets
.await?
.into_iter()
.copied()
.map(ExpandOutputAssetsInput::Asset)
.chain(
self.references
.await?
.into_iter()
.copied()
.map(ExpandOutputAssetsInput::Reference),
),
false,
)
.await?,
))
}
}
#[turbo_tasks::value(shared)]
pub struct EntryChunkGroupResult {
pub asset: ResolvedVc<Box<dyn OutputAsset>>,
......@@ -564,38 +675,28 @@ async fn relative_path_from_chunk_root_to_project_root(
}
#[turbo_tasks::function]
async fn root_chunk_group_assets(
fn root_chunk_group_assets(
chunking_context: Vc<Box<dyn ChunkingContext>>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
) -> Result<Vc<OutputAssetsWithReferenced>> {
let root_chunk_group = chunking_context
) -> Vc<OutputAssetsWithReferenced> {
chunking_context
.root_chunk_group(ident, chunk_group, module_graph)
.await?;
Ok(OutputAssetsWithReferenced {
assets: root_chunk_group.assets,
referenced_assets: root_chunk_group.referenced_assets,
}
.cell())
.output_assets_with_referenced()
}
#[turbo_tasks::function]
async fn evaluated_chunk_group_assets(
fn evaluated_chunk_group_assets(
chunking_context: Vc<Box<dyn ChunkingContext>>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<OutputAssetsWithReferenced>> {
let evaluated_chunk_group = chunking_context
) -> Vc<OutputAssetsWithReferenced> {
chunking_context
.evaluated_chunk_group(ident, chunk_group, module_graph, availability_info)
.await?;
Ok(OutputAssetsWithReferenced {
assets: evaluated_chunk_group.assets,
referenced_assets: evaluated_chunk_group.referenced_assets,
}
.cell())
.output_assets_with_referenced()
}
#[turbo_tasks::function]
......@@ -622,19 +723,14 @@ async fn entry_chunk_group_asset(
}
#[turbo_tasks::function]
async fn chunk_group_assets(
fn chunk_group_assets(
chunking_context: Vc<Box<dyn ChunkingContext>>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<OutputAssetsWithReferenced>> {
let chunk_group = chunking_context
) -> Vc<OutputAssetsWithReferenced> {
chunking_context
.chunk_group(ident, chunk_group, module_graph, availability_info)
.await?;
Ok(OutputAssetsWithReferenced {
assets: chunk_group.assets,
referenced_assets: chunk_group.referenced_assets,
}
.cell())
.output_assets_with_referenced()
}
......@@ -44,7 +44,7 @@ use crate::{
ModuleGraph,
module_batch::{ChunkableModuleOrBatch, ModuleBatchGroup},
},
output::OutputAssets,
output::{OutputAssets, OutputAssetsReference},
reference::ModuleReference,
resolve::ExportUsage,
};
......@@ -209,7 +209,7 @@ impl Chunks {
/// It usually contains multiple chunk items.
// TODO This could be simplified to and merged with [OutputChunk]
#[turbo_tasks::value_trait]
pub trait Chunk {
pub trait Chunk: OutputAssetsReference {
#[turbo_tasks::function]
fn ident(self: Vc<Self>) -> Vc<AssetIdent>;
#[turbo_tasks::function]
......@@ -218,12 +218,6 @@ pub trait Chunk {
// self.ident().path()
// }
/// Other [OutputAsset]s referenced from this [Chunk].
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
OutputAssets::empty()
}
#[turbo_tasks::function]
fn chunk_items(self: Vc<Self>) -> Vc<ChunkItems> {
ChunkItems(vec![]).cell()
......@@ -436,12 +430,13 @@ pub struct ChunkGroupContent {
}
#[turbo_tasks::value_trait]
pub trait ChunkItem {
pub trait ChunkItem: OutputAssetsReference {
/// The [AssetIdent] of the [Module] that this [ChunkItem] was created from.
/// For most chunk types this must uniquely identify the chunk item at
/// runtime as it's the source of the module id used at runtime.
#[turbo_tasks::function]
fn asset_ident(self: Vc<Self>) -> Vc<AssetIdent>;
/// A [AssetIdent] that uniquely identifies the content of this [ChunkItem].
/// It is usually identical to [ChunkItem::asset_ident] but can be
/// different when the chunk item content depends on available modules e. g.
......@@ -450,11 +445,6 @@ pub trait ChunkItem {
fn content_ident(self: Vc<Self>) -> Vc<AssetIdent> {
self.asset_ident()
}
/// A [ChunkItem] can reference OutputAssets, unlike [Module]s referencing other [Module]s.
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
OutputAssets::empty()
}
/// The type of chunk this item should be assembled into.
#[turbo_tasks::function]
......
......@@ -6,7 +6,10 @@ use super::{
Introspectable, IntrospectableChildren,
utils::{children_from_output_assets, content_to_details},
};
use crate::{asset::Asset, output::OutputAsset};
use crate::{
asset::Asset,
output::{OutputAsset, OutputAssetsReference},
};
#[turbo_tasks::value]
pub struct IntrospectableOutputAsset(ResolvedVc<Box<dyn OutputAsset>>);
......
......@@ -9,7 +9,7 @@ use super::{
use crate::{
asset::AssetContent,
chunk::{ChunkableModuleReference, ChunkingType},
output::OutputAssets,
output::OutputAssetsWithReferenced,
reference::{ModuleReference, ModuleReferences},
};
......@@ -121,11 +121,11 @@ pub async fn children_from_module_references(
#[turbo_tasks::function]
pub async fn children_from_output_assets(
references: Vc<OutputAssets>,
references: Vc<OutputAssetsWithReferenced>,
) -> Result<Vc<IntrospectableChildren>> {
let key = reference_ty();
let mut children = FxIndexSet::default();
let references = references.await?;
let references = references.expand_all_assets().await?;
for &reference in &*references {
children.insert((
key.clone(),
......
use anyhow::Result;
use either::Either;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, ValueToString, Vc};
use turbo_tasks::{
FxIndexSet, ResolvedVc, ValueToString, Vc,
graph::{AdjacencyMap, GraphTraversal},
};
use turbo_tasks_fs::FileSystemPath;
use crate::asset::Asset;
......@@ -8,10 +12,24 @@ use crate::asset::Asset;
#[turbo_tasks::value(transparent)]
pub struct OptionOutputAsset(Option<ResolvedVc<Box<dyn OutputAsset>>>);
#[turbo_tasks::value_trait]
pub trait OutputAssetsReference {
/// References to other [OutputAsset]s from this [OutputAssetReference].
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssetsWithReferenced> {
OutputAssetsWithReferenced {
assets: OutputAssets::empty_resolved(),
referenced_assets: OutputAssets::empty_resolved(),
references: OutputAssetsReferences::empty_resolved(),
}
.cell()
}
}
/// An asset that should be outputted, e. g. written to disk or served from a
/// server.
#[turbo_tasks::value_trait]
pub trait OutputAsset: Asset {
pub trait OutputAsset: Asset + OutputAssetsReference {
/// The identifier of the [OutputAsset]. It's expected to be unique and
/// capture all properties of the [OutputAsset].
#[turbo_tasks::function]
......@@ -24,12 +42,6 @@ pub trait OutputAsset: Asset {
self.path().to_string()
}
/// Other references [OutputAsset]s from this [OutputAsset].
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
OutputAssets::empty()
}
#[turbo_tasks::function]
fn size_bytes(self: Vc<Self>) -> Vc<Option<u64>> {
Vc::cell(None)
......@@ -37,15 +49,32 @@ pub trait OutputAsset: Asset {
}
#[turbo_tasks::value(transparent)]
pub struct OutputAssets(Vec<ResolvedVc<Box<dyn OutputAsset>>>);
pub struct OutputAssetsReferences(Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>);
#[turbo_tasks::value_impl]
impl OutputAssets {
impl OutputAssetsReferences {
#[turbo_tasks::function]
pub fn new(assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>) -> Vc<Self> {
Vc::cell(assets)
pub async fn concatenate(&self, other: Vc<Self>) -> Result<Vc<Self>> {
let mut references: FxIndexSet<_> = self.0.iter().copied().collect();
references.extend(other.await?.iter().copied());
Ok(Vc::cell(references.into_iter().collect()))
}
}
impl OutputAssetsReferences {
pub fn empty() -> Vc<Self> {
Vc::cell(vec![])
}
pub fn empty_resolved() -> ResolvedVc<Self> {
ResolvedVc::cell(vec![])
}
}
#[turbo_tasks::value(transparent)]
pub struct OutputAssets(Vec<ResolvedVc<Box<dyn OutputAsset>>>);
#[turbo_tasks::value_impl]
impl OutputAssets {
#[turbo_tasks::function]
pub async fn concatenate(&self, other: Vc<Self>) -> Result<Vc<Self>> {
let mut assets: FxIndexSet<_> = self.0.iter().copied().collect();
......@@ -65,7 +94,7 @@ impl OutputAssets {
impl OutputAssets {
pub fn empty() -> Vc<Self> {
Self::new(vec![])
Vc::cell(vec![])
}
pub fn empty_resolved() -> ResolvedVc<Self> {
......@@ -73,21 +102,186 @@ impl OutputAssets {
}
}
#[turbo_tasks::value(transparent)]
pub struct ExpandedOutputAssets(Vec<ResolvedVc<Box<dyn OutputAsset>>>);
/// A set of [OutputAsset]s
#[turbo_tasks::value(transparent)]
pub struct OutputAssetsSet(FxIndexSet<ResolvedVc<Box<dyn OutputAsset>>>);
#[turbo_tasks::value(shared)]
#[derive(Clone, Copy)]
#[derive(Clone)]
pub struct OutputAssetsWithReferenced {
/// Primary output assets. These are e. g. the chunks needed for a chunk group.
pub assets: ResolvedVc<OutputAssets>,
/// Secondary output assets that are referenced by the primary assets.
pub referenced_assets: ResolvedVc<OutputAssets>,
/// Secondary output assets that are referenced by the primary assets. These are unresolved
/// `OutputAssetsReference`s and need to be expanded to get the actual assets. These are e. g.
/// async loaders that reference other chunk groups.
pub references: ResolvedVc<OutputAssetsReferences>,
}
impl OutputAssetsWithReferenced {
async fn expand_assets(
&self,
inner_output_assets: bool,
) -> Result<Vec<ResolvedVc<Box<dyn OutputAsset>>>> {
expand_output_assets(
self.assets
.await?
.into_iter()
.chain(self.referenced_assets.await?.into_iter())
.map(|&asset| ExpandOutputAssetsInput::Asset(asset))
.chain(
self.references
.await?
.into_iter()
.map(|&reference| ExpandOutputAssetsInput::Reference(reference)),
),
inner_output_assets,
)
.await
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsWithReferenced {
#[turbo_tasks::function]
pub fn all_assets(&self) -> Vc<OutputAssets> {
self.assets.concatenate(*self.referenced_assets)
pub fn from_assets(assets: ResolvedVc<OutputAssets>) -> Vc<Self> {
OutputAssetsWithReferenced {
assets,
referenced_assets: OutputAssets::empty_resolved(),
references: OutputAssetsReferences::empty_resolved(),
}
.cell()
}
#[turbo_tasks::function]
pub async fn concatenate(&self, other: Vc<Self>) -> Result<Vc<Self>> {
Ok(Self {
assets: self
.assets
.concatenate(*other.await?.assets)
.to_resolved()
.await?,
referenced_assets: self
.referenced_assets
.concatenate(*other.await?.referenced_assets)
.to_resolved()
.await?,
references: self
.references
.concatenate(*other.await?.references)
.to_resolved()
.await?,
}
.cell())
}
/// Returns all assets, including referenced assets and nested assets.
#[turbo_tasks::function]
pub async fn expand_all_assets(&self) -> Result<Vc<ExpandedOutputAssets>> {
Ok(Vc::cell(self.expand_assets(true).await?))
}
/// Returns only direct referenced assets and does not include assets referenced indirectly by
/// them.
#[turbo_tasks::function]
pub async fn all_assets(&self) -> Result<Vc<OutputAssets>> {
Ok(Vc::cell(self.expand_assets(false).await?))
}
/// Returns only primary asset entries. Doesn't expand OutputAssets. Doesn't return referenced
/// assets.
#[turbo_tasks::function]
pub fn primary_assets(&self) -> Vc<OutputAssets> {
*self.assets
}
/// Returns only secondary referenced asset entries. Doesn't expand OutputAssets. Doesn't return
/// primary assets.
#[turbo_tasks::function]
pub async fn referenced_assets(&self) -> Result<Vc<OutputAssets>> {
Ok(Vc::cell(
expand_output_assets(
self.referenced_assets
.await?
.into_iter()
.copied()
.map(ExpandOutputAssetsInput::Asset)
.chain(
self.references
.await?
.into_iter()
.copied()
.map(ExpandOutputAssetsInput::Reference),
),
false,
)
.await?,
))
}
}
/// Computes the list of all chunk children of a given chunk.
async fn get_referenced_assets(
inner_output_assets: bool,
input: ExpandOutputAssetsInput,
) -> Result<impl Iterator<Item = ExpandOutputAssetsInput>> {
let refs = match input {
ExpandOutputAssetsInput::Asset(output_asset) => {
if !inner_output_assets {
return Ok(Either::Left(std::iter::empty()));
}
output_asset.references().await?
}
ExpandOutputAssetsInput::Reference(reference) => reference.references().await?,
};
let assets = refs
.assets
.await?
.into_iter()
.chain(refs.referenced_assets.await?.into_iter())
.map(|&asset| ExpandOutputAssetsInput::Asset(asset))
.chain(
refs.references
.await?
.into_iter()
.map(|&reference| ExpandOutputAssetsInput::Reference(reference)),
);
Ok(Either::Right(assets))
}
#[derive(PartialEq, Eq, Hash, Clone, Copy)]
pub enum ExpandOutputAssetsInput {
Asset(ResolvedVc<Box<dyn OutputAsset>>),
Reference(ResolvedVc<Box<dyn OutputAssetsReference>>),
}
pub async fn expand_output_assets(
inputs: impl Iterator<Item = ExpandOutputAssetsInput>,
inner_output_assets: bool,
) -> Result<Vec<ResolvedVc<Box<dyn OutputAsset>>>> {
let edges = AdjacencyMap::new()
.skip_duplicates()
.visit(inputs, async |input| {
get_referenced_assets(inner_output_assets, input).await
})
.await
.completed()?
.into_inner()
.into_postorder_topological();
let mut assets = Vec::new();
for input in edges {
match input {
ExpandOutputAssetsInput::Asset(asset) => {
assets.push(asset);
}
ExpandOutputAssetsInput::Reference(_) => {}
}
}
Ok(assets)
}
......@@ -3,7 +3,7 @@ use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
version::VersionedContent,
};
......@@ -28,15 +28,18 @@ impl ProxiedAsset {
}
#[turbo_tasks::value_impl]
impl OutputAsset for ProxiedAsset {
impl OutputAssetsReference for ProxiedAsset {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
self.asset.references()
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for ProxiedAsset {
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
self.asset.references()
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
}
}
......
......@@ -3,7 +3,7 @@ use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source::Source,
};
......@@ -15,6 +15,9 @@ pub struct RawOutput {
source: ResolvedVc<Box<dyn Source>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for RawOutput {}
#[turbo_tasks::value_impl]
impl OutputAsset for RawOutput {
#[turbo_tasks::function]
......
......@@ -7,7 +7,7 @@ use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
module::Module,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
reference::referenced_modules_and_affecting_sources,
};
......@@ -38,18 +38,9 @@ impl RebasedAsset {
}
#[turbo_tasks::value_impl]
impl OutputAsset for RebasedAsset {
impl OutputAssetsReference for RebasedAsset {
#[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> {
Ok(FileSystemPath::rebase(
self.module.ident().path().owned().await?,
self.input_dir.clone(),
self.output_dir.clone(),
))
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let references = referenced_modules_and_affecting_sources(*self.module)
.await?
.iter()
......@@ -62,7 +53,21 @@ impl OutputAsset for RebasedAsset {
})
.try_join()
.await?;
Ok(Vc::cell(references))
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for RebasedAsset {
#[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> {
Ok(FileSystemPath::rebase(
self.module.ident().path().owned().await?,
self.input_dir.clone(),
self.output_dir.clone(),
))
}
}
......
......@@ -4,13 +4,15 @@ use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexSet, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
graph::{AdjacencyMap, GraphTraversal},
};
use crate::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
module::{Module, Modules},
output::{OutputAsset, OutputAssets},
output::{
ExpandOutputAssetsInput, ExpandedOutputAssets, OutputAsset, OutputAssets,
expand_output_assets,
},
raw_module::RawModule,
resolve::{ExportUsage, ModuleResolveResult, RequestKey},
};
......@@ -331,28 +333,28 @@ pub async fn primary_chunkable_referenced_modules(
/// Walks the asset graph from multiple assets and collect all referenced
/// assets.
#[turbo_tasks::function]
pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<OutputAssets>> {
pub async fn all_assets_from_entries(
entries: Vc<OutputAssets>,
) -> Result<Vc<ExpandedOutputAssets>> {
Ok(Vc::cell(
AdjacencyMap::new()
.skip_duplicates()
.visit(entries.await?.iter().copied(), get_referenced_assets)
.await
.completed()?
.into_inner()
.into_postorder_topological()
.collect(),
expand_output_assets(
entries
.await?
.into_iter()
.map(|&asset| ExpandOutputAssetsInput::Asset(asset)),
true,
)
.await?,
))
}
/// Computes the list of all chunk children of a given chunk.
pub async fn get_referenced_assets(
asset: ResolvedVc<Box<dyn OutputAsset>>,
) -> Result<impl Iterator<Item = ResolvedVc<Box<dyn OutputAsset>>> + Send> {
Ok(asset
.references()
.await?
.iter()
.copied()
.collect::<Vec<_>>()
.into_iter())
/// Walks the asset graph from multiple assets and collect all referenced
/// assets.
#[turbo_tasks::function]
pub async fn all_assets_from_entry(
entry: ResolvedVc<Box<dyn OutputAsset>>,
) -> Result<Vc<ExpandedOutputAssets>> {
Ok(Vc::cell(
expand_output_assets(std::iter::once(ExpandOutputAssetsInput::Asset(entry)), true).await?,
))
}
......@@ -12,7 +12,7 @@ use crate::{
chunk::ChunkingContext,
ident::AssetIdent,
introspect::{Introspectable, IntrospectableChildren},
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source_map::{GenerateSourceMap, SourceMap},
};
......@@ -65,6 +65,9 @@ impl SourceMapAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for SourceMapAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for SourceMapAsset {
#[turbo_tasks::function]
......
......@@ -5,7 +5,7 @@ use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
module::Module,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
reference::referenced_modules_and_affecting_sources,
};
......@@ -24,14 +24,9 @@ impl TracedAsset {
}
#[turbo_tasks::value_impl]
impl OutputAsset for TracedAsset {
impl OutputAssetsReference for TracedAsset {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.module.ident().path()
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let references = referenced_modules_and_affecting_sources(*self.module)
.await?
.iter()
......@@ -42,7 +37,17 @@ impl OutputAsset for TracedAsset {
})
.try_join()
.await?;
Ok(Vc::cell(references))
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for TracedAsset {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.module.ident().path()
}
}
......
......@@ -3,7 +3,7 @@ use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
};
/// An [OutputAsset] that is created from some passed source code and can have a list of references
......@@ -43,15 +43,18 @@ impl VirtualOutputAsset {
}
#[turbo_tasks::value_impl]
impl OutputAsset for VirtualOutputAsset {
impl OutputAssetsReference for VirtualOutputAsset {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
OutputAssetsWithReferenced::from_assets(*self.references)
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for VirtualOutputAsset {
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
*self.references
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
}
}
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
ident::AssetIdent,
module::{Module, StyleModule, StyleType},
module_graph::ModuleGraph,
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
reference::{ModuleReference, ModuleReferences},
reference_type::ImportContext,
resolve::origin::ResolveOrigin,
......@@ -208,6 +208,27 @@ struct CssModuleChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for CssModuleChunkItem {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let mut references = Vec::new();
if let ParseCssResult::Ok { url_references, .. } = &*self.module.parse_css().await? {
for (_, reference) in url_references.await? {
if let ReferencedAsset::Some(asset) = *reference
.get_referenced_asset(*self.chunking_context)
.await?
{
references.push(asset);
}
}
}
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for CssModuleChunkItem {
#[turbo_tasks::function]
......@@ -229,22 +250,6 @@ impl ChunkItem for CssModuleChunkItem {
fn module(&self) -> Vc<Box<dyn Module>> {
Vc::upcast(*self.module)
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
let mut references = Vec::new();
if let ParseCssResult::Ok { url_references, .. } = &*self.module.parse_css().await? {
for (_, reference) in url_references.await? {
if let ReferencedAsset::Some(asset) = *reference
.get_referenced_asset(*self.chunking_context)
.await?
{
references.push(asset);
}
}
}
Ok(Vc::cell(references))
}
}
#[turbo_tasks::value_impl]
......
......@@ -4,11 +4,8 @@ pub mod source_map;
use std::fmt::Write;
use anyhow::{Result, bail};
use swc_core::common::pass::Either;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueDefault, ValueToString, Vc,
};
use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, ValueDefault, ValueToString, Vc};
use turbo_tasks_fs::{
File, FileSystem, FileSystemPath,
rope::{Rope, RopeBuilder},
......@@ -29,7 +26,7 @@ use turbopack_core::{
utils::{children_from_output_assets, content_to_details},
},
module::Module,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
reference_type::ImportContext,
server_fs::ServerFileSystem,
source_map::{
......@@ -251,6 +248,82 @@ pub struct CssChunkContent {
pub chunk_items: Vec<ResolvedVc<Box<dyn CssChunkItem>>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for CssChunk {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let content = this.content.await?;
let should_generate_single_item_chunks = content.chunk_items.len() > 1
&& *this
.chunking_context
.is_dynamic_chunk_content_loading_enabled()
.await?;
let references = content
.chunk_items
.iter()
.map(|item| async {
let refs = item.references().await?;
let single_css_chunk = if should_generate_single_item_chunks {
Some(ResolvedVc::upcast(
SingleItemCssChunk::new(*this.chunking_context, **item)
.to_resolved()
.await?,
))
} else {
None
};
Ok((
refs.assets.await?,
single_css_chunk,
refs.referenced_assets.await?,
refs.references.await?,
))
})
.try_join()
.await?;
let source_map = if *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?
{
Some(ResolvedVc::upcast(
CssChunkSourceMapAsset::new(self).to_resolved().await?,
))
} else {
None
};
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(
references
.iter()
.flat_map(|(assets, single_css_chunk, _, _)| {
assets
.iter()
.copied()
.chain(single_css_chunk.iter().copied())
})
.chain(source_map.into_iter())
.collect(),
),
referenced_assets: ResolvedVc::cell(
references
.iter()
.flat_map(|(_, _, referenced_assets, _)| referenced_assets.iter().copied())
.collect(),
),
references: ResolvedVc::cell(
references
.iter()
.flat_map(|(_, _, _, references)| references.iter().copied())
.collect(),
),
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl Chunk for CssChunk {
#[turbo_tasks::function]
......@@ -344,46 +417,6 @@ impl OutputAsset for CssChunk {
rcstr!(".css"),
))
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = self.await?;
let content = this.content.await?;
let should_generate_single_item_chunks = content.chunk_items.len() > 1
&& *this
.chunking_context
.is_dynamic_chunk_content_loading_enabled()
.await?;
let mut references = content
.chunk_items
.iter()
.map(|item| async {
let references = item.references().await?.into_iter().copied();
Ok(if should_generate_single_item_chunks {
Either::Left(
references.chain(std::iter::once(ResolvedVc::upcast(
SingleItemCssChunk::new(*this.chunking_context, **item)
.to_resolved()
.await?,
))),
)
} else {
Either::Right(references)
})
})
.try_flat_join()
.await?;
if *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?
{
references.push(ResolvedVc::upcast(
CssChunkSourceMapAsset::new(self).to_resolved().await?,
));
}
Ok(Vc::cell(references))
}
}
#[turbo_tasks::value_impl]
......@@ -427,7 +460,7 @@ pub struct CssChunkItemContent {
}
#[turbo_tasks::value_trait]
pub trait CssChunkItem: ChunkItem {
pub trait CssChunkItem: ChunkItem + OutputAssetsReference {
#[turbo_tasks::function]
fn content(self: Vc<Self>) -> Vc<CssChunkItemContent>;
}
......@@ -461,7 +494,7 @@ impl Introspectable for CssChunk {
#[turbo_tasks::function]
async fn children(self: Vc<Self>) -> Result<Vc<IntrospectableChildren>> {
let mut children = children_from_output_assets(OutputAsset::references(self))
let mut children = children_from_output_assets(OutputAssetsReference::references(self))
.owned()
.await?;
children.extend(
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
code_builder::{Code, CodeBuilder},
ident::AssetIdent,
introspect::Introspectable,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap},
};
......@@ -80,6 +80,29 @@ impl SingleItemCssChunk {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for SingleItemCssChunk {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let mut references = Vec::new();
if *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?
{
references.push(ResolvedVc::upcast(
SingleItemCssChunkSourceMapAsset::new(self)
.to_resolved()
.await?,
));
}
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl Chunk for SingleItemCssChunk {
#[turbo_tasks::function]
......@@ -107,24 +130,6 @@ impl OutputAsset for SingleItemCssChunk {
rcstr!(".single.css"),
))
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = self.await?;
let mut references = Vec::new();
if *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?
{
references.push(ResolvedVc::upcast(
SingleItemCssChunkSourceMapAsset::new(self)
.to_resolved()
.await?,
));
}
Ok(Vc::cell(references))
}
}
#[turbo_tasks::value_impl]
......
......@@ -5,7 +5,7 @@ use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::ChunkingContext,
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source_map::{GenerateSourceMap, SourceMap},
};
......@@ -25,6 +25,9 @@ impl SingleItemCssChunkSourceMapAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for SingleItemCssChunkSourceMapAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for SingleItemCssChunkSourceMapAsset {
#[turbo_tasks::function]
......
......@@ -5,7 +5,7 @@ use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::ChunkingContext,
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source_map::{GenerateSourceMap, SourceMap},
};
......@@ -25,6 +25,9 @@ impl CssChunkSourceMapAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for CssChunkSourceMapAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for CssChunkSourceMapAsset {
#[turbo_tasks::function]
......
......@@ -18,6 +18,7 @@ use turbopack_core::{
},
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::{ModuleReference, ModuleReferences},
reference_type::{CssReferenceSubType, ReferenceType},
resolve::{origin::ResolveOrigin, parse::Request},
......@@ -287,6 +288,9 @@ struct ModuleChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ModuleChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for ModuleChunkItem {
#[turbo_tasks::function]
......
......@@ -15,7 +15,7 @@ use turbopack_core::{
},
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
version::{Version, VersionedContent},
};
......@@ -41,15 +41,18 @@ pub struct DevHtmlAsset {
}
#[turbo_tasks::value_impl]
impl OutputAsset for DevHtmlAsset {
impl OutputAssetsReference for DevHtmlAsset {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
fn references(self: Vc<Self>) -> Vc<OutputAssetsWithReferenced> {
self.chunk_group()
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for DevHtmlAsset {
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
self.chunk_group().all_assets()
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
}
}
......@@ -176,6 +179,7 @@ impl DevHtmlAsset {
Ok((
asset_with_referenced.assets.await?,
asset_with_referenced.referenced_assets.await?,
asset_with_referenced.references.await?,
))
})
.try_join()
......@@ -183,14 +187,17 @@ impl DevHtmlAsset {
let mut all_assets = Vec::new();
let mut all_referenced_assets = Vec::new();
for (asset, referenced_asset) in all_chunk_groups {
let mut all_references = Vec::new();
for (asset, referenced_asset, reference) in all_chunk_groups {
all_assets.extend(asset);
all_referenced_assets.extend(referenced_asset);
all_references.extend(reference);
}
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(all_assets),
referenced_assets: ResolvedVc::cell(all_referenced_assets),
references: ResolvedVc::cell(all_references),
}
.cell())
}
......
......@@ -10,7 +10,7 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::Asset,
introspect::{Introspectable, IntrospectableChildren, output_asset::IntrospectableOutputAsset},
output::{OutputAsset, OutputAssetsSet},
output::{OutputAsset, OutputAssetsReference, OutputAssetsSet},
};
use super::{
......@@ -107,7 +107,8 @@ async fn expand(
) -> Result<FxIndexMap<RcStr, ResolvedVc<Box<dyn OutputAsset>>>> {
let mut map = FxIndexMap::default();
let mut assets = Vec::new();
let mut queue = VecDeque::with_capacity(32);
let mut queue: VecDeque<ResolvedVc<Box<dyn OutputAssetsReference>>> =
VecDeque::with_capacity(32);
let mut assets_set = FxHashSet::default();
let root_assets_with_path = root_assets
.iter()
......@@ -132,7 +133,7 @@ async fn expand(
}
assets_set.insert(root_asset);
if expanded {
queue.push_back(root_asset.references());
queue.push_back(ResolvedVc::upcast(root_asset));
}
}
}
......@@ -143,15 +144,24 @@ async fn expand(
for sub_path in sub_paths_buffer.into_iter().take(sub_paths) {
assets.push((sub_path, root_asset));
}
queue.push_back(root_asset.references());
queue.push_back(ResolvedVc::upcast(root_asset));
assets_set.insert(root_asset);
}
}
}
while let Some(references) = queue.pop_front() {
for asset in references.await?.iter() {
if assets_set.insert(*asset) {
while let Some(asset) = queue.pop_front() {
let refs = asset.references().await?;
for &reference in refs.references.await?.iter() {
queue.push_back(reference);
}
let ref_assets = refs
.assets
.await?
.into_iter()
.chain(refs.referenced_assets.await?.into_iter());
for &asset in ref_assets {
if assets_set.insert(asset) {
let path = asset.path().await?;
if let Some(sub_path) = root_path.get_path_to(&path) {
let (sub_paths_buffer, sub_paths) = get_sub_paths(sub_path);
......@@ -165,10 +175,10 @@ async fn expand(
true
};
if expanded {
queue.push_back(asset.references());
queue.push_back(ResolvedVc::upcast(asset));
}
for sub_path in sub_paths_buffer.into_iter().take(sub_paths) {
assets.push((sub_path, *asset));
assets.push((sub_path, asset));
}
}
}
......
......@@ -11,7 +11,7 @@ use turbopack_core::{
module_graph::{
ModuleGraph, chunk_group_info::ChunkGroup, module_batch::ChunkableModuleOrBatch,
},
output::{OutputAssets, OutputAssetsWithReferenced},
output::{OutputAssetsReference, OutputAssetsWithReferenced},
};
use crate::{
......@@ -50,6 +50,7 @@ impl AsyncLoaderChunkItem {
return Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
}
.cell());
}
......@@ -150,6 +151,14 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for AsyncLoaderChunkItem {
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssetsWithReferenced> {
self.chunk_group()
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for AsyncLoaderChunkItem {
#[turbo_tasks::function]
......@@ -164,11 +173,6 @@ impl ChunkItem for AsyncLoaderChunkItem {
Ok(ident.with_modifier(self.chunks_data().hash().await?.to_string().into()))
}
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
self.chunk_group().all_assets()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
......
......@@ -3,10 +3,9 @@ use serde::{Deserialize, Serialize};
use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, TryJoinIterExt, Vc, trace::TraceRawVcs};
use turbopack_core::{
chunk::{
ChunkItem, ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo,
ChunkItemOrBatchWithAsyncModuleInfo,
ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo, ChunkItemOrBatchWithAsyncModuleInfo,
},
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
};
use crate::chunk::EcmascriptChunkItemWithAsyncInfo;
......@@ -39,7 +38,7 @@ impl EcmascriptChunkItemOrBatchWithAsyncInfo {
})
}
pub fn references(&self) -> Vc<OutputAssets> {
pub fn references(&self) -> Vc<OutputAssetsWithReferenced> {
match self {
EcmascriptChunkItemOrBatchWithAsyncInfo::ChunkItem(item) => {
item.chunk_item.references()
......@@ -70,13 +69,23 @@ impl EcmascriptChunkBatchWithAsyncInfo {
}
#[turbo_tasks::function]
pub async fn references(&self) -> Result<Vc<OutputAssets>> {
pub async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let mut output_assets = Vec::new();
let mut referenced_output_assets = Vec::new();
let mut references = Vec::new();
// We expect most references to be empty, and avoiding try_join to avoid allocating the Vec
for item in &self.chunk_items {
references.extend(item.chunk_item.references().await?.into_iter().copied());
let r = item.chunk_item.references().await?;
output_assets.extend(r.assets.await?);
referenced_output_assets.extend(r.referenced_assets.await?);
references.extend(r.references.await?);
}
Ok(Vc::cell(references))
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(output_assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
references: ResolvedVc::cell(references),
}
.cell())
}
}
......
......@@ -17,6 +17,7 @@ use turbopack_core::{
code_builder::{Code, CodeBuilder},
error::PrettyPrintError,
issue::{IssueExt, IssueSeverity, StyledString, code_gen::CodeGenerationIssue},
output::OutputAssetsReference,
source_map::utils::{absolute_fileify_source_map, relative_fileify_source_map},
};
......@@ -223,7 +224,7 @@ impl EcmascriptChunkItemWithAsyncInfo {
}
#[turbo_tasks::value_trait]
pub trait EcmascriptChunkItem: ChunkItem {
pub trait EcmascriptChunkItem: ChunkItem + OutputAssetsReference {
#[turbo_tasks::function]
fn content(self: Vc<Self>) -> Vc<EcmascriptChunkItemContent>;
#[turbo_tasks::function]
......
......@@ -10,7 +10,7 @@ use std::fmt::Write;
use anyhow::Result;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks_fs::FileSystem;
use turbopack_core::{
chunk::{Chunk, ChunkItem, ChunkItems, ChunkingContext, ModuleIds},
......@@ -19,7 +19,7 @@ use turbopack_core::{
Introspectable, IntrospectableChildren, module::IntrospectableModule,
utils::children_from_output_assets,
},
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
server_fs::ServerFileSystem,
};
......@@ -66,6 +66,48 @@ impl EcmascriptChunk {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptChunk {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let content = self.content.await?;
let references = content
.chunk_items
.iter()
.map(async |with_info| {
let r = with_info.references().await?;
Ok((
r.assets.await?,
r.referenced_assets.await?,
r.references.await?,
))
})
.try_join()
.await?;
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(
references
.iter()
.flat_map(|(assets, _, _)| assets.into_iter().copied())
.collect(),
),
referenced_assets: ResolvedVc::cell(
references
.iter()
.flat_map(|(_, referenced_assets, _)| referenced_assets.into_iter().copied())
.collect(),
),
references: ResolvedVc::cell(
references
.iter()
.flat_map(|(_, _, references)| references.into_iter().copied())
.collect(),
),
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl Chunk for EcmascriptChunk {
#[turbo_tasks::function]
......@@ -127,18 +169,6 @@ impl Chunk for EcmascriptChunk {
*self.chunking_context
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
let content = self.content.await?;
let referenced_output_assets = content
.chunk_items
.iter()
.map(async |with_info| Ok(with_info.references().await?.into_iter().copied()))
.try_flat_join()
.await?;
Ok(Vc::cell(referenced_output_assets))
}
#[turbo_tasks::function]
fn chunk_items(&self) -> Vc<ChunkItems> {
self.content.included_chunk_items()
......
......@@ -10,6 +10,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
source::Source,
};
......@@ -87,6 +88,9 @@ struct InlinedBytesJsChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for InlinedBytesJsChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for InlinedBytesJsChunkItem {
#[turbo_tasks::function]
......
......@@ -100,6 +100,7 @@ use turbopack_core::{
ident::AssetIdent,
module::{Module, OptionModule},
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::ModuleReferences,
reference_type::InnerAssets,
resolve::{
......@@ -869,6 +870,9 @@ struct ModuleChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ModuleChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for ModuleChunkItem {
#[turbo_tasks::function]
......
......@@ -83,6 +83,7 @@ impl ManifestAsyncModule {
return Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
}
.cell());
}
......@@ -125,7 +126,7 @@ impl Module for ManifestAsyncModule {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> {
let assets = self.chunk_group().all_assets().await?;
let assets = self.chunk_group().expand_all_assets().await?;
Ok(Vc::cell(
assets
......
......@@ -5,7 +5,7 @@ use turbopack_core::{
chunk::{ChunkData, ChunkItem, ChunkType, ChunkingContext, ChunksData},
ident::AssetIdent,
module::Module,
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
};
use super::chunk_asset::ManifestAsyncModule;
......@@ -64,6 +64,14 @@ impl EcmascriptChunkItem for ManifestChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ManifestChunkItem {
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
self.manifest.chunk_group()
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for ManifestChunkItem {
#[turbo_tasks::function]
......@@ -76,11 +84,6 @@ impl ChunkItem for ManifestChunkItem {
self.manifest.content_ident()
}
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
self.manifest.chunk_group().all_assets()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
......
......@@ -12,7 +12,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
};
use super::chunk_asset::ManifestAsyncModule;
......@@ -80,6 +80,14 @@ impl ManifestLoaderChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for ManifestLoaderChunkItem {
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
self.manifest.manifest_chunk_group()
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for ManifestLoaderChunkItem {
#[turbo_tasks::function]
......@@ -92,11 +100,6 @@ impl ChunkItem for ManifestLoaderChunkItem {
self.manifest.content_ident().with_modifier(modifier())
}
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
self.manifest.manifest_chunk_group().all_assets()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
......
......@@ -9,6 +9,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::ModuleReferences,
};
......@@ -111,6 +112,9 @@ struct MergedEcmascriptModuleChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for MergedEcmascriptModuleChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for MergedEcmascriptModuleChunkItem {
#[turbo_tasks::function]
......
......@@ -11,6 +11,7 @@ use turbopack_core::{
ident::{AssetIdent, Layer},
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
raw_module::RawModule,
reference::{ModuleReference, ModuleReferences, TracedModuleReference},
reference_type::ReferenceType,
......@@ -372,6 +373,9 @@ pub struct CachedExternalModuleChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for CachedExternalModuleChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for CachedExternalModuleChunkItem {
#[turbo_tasks::function]
......
......@@ -30,6 +30,7 @@ use turbopack_core::{
issue::IssueSource,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::{ModuleReference, ModuleReferences},
reference_type::CommonJsReferenceSubType,
resolve::{ModuleResolveResult, origin::ResolveOrigin, parse::Request},
......@@ -469,6 +470,9 @@ pub struct RequireContextChunkItem {
map: ResolvedVc<RequireContextMap>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for RequireContextChunkItem {}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for RequireContextChunkItem {
#[turbo_tasks::function]
......
......@@ -4,6 +4,7 @@ use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkingContext},
ident::AssetIdent,
module::Module,
output::OutputAssetsReference,
};
use super::module::EcmascriptModuleFacadeModule;
......@@ -53,6 +54,9 @@ impl EcmascriptChunkItem for EcmascriptModuleFacadeChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptModuleFacadeChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for EcmascriptModuleFacadeChunkItem {
#[turbo_tasks::function]
......
......@@ -4,6 +4,7 @@ use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkingContext},
ident::AssetIdent,
module::Module,
output::OutputAssetsReference,
};
use super::module::EcmascriptModuleLocalsModule;
......@@ -53,6 +54,9 @@ impl EcmascriptChunkItem for EcmascriptModuleLocalsChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptModuleLocalsChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for EcmascriptModuleLocalsChunkItem {
#[turbo_tasks::function]
......
......@@ -6,6 +6,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
};
use super::asset::EcmascriptModulePartAsset;
......@@ -59,6 +60,9 @@ impl EcmascriptChunkItem for EcmascriptModulePartChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for EcmascriptModulePartChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for EcmascriptModulePartChunkItem {
#[turbo_tasks::function]
......@@ -91,6 +95,9 @@ pub(super) struct SideEffectsModuleChunkItem {
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for SideEffectsModuleChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for SideEffectsModuleChunkItem {
#[turbo_tasks::function]
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::{OutputAssets, OutputAssetsWithReferenced},
output::{OutputAssetsReference, OutputAssetsWithReferenced},
};
use super::module::WorkerLoaderModule;
......@@ -80,6 +80,14 @@ impl EcmascriptChunkItem for WorkerLoaderChunkItem {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for WorkerLoaderChunkItem {
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssetsWithReferenced> {
self.chunk_group()
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for WorkerLoaderChunkItem {
#[turbo_tasks::function]
......@@ -92,11 +100,6 @@ impl ChunkItem for WorkerLoaderChunkItem {
self.module.ident()
}
#[turbo_tasks::function]
fn references(self: Vc<Self>) -> Vc<OutputAssets> {
self.chunk_group().all_assets()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
......
......@@ -22,6 +22,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::OutputAssetsReference,
source::Source,
};
use turbopack_ecmascript::{
......@@ -95,6 +96,9 @@ struct JsonChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for JsonChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for JsonChunkItem {
#[turbo_tasks::function]
......
......@@ -20,7 +20,7 @@ use turbopack_core::{
file_source::FileSource,
ident::Layer,
issue::{IssueReporter, IssueSeverity, handle_issues},
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
reference::all_assets_from_entries,
reference_type::ReferenceType,
traced_asset::TracedAsset,
......@@ -150,7 +150,7 @@ async fn to_graph(asset: ResolvedVc<Box<dyn OutputAsset>>, max_depth: usize) ->
let mut result = vec![];
while let Some((depth, asset)) = queue.pop() {
let references = asset.references().await?;
let references = asset.references().all_assets().await?;
let mut indent = String::new();
for _ in 0..depth {
indent.push_str(" ");
......
......@@ -2,15 +2,12 @@
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
use std::{iter::once, thread::available_parallelism};
use std::thread::available_parallelism;
use anyhow::{Result, bail};
use rustc_hash::FxHashMap;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
FxIndexSet, ResolvedVc, TryJoinIterExt, Vc,
graph::{AdjacencyMap, GraphTraversal},
};
use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, Vc};
use turbo_tasks_env::ProcessEnv;
use turbo_tasks_fs::{File, FileSystemPath, to_sys_path};
use turbopack_core::{
......@@ -19,7 +16,7 @@ use turbopack_core::{
chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAsset, EvaluatableAssets},
module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroupEntry},
output::{OutputAsset, OutputAssets, OutputAssetsSet},
output::{ExpandOutputAssetsInput, OutputAsset, OutputAssets, expand_output_assets},
source_map::GenerateSourceMap,
virtual_output::VirtualOutputAsset,
};
......@@ -50,15 +47,6 @@ async fn emit(
Ok(())
}
/// List of the all assets of the "internal" subgraph and a list of boundary
/// assets that are not considered "internal" ("external")
#[derive(Debug)]
#[turbo_tasks::value]
struct SeparatedAssets {
internal_assets: ResolvedVc<OutputAssetsSet>,
external_asset_entrypoints: ResolvedVc<OutputAssetsSet>,
}
/// Extracts the subgraph of "internal" assets (assets within the passes
/// directory). Also lists all boundary assets that are not part of the
/// "internal" subgraph.
......@@ -66,13 +54,25 @@ struct SeparatedAssets {
async fn internal_assets(
intermediate_asset: ResolvedVc<Box<dyn OutputAsset>>,
intermediate_output_path: FileSystemPath,
) -> Result<Vc<OutputAssetsSet>> {
Ok(
*separate_assets_operation(intermediate_asset, intermediate_output_path)
.read_strongly_consistent()
.await?
.internal_assets,
) -> Result<Vc<OutputAssets>> {
let all_assets = expand_output_assets(
std::iter::once(ExpandOutputAssetsInput::Asset(intermediate_asset)),
true,
)
.await?;
let internal_assets = all_assets
.into_iter()
.map(async |asset| {
let path = asset.path().await?;
if path.is_inside_ref(&intermediate_output_path) {
Ok(Some(asset))
} else {
Ok(None)
}
})
.try_flat_join()
.await?;
Ok(Vc::cell(internal_assets))
}
#[turbo_tasks::value(transparent)]
......@@ -100,93 +100,6 @@ async fn internal_assets_for_source_mapping(
Ok(Vc::cell(internal_assets_for_source_mapping))
}
/// Returns a set of "external" assets on the boundary of the "internal"
/// subgraph
#[turbo_tasks::function]
pub async fn external_asset_entrypoints(
module: Vc<Box<dyn EvaluatableAsset>>,
runtime_entries: Vc<EvaluatableAssets>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
intermediate_output_path: FileSystemPath,
) -> Result<Vc<OutputAssetsSet>> {
Ok(*separate_assets_operation(
get_intermediate_asset(chunking_context, module, runtime_entries)
.to_resolved()
.await?,
intermediate_output_path,
)
.read_strongly_consistent()
.await?
.external_asset_entrypoints)
}
/// Splits the asset graph into "internal" assets and boundaries to "external"
/// assets.
#[turbo_tasks::function(operation)]
async fn separate_assets_operation(
intermediate_asset: ResolvedVc<Box<dyn OutputAsset>>,
intermediate_output_path: FileSystemPath,
) -> Result<Vc<SeparatedAssets>> {
let intermediate_output_path = intermediate_output_path.clone();
#[derive(PartialEq, Eq, Hash, Clone, Copy)]
enum Type {
Internal(ResolvedVc<Box<dyn OutputAsset>>),
External(ResolvedVc<Box<dyn OutputAsset>>),
}
let get_asset_children = |asset| {
let intermediate_output_path = intermediate_output_path.clone();
async move {
let Type::Internal(asset) = asset else {
return Ok(Vec::new());
};
asset
.references()
.await?
.iter()
.map(|asset| async {
// Assets within the output directory are considered as "internal" and all
// others as "external". We follow references on "internal" assets, but do not
// look into references of "external" assets, since there are no "internal"
// assets behind "externals"
if asset.path().await?.is_inside_ref(&intermediate_output_path) {
Ok(Type::Internal(*asset))
} else {
Ok(Type::External(*asset))
}
})
.try_join()
.await
}
};
let graph = AdjacencyMap::new()
.skip_duplicates()
.visit(once(Type::Internal(intermediate_asset)), get_asset_children)
.await
.completed()?
.into_inner();
let mut internal_assets = FxIndexSet::default();
let mut external_asset_entrypoints = FxIndexSet::default();
for item in graph.into_postorder_topological() {
match item {
Type::Internal(asset) => {
internal_assets.insert(asset);
}
Type::External(asset) => {
external_asset_entrypoints.insert(asset);
}
}
}
Ok(SeparatedAssets {
internal_assets: ResolvedVc::cell(internal_assets),
external_asset_entrypoints: ResolvedVc::cell(external_asset_entrypoints),
}
.cell())
}
/// Emit a basic package.json that sets the type of the package to commonjs.
/// Currently code generated for Node is CommonJS, while authored code may be
/// ESM, for example.
......
......@@ -439,6 +439,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
modules,
......@@ -457,6 +458,7 @@ impl ChunkingContext for NodeJsChunkingContext {
Ok(ChunkGroupResult {
assets: ResolvedVc::cell(assets),
referenced_assets: ResolvedVc::cell(referenced_output_assets),
references: ResolvedVc::cell(references),
availability_info,
}
.cell())
......@@ -489,6 +491,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let MakeChunkGroupResult {
chunks,
mut referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
......@@ -520,6 +523,7 @@ impl ChunkingContext for NodeJsChunkingContext {
evaluatable_assets,
*module,
Vc::cell(referenced_output_assets),
Vc::cell(references),
module_graph,
*self,
)
......
......@@ -6,7 +6,7 @@ use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{Chunk, ChunkingContext},
introspect::{Introspectable, IntrospectableChildren},
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, SourceMapAsset},
version::VersionedContent,
};
......@@ -75,36 +75,43 @@ impl EcmascriptBuildNodeChunk {
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBuildNodeChunk {
#[turbo_tasks::function]
async fn path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let ident = this.chunk.ident().with_modifier(modifier());
Ok(this
.chunking_context
.chunk_path(Some(Vc::upcast(self)), ident, None, rcstr!(".js")))
}
impl OutputAssetsReference for EcmascriptBuildNodeChunk {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let chunk_references = this.chunk.references().await?;
let include_source_map = *this
.chunking_context
.reference_chunk_source_maps(Vc::upcast(self))
.await?;
let mut references =
Vec::with_capacity(chunk_references.len() + if include_source_map { 1 } else { 0 });
let ref_assets = chunk_references.assets.await?;
let mut assets =
Vec::with_capacity(ref_assets.len() + if include_source_map { 1 } else { 0 });
for reference in &*chunk_references {
references.push(*reference);
}
assets.extend(ref_assets.iter().copied());
if include_source_map {
references.push(ResolvedVc::upcast(self.source_map().to_resolved().await?));
assets.push(ResolvedVc::upcast(self.source_map().to_resolved().await?));
}
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(assets),
referenced_assets: chunk_references.referenced_assets,
references: chunk_references.references,
}
.cell())
}
}
Ok(Vc::cell(references))
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBuildNodeChunk {
#[turbo_tasks::function]
async fn path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let ident = this.chunk.ident().with_modifier(modifier());
Ok(this
.chunking_context
.chunk_path(Some(Vc::upcast(self)), ident, None, rcstr!(".js")))
}
}
......
......@@ -10,7 +10,10 @@ use turbopack_core::{
chunk::{ChunkingContext, EvaluatableAssets, ModuleChunkItemIdExt},
code_builder::{Code, CodeBuilder},
module_graph::ModuleGraph,
output::{OutputAsset, OutputAssets},
output::{
OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsReferences,
OutputAssetsWithReferenced,
},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, SourceMapAsset},
};
use turbopack_ecmascript::{chunk::EcmascriptChunkPlaceable, utils::StringifyJs};
......@@ -27,6 +30,7 @@ pub(crate) struct EcmascriptBuildNodeEntryChunk {
evaluatable_assets: ResolvedVc<EvaluatableAssets>,
exported_module: ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>,
referenced_output_assets: ResolvedVc<OutputAssets>,
references: ResolvedVc<OutputAssetsReferences>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<NodeJsChunkingContext>,
}
......@@ -41,6 +45,7 @@ impl EcmascriptBuildNodeEntryChunk {
evaluatable_assets: ResolvedVc<EvaluatableAssets>,
exported_module: ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>,
referenced_output_assets: ResolvedVc<OutputAssets>,
references: ResolvedVc<OutputAssetsReferences>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<NodeJsChunkingContext>,
) -> Vc<Self> {
......@@ -50,6 +55,7 @@ impl EcmascriptBuildNodeEntryChunk {
evaluatable_assets,
exported_module,
referenced_output_assets,
references,
module_graph,
chunking_context,
}
......@@ -165,16 +171,11 @@ impl ValueToString for EcmascriptBuildNodeEntryChunk {
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBuildNodeEntryChunk {
impl OutputAssetsReference for EcmascriptBuildNodeEntryChunk {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let mut references = vec![ResolvedVc::upcast(
let mut assets = vec![ResolvedVc::upcast(
self.runtime_chunk().to_resolved().await?,
)];
......@@ -183,18 +184,26 @@ impl OutputAsset for EcmascriptBuildNodeEntryChunk {
.reference_chunk_source_maps(Vc::upcast(self))
.await?
{
references.push(ResolvedVc::upcast(self.source_map().to_resolved().await?))
assets.push(ResolvedVc::upcast(self.source_map().to_resolved().await?))
}
let other_chunks = this.other_chunks.await?;
references.extend(other_chunks.iter().copied());
assets.extend(other_chunks.iter().copied());
let referenced_output_assets = this.referenced_output_assets.await?;
for &referenced_output_asset in &*referenced_output_assets {
references.push(referenced_output_asset);
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(assets),
referenced_assets: this.referenced_output_assets,
references: this.references,
}
.cell())
}
}
Ok(Vc::cell(references))
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBuildNodeEntryChunk {
#[turbo_tasks::function]
fn path(&self) -> Vc<FileSystemPath> {
self.path.clone().cell()
}
}
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
chunk::ChunkingContext,
code_builder::{Code, CodeBuilder},
ident::AssetIdent,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, SourceMapAsset},
};
use turbopack_ecmascript::utils::StringifyJs;
......@@ -120,19 +120,9 @@ impl ValueToString for EcmascriptBuildNodeRuntimeChunk {
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBuildNodeRuntimeChunk {
#[turbo_tasks::function]
async fn path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let ident = self.ident_for_path();
Ok(this
.chunking_context
.chunk_path(Some(Vc::upcast(self)), ident, None, rcstr!(".js")))
}
impl OutputAssetsReference for EcmascriptBuildNodeRuntimeChunk {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?;
let mut references = vec![];
......@@ -144,7 +134,22 @@ impl OutputAsset for EcmascriptBuildNodeRuntimeChunk {
references.push(ResolvedVc::upcast(self.source_map().to_resolved().await?))
}
Ok(Vc::cell(references))
Ok(OutputAssetsWithReferenced::from_assets(Vc::cell(
references,
)))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for EcmascriptBuildNodeRuntimeChunk {
#[turbo_tasks::function]
async fn path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let ident = self.ident_for_path();
Ok(this
.chunking_context
.chunk_path(Some(Vc::upcast(self)), ident, None, rcstr!(".js")))
}
}
......
......@@ -7,7 +7,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
source::Source,
};
use turbopack_ecmascript::{
......@@ -106,15 +106,20 @@ struct StaticUrlJsChunkItem {
}
#[turbo_tasks::value_impl]
impl ChunkItem for StaticUrlJsChunkItem {
impl OutputAssetsReference for StaticUrlJsChunkItem {
#[turbo_tasks::function]
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
OutputAssetsWithReferenced::from_assets(Vc::cell(vec![ResolvedVc::upcast(
self.static_asset,
)]))
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for StaticUrlJsChunkItem {
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
Vc::cell(vec![ResolvedVc::upcast(self.static_asset)])
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
......
......@@ -2,7 +2,7 @@ use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::{Asset, AssetContent},
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source::Source,
};
......@@ -26,6 +26,9 @@ impl FixedStaticAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for FixedStaticAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for FixedStaticAsset {
#[turbo_tasks::function]
......
......@@ -5,7 +5,7 @@ use turbo_tasks_fs::{FileContent, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::ChunkingContext,
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source::Source,
};
#[turbo_tasks::value]
......@@ -31,6 +31,9 @@ impl StaticOutputAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for StaticOutputAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for StaticOutputAsset {
#[turbo_tasks::function]
......
......@@ -50,7 +50,7 @@ use turbopack_core::{
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
export_usage::compute_export_usage_info,
},
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
reference_type::{EntryReferenceSubType, ReferenceType},
source::Source,
};
......@@ -541,13 +541,14 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
.asset,
]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
}
.cell()
}
};
let mut seen = FxHashSet::default();
let mut queue: VecDeque<_> = chunks.all_assets().await?.iter().copied().collect();
let mut queue: VecDeque<_> = chunks.expand_all_assets().await?.iter().copied().collect();
let output_path = project_path.clone();
while let Some(asset) = queue.pop_front() {
......@@ -583,7 +584,7 @@ async fn walk_asset(
diff(path.clone(), asset.content()).await?;
}
queue.extend(asset.references().await?.iter().copied());
queue.extend(asset.references().all_assets().await?.iter().copied());
Ok(())
}
......
......@@ -3,11 +3,11 @@ use std::{fs, path::PathBuf};
use criterion::{Bencher, BenchmarkId, Criterion};
use regex::Regex;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TurboTasks, Vc, apply_effects};
use turbo_tasks::{TurboTasks, Vc, apply_effects};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, NullFileSystem};
use turbopack::{
ModuleAssetContext, emit_with_completion_operation,
ModuleAssetContext, emit_assets_into_dir_operation,
module_options::{EcmascriptOptionsContext, ModuleOptionsContext},
};
use turbopack_core::{
......@@ -17,6 +17,7 @@ use turbopack_core::{
file_source::FileSource,
ident::Layer,
rebase::RebasedAsset,
reference::all_assets_from_entry,
reference_type::ReferenceType,
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......@@ -117,12 +118,12 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) {
let module = module_asset_context
.process(Vc::upcast(source), ReferenceType::Undefined)
.module();
let rebased = RebasedAsset::new(module, input_dir, output_dir.clone())
let rebased = RebasedAsset::new(module, input_dir, output_dir.clone());
let assets = all_assets_from_entry(Vc::upcast(rebased))
.to_resolved()
.await?;
let emit_op =
emit_with_completion_operation(ResolvedVc::upcast(rebased), output_dir);
let emit_op = emit_assets_into_dir_operation(assets, output_dir);
emit_op.read_strongly_consistent().await?;
apply_effects(emit_op).await?;
......
......@@ -35,7 +35,7 @@ use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{
ModuleAssetContext,
ecmascript::AnalyzeMode,
emit_with_completion_operation,
emit_assets_into_dir_operation,
module_options::{
CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext,
TypescriptTransformOptions,
......@@ -47,8 +47,9 @@ use turbopack_core::{
environment::{Environment, ExecutionEnvironment, NodeJsEnvironment},
file_source::FileSource,
ident::Layer,
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
rebase::RebasedAsset,
reference::all_assets_from_entry,
reference_type::ReferenceType,
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......@@ -407,8 +408,11 @@ async fn node_file_trace_operation(
let rebased = RebasedAsset::new(module, input_dir.clone(), output_dir.clone())
.to_resolved()
.await?;
let assets = all_assets_from_entry(Vc::upcast(*rebased))
.to_resolved()
.await?;
let emit_op = emit_with_completion_operation(ResolvedVc::upcast(rebased), output_dir.clone());
let emit_op = emit_assets_into_dir_operation(assets, output_dir.clone());
emit_op.read_strongly_consistent().await?;
apply_effects(emit_op).await?;
......@@ -746,7 +750,7 @@ async fn print_graph(asset: ResolvedVc<Box<dyn OutputAsset>>) -> Result<()> {
let mut queue = Vec::new();
queue.push((0, asset));
while let Some((depth, asset)) = queue.pop() {
let references = asset.references().await?;
let references = asset.references().all_assets().await?;
let mut indent = String::new();
for _ in 0..depth {
indent.push_str(" ");
......
......@@ -12,7 +12,7 @@ use turbopack_core::{
ident::AssetIdent,
module::{Module, OptionModule},
module_graph::ModuleGraph,
output::OutputAssets,
output::{OutputAssetsReference, OutputAssetsWithReferenced},
reference::{ModuleReferences, SingleChunkableModuleReference},
reference_type::ReferenceType,
resolve::{ExportUsage, origin::ResolveOrigin, parse::Request},
......@@ -207,16 +207,19 @@ struct ModuleChunkItem {
}
#[turbo_tasks::value_impl]
impl ChunkItem for ModuleChunkItem {
impl OutputAssetsReference for ModuleChunkItem {
#[turbo_tasks::function]
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
async fn references(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let loader_references = self.module.loader().references().await?;
references_to_output_assets(&*loader_references).await
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for ModuleChunkItem {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
let loader_references = self.module.loader().references().await?;
references_to_output_assets(&*loader_references).await
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
......
......@@ -5,7 +5,7 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::ChunkingContext,
output::OutputAsset,
output::{OutputAsset, OutputAssetsReference},
source::Source,
};
......@@ -33,6 +33,9 @@ impl WebAssemblyAsset {
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for WebAssemblyAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for WebAssemblyAsset {
#[turbo_tasks::function]
......
......@@ -8,7 +8,7 @@ use turbopack_core::{
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
output::{OutputAsset, OutputAssets},
output::{OutputAsset, OutputAssetsReference, OutputAssetsWithReferenced},
source::Source,
};
use turbopack_ecmascript::{
......@@ -104,15 +104,18 @@ struct RawModuleChunkItem {
}
#[turbo_tasks::value_impl]
impl ChunkItem for RawModuleChunkItem {
impl OutputAssetsReference for RawModuleChunkItem {
#[turbo_tasks::function]
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
fn references(&self) -> Vc<OutputAssetsWithReferenced> {
OutputAssetsWithReferenced::from_assets(Vc::cell(vec![ResolvedVc::upcast(self.wasm_asset)]))
}
}
#[turbo_tasks::value_impl]
impl ChunkItem for RawModuleChunkItem {
#[turbo_tasks::function]
fn references(&self) -> Result<Vc<OutputAssets>> {
Ok(Vc::cell(vec![ResolvedVc::upcast(self.wasm_asset)]))
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
......
......@@ -12,7 +12,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ReadConsistency, TurboTasks, UpdateInfo, Vc, util::FormatDuration};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::emit_with_completion;
use turbopack::emit_assets_into_dir;
use turbopack_core::{
PROJECT_FILESYSTEM_NAME,
compile_time_info::CompileTimeInfo,
......@@ -21,6 +21,7 @@ use turbopack_core::{
file_source::FileSource,
ident::Layer,
rebase::RebasedAsset,
reference::all_assets_from_entry,
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......@@ -68,7 +69,8 @@ async fn main() -> Result<()> {
)
.module();
let rebased = RebasedAsset::new(module, input, output.clone());
emit_with_completion(Vc::upcast(rebased), output).await?;
let assets = all_assets_from_entry(Vc::upcast(rebased));
emit_assets_into_dir(assets, output).await?;
anyhow::Ok::<Vc<()>>(Default::default())
})
......
use anyhow::Result;
use rustc_hash::FxHashSet;
use turbo_tasks::{ResolvedVc, Vc};
use turbopack_core::output::OutputAsset;
#[turbo_tasks::value(shared)]
pub enum AggregatedGraph {
Leaf(ResolvedVc<Box<dyn OutputAsset>>),
Node {
depth: usize,
content: FxHashSet<ResolvedVc<AggregatedGraph>>,
references: FxHashSet<ResolvedVc<AggregatedGraph>>,
},
}
#[turbo_tasks::value_impl]
impl AggregatedGraph {
#[turbo_tasks::function]
fn leaf(asset: ResolvedVc<Box<dyn OutputAsset>>) -> Vc<Self> {
Self::cell(AggregatedGraph::Leaf(asset))
}
}
impl AggregatedGraph {
fn depth(&self) -> usize {
match self {
AggregatedGraph::Leaf(_) => 0,
AggregatedGraph::Node { depth, .. } => *depth,
}
}
}
#[turbo_tasks::value_impl]
impl AggregatedGraph {
#[turbo_tasks::function]
pub async fn content(self: Vc<Self>) -> Result<Vc<AggregatedGraphNodeContent>> {
Ok(match *self.await? {
AggregatedGraph::Leaf(asset) => AggregatedGraphNodeContent::Asset(asset).cell(),
AggregatedGraph::Node { ref content, .. } => {
AggregatedGraphNodeContent::Children(content.clone()).cell()
}
})
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<AggregatedGraphsSet>> {
Ok(match &*self.await? {
AggregatedGraph::Leaf(asset) => {
let mut refs = FxHashSet::default();
for reference in asset.references().await? {
if asset != reference {
refs.insert(AggregatedGraph::leaf(**reference).to_resolved().await?);
}
}
AggregatedGraphsSet { set: refs }.cell()
}
AggregatedGraph::Node { references, .. } => {
let mut set = FxHashSet::default();
for item in references
.iter()
.map(|&reference| aggregate_more(*reference))
.collect::<Vec<_>>()
.into_iter()
{
set.insert(item.to_resolved().await?);
}
AggregatedGraphsSet { set }.cell()
}
})
}
#[turbo_tasks::function]
async fn cost(self: Vc<Self>) -> Result<Vc<AggregationCost>> {
Ok(match *self.await? {
AggregatedGraph::Leaf(asset) => AggregationCost(asset.references().await?.len()).cell(),
AggregatedGraph::Node { ref references, .. } => {
AggregationCost(references.len()).cell()
}
})
}
#[turbo_tasks::function]
async fn valued_references(self: Vc<Self>) -> Result<Vc<AggregatedGraphsValuedReferences>> {
let self_cost = self.cost().await?.0;
let mut inner = FxHashSet::default();
let mut outer = FxHashSet::default();
let mut references = FxHashSet::default();
for (reference, cost) in self
.references()
.await?
.set
.iter()
.map(|&reference| (reference, reference.cost()))
.collect::<Vec<_>>()
{
let cost = cost.await?.0;
if cost == 0 {
inner.insert(reference);
} else if cost > self_cost {
references.insert(reference);
} else {
outer.insert(reference);
}
}
Ok(AggregatedGraphsValuedReferences {
inner,
outer,
references,
}
.cell())
}
}
#[turbo_tasks::function]
pub async fn aggregate(asset: Vc<Box<dyn OutputAsset>>) -> Result<Vc<AggregatedGraph>> {
let mut current = AggregatedGraph::leaf(asset);
loop {
if current.references().await?.set.is_empty() {
return Ok(current);
}
current = aggregate_more(current);
}
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Hash, Debug)]
struct AggregationCost(usize);
#[turbo_tasks::function]
async fn aggregate_more(node: ResolvedVc<AggregatedGraph>) -> Result<Vc<AggregatedGraph>> {
let node_data = node.await?;
let depth = node_data.depth();
let mut in_progress = FxHashSet::default();
let mut content = FxHashSet::default();
let mut references = FxHashSet::default();
in_progress.insert(node);
// only one kind of aggregation can't eliminate cycles with that
// number of nodes. Alternating the aggregation will get rid of all
// cycles
let aggregation = if depth > 0 && depth.is_multiple_of(2) {
3
} else {
2
};
for _ in 0..aggregation {
for &node in in_progress.iter() {
content.insert(node);
}
let valued_refs = in_progress
.drain()
.map(|node| node.valued_references())
.collect::<Vec<_>>();
for valued_refs in valued_refs {
let valued_refs = valued_refs.await?;
for &reference in valued_refs.inner.iter() {
content.insert(reference);
}
for &reference in valued_refs.references.iter() {
if content.contains(&reference) {
continue;
}
references.insert(reference);
}
for &reference in valued_refs.outer.iter() {
if content.contains(&reference) {
continue;
}
references.remove(&reference);
in_progress.insert(reference);
}
}
}
for node in in_progress.into_iter() {
references.insert(node);
}
Ok(AggregatedGraph::Node {
depth: depth + 1,
content,
references,
}
.cell())
}
#[turbo_tasks::value(shared)]
struct AggregatedGraphsSet {
pub set: FxHashSet<ResolvedVc<AggregatedGraph>>,
}
#[turbo_tasks::value(shared)]
pub enum AggregatedGraphNodeContent {
Asset(ResolvedVc<Box<dyn OutputAsset>>),
Children(FxHashSet<ResolvedVc<AggregatedGraph>>),
}
#[turbo_tasks::value(shared)]
struct AggregatedGraphsValuedReferences {
pub inner: FxHashSet<ResolvedVc<AggregatedGraph>>,
pub outer: FxHashSet<ResolvedVc<AggregatedGraph>>,
pub references: FxHashSet<ResolvedVc<AggregatedGraph>>,
}
......@@ -9,7 +9,6 @@
pub mod evaluate_context;
pub mod global_module_ids;
mod graph;
pub mod module_options;
pub mod transition;
......@@ -21,11 +20,10 @@ use ecmascript::{
references::{FollowExportsResult, follow_reexports},
side_effect_optimization::facade::module::EcmascriptModuleFacadeModule,
};
use graph::{AggregatedGraph, AggregatedGraphNodeContent, aggregate};
use module_options::{ModuleOptions, ModuleOptionsContext, ModuleRuleEffect, ModuleType};
use tracing::{Instrument, field::Empty};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks_fs::{
FileSystemPath,
glob::{Glob, GlobOptions},
......@@ -40,7 +38,7 @@ use turbopack_core::{
issue::{IssueExt, IssueSource, module::ModuleIssue},
module::Module,
node_addon_module::NodeAddonModule,
output::OutputAsset,
output::{ExpandedOutputAssets, OutputAsset},
raw_module::RawModule,
reference_type::{
CssReferenceSubType, EcmaScriptModulesReferenceSubType, ImportContext, ImportWithType,
......@@ -973,57 +971,6 @@ impl AssetContext for ModuleAssetContext {
}
}
#[turbo_tasks::function]
pub async fn emit_with_completion(
asset: Vc<Box<dyn OutputAsset>>,
output_dir: FileSystemPath,
) -> Result<()> {
emit_assets_aggregated(asset, output_dir)
.as_side_effect()
.await
}
#[turbo_tasks::function(operation)]
pub fn emit_with_completion_operation(
asset: ResolvedVc<Box<dyn OutputAsset>>,
output_dir: FileSystemPath,
) -> Vc<()> {
emit_with_completion(*asset, output_dir)
}
#[turbo_tasks::function]
async fn emit_assets_aggregated(
asset: Vc<Box<dyn OutputAsset>>,
output_dir: FileSystemPath,
) -> Result<()> {
let aggregated = aggregate(asset);
emit_aggregated_assets(aggregated, output_dir)
.as_side_effect()
.await
}
#[turbo_tasks::function]
async fn emit_aggregated_assets(
aggregated: Vc<AggregatedGraph>,
output_dir: FileSystemPath,
) -> Result<()> {
match &*aggregated.content().await? {
AggregatedGraphNodeContent::Asset(asset) => {
emit_asset_into_dir(**asset, output_dir)
.as_side_effect()
.await?;
}
AggregatedGraphNodeContent::Children(children) => {
for aggregated in children {
emit_aggregated_assets(**aggregated, output_dir.clone())
.as_side_effect()
.await?;
}
}
}
Ok(())
}
#[turbo_tasks::function]
pub async fn emit_asset(asset: Vc<Box<dyn OutputAsset>>) -> Result<()> {
asset
......@@ -1047,6 +994,32 @@ pub async fn emit_asset_into_dir(
Ok(())
}
#[turbo_tasks::function]
pub async fn emit_assets_into_dir(
assets: Vc<ExpandedOutputAssets>,
output_dir: FileSystemPath,
) -> Result<()> {
let assets = assets.await?;
let paths = assets.iter().map(|&asset| asset.path()).try_join().await?;
for (&asset, path) in assets.iter().zip(paths.iter()) {
if path.is_inside_ref(&output_dir) {
emit_asset(*asset).as_side_effect().await?;
}
}
Ok(())
}
#[turbo_tasks::function(operation)]
pub async fn emit_assets_into_dir_operation(
assets: ResolvedVc<ExpandedOutputAssets>,
output_dir: FileSystemPath,
) -> Result<Vc<()>> {
emit_assets_into_dir(*assets, output_dir)
.as_side_effect()
.await?;
Ok(Vc::cell(()))
}
/// Replaces the externals in the result with `ExternalModuleAsset` instances.
pub async fn replace_external(
name: &RcStr,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment