Unverified Commit b94dfd0a authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: Single-graph-traversal and migrate next/dynamic (#73222)

Closes PACK-3535

- Adds some infrastructure for the single-graph-traversal stuff, ~~focusing on dev with layout segment optimization for now~~
- Migrate next/dynamic collection to it
- This shouldn't make any observable change to the output
parent 778ee4fc
...@@ -3951,9 +3951,11 @@ name = "next-api" ...@@ -3951,9 +3951,11 @@ name = "next-api"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"auto-hash-map",
"futures", "futures",
"indexmap 2.5.0", "indexmap 2.5.0",
"next-core", "next-core",
"petgraph",
"regex", "regex",
"serde", "serde",
"serde_json", "serde_json",
...@@ -4633,6 +4635,8 @@ checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" ...@@ -4633,6 +4635,8 @@ checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
dependencies = [ dependencies = [
"fixedbitset", "fixedbitset",
"indexmap 1.9.3", "indexmap 1.9.3",
"serde",
"serde_derive",
] ]
[[package]] [[package]]
......
...@@ -14,9 +14,11 @@ workspace = true ...@@ -14,9 +14,11 @@ workspace = true
[dependencies] [dependencies]
anyhow = { workspace = true, features = ["backtrace"] } anyhow = { workspace = true, features = ["backtrace"] }
auto-hash-map = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
next-core = { workspace = true } next-core = { workspace = true }
petgraph = { workspace = true, features = ["serde-1"]}
regex = { workspace = true } regex = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
......
...@@ -40,8 +40,8 @@ use serde::{Deserialize, Serialize}; ...@@ -40,8 +40,8 @@ use serde::{Deserialize, Serialize};
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
fxindexmap, fxindexset, trace::TraceRawVcs, Completion, FxIndexMap, FxIndexSet, ResolvedVc, fxindexmap, fxindexset, trace::TraceRawVcs, Completion, FxIndexSet, ResolvedVc, TryJoinIterExt,
TryJoinIterExt, Value, ValueToString, Vc, Value, ValueToString, Vc,
}; };
use turbo_tasks_env::{CustomProcessEnv, ProcessEnv}; use turbo_tasks_env::{CustomProcessEnv, ProcessEnv};
use turbo_tasks_fs::{File, FileContent, FileSystemPath}; use turbo_tasks_fs::{File, FileContent, FileSystemPath};
...@@ -69,12 +69,10 @@ use turbopack_core::{ ...@@ -69,12 +69,10 @@ use turbopack_core::{
use turbopack_ecmascript::resolve::cjs_resolve; use turbopack_ecmascript::resolve::cjs_resolve;
use crate::{ use crate::{
dynamic_imports::{ dynamic_imports::{collect_chunk_group, collect_evaluated_chunk_group},
collect_chunk_group, collect_evaluated_chunk_group, collect_next_dynamic_imports,
VisitedDynamicImportModules,
},
font::create_font_manifest, font::create_font_manifest,
loadable_manifest::create_react_loadable_manifest, loadable_manifest::create_react_loadable_manifest,
module_graph::get_reduced_graphs_for_endpoint,
nft_json::NftJsonAsset, nft_json::NftJsonAsset,
paths::{ paths::{
all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root,
...@@ -912,7 +910,7 @@ impl AppEndpoint { ...@@ -912,7 +910,7 @@ impl AppEndpoint {
None None
}; };
let (client_dynamic_imports, client_references, client_references_chunks) = let (next_dynamic_imports, client_references, client_references_chunks) =
if process_client_components { if process_client_components {
let client_shared_chunk_group = get_app_client_shared_chunk_group( let client_shared_chunk_group = get_app_client_shared_chunk_group(
AssetIdent::from_path(this.app_project.project().project_path()) AssetIdent::from_path(this.app_project.project().project_path())
...@@ -933,6 +931,15 @@ impl AppEndpoint { ...@@ -933,6 +931,15 @@ impl AppEndpoint {
} }
let client_shared_availability_info = client_shared_chunk_group.availability_info; let client_shared_availability_info = client_shared_chunk_group.availability_info;
let reduced_graphs = get_reduced_graphs_for_endpoint(
this.app_project.project(),
*rsc_entry,
Vc::upcast(this.app_project.client_module_context()),
);
let next_dynamic_imports = reduced_graphs
.get_next_dynamic_imports_for_endpoint(*rsc_entry)
.await?;
let client_references = { let client_references = {
let ServerEntries { let ServerEntries {
server_component_entries, server_component_entries,
...@@ -961,32 +968,6 @@ impl AppEndpoint { ...@@ -961,32 +968,6 @@ impl AppEndpoint {
}; };
let client_references_cell = client_references.clone().cell(); let client_references_cell = client_references.clone().cell();
let client_dynamic_imports = {
let mut client_dynamic_imports = FxIndexMap::default();
let mut visited_modules = VisitedDynamicImportModules::empty();
for refs in client_references
.client_references_by_server_component
.values()
{
let result = collect_next_dynamic_imports(
refs.iter().map(|v| **v).collect(),
Vc::upcast(this.app_project.client_module_context()),
visited_modules,
)
.await?;
client_dynamic_imports.extend(
result
.client_dynamic_imports
.iter()
.map(|(k, v)| (*k, v.clone())),
);
visited_modules = *result.visited_modules;
}
client_dynamic_imports
};
let client_references_chunks = get_app_client_references_chunks( let client_references_chunks = get_app_client_references_chunks(
client_references_cell, client_references_cell,
client_chunking_context, client_chunking_context,
...@@ -1130,7 +1111,7 @@ impl AppEndpoint { ...@@ -1130,7 +1111,7 @@ impl AppEndpoint {
} }
( (
Some(client_dynamic_imports), Some(next_dynamic_imports),
Some(client_references_cell), Some(client_references_cell),
Some(client_references_chunks), Some(client_references_chunks),
) )
...@@ -1310,19 +1291,11 @@ impl AppEndpoint { ...@@ -1310,19 +1291,11 @@ impl AppEndpoint {
.await?; .await?;
server_assets.insert(app_paths_manifest_output); server_assets.insert(app_paths_manifest_output);
// create react-loadable-manifest for next/dynamic
let mut dynamic_import_modules = collect_next_dynamic_imports(
vec![*ResolvedVc::upcast(app_entry.rsc_entry)],
Vc::upcast(this.app_project.client_module_context()),
VisitedDynamicImportModules::empty(),
)
.await?
.client_dynamic_imports
.clone();
dynamic_import_modules.extend(client_dynamic_imports.into_iter().flatten());
let dynamic_import_entries = collect_evaluated_chunk_group( let dynamic_import_entries = collect_evaluated_chunk_group(
Vc::upcast(client_chunking_context), Vc::upcast(client_chunking_context),
dynamic_import_modules, next_dynamic_imports
.as_deref()
.unwrap_or(&Default::default()),
) )
.await?; .await?;
let loadable_manifest_output = create_react_loadable_manifest( let loadable_manifest_output = create_react_loadable_manifest(
...@@ -1369,18 +1342,11 @@ impl AppEndpoint { ...@@ -1369,18 +1342,11 @@ impl AppEndpoint {
// create react-loadable-manifest for next/dynamic // create react-loadable-manifest for next/dynamic
let availability_info = Value::new(AvailabilityInfo::Root); let availability_info = Value::new(AvailabilityInfo::Root);
let mut dynamic_import_modules = collect_next_dynamic_imports(
vec![*ResolvedVc::upcast(app_entry.rsc_entry)],
Vc::upcast(this.app_project.client_module_context()),
VisitedDynamicImportModules::empty(),
)
.await?
.client_dynamic_imports
.clone();
dynamic_import_modules.extend(client_dynamic_imports.into_iter().flatten());
let dynamic_import_entries = collect_chunk_group( let dynamic_import_entries = collect_chunk_group(
Vc::upcast(client_chunking_context), Vc::upcast(client_chunking_context),
dynamic_import_modules, next_dynamic_imports
.as_deref()
.unwrap_or(&Default::default()),
availability_info, availability_info,
) )
.await?; .await?;
......
This diff is collapsed.
...@@ -12,6 +12,7 @@ pub mod global_module_id_strategy; ...@@ -12,6 +12,7 @@ pub mod global_module_id_strategy;
mod instrumentation; mod instrumentation;
mod loadable_manifest; mod loadable_manifest;
mod middleware; mod middleware;
mod module_graph;
mod nft_json; mod nft_json;
mod pages; mod pages;
pub mod paths; pub mod paths;
......
This diff is collapsed.
...@@ -63,12 +63,10 @@ use turbopack_ecmascript::resolve::esm_resolve; ...@@ -63,12 +63,10 @@ use turbopack_ecmascript::resolve::esm_resolve;
use turbopack_nodejs::NodeJsChunkingContext; use turbopack_nodejs::NodeJsChunkingContext;
use crate::{ use crate::{
dynamic_imports::{ dynamic_imports::{collect_chunk_group, collect_evaluated_chunk_group, DynamicImportedChunks},
collect_chunk_group, collect_evaluated_chunk_group, collect_next_dynamic_imports,
DynamicImportedChunks, VisitedDynamicImportModules,
},
font::create_font_manifest, font::create_font_manifest,
loadable_manifest::create_react_loadable_manifest, loadable_manifest::create_react_loadable_manifest,
module_graph::get_reduced_graphs_for_endpoint,
nft_json::NftJsonAsset, nft_json::NftJsonAsset,
paths::{ paths::{
all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root,
...@@ -846,14 +844,14 @@ impl PageEndpoint { ...@@ -846,14 +844,14 @@ impl PageEndpoint {
runtime, runtime,
} = *self.internal_ssr_chunk_module().await?; } = *self.internal_ssr_chunk_module().await?;
let dynamic_import_modules = collect_next_dynamic_imports( let reduced_graphs = get_reduced_graphs_for_endpoint(
vec![*ResolvedVc::upcast(ssr_module)], this.pages_project.project(),
this.pages_project.client_module_context(), *ssr_module,
VisitedDynamicImportModules::empty(), Vc::upcast(this.pages_project.client_module_context()),
) );
.await? let next_dynamic_imports = reduced_graphs
.client_dynamic_imports .get_next_dynamic_imports_for_endpoint(*ssr_module)
.clone(); .await?;
let is_edge = matches!(runtime, NextRuntime::Edge); let is_edge = matches!(runtime, NextRuntime::Edge);
if is_edge { if is_edge {
...@@ -876,7 +874,7 @@ impl PageEndpoint { ...@@ -876,7 +874,7 @@ impl PageEndpoint {
this.pages_project.project().client_chunking_context(); this.pages_project.project().client_chunking_context();
let dynamic_import_entries = collect_evaluated_chunk_group( let dynamic_import_entries = collect_evaluated_chunk_group(
Vc::upcast(client_chunking_context), Vc::upcast(client_chunking_context),
dynamic_import_modules, &next_dynamic_imports,
) )
.await? .await?
.to_resolved() .to_resolved()
...@@ -911,7 +909,7 @@ impl PageEndpoint { ...@@ -911,7 +909,7 @@ impl PageEndpoint {
this.pages_project.project().client_chunking_context(); this.pages_project.project().client_chunking_context();
let dynamic_import_entries = collect_chunk_group( let dynamic_import_entries = collect_chunk_group(
Vc::upcast(client_chunking_context), Vc::upcast(client_chunking_context),
dynamic_import_modules, &next_dynamic_imports,
Value::new(AvailabilityInfo::Root), Value::new(AvailabilityInfo::Root),
) )
.await? .await?
......
...@@ -47,7 +47,7 @@ use turbopack_core::{ ...@@ -47,7 +47,7 @@ use turbopack_core::{
diagnostics::DiagnosticExt, diagnostics::DiagnosticExt,
file_source::FileSource, file_source::FileSource,
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString}, issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
module::Modules, module::{Module, Modules},
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
resolve::{find_context_file, FindContextFileResult}, resolve::{find_context_file, FindContextFileResult},
source_map::OptionSourceMap, source_map::OptionSourceMap,
...@@ -68,7 +68,7 @@ use crate::{ ...@@ -68,7 +68,7 @@ use crate::{
instrumentation::InstrumentationEndpoint, instrumentation::InstrumentationEndpoint,
middleware::MiddlewareEndpoint, middleware::MiddlewareEndpoint,
pages::PagesProject, pages::PagesProject,
route::{Endpoint, Route}, route::{AppPageRoute, Endpoint, Route},
versioned_content_map::{OutputAssetsOperation, VersionedContentMap}, versioned_content_map::{OutputAssetsOperation, VersionedContentMap},
}; };
...@@ -666,6 +666,75 @@ impl Project { ...@@ -666,6 +666,75 @@ impl Project {
get_client_compile_time_info(self.browserslist_query.clone(), self.define_env.client()) get_client_compile_time_info(self.browserslist_query.clone(), self.define_env.client())
} }
#[turbo_tasks::function]
pub async fn get_all_entries(self: Vc<Self>) -> Result<Vc<Modules>> {
let mut modules = Vec::new();
async fn add_endpoint(
endpoint: Vc<Box<dyn Endpoint>>,
modules: &mut Vec<ResolvedVc<Box<dyn Module>>>,
) -> Result<()> {
let root_modules = endpoint.root_modules().await?;
modules.extend(root_modules.iter().copied());
Ok(())
}
modules.extend(self.client_main_modules().await?.iter().copied());
let entrypoints = self.entrypoints().await?;
modules.extend(self.client_main_modules().await?.iter().copied());
add_endpoint(*entrypoints.pages_error_endpoint, &mut modules).await?;
add_endpoint(*entrypoints.pages_app_endpoint, &mut modules).await?;
add_endpoint(*entrypoints.pages_document_endpoint, &mut modules).await?;
if let Some(middleware) = &entrypoints.middleware {
add_endpoint(middleware.endpoint, &mut modules).await?;
}
if let Some(instrumentation) = &entrypoints.instrumentation {
let node_js = instrumentation.node_js;
let edge = instrumentation.edge;
add_endpoint(node_js, &mut modules).await?;
add_endpoint(edge, &mut modules).await?;
}
for (_, route) in entrypoints.routes.iter() {
match route {
Route::Page {
html_endpoint,
data_endpoint: _,
} => {
add_endpoint(**html_endpoint, &mut modules).await?;
}
Route::PageApi { endpoint } => {
add_endpoint(**endpoint, &mut modules).await?;
}
Route::AppPage(page_routes) => {
for AppPageRoute {
original_name: _,
html_endpoint,
rsc_endpoint: _,
} in page_routes
{
add_endpoint(*html_endpoint, &mut modules).await?;
}
}
Route::AppRoute {
original_name: _,
endpoint,
} => {
add_endpoint(**endpoint, &mut modules).await?;
}
Route::Conflict => {
tracing::info!("WARN: conflict");
}
}
}
Ok(Vc::cell(modules))
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub(super) async fn server_compile_time_info(self: Vc<Self>) -> Result<Vc<CompileTimeInfo>> { pub(super) async fn server_compile_time_info(self: Vc<Self>) -> Result<Vc<CompileTimeInfo>> {
let this = self.await?; let this = self.await?;
......
#![allow(rustdoc::private_intra_doc_links)] #![allow(rustdoc::private_intra_doc_links)]
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{ChunkGroupType, ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
ident::AssetIdent, ident::AssetIdent,
module::Module, module::Module,
reference::{ModuleReferences, SingleModuleReference}, reference::{ModuleReference, ModuleReferences},
resolve::ModuleResolveResult,
}; };
use turbopack_ecmascript::chunk::EcmascriptChunkPlaceable; use turbopack_ecmascript::chunk::EcmascriptChunkPlaceable;
...@@ -70,20 +72,20 @@ impl Module for EcmascriptClientReferenceModule { ...@@ -70,20 +72,20 @@ impl Module for EcmascriptClientReferenceModule {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(&self) -> Result<Vc<ModuleReferences>> { async fn references(&self) -> Result<Vc<ModuleReferences>> {
let client_module = ResolvedVc::upcast(self.client_module);
let ssr_module = ResolvedVc::upcast(self.ssr_module);
Ok(Vc::cell(vec![ Ok(Vc::cell(vec![
ResolvedVc::upcast( ResolvedVc::upcast(
SingleModuleReference::new( EcmascriptClientReference::new(
*client_module, *ResolvedVc::upcast(self.client_module),
ChunkGroupType::Evaluated,
ecmascript_client_reference_client_ref_modifier(), ecmascript_client_reference_client_ref_modifier(),
) )
.to_resolved() .to_resolved()
.await?, .await?,
), ),
ResolvedVc::upcast( ResolvedVc::upcast(
SingleModuleReference::new( EcmascriptClientReference::new(
*ssr_module, *ResolvedVc::upcast(self.ssr_module),
ChunkGroupType::Entry,
ecmascript_client_reference_ssr_ref_modifier(), ecmascript_client_reference_ssr_ref_modifier(),
) )
.to_resolved() .to_resolved()
...@@ -101,3 +103,55 @@ impl Asset for EcmascriptClientReferenceModule { ...@@ -101,3 +103,55 @@ impl Asset for EcmascriptClientReferenceModule {
bail!("EcmascriptClientReferenceModule has no content") bail!("EcmascriptClientReferenceModule has no content")
} }
} }
#[turbo_tasks::value]
pub(crate) struct EcmascriptClientReference {
module: ResolvedVc<Box<dyn Module>>,
ty: ChunkGroupType,
description: ResolvedVc<RcStr>,
}
#[turbo_tasks::value_impl]
impl EcmascriptClientReference {
#[turbo_tasks::function]
pub fn new(
module: ResolvedVc<Box<dyn Module>>,
ty: ChunkGroupType,
description: ResolvedVc<RcStr>,
) -> Vc<Self> {
Self::cell(EcmascriptClientReference {
module,
ty,
description,
})
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for EcmascriptClientReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Isolated {
_ty: self.ty,
// TODO use proper values here
_merge_tag: None,
_chunking_context: None,
}))
}
}
#[turbo_tasks::value_impl]
impl ModuleReference for EcmascriptClientReference {
#[turbo_tasks::function]
fn resolve_reference(&self) -> Vc<ModuleResolveResult> {
ModuleResolveResult::module(self.module).cell()
}
}
#[turbo_tasks::value_impl]
impl ValueToString for EcmascriptClientReference {
#[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> {
*self.description
}
}
...@@ -7,12 +7,14 @@ use turbo_tasks::{ResolvedVc, Value, ValueToString, Vc}; ...@@ -7,12 +7,14 @@ use turbo_tasks::{ResolvedVc, Value, ValueToString, Vc};
use turbo_tasks_fs::File; use turbo_tasks_fs::File;
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkableModule, ChunkingContext}, chunk::{
AsyncModuleInfo, ChunkGroupType, ChunkItem, ChunkType, ChunkableModule, ChunkingContext,
},
code_builder::CodeBuilder, code_builder::CodeBuilder,
context::AssetContext, context::AssetContext,
ident::AssetIdent, ident::AssetIdent,
module::Module, module::Module,
reference::{ModuleReferences, SingleModuleReference}, reference::ModuleReferences,
reference_type::ReferenceType, reference_type::ReferenceType,
virtual_source::VirtualSource, virtual_source::VirtualSource,
}; };
...@@ -24,7 +26,10 @@ use turbopack_ecmascript::{ ...@@ -24,7 +26,10 @@ use turbopack_ecmascript::{
utils::StringifyJs, utils::StringifyJs,
}; };
use super::ecmascript_client_reference_module::EcmascriptClientReferenceModule; use crate::next_client_reference::{
ecmascript_client_reference::ecmascript_client_reference_module::EcmascriptClientReference,
EcmascriptClientReferenceModule,
};
/// A [`EcmascriptClientReferenceProxyModule`] is used in RSC to represent /// A [`EcmascriptClientReferenceProxyModule`] is used in RSC to represent
/// a client or SSR asset. /// a client or SSR asset.
...@@ -186,13 +191,18 @@ impl Module for EcmascriptClientReferenceProxyModule { ...@@ -186,13 +191,18 @@ impl Module for EcmascriptClientReferenceProxyModule {
.await? .await?
.iter() .iter()
.copied() .copied()
// TODO this will break once ChunkingType::Isolated is properly implemented.
//
// We should instead merge EcmascriptClientReferenceProxyModule and
// EcmascriptClientReferenceModule into a single module
.chain(once(ResolvedVc::upcast( .chain(once(ResolvedVc::upcast(
SingleModuleReference::new( EcmascriptClientReference::new(
Vc::upcast(EcmascriptClientReferenceModule::new( Vc::upcast(EcmascriptClientReferenceModule::new(
**server_module_ident, **server_module_ident,
**client_module, **client_module,
**ssr_module, **ssr_module,
)), )),
ChunkGroupType::Entry,
client_reference_description(), client_reference_description(),
) )
.to_resolved() .to_resolved()
......
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet, VecDeque};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_tasks_macros::{TraceRawVcs, ValueDebugFormat}; use turbo_tasks_macros::{TraceRawVcs, ValueDebugFormat};
...@@ -88,6 +88,21 @@ where ...@@ -88,6 +88,21 @@ where
} }
} }
/// Returns an owned iterator over all edges (node pairs) in breadth first order,
/// starting from the roots.
pub fn into_breadth_first_edges(self) -> IntoBreadthFirstEdges<T> {
IntoBreadthFirstEdges {
adjacency_map: self.adjacency_map,
stack: self
.roots
.into_iter()
.rev()
.map(|root| (None, root))
.collect(),
visited: HashSet::new(),
}
}
/// Returns an iterator over the nodes in reverse topological order, /// Returns an iterator over the nodes in reverse topological order,
/// starting from the roots. /// starting from the roots.
pub fn reverse_topological(&self) -> ReverseTopologicalIter<T> { pub fn reverse_topological(&self) -> ReverseTopologicalIter<T> {
...@@ -174,6 +189,42 @@ where ...@@ -174,6 +189,42 @@ where
} }
} }
pub struct IntoBreadthFirstEdges<T>
where
T: Eq + std::hash::Hash + Clone,
{
adjacency_map: HashMap<T, Vec<T>>,
stack: VecDeque<(Option<T>, T)>,
visited: HashSet<T>,
}
impl<T> Iterator for IntoBreadthFirstEdges<T>
where
T: Eq + std::hash::Hash + Clone,
{
type Item = (Option<T>, T);
fn next(&mut self) -> Option<Self::Item> {
let (parent, current) = self.stack.pop_front()?;
let Some(neighbors) = self.adjacency_map.get(&current) else {
self.visited.insert(current.clone());
return Some((parent, current));
};
if self.visited.insert(current.clone()) {
self.stack.extend(
neighbors
.iter()
.rev()
.map(|neighbor| (Some(current.clone()), neighbor.clone())),
);
}
Some((parent, current))
}
}
/// An iterator over the nodes of a graph in reverse topological order, starting /// An iterator over the nodes of a graph in reverse topological order, starting
/// from the roots. /// from the roots.
pub struct ReverseTopologicalIter<'graph, T> pub struct ReverseTopologicalIter<'graph, T>
......
...@@ -22,8 +22,6 @@ use crate::{ ...@@ -22,8 +22,6 @@ use crate::{
Copy, Copy,
PartialEq, PartialEq,
Eq, Eq,
PartialOrd,
Ord,
Hash, Hash,
Serialize, Serialize,
Deserialize, Deserialize,
...@@ -36,6 +34,24 @@ pub enum MinifyType { ...@@ -36,6 +34,24 @@ pub enum MinifyType {
NoMinify, NoMinify,
} }
#[derive(
Debug,
TaskInput,
Clone,
Copy,
PartialEq,
Eq,
Hash,
Serialize,
Deserialize,
TraceRawVcs,
DeterministicHash,
)]
pub enum ChunkGroupType {
Entry,
Evaluated,
}
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct ChunkGroupResult { pub struct ChunkGroupResult {
pub assets: ResolvedVc<OutputAssets>, pub assets: ResolvedVc<OutputAssets>,
......
...@@ -34,7 +34,8 @@ use turbo_tasks_hash::DeterministicHash; ...@@ -34,7 +34,8 @@ use turbo_tasks_hash::DeterministicHash;
use self::{availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItems}; use self::{availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItems};
pub use self::{ pub use self::{
chunking_context::{ chunking_context::{
ChunkGroupResult, ChunkingContext, ChunkingContextExt, EntryChunkGroupResult, MinifyType, ChunkGroupResult, ChunkGroupType, ChunkingContext, ChunkingContextExt,
EntryChunkGroupResult, MinifyType,
}, },
data::{ChunkData, ChunkDataOption, ChunksData}, data::{ChunkData, ChunkDataOption, ChunksData},
evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets}, evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets},
...@@ -154,7 +155,6 @@ pub trait OutputChunk: Asset { ...@@ -154,7 +155,6 @@ pub trait OutputChunk: Asset {
/// Specifies how a chunk interacts with other chunks when building a chunk /// Specifies how a chunk interacts with other chunks when building a chunk
/// group /// group
#[derive( #[derive(
Copy,
Debug, Debug,
Default, Default,
Clone, Clone,
...@@ -177,6 +177,14 @@ pub enum ChunkingType { ...@@ -177,6 +177,14 @@ pub enum ChunkingType {
/// An async loader is placed into the referencing chunk and loads the /// An async loader is placed into the referencing chunk and loads the
/// separate chunk group in which the module is placed. /// separate chunk group in which the module is placed.
Async, Async,
/// Create a new chunk group in a separate context, merging references with the same tag into a
/// single chunk group. It does not inherit the available modules from the parent.
// TODO implement
Isolated {
_ty: ChunkGroupType,
_merge_tag: Option<RcStr>,
_chunking_context: Option<ResolvedVc<Box<dyn ChunkingContext>>>,
},
/// Module not placed in chunk group, but its references are still followed and placed into the /// Module not placed in chunk group, but its references are still followed and placed into the
/// chunk group. /// chunk group.
Passthrough, Passthrough,
...@@ -369,7 +377,7 @@ async fn graph_node_to_referenced_nodes( ...@@ -369,7 +377,7 @@ async fn graph_node_to_referenced_nodes(
}]); }]);
}; };
let Some(chunking_type) = *chunkable_module_reference.chunking_type().await? else { let Some(chunking_type) = &*chunkable_module_reference.chunking_type().await? else {
return Ok(vec![ChunkGraphEdge { return Ok(vec![ChunkGraphEdge {
key: None, key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference), node: ChunkContentGraphNode::ExternalModuleReference(reference),
...@@ -384,7 +392,7 @@ async fn graph_node_to_referenced_nodes( ...@@ -384,7 +392,7 @@ async fn graph_node_to_referenced_nodes(
.await? .await?
.into_iter() .into_iter()
.map(|&module| async move { .map(|&module| async move {
if chunking_type == ChunkingType::Traced { if matches!(chunking_type, ChunkingType::Traced) {
if *chunking_context.is_tracing_enabled().await? { if *chunking_context.is_tracing_enabled().await? {
return Ok(( return Ok((
Some(ChunkGraphEdge { Some(ChunkGraphEdge {
...@@ -492,6 +500,10 @@ async fn graph_node_to_referenced_nodes( ...@@ -492,6 +500,10 @@ async fn graph_node_to_referenced_nodes(
)) ))
} }
} }
ChunkingType::Isolated { .. } => {
// TODO implement
Ok((None, None))
}
ChunkingType::Traced => { ChunkingType::Traced => {
bail!("unreachable ChunkingType::Traced"); bail!("unreachable ChunkingType::Traced");
} }
......
...@@ -38,6 +38,11 @@ fn passthrough_reference_ty() -> Vc<RcStr> { ...@@ -38,6 +38,11 @@ fn passthrough_reference_ty() -> Vc<RcStr> {
Vc::cell("passthrough reference".into()) Vc::cell("passthrough reference".into())
} }
#[turbo_tasks::function]
fn isolated_reference_ty() -> Vc<RcStr> {
Vc::cell("isolated reference".into())
}
#[turbo_tasks::function] #[turbo_tasks::function]
fn traced_reference_ty() -> Vc<RcStr> { fn traced_reference_ty() -> Vc<RcStr> {
Vc::cell("traced reference".into()) Vc::cell("traced reference".into())
...@@ -81,6 +86,7 @@ pub async fn children_from_module_references( ...@@ -81,6 +86,7 @@ pub async fn children_from_module_references(
key = parallel_inherit_async_reference_ty() key = parallel_inherit_async_reference_ty()
} }
Some(ChunkingType::Async) => key = async_reference_ty(), Some(ChunkingType::Async) => key = async_reference_ty(),
Some(ChunkingType::Isolated { .. }) => key = isolated_reference_ty(),
Some(ChunkingType::Passthrough) => key = passthrough_reference_ty(), Some(ChunkingType::Passthrough) => key = passthrough_reference_ty(),
Some(ChunkingType::Traced) => key = traced_reference_ty(), Some(ChunkingType::Traced) => key = traced_reference_ty(),
} }
......
...@@ -4,7 +4,7 @@ use anyhow::Result; ...@@ -4,7 +4,7 @@ use anyhow::Result;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal}, graph::{AdjacencyMap, GraphTraversal},
FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
}; };
use crate::{ use crate::{
...@@ -270,6 +270,45 @@ pub async fn primary_referenced_modules(module: Vc<Box<dyn Module>>) -> Result<V ...@@ -270,6 +270,45 @@ pub async fn primary_referenced_modules(module: Vc<Box<dyn Module>>) -> Result<V
Ok(Vc::cell(modules)) Ok(Vc::cell(modules))
} }
type ModulesVec = Vec<ResolvedVc<Box<dyn Module>>>;
#[turbo_tasks::value(transparent)]
pub struct ModulesWithChunkingType(Vec<(ChunkingType, ModulesVec)>);
/// Aggregates all primary [Module]s referenced by an [Module] via [ChunkableModuleReference]s.
/// This does not include transitively references [Module]s, only includes
/// primary [Module]s referenced.
///
/// [Module]: crate::module::Module
#[turbo_tasks::function]
pub async fn primary_chunkable_referenced_modules(
module: Vc<Box<dyn Module>>,
) -> Result<Vc<ModulesWithChunkingType>> {
let modules = module
.references()
.await?
.iter()
.map(|reference| async {
if let Some(reference) =
ResolvedVc::try_downcast::<Box<dyn ChunkableModuleReference>>(*reference).await?
{
if let Some(chunking_type) = &*reference.chunking_type().await? {
let resolved = reference
.resolve_reference()
.resolve()
.await?
.primary_modules()
.await?
.clone_value();
return Ok(Some((chunking_type.clone(), resolved)));
}
}
Ok(None)
})
.try_flat_join()
.await?;
Ok(Vc::cell(modules))
}
/// Walks the asset graph from multiple assets and collect all referenced /// Walks the asset graph from multiple assets and collect all referenced
/// assets. /// assets.
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -83,7 +83,7 @@ async fn get_inherit_async_referenced_asset( ...@@ -83,7 +83,7 @@ async fn get_inherit_async_referenced_asset(
let Some(r) = Vc::try_resolve_downcast::<Box<dyn ChunkableModuleReference>>(r).await? else { let Some(r) = Vc::try_resolve_downcast::<Box<dyn ChunkableModuleReference>>(r).await? else {
return Ok(None); return Ok(None);
}; };
let Some(ty) = *r.chunking_type().await? else { let Some(ty) = &*r.chunking_type().await? else {
return Ok(None); return Ok(None);
}; };
if !matches!(ty, ChunkingType::ParallelInheritAsync) { if !matches!(ty, ChunkingType::ParallelInheritAsync) {
......
use anyhow::Result; use anyhow::Result;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{ChunkableModule, ChunkingContext}, chunk::{
ChunkGroupType, ChunkableModule, ChunkableModuleReference, ChunkingContext, ChunkingType,
ChunkingTypeOption,
},
ident::AssetIdent, ident::AssetIdent,
module::Module, module::Module,
reference::{ModuleReferences, SingleModuleReference}, reference::{ModuleReference, ModuleReferences},
resolve::ModuleResolveResult,
}; };
use super::chunk_item::WorkerLoaderChunkItem; use super::chunk_item::WorkerLoaderChunkItem;
...@@ -36,11 +40,6 @@ impl WorkerLoaderModule { ...@@ -36,11 +40,6 @@ impl WorkerLoaderModule {
} }
} }
#[turbo_tasks::function]
fn inner_module_reference_description() -> Vc<RcStr> {
Vc::cell("worker module".into())
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Module for WorkerLoaderModule { impl Module for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -51,12 +50,9 @@ impl Module for WorkerLoaderModule { ...@@ -51,12 +50,9 @@ impl Module for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> { async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> {
Ok(Vc::cell(vec![ResolvedVc::upcast( Ok(Vc::cell(vec![ResolvedVc::upcast(
SingleModuleReference::new( WorkerModuleReference::new(*ResolvedVc::upcast(self.await?.inner))
*ResolvedVc::upcast(self.await?.inner), .to_resolved()
inner_module_reference_description(), .await?,
)
.to_resolved()
.await?,
)])) )]))
} }
} }
...@@ -85,3 +81,44 @@ impl ChunkableModule for WorkerLoaderModule { ...@@ -85,3 +81,44 @@ impl ChunkableModule for WorkerLoaderModule {
) )
} }
} }
#[turbo_tasks::value]
struct WorkerModuleReference {
module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl]
impl WorkerModuleReference {
#[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(WorkerModuleReference { module })
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for WorkerModuleReference {
#[turbo_tasks::function]
fn chunking_type(self: Vc<Self>) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Isolated {
_ty: ChunkGroupType::Evaluated,
_merge_tag: None,
_chunking_context: None,
}))
}
}
#[turbo_tasks::value_impl]
impl ModuleReference for WorkerModuleReference {
#[turbo_tasks::function]
fn resolve_reference(&self) -> Vc<ModuleResolveResult> {
ModuleResolveResult::module(self.module).cell()
}
}
#[turbo_tasks::value_impl]
impl ValueToString for WorkerModuleReference {
#[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> {
Vc::cell("worker module".into())
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment