Unverified Commit e431f9e6 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: migrate global module ids to single-graph (#73752)

parent bc3e597a
...@@ -4412,6 +4412,7 @@ dependencies = [ ...@@ -4412,6 +4412,7 @@ dependencies = [
"turbo-tasks-build", "turbo-tasks-build",
"turbo-tasks-env", "turbo-tasks-env",
"turbo-tasks-fs", "turbo-tasks-fs",
"turbo-tasks-hash",
"turbo-tasks-memory", "turbo-tasks-memory",
"turbopack", "turbopack",
"turbopack-browser", "turbopack-browser",
...@@ -9377,8 +9378,10 @@ dependencies = [ ...@@ -9377,8 +9378,10 @@ dependencies = [
"regex", "regex",
"rstest", "rstest",
"rstest_reuse", "rstest_reuse",
"rustc-hash 1.1.0",
"serde", "serde",
"serde_json", "serde_json",
"smallvec",
"tokio", "tokio",
"tracing", "tracing",
"turbo-rcstr", "turbo-rcstr",
...@@ -9386,6 +9389,7 @@ dependencies = [ ...@@ -9386,6 +9389,7 @@ dependencies = [
"turbo-tasks-build", "turbo-tasks-build",
"turbo-tasks-env", "turbo-tasks-env",
"turbo-tasks-fs", "turbo-tasks-fs",
"turbo-tasks-hash",
"turbo-tasks-malloc", "turbo-tasks-malloc",
"turbo-tasks-memory", "turbo-tasks-memory",
"turbopack-core", "turbopack-core",
......
...@@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true } ...@@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-env = { workspace = true } turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbo-tasks-memory = { workspace = true } turbo-tasks-memory = { workspace = true }
turbopack = { workspace = true } turbopack = { workspace = true }
turbopack-browser = { workspace = true } turbopack-browser = { workspace = true }
......
use anyhow::Result;
use turbo_tasks::Vc;
use turbopack_core::chunk::module_id_strategies::{GlobalModuleIdStrategy, ModuleIdStrategy};
use turbopack_ecmascript::global_module_id_strategy::{
children_modules_idents, merge_preprocessed_module_ids, PreprocessedChildrenIdents,
};
use crate::{
project::Project,
route::{Endpoint, Route},
};
#[turbo_tasks::value]
pub struct GlobalModuleIdStrategyBuilder;
// NOTE(LichuAcu) To access all entrypoints, we need to access an instance of `Project`, but
// `Project` is not available in `turbopack-core`, so we need need this
// `GlobalModuleIdStrategyBuilder` in `next-api`.
#[turbo_tasks::value_impl]
impl GlobalModuleIdStrategyBuilder {
#[turbo_tasks::function]
pub async fn build(project: Vc<Project>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
let mut preprocessed_module_ids = Vec::new();
preprocessed_module_ids.push(children_modules_idents(project.client_main_modules()));
let entrypoints = project.entrypoints().await?;
preprocessed_module_ids.push(preprocess_module_ids(*entrypoints.pages_error_endpoint));
preprocessed_module_ids.push(preprocess_module_ids(*entrypoints.pages_app_endpoint));
preprocessed_module_ids.push(preprocess_module_ids(*entrypoints.pages_document_endpoint));
if let Some(middleware) = &entrypoints.middleware {
preprocessed_module_ids.push(preprocess_module_ids(*middleware.endpoint));
}
if let Some(instrumentation) = &entrypoints.instrumentation {
preprocessed_module_ids.push(preprocess_module_ids(*instrumentation.node_js));
preprocessed_module_ids.push(preprocess_module_ids(*instrumentation.edge));
}
for (_, route) in entrypoints.routes.iter() {
match route {
Route::Page {
html_endpoint,
data_endpoint,
} => {
preprocessed_module_ids.push(preprocess_module_ids(**html_endpoint));
preprocessed_module_ids.push(preprocess_module_ids(**data_endpoint));
}
Route::PageApi { endpoint } => {
preprocessed_module_ids.push(preprocess_module_ids(**endpoint));
}
Route::AppPage(page_routes) => {
for page_route in page_routes {
preprocessed_module_ids
.push(preprocess_module_ids(*page_route.html_endpoint));
preprocessed_module_ids
.push(preprocess_module_ids(*page_route.rsc_endpoint));
}
}
Route::AppRoute {
original_name: _,
endpoint,
} => {
preprocessed_module_ids.push(preprocess_module_ids(**endpoint));
}
Route::Conflict => {
tracing::info!("WARN: conflict");
}
}
}
let module_id_map = merge_preprocessed_module_ids(preprocessed_module_ids).await?;
Ok(Vc::upcast(
GlobalModuleIdStrategy::new(module_id_map).await?,
))
}
}
// NOTE(LichuAcu) We can't move this function to `turbopack-core` because we need access to
// `Endpoint`, which is not available there.
#[turbo_tasks::function]
fn preprocess_module_ids(endpoint: Vc<Box<dyn Endpoint>>) -> Vc<PreprocessedChildrenIdents> {
let root_modules = endpoint.root_modules();
children_modules_idents(root_modules)
}
...@@ -9,7 +9,6 @@ mod dynamic_imports; ...@@ -9,7 +9,6 @@ mod dynamic_imports;
mod empty; mod empty;
pub mod entrypoints; pub mod entrypoints;
mod font; mod font;
pub mod global_module_id_strategy;
mod instrumentation; mod instrumentation;
mod loadable_manifest; mod loadable_manifest;
mod middleware; mod middleware;
......
...@@ -86,6 +86,11 @@ pub struct PagesProject { ...@@ -86,6 +86,11 @@ pub struct PagesProject {
project: ResolvedVc<Project>, project: ResolvedVc<Project>,
} }
#[turbo_tasks::function]
fn client_layer() -> Vc<RcStr> {
Vc::cell("client".into())
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl PagesProject { impl PagesProject {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -312,7 +317,7 @@ impl PagesProject { ...@@ -312,7 +317,7 @@ impl PagesProject {
self.project().client_compile_time_info(), self.project().client_compile_time_info(),
self.client_module_options_context(), self.client_module_options_context(),
self.client_resolve_options_context(), self.client_resolve_options_context(),
Vc::cell("client".into()), client_layer(),
) )
} }
...@@ -351,7 +356,7 @@ impl PagesProject { ...@@ -351,7 +356,7 @@ impl PagesProject {
self.project().client_compile_time_info(), self.project().client_compile_time_info(),
self.client_module_options_context(), self.client_module_options_context(),
self.client_resolve_options_context(), self.client_resolve_options_context(),
Vc::cell("client".into()), client_layer(),
)) ))
} }
......
...@@ -34,7 +34,8 @@ use turbo_tasks::{ ...@@ -34,7 +34,8 @@ use turbo_tasks::{
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem};
use turbopack::{ use turbopack::{
evaluate_context::node_build_environment, transition::TransitionOptions, ModuleAssetContext, evaluate_context::node_build_environment, global_module_ids::get_global_module_id_strategy,
transition::TransitionOptions, ModuleAssetContext,
}; };
use turbopack_core::{ use turbopack_core::{
changed::content_changed, changed::content_changed,
...@@ -68,7 +69,6 @@ use crate::{ ...@@ -68,7 +69,6 @@ use crate::{
build, build,
empty::EmptyEndpoint, empty::EmptyEndpoint,
entrypoints::Entrypoints, entrypoints::Entrypoints,
global_module_id_strategy::GlobalModuleIdStrategyBuilder,
instrumentation::InstrumentationEndpoint, instrumentation::InstrumentationEndpoint,
middleware::MiddlewareEndpoint, middleware::MiddlewareEndpoint,
pages::PagesProject, pages::PagesProject,
...@@ -1576,16 +1576,25 @@ impl Project { ...@@ -1576,16 +1576,25 @@ impl Project {
/// Gets the module id strategy for the project. /// Gets the module id strategy for the project.
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn module_id_strategy(self: Vc<Self>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> { pub async fn module_id_strategy(self: Vc<Self>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
let module_id_strategy = self.next_config().module_id_strategy_config(); let module_id_strategy = if let Some(module_id_strategy) =
match *module_id_strategy.await? { &*self.next_config().module_id_strategy_config().await?
Some(ModuleIdStrategyConfig::Named) => Ok(Vc::upcast(DevModuleIdStrategy::new())), {
Some(ModuleIdStrategyConfig::Deterministic) => { *module_id_strategy
Ok(Vc::upcast(GlobalModuleIdStrategyBuilder::build(self))) } else {
match *self.next_mode().await? {
NextMode::Development => ModuleIdStrategyConfig::Named,
NextMode::Build => ModuleIdStrategyConfig::Deterministic,
}
};
match module_id_strategy {
ModuleIdStrategyConfig::Named => Ok(Vc::upcast(DevModuleIdStrategy::new())),
ModuleIdStrategyConfig::Deterministic => {
let module_graphs = self.whole_app_module_graphs().await?;
Ok(Vc::upcast(get_global_module_id_strategy(
*module_graphs.full,
)))
} }
None => match *self.next_mode().await? {
NextMode::Development => Ok(Vc::upcast(DevModuleIdStrategy::new())),
NextMode::Build => Ok(Vc::upcast(DevModuleIdStrategy::new())),
},
} }
} }
} }
......
...@@ -567,7 +567,7 @@ pub enum LoaderItem { ...@@ -567,7 +567,7 @@ pub enum LoaderItem {
} }
#[turbo_tasks::value(operation)] #[turbo_tasks::value(operation)]
#[derive(Clone, Debug)] #[derive(Copy, Clone, Debug)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub enum ModuleIdStrategy { pub enum ModuleIdStrategy {
Named, Named,
...@@ -1483,11 +1483,11 @@ impl NextConfig { ...@@ -1483,11 +1483,11 @@ impl NextConfig {
.experimental .experimental
.turbo .turbo
.as_ref() .as_ref()
.and_then(|t| t.module_id_strategy.as_ref()) .and_then(|t| t.module_id_strategy)
else { else {
return Vc::cell(None); return Vc::cell(None);
}; };
Vc::cell(Some(module_id_strategy.clone())) Vc::cell(Some(module_id_strategy))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -9,7 +9,7 @@ use turbopack_core::{ ...@@ -9,7 +9,7 @@ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext}, chunk::{ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext},
ident::AssetIdent, ident::AssetIdent,
module::{Module, Modules}, module::Module,
module_graph::ModuleGraph, module_graph::ModuleGraph,
reference::ModuleReferences, reference::ModuleReferences,
}; };
...@@ -18,17 +18,11 @@ use turbopack_ecmascript::{ ...@@ -18,17 +18,11 @@ use turbopack_ecmascript::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable,
EcmascriptChunkType, EcmascriptExports, EcmascriptChunkType, EcmascriptExports,
}, },
references::{ references::esm::{EsmExport, EsmExports},
esm::{EsmExport, EsmExports},
external_module::IncludeIdentModule,
},
utils::StringifyJs, utils::StringifyJs,
}; };
use super::server_component_reference::NextServerComponentModuleReference; use super::server_component_reference::NextServerComponentModuleReference;
use crate::next_app::app_client_references_chunks::{
client_modules_modifier, ssr_modules_modifier,
};
#[turbo_tasks::function] #[turbo_tasks::function]
fn modifier() -> Vc<RcStr> { fn modifier() -> Vc<RcStr> {
...@@ -68,22 +62,6 @@ impl Module for NextServerComponentModule { ...@@ -68,22 +62,6 @@ impl Module for NextServerComponentModule {
.await?, .await?,
)])) )]))
} }
#[turbo_tasks::function]
async fn additional_layers_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
let base_ident = self.ident();
let ssr_entry_module = ResolvedVc::upcast(
IncludeIdentModule::new(base_ident.with_modifier(ssr_modules_modifier()))
.to_resolved()
.await?,
);
let client_entry_module = ResolvedVc::upcast(
IncludeIdentModule::new(base_ident.with_modifier(client_modules_modifier()))
.to_resolved()
.await?,
);
Ok(Vc::cell(vec![ssr_entry_module, client_entry_module]))
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
......
...@@ -580,16 +580,19 @@ describe('Production Usage', () => { ...@@ -580,16 +580,19 @@ describe('Production Usage', () => {
const resources: Set<string> = new Set() const resources: Set<string> = new Set()
const manifestKey = Object.keys(reactLoadableManifest).find((item) => { let manifestKey: string
return item if (process.env.TURBOPACK) {
.replace(/\\/g, '/') // the key is an arbitrary and changing number for Turbopack prod, but each page has its own manifest
.endsWith( expect(Object.keys(reactLoadableManifest).length).toBe(1)
process.env.TURBOPACK manifestKey = Object.keys(reactLoadableManifest)[0]
? 'components/dynamic-css/with-css.js [client] (ecmascript, next/dynamic entry)' expect(manifestKey).toBeString()
: 'dynamic/css.js -> ../../components/dynamic-css/with-css' } else {
) manifestKey = Object.keys(reactLoadableManifest).find((item) =>
}) item
expect(manifestKey).toBeString() .replace(/\\/g, '/')
.endsWith('dynamic/css.js -> ../../components/dynamic-css/with-css')
)
}
// test dynamic chunk // test dynamic chunk
reactLoadableManifest[manifestKey].files.forEach((f) => { reactLoadableManifest[manifestKey].files.forEach((f) => {
......
use anyhow::Result; use anyhow::{bail, Result};
use turbo_rcstr::RcStr; use rustc_hash::FxHashMap;
use turbo_tasks::{FxIndexMap, ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_hash::hash_xxh3_hash64; use turbo_tasks_hash::hash_xxh3_hash64;
use super::ModuleId; use super::ModuleId;
use crate::ident::AssetIdent; use crate::{
ident::AssetIdent,
issue::{module::ModuleIssue, IssueExt, StyledString},
};
#[turbo_tasks::value_trait] #[turbo_tasks::value_trait]
pub trait ModuleIdStrategy { pub trait ModuleIdStrategy {
...@@ -32,25 +35,43 @@ impl ModuleIdStrategy for DevModuleIdStrategy { ...@@ -32,25 +35,43 @@ impl ModuleIdStrategy for DevModuleIdStrategy {
} }
} }
#[turbo_tasks::value] #[turbo_tasks::value(shared)]
pub struct GlobalModuleIdStrategy { pub struct GlobalModuleIdStrategy {
module_id_map: FxIndexMap<RcStr, ModuleId>, pub module_id_map: FxHashMap<ResolvedVc<AssetIdent>, u64>,
}
impl GlobalModuleIdStrategy {
pub async fn new(module_id_map: FxIndexMap<RcStr, ModuleId>) -> Result<Vc<Self>> {
Ok(GlobalModuleIdStrategy { module_id_map }.cell())
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ModuleIdStrategy for GlobalModuleIdStrategy { impl ModuleIdStrategy for GlobalModuleIdStrategy {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn get_module_id(&self, ident: Vc<AssetIdent>) -> Result<Vc<ModuleId>> { async fn get_module_id(&self, ident: ResolvedVc<AssetIdent>) -> Result<Vc<ModuleId>> {
let ident_string = ident.to_string().await?.clone_value(); if let Some(module_id) = self.module_id_map.get(&ident) {
if let Some(module_id) = self.module_id_map.get(&ident_string) { const JS_MAX_SAFE_INTEGER: u64 = (1u64 << 53) - 1;
return Ok(module_id.clone().cell()); if *module_id > JS_MAX_SAFE_INTEGER {
bail!("Numeric module id is too large: {}", module_id);
}
return Ok(ModuleId::Number(*module_id).cell());
} }
let ident_string = ident.to_string().await?;
if !ident_string.ends_with("[app-client] (ecmascript, next/dynamic entry)") {
// TODO: This shouldn't happen, but is a temporary workaround to ignore next/dynamic
// imports of a server component from another server component.
ModuleIssue {
ident,
title: StyledString::Text(
format!("ModuleId not found for ident: {:?}", ident_string).into(),
)
.resolved_cell(),
description: StyledString::Text(
format!("ModuleId not found for ident: {:?}", ident_string).into(),
)
.resolved_cell(),
}
.resolved_cell()
.emit();
}
Ok(ModuleId::String( Ok(ModuleId::String(
hash_xxh3_hash64(ident.to_string().await?) hash_xxh3_hash64(ident.to_string().await?)
.to_string() .to_string()
......
...@@ -28,8 +28,15 @@ impl FileSource { ...@@ -28,8 +28,15 @@ impl FileSource {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new_with_query(path: ResolvedVc<FileSystemPath>, query: ResolvedVc<RcStr>) -> Vc<Self> { pub async fn new_with_query(
Self::cell(FileSource { path, query }) path: ResolvedVc<FileSystemPath>,
query: ResolvedVc<RcStr>,
) -> Result<Vc<Self>> {
if query.await?.is_empty() {
Ok(Self::new(*path))
} else {
Ok(Self::cell(FileSource { path, query }))
}
} }
} }
......
...@@ -16,10 +16,6 @@ pub trait Module: Asset { ...@@ -16,10 +16,6 @@ pub trait Module: Asset {
ModuleReferences::empty() ModuleReferences::empty()
} }
fn additional_layers_modules(self: Vc<Self>) -> Vc<Modules> {
Vc::cell(vec![])
}
/// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc. /// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc.
fn is_self_async(self: Vc<Self>) -> Vc<bool> { fn is_self_async(self: Vc<Self>) -> Vc<bool> {
Vc::cell(false) Vc::cell(false)
......
...@@ -731,6 +731,41 @@ impl ModuleGraph { ...@@ -731,6 +731,41 @@ impl ModuleGraph {
Ok(()) Ok(())
} }
/// Traverses all edges exactly once (in an unspecified order) and calls the visitor with the
/// edge source and target.
///
/// This means that target nodes can be revisited (once per incoming edge).
///
/// * `visitor` - Called before visiting the children of a node.
/// - Receives (originating &SingleModuleGraphNode, edge &ChunkingType), target
/// &SingleModuleGraphNode
pub async fn traverse_all_edges_unordered(
&self,
mut visitor: impl FnMut(
(&'_ SingleModuleGraphModuleNode, &'_ ChunkingType),
&'_ SingleModuleGraphModuleNode,
) -> Result<()>,
) -> Result<()> {
let graphs = self.get_graphs().await?;
for graph in &graphs {
let graph = &graph.graph;
for edge in graph.edge_references() {
let source = match graph.node_weight(edge.source()).unwrap() {
SingleModuleGraphNode::Module(node) => node,
SingleModuleGraphNode::VisitedModule { .. } => unreachable!(),
};
let target = match graph.node_weight(edge.target()).unwrap() {
SingleModuleGraphNode::Module(node) => node,
SingleModuleGraphNode::VisitedModule { idx } => get_node!(graphs, idx),
};
visitor((source, edge.weight()), target)?;
}
}
Ok(())
}
/// Traverses all reachable edges in topological order. The preorder visitor can be used to /// Traverses all reachable edges in topological order. The preorder visitor can be used to
/// forward state down the graph, and to skip subgraphs /// forward state down the graph, and to skip subgraphs
/// ///
......
...@@ -19,7 +19,7 @@ use crate::{ ...@@ -19,7 +19,7 @@ use crate::{
data::EcmascriptChunkData, EcmascriptChunkItem, EcmascriptChunkItemContent, data::EcmascriptChunkData, EcmascriptChunkItem, EcmascriptChunkItemContent,
EcmascriptChunkPlaceable, EcmascriptChunkType, EcmascriptChunkPlaceable, EcmascriptChunkType,
}, },
utils::StringifyJs, utils::{StringifyJs, StringifyModuleId},
}; };
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
...@@ -102,7 +102,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem { ...@@ -102,7 +102,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
}}); }});
}}); }});
"#, "#,
id = StringifyJs(id), id = StringifyModuleId(id),
} }
} }
(Some(id), false) => { (Some(id), false) => {
...@@ -115,7 +115,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem { ...@@ -115,7 +115,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
}}); }});
"#, "#,
chunks = StringifyJs(&chunks_data), chunks = StringifyJs(&chunks_data),
id = StringifyJs(id), id = StringifyModuleId(id),
} }
} }
(None, true) => { (None, true) => {
......
...@@ -13,7 +13,7 @@ use turbopack_core::{ ...@@ -13,7 +13,7 @@ use turbopack_core::{
use crate::async_chunk::chunk_item::AsyncLoaderChunkItem; use crate::async_chunk::chunk_item::AsyncLoaderChunkItem;
#[turbo_tasks::function] #[turbo_tasks::function]
fn modifier() -> Vc<RcStr> { fn async_loader_modifier() -> Vc<RcStr> {
Vc::cell("async loader".into()) Vc::cell("async loader".into())
} }
...@@ -43,7 +43,7 @@ impl AsyncLoaderModule { ...@@ -43,7 +43,7 @@ impl AsyncLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn asset_ident_for(module: Vc<Box<dyn ChunkableModule>>) -> Vc<AssetIdent> { pub fn asset_ident_for(module: Vc<Box<dyn ChunkableModule>>) -> Vc<AssetIdent> {
module.ident().with_modifier(modifier()) module.ident().with_modifier(async_loader_modifier())
} }
} }
......
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal},
FxIndexMap, FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc,
};
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::{
chunk::ModuleId,
module::{Module, Modules},
reference::ModuleReference,
};
use crate::references::esm::EsmAsyncAssetReference;
#[turbo_tasks::value]
pub struct PreprocessedChildrenIdents {
// ident.to_string() -> full hash
// We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids`
// if this endpoint did not change.
modules_idents: FxIndexMap<RcStr, u64>,
}
#[derive(Clone, Hash)]
#[turbo_tasks::value(shared)]
pub enum ReferencedModule {
Module(ResolvedVc<Box<dyn Module>>),
AsyncLoaderModule(ResolvedVc<Box<dyn Module>>),
}
impl ReferencedModule {
fn module(&self) -> Vc<Box<dyn Module>> {
match *self {
ReferencedModule::Module(module) => *module,
ReferencedModule::AsyncLoaderModule(module) => *module,
}
}
}
#[turbo_tasks::value(transparent)]
pub struct ReferencedModules(Vec<ResolvedVc<ReferencedModule>>);
#[turbo_tasks::function]
async fn referenced_modules(module: Vc<Box<dyn Module>>) -> Result<Vc<ReferencedModules>> {
let references = module.references().await?;
// TODO(LichuAcu): Reduce type complexity
#[allow(clippy::type_complexity)]
type ModulesAndAsyncLoaders = Vec<(
Vec<ResolvedVc<Box<dyn Module>>>,
Option<ResolvedVc<Box<dyn Module>>>,
)>;
let modules_and_async_loaders: ModulesAndAsyncLoaders = references
.iter()
.map(|reference| async move {
let async_loader =
if ResolvedVc::try_downcast_type::<EsmAsyncAssetReference>(*reference)
.await?
.is_some()
{
*reference
.resolve_reference()
.resolve()
.await?
.first_module()
.await?
} else {
None
};
let modules = reference
.resolve_reference()
.resolve()
.await?
.primary_modules()
.await?
.clone_value();
Ok((modules, async_loader))
})
.try_join()
.await?;
let mut set = FxIndexSet::default();
let mut modules = Vec::new();
for (module_list, async_loader) in modules_and_async_loaders {
for module in module_list {
if set.insert(module) {
modules.push(ReferencedModule::Module(module).resolved_cell());
}
}
if let Some(async_loader_module) = async_loader {
if set.insert(async_loader_module) {
modules
.push(ReferencedModule::AsyncLoaderModule(async_loader_module).resolved_cell());
}
}
}
Ok(Vc::cell(modules))
}
pub async fn get_children_modules(
parent: ResolvedVc<ReferencedModule>,
) -> Result<impl Iterator<Item = ResolvedVc<ReferencedModule>> + Send> {
let parent_module = parent.await?.module();
let mut modules = referenced_modules(parent_module).await?.clone_value();
for module in parent_module.additional_layers_modules().await? {
modules.push(ReferencedModule::Module(*module).resolved_cell());
}
Ok(modules.into_iter())
}
// NOTE(LichuAcu) Called on endpoint.root_modules(). It would probably be better if this was called
// directly on `Endpoint`, but such struct is not available in turbopack-core. The whole function
// could be moved to `next-api`, but it would require adding turbo-tasks-hash to `next-api`,
// making it heavier.
#[turbo_tasks::function]
pub async fn children_modules_idents(
root_modules: Vc<Modules>,
) -> Result<Vc<PreprocessedChildrenIdents>> {
let children_modules_iter = AdjacencyMap::new()
.skip_duplicates()
.visit(
root_modules
.await?
.iter()
.map(|module| ReferencedModule::Module(*module).resolved_cell())
.collect::<Vec<_>>(),
get_children_modules,
)
.await
.completed()?
.into_inner()
.into_reverse_topological();
// module_id -> full hash
let mut modules_idents = FxIndexMap::default();
for child_module in children_modules_iter {
match *child_module.await? {
ReferencedModule::Module(module) => {
let module_ident = module.ident();
let ident_str = module_ident.to_string().await?.clone_value();
let hash = hash_xxh3_hash64(&ident_str);
modules_idents.insert(ident_str, hash);
}
ReferencedModule::AsyncLoaderModule(async_loader_module) => {
let loader_ident = async_loader_module
.ident()
.with_modifier(Vc::cell("async loader".into()));
let loader_ident_str = loader_ident.to_string().await?.clone_value();
let loader_hash = hash_xxh3_hash64(&loader_ident_str);
modules_idents.insert(loader_ident_str, loader_hash);
let loaded_client_ident = async_loader_module
.ident()
.with_layer(Vc::cell("app-client".into()));
let loaded_client_ident_str = loaded_client_ident.to_string().await?.clone_value();
let loaded_client_hash = hash_xxh3_hash64(&loaded_client_ident_str);
modules_idents.insert(loaded_client_ident_str, loaded_client_hash);
}
}
}
Ok(PreprocessedChildrenIdents { modules_idents }.cell())
}
const JS_MAX_SAFE_INTEGER: u64 = (1u64 << 53) - 1;
// Note(LichuAcu): This could be split into two functions: one that merges the preprocessed module
// ids and another that generates the final, optimized module ids. Thoughts?
pub async fn merge_preprocessed_module_ids(
preprocessed_module_ids: Vec<Vc<PreprocessedChildrenIdents>>,
) -> Result<FxIndexMap<RcStr, ModuleId>> {
let mut merged_module_ids = FxIndexMap::default();
for preprocessed_module_ids in preprocessed_module_ids {
for (module_ident, full_hash) in &preprocessed_module_ids.await?.modules_idents {
merged_module_ids.insert(module_ident.clone(), *full_hash);
}
}
// 5% fill rate, as done in Webpack
// https://github.com/webpack/webpack/blob/27cf3e59f5f289dfc4d76b7a1df2edbc4e651589/lib/ids/IdHelpers.js#L366-L405
let optimal_range = merged_module_ids.len() * 20;
let digit_mask = std::cmp::min(
10u64.pow((optimal_range as f64).log10().ceil() as u32),
JS_MAX_SAFE_INTEGER,
);
let mut module_id_map = FxIndexMap::default();
let mut used_ids = FxIndexSet::default();
for (module_ident, full_hash) in merged_module_ids.iter() {
let mut trimmed_hash = full_hash % digit_mask;
let mut i = 1;
while used_ids.contains(&trimmed_hash) {
// If the id is already used, seek to find another available id.
trimmed_hash = hash_xxh3_hash64(full_hash + i) % digit_mask;
i += 1;
}
used_ids.insert(trimmed_hash);
module_id_map.insert(module_ident.clone(), ModuleId::Number(trimmed_hash));
}
Ok(module_id_map)
}
...@@ -16,7 +16,6 @@ pub mod chunk; ...@@ -16,7 +16,6 @@ pub mod chunk;
pub mod chunk_group_files_asset; pub mod chunk_group_files_asset;
pub mod code_gen; pub mod code_gen;
mod errors; mod errors;
pub mod global_module_id_strategy;
pub mod magic_identifier; pub mod magic_identifier;
pub mod manifest; pub mod manifest;
pub mod minify; pub mod minify;
...@@ -253,6 +252,19 @@ pub struct EcmascriptModuleAsset { ...@@ -253,6 +252,19 @@ pub struct EcmascriptModuleAsset {
#[turbo_tasks(debug_ignore)] #[turbo_tasks(debug_ignore)]
last_successful_parse: turbo_tasks::TransientState<ReadRef<ParseResult>>, last_successful_parse: turbo_tasks::TransientState<ReadRef<ParseResult>>,
} }
impl core::fmt::Debug for EcmascriptModuleAsset {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
f.debug_struct("EcmascriptModuleAsset")
.field("source", &self.source)
.field("asset_context", &self.asset_context)
.field("ty", &self.ty)
.field("transforms", &self.transforms)
.field("options", &self.options)
.field("compile_time_info", &self.compile_time_info)
.field("inner_assets", &self.inner_assets)
.finish()
}
}
#[turbo_tasks::value_trait] #[turbo_tasks::value_trait]
pub trait EcmascriptParsable { pub trait EcmascriptParsable {
......
...@@ -20,7 +20,7 @@ use crate::{ ...@@ -20,7 +20,7 @@ use crate::{
data::EcmascriptChunkData, EcmascriptChunkItem, EcmascriptChunkItemContent, data::EcmascriptChunkData, EcmascriptChunkItem, EcmascriptChunkItemContent,
EcmascriptChunkPlaceable, EcmascriptChunkType, EcmascriptChunkPlaceable, EcmascriptChunkType,
}, },
utils::StringifyJs, utils::{StringifyJs, StringifyModuleId},
}; };
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -197,8 +197,8 @@ impl EcmascriptChunkItem for ManifestLoaderChunkItem { ...@@ -197,8 +197,8 @@ impl EcmascriptChunkItem for ManifestLoaderChunkItem {
.map(|chunk_data| EcmascriptChunkData::new(chunk_data)) .map(|chunk_data| EcmascriptChunkData::new(chunk_data))
.collect::<Vec<_>>() .collect::<Vec<_>>()
), ),
item_id = StringifyJs(item_id), item_id = StringifyModuleId(item_id),
dynamic_id = StringifyJs(dynamic_id), dynamic_id = StringifyModuleId(dynamic_id),
)?; )?;
Ok(EcmascriptChunkItemContent { Ok(EcmascriptChunkItemContent {
......
use anyhow::Result; use anyhow::Result;
use turbo_tasks::{ResolvedVc, ValueDefault, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueDefault, Vc};
use turbo_tasks_fs::rope::RopeBuilder; use turbo_tasks_fs::rope::RopeBuilder;
use turbopack_core::{ use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkingContext}, chunk::{
AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext,
},
ident::AssetIdent, ident::AssetIdent,
module::Module, module::Module,
module_graph::ModuleGraph, module_graph::ModuleGraph,
...@@ -16,7 +18,7 @@ use crate::{ ...@@ -16,7 +18,7 @@ use crate::{
}, },
references::async_module::AsyncModuleOptions, references::async_module::AsyncModuleOptions,
tree_shake::side_effect_module::SideEffectsModule, tree_shake::side_effect_module::SideEffectsModule,
utils::StringifyJs, utils::StringifyModuleId,
EcmascriptModuleContent, EcmascriptModuleContent,
}; };
...@@ -113,6 +115,7 @@ impl ChunkItem for EcmascriptModulePartChunkItem { ...@@ -113,6 +115,7 @@ impl ChunkItem for EcmascriptModulePartChunkItem {
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub(super) struct SideEffectsModuleChunkItem { pub(super) struct SideEffectsModuleChunkItem {
pub module: ResolvedVc<SideEffectsModule>, pub module: ResolvedVc<SideEffectsModule>,
pub module_graph: ResolvedVc<ModuleGraph>,
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
} }
...@@ -167,7 +170,12 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem { ...@@ -167,7 +170,12 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
format!( format!(
"{}__turbopack_import__({});\n", "{}__turbopack_import__({});\n",
if need_await { "await " } else { "" }, if need_await { "await " } else { "" },
StringifyJs(&*side_effect.ident().to_string().await?) StringifyModuleId(
&*side_effect
.as_chunk_item(*self.module_graph, *self.chunking_context)
.id()
.await?
)
) )
.as_bytes(), .as_bytes(),
); );
...@@ -176,7 +184,13 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem { ...@@ -176,7 +184,13 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
code.push_bytes( code.push_bytes(
format!( format!(
"__turbopack_export_namespace__(__turbopack_import__({}));\n", "__turbopack_export_namespace__(__turbopack_import__({}));\n",
StringifyJs(&*module.resolved_as.ident().to_string().await?) StringifyModuleId(
&*module
.resolved_as
.as_chunk_item(*self.module_graph, *self.chunking_context)
.id()
.await?
)
) )
.as_bytes(), .as_bytes(),
); );
......
...@@ -133,12 +133,13 @@ impl ChunkableModule for SideEffectsModule { ...@@ -133,12 +133,13 @@ impl ChunkableModule for SideEffectsModule {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn as_chunk_item( async fn as_chunk_item(
self: Vc<Self>, self: Vc<Self>,
_module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
) -> Result<Vc<Box<dyn turbopack_core::chunk::ChunkItem>>> { ) -> Result<Vc<Box<dyn turbopack_core::chunk::ChunkItem>>> {
Ok(Vc::upcast( Ok(Vc::upcast(
SideEffectsModuleChunkItem { SideEffectsModuleChunkItem {
module: self.to_resolved().await?, module: self.to_resolved().await?,
module_graph,
chunking_context, chunking_context,
} }
.cell(), .cell(),
......
...@@ -78,6 +78,23 @@ pub fn module_id_to_lit(module_id: &ModuleId) -> Expr { ...@@ -78,6 +78,23 @@ pub fn module_id_to_lit(module_id: &ModuleId) -> Expr {
}) })
} }
pub struct StringifyModuleId<'a>(pub &'a ModuleId);
impl std::fmt::Display for StringifyModuleId<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.0 {
ModuleId::Number(n) => {
if *n <= JS_MAX_SAFE_INTEGER {
n.fmt(f)
} else {
write!(f, "\"{n}\"")
}
}
ModuleId::String(s) => StringifyJs(s).fmt(f),
}
}
}
pub struct StringifyJs<'a, T>(pub &'a T) pub struct StringifyJs<'a, T>(pub &'a T)
where where
T: ?Sized; T: ?Sized;
......
...@@ -21,6 +21,8 @@ anyhow = { workspace = true } ...@@ -21,6 +21,8 @@ anyhow = { workspace = true }
indexmap = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] }
lazy_static = { workspace = true } lazy_static = { workspace = true }
regex = { workspace = true } regex = { workspace = true }
rustc-hash = { workspace = true }
smallvec = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
...@@ -30,6 +32,7 @@ turbo-rcstr = { workspace = true } ...@@ -30,6 +32,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-env = { workspace = true } turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbopack-core = { workspace = true } turbopack-core = { workspace = true }
turbopack-css = { workspace = true } turbopack-css = { workspace = true }
turbopack-ecmascript = { workspace = true } turbopack-ecmascript = { workspace = true }
......
use anyhow::{Context, Result};
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::{
chunk::{module_id_strategies::GlobalModuleIdStrategy, ChunkableModule, ChunkingType},
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
};
use turbopack_ecmascript::async_chunk::module::AsyncLoaderModule;
#[turbo_tasks::function]
pub async fn get_global_module_id_strategy(
module_graph: ResolvedVc<ModuleGraph>,
) -> Result<Vc<GlobalModuleIdStrategy>> {
let span = tracing::info_span!("compute module id map");
async move {
let module_graph = module_graph.await?;
let graphs = module_graph.graphs.iter().try_join().await?;
// All modules in the graph
let module_idents = graphs
.iter()
.flat_map(|graph| graph.iter_nodes())
.map(|m| m.module.ident());
// And additionally, all the modules that are inserted by chunking (i.e. async loaders)
let mut async_idents = vec![];
module_graph
.traverse_all_edges_unordered(|parent, current| {
if let (_, &ChunkingType::Async) = parent {
let module =
ResolvedVc::try_sidecast_sync::<Box<dyn ChunkableModule>>(current.module)
.context("expected chunkable module for async reference")?;
async_idents.push(AsyncLoaderModule::asset_ident_for(*module));
}
Ok(())
})
.await?;
let mut module_id_map = module_idents
.chain(async_idents.into_iter())
.map(|ident| async move {
let ident = ident.to_resolved().await?;
let ident_str = ident.to_string().await?;
let hash = hash_xxh3_hash64(&ident_str);
Ok((ident, (ident_str, hash)))
})
.try_join()
.await?
.into_iter()
.collect::<FxHashMap<_, _>>();
finalize_module_ids(&mut module_id_map);
Ok(GlobalModuleIdStrategy {
module_id_map: module_id_map
.into_iter()
.map(|(ident, (_, hash))| (ident, hash))
.collect(),
}
.cell())
}
.instrument(span)
.await
}
const JS_MAX_SAFE_INTEGER: u64 = (1u64 << 53) - 1;
/// Shorten hashes and handle any collisions.
fn finalize_module_ids(
merged_module_ids: &mut FxHashMap<ResolvedVc<AssetIdent>, (ReadRef<RcStr>, u64)>,
) {
// 5% fill rate, as done in Webpack
// https://github.com/webpack/webpack/blob/27cf3e59f5f289dfc4d76b7a1df2edbc4e651589/lib/ids/IdHelpers.js#L366-L405
let optimal_range = merged_module_ids.len() * 20;
let digit_mask = std::cmp::min(
10u64.pow((optimal_range as f64).log10().ceil() as u32),
JS_MAX_SAFE_INTEGER,
);
let mut used_ids =
FxHashMap::<u64, SmallVec<[(ResolvedVc<AssetIdent>, ReadRef<RcStr>); 1]>>::default();
// Run in multiple passes, to not depend on the order of the `merged_module_ids` (i.e. the order
// of imports). Hashes could still change if modules are added or removed.
// Find pass: shorten hashes, potentially causing (more) collisions
for (ident, (ident_str, full_hash)) in merged_module_ids.iter_mut() {
let first_pass_hash = *full_hash % digit_mask;
used_ids
.entry(first_pass_hash)
.or_default()
.push((*ident, ident_str.clone()));
*full_hash = first_pass_hash;
}
// Filter conflicts
let mut conflicting_hashes = used_ids
.iter()
.filter(|(_, list)| (list.len() > 1))
.map(|(hash, _)| *hash)
.collect::<Vec<_>>();
conflicting_hashes.sort();
// Second pass over the conflicts to resolve them
for hash in conflicting_hashes.into_iter() {
let list = used_ids.get_mut(&hash).unwrap();
// Take the vector but keep the (empty) entry, so that the "contains_key" check below works
let mut list = std::mem::take(list);
list.sort_by(|a, b| a.1.cmp(&b.1));
// Skip the first one, one module can keep the original hash
for (ident, _) in list.into_iter().skip(1) {
let hash = &mut merged_module_ids.get_mut(&ident).unwrap().1;
// the original algorithm since all that runs in deterministic order now
let mut i = 1;
let mut trimmed_hash;
loop {
// If the id is already used, find the next available hash.
trimmed_hash = hash_xxh3_hash64((*hash, i)) % digit_mask;
if !used_ids.contains_key(&trimmed_hash) {
break;
}
i += 1;
}
// At this point, we don't care about the values anymore, just the keys
used_ids.entry(trimmed_hash).or_default();
*hash = trimmed_hash;
}
}
}
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
pub mod evaluate_context; pub mod evaluate_context;
pub mod global_module_ids;
mod graph; mod graph;
pub mod module_options; pub mod module_options;
pub mod transition; pub mod transition;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment