Unverified Commit 9ecd1a2b authored by lichu acuña's avatar lichu acuña Committed by GitHub

Refactored module IDs optimization (#68846)

### What?

Refactored module ID strategies implementation to walk modules graph individually for each endpoint.

Based on feedback from [next.js#68408](https://github.com/vercel/next.js/pull/68408) and [turbo#8912](https://github.com/vercel/turbo/pull/8912).

Comments marked with `NOTE(LichuAcu)` are intended to make reviewing easier and will be removed before merging.
parent ba995993
...@@ -53,12 +53,15 @@ use turbopack_core::{ ...@@ -53,12 +53,15 @@ use turbopack_core::{
}, },
file_source::FileSource, file_source::FileSource,
ident::AssetIdent, ident::AssetIdent,
module::Module, issue::IssueSeverity,
module::{Module, Modules},
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
raw_output::RawOutput, raw_output::RawOutput,
resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern},
source::Source, source::Source,
virtual_output::VirtualOutputAsset, virtual_output::VirtualOutputAsset,
}; };
use turbopack_ecmascript::resolve::cjs_resolve;
use crate::{ use crate::{
dynamic_imports::{ dynamic_imports::{
...@@ -554,6 +557,30 @@ impl AppProject { ...@@ -554,6 +557,30 @@ impl AppProject {
.collect(), .collect(),
)) ))
} }
#[turbo_tasks::function]
pub async fn client_main_module(self: Vc<Self>) -> Result<Vc<Box<dyn Module>>> {
let client_module_context = Vc::upcast(self.client_module_context());
let client_main_module = cjs_resolve(
Vc::upcast(PlainResolveOrigin::new(
client_module_context,
self.project().project_path().join("_".into()),
)),
Request::parse(Value::new(Pattern::Constant(
"next/dist/client/app-next-turbopack.js".into(),
))),
None,
IssueSeverity::Error.cell(),
)
.resolve()
.await?
.first_module()
.await?
.context("expected Next.js client runtime to resolve to a module")?;
Ok(client_main_module)
}
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -719,6 +746,24 @@ impl AppEndpoint { ...@@ -719,6 +746,24 @@ impl AppEndpoint {
)) ))
} }
#[turbo_tasks::function]
async fn app_endpoint_entry(self: Vc<Self>) -> Result<Vc<AppEntry>> {
let this = self.await?;
let next_config = self.await?.app_project.project().next_config();
let app_entry = match this.ty {
AppEndpointType::Page { loader_tree, .. } => self.app_page_entry(loader_tree),
AppEndpointType::Route { path, root_layouts } => {
self.app_route_entry(path, root_layouts, next_config)
}
AppEndpointType::Metadata { metadata } => {
self.app_metadata_entry(metadata, next_config)
}
};
Ok(app_entry)
}
#[turbo_tasks::function] #[turbo_tasks::function]
fn output_assets(self: Vc<Self>) -> Vc<OutputAssets> { fn output_assets(self: Vc<Self>) -> Vc<OutputAssets> {
self.output().output_assets() self.output().output_assets()
...@@ -728,24 +773,15 @@ impl AppEndpoint { ...@@ -728,24 +773,15 @@ impl AppEndpoint {
async fn output(self: Vc<Self>) -> Result<Vc<AppEndpointOutput>> { async fn output(self: Vc<Self>) -> Result<Vc<AppEndpointOutput>> {
let this = self.await?; let this = self.await?;
let next_config = self.await?.app_project.project().next_config(); let app_entry = self.app_endpoint_entry().await?;
let (app_entry, process_client, process_ssr) = match this.ty {
AppEndpointType::Page { ty, loader_tree } => ( let (process_client, process_ssr) = match this.ty {
self.app_page_entry(loader_tree), AppEndpointType::Page { ty, .. } => (true, matches!(ty, AppPageEndpointType::Html)),
true,
matches!(ty, AppPageEndpointType::Html),
),
// NOTE(alexkirsz) For routes, technically, a lot of the following code is not needed, // NOTE(alexkirsz) For routes, technically, a lot of the following code is not needed,
// as we know we won't have any client references. However, for now, for simplicity's // as we know we won't have any client references. However, for now, for simplicity's
// sake, we just do the same thing as for pages. // sake, we just do the same thing as for pages.
AppEndpointType::Route { path, root_layouts } => ( AppEndpointType::Route { .. } => (false, false),
self.app_route_entry(path, root_layouts, next_config), AppEndpointType::Metadata { .. } => (false, false),
false,
false,
),
AppEndpointType::Metadata { metadata } => {
(self.app_metadata_entry(metadata, next_config), false, false)
}
}; };
let node_root = this.app_project.project().node_root(); let node_root = this.app_project.project().node_root();
...@@ -760,8 +796,6 @@ impl AppEndpoint { ...@@ -760,8 +796,6 @@ impl AppEndpoint {
// assets to add to the middleware manifest (to be loaded in the edge runtime). // assets to add to the middleware manifest (to be loaded in the edge runtime).
let mut middleware_assets = vec![]; let mut middleware_assets = vec![];
let app_entry = app_entry.await?;
let runtime = app_entry.config.await?.runtime.unwrap_or_default(); let runtime = app_entry.config.await?.runtime.unwrap_or_default();
let rsc_entry = app_entry.rsc_entry; let rsc_entry = app_entry.rsc_entry;
...@@ -1334,6 +1368,12 @@ impl Endpoint for AppEndpoint { ...@@ -1334,6 +1368,12 @@ impl Endpoint for AppEndpoint {
.project() .project()
.client_changed(self.output().client_assets())) .client_changed(self.output().client_assets()))
} }
#[turbo_tasks::function]
async fn root_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
let rsc_entry = self.app_endpoint_entry().await?.rsc_entry;
Ok(Vc::cell(vec![rsc_entry]))
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
......
use anyhow::Result;
use turbo_tasks::Vc;
use turbopack_core::chunk::{
global_module_id_strategy::{
children_modules_idents, merge_preprocessed_module_ids, PreprocessedChildrenIdents,
},
module_id_strategies::{GlobalModuleIdStrategy, ModuleIdStrategy},
};
use crate::{
project::Project,
route::{Endpoint, Route},
};
#[turbo_tasks::value]
pub struct GlobalModuleIdStrategyBuilder;
// NOTE(LichuAcu) To access all entrypoints, we need to access an instance of `Project`, but
// `Project` is not available in `turbopack-core`, so we need need this
// `GlobalModuleIdStrategyBuilder` in `next-api`.
#[turbo_tasks::value_impl]
impl GlobalModuleIdStrategyBuilder {
#[turbo_tasks::function]
pub async fn build(project: Vc<Project>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
let mut preprocessed_module_ids = Vec::new();
preprocessed_module_ids.push(children_modules_idents(project.client_main_modules()));
let entrypoints = project.entrypoints().await?;
preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_error_endpoint));
preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_app_endpoint));
preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_document_endpoint));
for (_, route) in entrypoints.routes.iter() {
match route {
Route::Page {
html_endpoint,
data_endpoint,
} => {
preprocessed_module_ids.push(preprocess_module_ids(*html_endpoint));
preprocessed_module_ids.push(preprocess_module_ids(*data_endpoint));
}
Route::PageApi { endpoint } => {
preprocessed_module_ids.push(preprocess_module_ids(*endpoint));
}
Route::AppPage(page_routes) => {
for page_route in page_routes {
preprocessed_module_ids
.push(preprocess_module_ids(page_route.html_endpoint));
preprocessed_module_ids
.push(preprocess_module_ids(page_route.rsc_endpoint));
}
}
Route::AppRoute {
original_name: _,
endpoint,
} => {
preprocessed_module_ids.push(preprocess_module_ids(*endpoint));
}
Route::Conflict => {
tracing::info!("WARN: conflict");
}
}
}
let module_id_map = merge_preprocessed_module_ids(preprocessed_module_ids).await?;
Ok(Vc::upcast(
GlobalModuleIdStrategy::new(module_id_map).await?,
))
}
}
// NOTE(LichuAcu) We can't move this function to `turbopack-core` because we need access to
// `Endpoint`, which is not available there.
#[turbo_tasks::function]
async fn preprocess_module_ids(
endpoint: Vc<Box<dyn Endpoint>>,
) -> Result<Vc<PreprocessedChildrenIdents>> {
let root_modules = endpoint.root_modules();
Ok(children_modules_idents(root_modules))
}
...@@ -15,7 +15,7 @@ use turbopack_core::{ ...@@ -15,7 +15,7 @@ use turbopack_core::{
EntryChunkGroupResult, EntryChunkGroupResult,
}, },
context::AssetContext, context::AssetContext,
module::Module, module::{Module, Modules},
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
reference_type::{EntryReferenceSubType, ReferenceType}, reference_type::{EntryReferenceSubType, ReferenceType},
source::Source, source::Source,
...@@ -236,4 +236,9 @@ impl Endpoint for InstrumentationEndpoint { ...@@ -236,4 +236,9 @@ impl Endpoint for InstrumentationEndpoint {
fn client_changed(self: Vc<Self>) -> Vc<Completion> { fn client_changed(self: Vc<Self>) -> Vc<Completion> {
Completion::immutable() Completion::immutable()
} }
#[turbo_tasks::function]
fn root_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
Err(anyhow::anyhow!("Not implemented."))
}
} }
...@@ -6,6 +6,7 @@ mod app; ...@@ -6,6 +6,7 @@ mod app;
mod dynamic_imports; mod dynamic_imports;
pub mod entrypoints; pub mod entrypoints;
mod font; mod font;
pub mod global_module_id_strategy;
mod instrumentation; mod instrumentation;
mod loadable_manifest; mod loadable_manifest;
mod middleware; mod middleware;
......
...@@ -14,7 +14,7 @@ use turbopack_core::{ ...@@ -14,7 +14,7 @@ use turbopack_core::{
asset::AssetContent, asset::AssetContent,
chunk::{availability_info::AvailabilityInfo, ChunkingContextExt}, chunk::{availability_info::AvailabilityInfo, ChunkingContextExt},
context::AssetContext, context::AssetContext,
module::Module, module::{Module, Modules},
output::OutputAssets, output::OutputAssets,
reference_type::{EntryReferenceSubType, ReferenceType}, reference_type::{EntryReferenceSubType, ReferenceType},
source::Source, source::Source,
...@@ -237,4 +237,9 @@ impl Endpoint for MiddlewareEndpoint { ...@@ -237,4 +237,9 @@ impl Endpoint for MiddlewareEndpoint {
fn client_changed(self: Vc<Self>) -> Vc<Completion> { fn client_changed(self: Vc<Self>) -> Vc<Completion> {
Completion::immutable() Completion::immutable()
} }
#[turbo_tasks::function]
fn root_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
Err(anyhow::anyhow!("Not implemented."))
}
} }
This diff is collapsed.
...@@ -34,12 +34,16 @@ use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSyst ...@@ -34,12 +34,16 @@ use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSyst
use turbopack::{evaluate_context::node_build_environment, ModuleAssetContext}; use turbopack::{evaluate_context::node_build_environment, ModuleAssetContext};
use turbopack_core::{ use turbopack_core::{
changed::content_changed, changed::content_changed,
chunk::ChunkingContext, chunk::{
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
ChunkingContext,
},
compile_time_info::CompileTimeInfo, compile_time_info::CompileTimeInfo,
context::AssetContext, context::AssetContext,
diagnostics::DiagnosticExt, diagnostics::DiagnosticExt,
file_source::FileSource, file_source::FileSource,
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString}, issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
module::Modules,
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
resolve::{find_context_file, FindContextFileResult}, resolve::{find_context_file, FindContextFileResult},
source::Source, source::Source,
...@@ -54,6 +58,7 @@ use crate::{ ...@@ -54,6 +58,7 @@ use crate::{
app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME}, app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME},
build, build,
entrypoints::Entrypoints, entrypoints::Entrypoints,
global_module_id_strategy::GlobalModuleIdStrategyBuilder,
instrumentation::InstrumentationEndpoint, instrumentation::InstrumentationEndpoint,
middleware::MiddlewareEndpoint, middleware::MiddlewareEndpoint,
pages::PagesProject, pages::PagesProject,
...@@ -443,7 +448,7 @@ impl Issue for ConflictIssue { ...@@ -443,7 +448,7 @@ impl Issue for ConflictIssue {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Project { impl Project {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn app_project(self: Vc<Self>) -> Result<Vc<OptionAppProject>> { pub async fn app_project(self: Vc<Self>) -> Result<Vc<OptionAppProject>> {
let app_dir = find_app_dir(self.project_path()).await?; let app_dir = find_app_dir(self.project_path()).await?;
Ok(Vc::cell( Ok(Vc::cell(
...@@ -452,7 +457,7 @@ impl Project { ...@@ -452,7 +457,7 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn pages_project(self: Vc<Self>) -> Result<Vc<PagesProject>> { pub async fn pages_project(self: Vc<Self>) -> Result<Vc<PagesProject>> {
Ok(PagesProject::new(self)) Ok(PagesProject::new(self))
} }
...@@ -621,6 +626,7 @@ impl Project { ...@@ -621,6 +626,7 @@ impl Project {
self.next_config().computed_asset_prefix(), self.next_config().computed_asset_prefix(),
self.client_compile_time_info().environment(), self.client_compile_time_info().environment(),
self.next_mode(), self.next_mode(),
self.module_id_strategy(),
)) ))
} }
...@@ -637,6 +643,7 @@ impl Project { ...@@ -637,6 +643,7 @@ impl Project {
self.client_relative_path(), self.client_relative_path(),
self.next_config().computed_asset_prefix(), self.next_config().computed_asset_prefix(),
self.server_compile_time_info().environment(), self.server_compile_time_info().environment(),
self.module_id_strategy(),
) )
} else { } else {
get_server_chunking_context( get_server_chunking_context(
...@@ -644,6 +651,7 @@ impl Project { ...@@ -644,6 +651,7 @@ impl Project {
self.project_path(), self.project_path(),
self.node_root(), self.node_root(),
self.server_compile_time_info().environment(), self.server_compile_time_info().environment(),
self.module_id_strategy(),
) )
} }
} }
...@@ -661,6 +669,7 @@ impl Project { ...@@ -661,6 +669,7 @@ impl Project {
self.client_relative_path(), self.client_relative_path(),
self.next_config().computed_asset_prefix(), self.next_config().computed_asset_prefix(),
self.edge_compile_time_info().environment(), self.edge_compile_time_info().environment(),
self.module_id_strategy(),
) )
} else { } else {
get_edge_chunking_context( get_edge_chunking_context(
...@@ -668,6 +677,7 @@ impl Project { ...@@ -668,6 +677,7 @@ impl Project {
self.project_path(), self.project_path(),
self.node_root(), self.node_root(),
self.edge_compile_time_info().environment(), self.edge_compile_time_info().environment(),
self.module_id_strategy(),
) )
} }
} }
...@@ -1161,6 +1171,29 @@ impl Project { ...@@ -1161,6 +1171,29 @@ impl Project {
let path = self.client_root(); let path = self.client_root();
any_output_changed(roots, path, false) any_output_changed(roots, path, false)
} }
#[turbo_tasks::function]
pub async fn client_main_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
let pages_project = self.pages_project();
let mut modules = vec![pages_project.client_main_module()];
if let Some(app_project) = *self.app_project().await? {
modules.push(app_project.client_main_module());
}
Ok(Vc::cell(modules))
}
/// Get the module id strategy for the project.
/// In production mode, we use the global module id strategy with optimized ids.
/// In development mode, we use a standard module id strategy with no modifications.
#[turbo_tasks::function]
pub async fn module_id_strategy(self: Vc<Self>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
match *self.next_mode().await? {
NextMode::Build => Ok(Vc::upcast(GlobalModuleIdStrategyBuilder::build(self))),
NextMode::Development => Ok(Vc::upcast(DevModuleIdStrategy::new())),
}
}
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -2,6 +2,7 @@ use anyhow::Result; ...@@ -2,6 +2,7 @@ use anyhow::Result;
use indexmap::IndexMap; use indexmap::IndexMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_tasks::{debug::ValueDebugFormat, trace::TraceRawVcs, Completion, RcStr, Vc}; use turbo_tasks::{debug::ValueDebugFormat, trace::TraceRawVcs, Completion, RcStr, Vc};
use turbopack_core::module::Modules;
use crate::paths::ServerPath; use crate::paths::ServerPath;
...@@ -75,6 +76,7 @@ pub trait Endpoint { ...@@ -75,6 +76,7 @@ pub trait Endpoint {
fn write_to_disk(self: Vc<Self>) -> Vc<WrittenEndpoint>; fn write_to_disk(self: Vc<Self>) -> Vc<WrittenEndpoint>;
fn server_changed(self: Vc<Self>) -> Vc<Completion>; fn server_changed(self: Vc<Self>) -> Vc<Completion>;
fn client_changed(self: Vc<Self>) -> Vc<Completion>; fn client_changed(self: Vc<Self>) -> Vc<Completion>;
fn root_modules(self: Vc<Self>) -> Vc<Modules>;
} }
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
......
...@@ -14,7 +14,7 @@ use turbopack::{ ...@@ -14,7 +14,7 @@ use turbopack::{
}; };
use turbopack_browser::{react_refresh::assert_can_resolve_react_refresh, BrowserChunkingContext}; use turbopack_browser::{react_refresh::assert_can_resolve_react_refresh, BrowserChunkingContext};
use turbopack_core::{ use turbopack_core::{
chunk::ChunkingContext, chunk::{module_id_strategies::ModuleIdStrategy, ChunkingContext},
compile_time_info::{ compile_time_info::{
CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReference, CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReference,
FreeVarReferences, FreeVarReferences,
...@@ -353,6 +353,7 @@ pub async fn get_client_chunking_context( ...@@ -353,6 +353,7 @@ pub async fn get_client_chunking_context(
asset_prefix: Vc<Option<RcStr>>, asset_prefix: Vc<Option<RcStr>>,
environment: Vc<Environment>, environment: Vc<Environment>,
mode: Vc<NextMode>, mode: Vc<NextMode>,
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
) -> Result<Vc<Box<dyn ChunkingContext>>> { ) -> Result<Vc<Box<dyn ChunkingContext>>> {
let next_mode = mode.await?; let next_mode = mode.await?;
let mut builder = BrowserChunkingContext::builder( let mut builder = BrowserChunkingContext::builder(
...@@ -366,7 +367,8 @@ pub async fn get_client_chunking_context( ...@@ -366,7 +367,8 @@ pub async fn get_client_chunking_context(
) )
.chunk_base_path(asset_prefix) .chunk_base_path(asset_prefix)
.minify_type(next_mode.minify_type()) .minify_type(next_mode.minify_type())
.asset_base_path(asset_prefix); .asset_base_path(asset_prefix)
.module_id_strategy(module_id_strategy);
if next_mode.is_development() { if next_mode.is_development() {
builder = builder.hot_module_replacement(); builder = builder.hot_module_replacement();
......
...@@ -6,7 +6,7 @@ use turbo_tasks_fs::FileSystemPath; ...@@ -6,7 +6,7 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack::resolve_options_context::ResolveOptionsContext; use turbopack::resolve_options_context::ResolveOptionsContext;
use turbopack_browser::BrowserChunkingContext; use turbopack_browser::BrowserChunkingContext;
use turbopack_core::{ use turbopack_core::{
chunk::ChunkingContext, chunk::{module_id_strategies::ModuleIdStrategy, ChunkingContext},
compile_time_info::{ compile_time_info::{
CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReference, CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReference,
FreeVarReferences, FreeVarReferences,
...@@ -178,6 +178,7 @@ pub async fn get_edge_chunking_context_with_client_assets( ...@@ -178,6 +178,7 @@ pub async fn get_edge_chunking_context_with_client_assets(
client_root: Vc<FileSystemPath>, client_root: Vc<FileSystemPath>,
asset_prefix: Vc<Option<RcStr>>, asset_prefix: Vc<Option<RcStr>>,
environment: Vc<Environment>, environment: Vc<Environment>,
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
) -> Result<Vc<Box<dyn ChunkingContext>>> { ) -> Result<Vc<Box<dyn ChunkingContext>>> {
let output_root = node_root.join("server/edge".into()); let output_root = node_root.join("server/edge".into());
let next_mode = mode.await?; let next_mode = mode.await?;
...@@ -193,6 +194,7 @@ pub async fn get_edge_chunking_context_with_client_assets( ...@@ -193,6 +194,7 @@ pub async fn get_edge_chunking_context_with_client_assets(
) )
.asset_base_path(asset_prefix) .asset_base_path(asset_prefix)
.minify_type(next_mode.minify_type()) .minify_type(next_mode.minify_type())
.module_id_strategy(module_id_strategy)
.build(), .build(),
)) ))
} }
...@@ -203,6 +205,7 @@ pub async fn get_edge_chunking_context( ...@@ -203,6 +205,7 @@ pub async fn get_edge_chunking_context(
project_path: Vc<FileSystemPath>, project_path: Vc<FileSystemPath>,
node_root: Vc<FileSystemPath>, node_root: Vc<FileSystemPath>,
environment: Vc<Environment>, environment: Vc<Environment>,
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
) -> Result<Vc<Box<dyn ChunkingContext>>> { ) -> Result<Vc<Box<dyn ChunkingContext>>> {
let output_root = node_root.join("server/edge".into()); let output_root = node_root.join("server/edge".into());
let next_mode = mode.await?; let next_mode = mode.await?;
...@@ -222,6 +225,7 @@ pub async fn get_edge_chunking_context( ...@@ -222,6 +225,7 @@ pub async fn get_edge_chunking_context(
// asset from the output directory. // asset from the output directory.
.asset_base_path(Vc::cell(Some("blob:server/edge/".into()))) .asset_base_path(Vc::cell(Some("blob:server/edge/".into())))
.minify_type(next_mode.minify_type()) .minify_type(next_mode.minify_type())
.module_id_strategy(module_id_strategy)
.build(), .build(),
)) ))
} }
...@@ -14,6 +14,7 @@ use turbopack::{ ...@@ -14,6 +14,7 @@ use turbopack::{
transition::Transition, transition::Transition,
}; };
use turbopack_core::{ use turbopack_core::{
chunk::module_id_strategies::ModuleIdStrategy,
compile_time_info::{ compile_time_info::{
CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReferences, CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReferences,
}, },
...@@ -922,6 +923,7 @@ pub async fn get_server_chunking_context_with_client_assets( ...@@ -922,6 +923,7 @@ pub async fn get_server_chunking_context_with_client_assets(
client_root: Vc<FileSystemPath>, client_root: Vc<FileSystemPath>,
asset_prefix: Vc<Option<RcStr>>, asset_prefix: Vc<Option<RcStr>>,
environment: Vc<Environment>, environment: Vc<Environment>,
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
) -> Result<Vc<NodeJsChunkingContext>> { ) -> Result<Vc<NodeJsChunkingContext>> {
let next_mode = mode.await?; let next_mode = mode.await?;
// TODO(alexkirsz) This should return a trait that can be implemented by the // TODO(alexkirsz) This should return a trait that can be implemented by the
...@@ -938,6 +940,7 @@ pub async fn get_server_chunking_context_with_client_assets( ...@@ -938,6 +940,7 @@ pub async fn get_server_chunking_context_with_client_assets(
) )
.asset_prefix(asset_prefix) .asset_prefix(asset_prefix)
.minify_type(next_mode.minify_type()) .minify_type(next_mode.minify_type())
.module_id_strategy(module_id_strategy)
.build()) .build())
} }
...@@ -947,6 +950,7 @@ pub async fn get_server_chunking_context( ...@@ -947,6 +950,7 @@ pub async fn get_server_chunking_context(
project_path: Vc<FileSystemPath>, project_path: Vc<FileSystemPath>,
node_root: Vc<FileSystemPath>, node_root: Vc<FileSystemPath>,
environment: Vc<Environment>, environment: Vc<Environment>,
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
) -> Result<Vc<NodeJsChunkingContext>> { ) -> Result<Vc<NodeJsChunkingContext>> {
let next_mode = mode.await?; let next_mode = mode.await?;
// TODO(alexkirsz) This should return a trait that can be implemented by the // TODO(alexkirsz) This should return a trait that can be implemented by the
...@@ -962,5 +966,6 @@ pub async fn get_server_chunking_context( ...@@ -962,5 +966,6 @@ pub async fn get_server_chunking_context(
next_mode.runtime_type(), next_mode.runtime_type(),
) )
.minify_type(next_mode.minify_type()) .minify_type(next_mode.minify_type())
.module_id_strategy(module_id_strategy)
.build()) .build())
} }
module.exports = { module.exports = {
foo: 'bar', foo: 'external-package content',
} }
module.exports = { module.exports = {
bar: 'baz', bar: 'opted-out-external-package content',
} }
...@@ -27,10 +27,8 @@ describe('bundle pages externals with config.bundlePagesRouterDependencies', () ...@@ -27,10 +27,8 @@ describe('bundle pages externals with config.bundlePagesRouterDependencies', ()
allBundles += output allBundles += output
} }
// we don't know the name of the minified `__turbopack_external_require__`, so we just check the arguments. // we don't know the name of the minified `__turbopack_external_require__`, so we just check the content.
expect(allBundles).not.toContain( expect(allBundles).toContain('"external-package content"')
'"[externals]/ [external] (external-package, cjs)"'
)
} else { } else {
const output = await fs.readFile( const output = await fs.readFile(
join(appDir, '.next/server/pages/index.js'), join(appDir, '.next/server/pages/index.js'),
...@@ -53,9 +51,9 @@ describe('bundle pages externals with config.bundlePagesRouterDependencies', () ...@@ -53,9 +51,9 @@ describe('bundle pages externals with config.bundlePagesRouterDependencies', ()
allBundles += output allBundles += output
} }
// we don't know the name of the minified `__turbopack_external_require__`, so we just check the arguments. // we don't know the name of the minified `__turbopack_external_require__`, so we just check the content.
expect(allBundles).toContain( expect(allBundles).not.toContain(
'"[externals]/ [external] (opted-out-external-package, cjs)"' '"opted-out-external-package content"'
) )
} else { } else {
const output = await fs.readFile( const output = await fs.readFile(
......
...@@ -6,6 +6,7 @@ use turbopack_core::{ ...@@ -6,6 +6,7 @@ use turbopack_core::{
chunk::{ chunk::{
availability_info::AvailabilityInfo, availability_info::AvailabilityInfo,
chunk_group::{make_chunk_group, MakeChunkGroupResult}, chunk_group::{make_chunk_group, MakeChunkGroupResult},
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext, Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext,
EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId,
}, },
...@@ -77,6 +78,11 @@ impl BrowserChunkingContextBuilder { ...@@ -77,6 +78,11 @@ impl BrowserChunkingContextBuilder {
self self
} }
pub fn module_id_strategy(mut self, module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>) -> Self {
self.chunking_context.module_id_strategy = module_id_strategy;
self
}
pub fn build(self) -> Vc<BrowserChunkingContext> { pub fn build(self) -> Vc<BrowserChunkingContext> {
BrowserChunkingContext::new(Value::new(self.chunking_context)) BrowserChunkingContext::new(Value::new(self.chunking_context))
} }
...@@ -122,6 +128,8 @@ pub struct BrowserChunkingContext { ...@@ -122,6 +128,8 @@ pub struct BrowserChunkingContext {
minify_type: MinifyType, minify_type: MinifyType,
/// Whether to use manifest chunks for lazy compilation /// Whether to use manifest chunks for lazy compilation
manifest_chunks: bool, manifest_chunks: bool,
/// The module id strategy to use
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
} }
impl BrowserChunkingContext { impl BrowserChunkingContext {
...@@ -151,6 +159,7 @@ impl BrowserChunkingContext { ...@@ -151,6 +159,7 @@ impl BrowserChunkingContext {
runtime_type, runtime_type,
minify_type: MinifyType::NoMinify, minify_type: MinifyType::NoMinify,
manifest_chunks: false, manifest_chunks: false,
module_id_strategy: Vc::upcast(DevModuleIdStrategy::new()),
}, },
} }
} }
...@@ -479,6 +488,14 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -479,6 +488,14 @@ impl ChunkingContext for BrowserChunkingContext {
bail!("Browser chunking context does not support entry chunk groups") bail!("Browser chunking context does not support entry chunk groups")
} }
#[turbo_tasks::function]
async fn chunk_item_id_from_ident(
self: Vc<Self>,
ident: Vc<AssetIdent>,
) -> Result<Vc<ModuleId>> {
Ok(self.await?.module_id_strategy.get_module_id(ident))
}
#[turbo_tasks::function] #[turbo_tasks::function]
async fn async_loader_chunk_item( async fn async_loader_chunk_item(
self: Vc<Self>, self: Vc<Self>,
......
...@@ -17,7 +17,7 @@ async fn get_referenced_output_assets( ...@@ -17,7 +17,7 @@ async fn get_referenced_output_assets(
Ok(parent.references().await?.clone_value().into_iter()) Ok(parent.references().await?.clone_value().into_iter())
} }
async fn get_referenced_modules( pub async fn get_referenced_modules(
parent: Vc<Box<dyn Module>>, parent: Vc<Box<dyn Module>>,
) -> Result<impl Iterator<Item = Vc<Box<dyn Module>>> + Send> { ) -> Result<impl Iterator<Item = Vc<Box<dyn Module>>> + Send> {
Ok(primary_referenced_modules(parent) Ok(primary_referenced_modules(parent)
......
use anyhow::Result; use anyhow::Result;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_tasks::{trace::TraceRawVcs, RcStr, TaskInput, Upcast, Value, ValueToString, Vc}; use turbo_tasks::{trace::TraceRawVcs, RcStr, TaskInput, Upcast, Value, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::DeterministicHash; use turbo_tasks_hash::DeterministicHash;
...@@ -116,9 +116,7 @@ pub trait ChunkingContext { ...@@ -116,9 +116,7 @@ pub trait ChunkingContext {
async fn chunk_item_id_from_ident( async fn chunk_item_id_from_ident(
self: Vc<Self>, self: Vc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
) -> Result<Vc<ModuleId>> { ) -> Result<Vc<ModuleId>>;
Ok(ModuleId::String(ident.to_string().await?.clone_value()).cell())
}
fn chunk_item_id(self: Vc<Self>, chunk_item: Vc<Box<dyn ChunkItem>>) -> Vc<ModuleId> { fn chunk_item_id(self: Vc<Self>, chunk_item: Vc<Box<dyn ChunkItem>>) -> Vc<ModuleId> {
self.chunk_item_id_from_ident(chunk_item.asset_ident()) self.chunk_item_id_from_ident(chunk_item.asset_ident())
......
use std::collections::{HashMap, HashSet};
use anyhow::Result;
use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal},
ValueToString, Vc,
};
use turbo_tasks_hash::hash_xxh3_hash64;
use crate::{
changed::get_referenced_modules,
chunk::ModuleId,
ident::AssetIdent,
module::{Module, Modules},
};
#[turbo_tasks::value]
pub struct PreprocessedChildrenIdents {
// module_id -> full hash
// We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids`
// if this endpoint did not change.
modules_idents: HashMap<AssetIdent, u64>,
}
// NOTE(LichuAcu) Called on endpoint.root_modules(). It would probably be better if this was called
// directly on `Endpoint`, but such struct is not available in turbopack-core. The whole function
// could be moved to `next-api`, but it would require adding turbo-tasks-hash to `next-api`,
// making it heavier.
#[turbo_tasks::function]
pub async fn children_modules_idents(
root_modules: Vc<Modules>,
) -> Result<Vc<PreprocessedChildrenIdents>> {
let children_modules_iter = AdjacencyMap::new()
.skip_duplicates()
.visit(root_modules.await?.iter().copied(), get_referenced_modules)
.await
.completed()?
.into_inner()
.into_reverse_topological();
// module_id -> full hash
let mut modules_idents = HashMap::new();
for module in children_modules_iter {
let module_ident = module.ident();
let hash = hash_xxh3_hash64(module_ident.to_string().await?);
modules_idents.insert(module_ident.await?.clone_value(), hash);
}
Ok(PreprocessedChildrenIdents { modules_idents }.cell())
}
// Note(LichuAcu): This could be split into two functions: one that merges the preprocessed module
// ids and another that generates the final, optimized module ids. Thoughts?
pub async fn merge_preprocessed_module_ids(
prepared_module_ids: Vec<Vc<PreprocessedChildrenIdents>>,
) -> Result<HashMap<AssetIdent, Vc<ModuleId>>> {
let mut module_id_map: HashMap<AssetIdent, Vc<ModuleId>> = HashMap::new();
let mut used_ids: HashSet<u64> = HashSet::new();
for prepared_module_ids in prepared_module_ids {
for (module_ident, full_hash) in prepared_module_ids.await?.modules_idents.iter() {
process_module(
module_ident.clone(),
*full_hash,
&mut module_id_map,
&mut used_ids,
)
.await?;
}
}
Ok(module_id_map)
}
pub async fn process_module(
module_ident: AssetIdent,
full_hash: u64,
id_map: &mut HashMap<AssetIdent, Vc<ModuleId>>,
used_ids: &mut HashSet<u64>,
) -> Result<()> {
if id_map.contains_key(&module_ident) {
return Ok(());
}
let mut masked_hash = full_hash & 0xF;
let mut mask = 0xF;
while used_ids.contains(&masked_hash) {
if mask == 0xFFFFFFFFFFFFFFFF {
return Err(anyhow::anyhow!("This is a... 64-bit hash collision?"));
}
mask = (mask << 4) | 0xF;
masked_hash = full_hash & mask;
}
let hashed_module_id = ModuleId::String(masked_hash.to_string().into());
id_map.insert(module_ident, hashed_module_id.cell());
used_ids.insert(masked_hash);
Ok(())
}
...@@ -6,6 +6,8 @@ pub(crate) mod chunking_context; ...@@ -6,6 +6,8 @@ pub(crate) mod chunking_context;
pub(crate) mod containment_tree; pub(crate) mod containment_tree;
pub(crate) mod data; pub(crate) mod data;
pub(crate) mod evaluate; pub(crate) mod evaluate;
pub mod global_module_id_strategy;
pub mod module_id_strategies;
pub mod optimize; pub mod optimize;
use std::{ use std::{
......
use std::collections::HashMap;
use anyhow::Result;
use turbo_tasks::{ValueToString, Vc};
use turbo_tasks_hash::hash_xxh3_hash64;
use super::ModuleId;
use crate::ident::AssetIdent;
#[turbo_tasks::value_trait]
pub trait ModuleIdStrategy {
fn get_module_id(self: Vc<Self>, ident: Vc<AssetIdent>) -> Vc<ModuleId>;
}
#[turbo_tasks::value]
pub struct DevModuleIdStrategy;
impl DevModuleIdStrategy {
pub fn new() -> Vc<Self> {
DevModuleIdStrategy {}.cell()
}
}
#[turbo_tasks::value_impl]
impl ModuleIdStrategy for DevModuleIdStrategy {
#[turbo_tasks::function]
async fn get_module_id(self: Vc<Self>, ident: Vc<AssetIdent>) -> Result<Vc<ModuleId>> {
Ok(ModuleId::String(ident.to_string().await?.clone_value()).cell())
}
}
#[turbo_tasks::value]
pub struct GlobalModuleIdStrategy {
module_id_map: HashMap<AssetIdent, Vc<ModuleId>>,
}
impl GlobalModuleIdStrategy {
pub async fn new(module_id_map: HashMap<AssetIdent, Vc<ModuleId>>) -> Result<Vc<Self>> {
Ok(GlobalModuleIdStrategy { module_id_map }.cell())
}
}
#[turbo_tasks::value_impl]
impl ModuleIdStrategy for GlobalModuleIdStrategy {
#[turbo_tasks::function]
async fn get_module_id(self: Vc<Self>, ident: Vc<AssetIdent>) -> Result<Vc<ModuleId>> {
if let Some(module_id) = self.await?.module_id_map.get(&ident.await?.clone_value()) {
// dbg!(format!("Hit {}", ident.to_string().await?));
return Ok(*module_id);
}
// dbg!(format!("Miss {}", ident.to_string().await?));
Ok(ModuleId::String(
hash_xxh3_hash64(ident.to_string().await?)
.to_string()
.into(),
)
.cell())
}
}
...@@ -8,6 +8,7 @@ use turbopack_core::{ ...@@ -8,6 +8,7 @@ use turbopack_core::{
chunk::{ chunk::{
availability_info::AvailabilityInfo, availability_info::AvailabilityInfo,
chunk_group::{make_chunk_group, MakeChunkGroupResult}, chunk_group::{make_chunk_group, MakeChunkGroupResult},
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext, Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext,
EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId,
}, },
...@@ -53,6 +54,11 @@ impl NodeJsChunkingContextBuilder { ...@@ -53,6 +54,11 @@ impl NodeJsChunkingContextBuilder {
self self
} }
pub fn module_id_strategy(mut self, module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>) -> Self {
self.chunking_context.module_id_strategy = module_id_strategy;
self
}
/// Builds the chunking context. /// Builds the chunking context.
pub fn build(self) -> Vc<NodeJsChunkingContext> { pub fn build(self) -> Vc<NodeJsChunkingContext> {
NodeJsChunkingContext::new(Value::new(self.chunking_context)) NodeJsChunkingContext::new(Value::new(self.chunking_context))
...@@ -84,6 +90,8 @@ pub struct NodeJsChunkingContext { ...@@ -84,6 +90,8 @@ pub struct NodeJsChunkingContext {
minify_type: MinifyType, minify_type: MinifyType,
/// Whether to use manifest chunks for lazy compilation /// Whether to use manifest chunks for lazy compilation
manifest_chunks: bool, manifest_chunks: bool,
/// The strategy to use for generating module ids
module_id_strategy: Vc<Box<dyn ModuleIdStrategy>>,
} }
impl NodeJsChunkingContext { impl NodeJsChunkingContext {
...@@ -109,6 +117,7 @@ impl NodeJsChunkingContext { ...@@ -109,6 +117,7 @@ impl NodeJsChunkingContext {
runtime_type, runtime_type,
minify_type: MinifyType::NoMinify, minify_type: MinifyType::NoMinify,
manifest_chunks: false, manifest_chunks: false,
module_id_strategy: Vc::upcast(DevModuleIdStrategy::new()),
}, },
} }
} }
...@@ -349,6 +358,14 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -349,6 +358,14 @@ impl ChunkingContext for NodeJsChunkingContext {
bail!("the build chunking context does not support evaluated chunk groups") bail!("the build chunking context does not support evaluated chunk groups")
} }
#[turbo_tasks::function]
async fn chunk_item_id_from_ident(
self: Vc<Self>,
ident: Vc<AssetIdent>,
) -> Result<Vc<ModuleId>> {
Ok(self.await?.module_id_strategy.get_module_id(ident))
}
#[turbo_tasks::function] #[turbo_tasks::function]
async fn async_loader_chunk_item( async fn async_loader_chunk_item(
self: Vc<Self>, self: Vc<Self>,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment