Unverified Commit e91660d1 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: scope hoisting (#79459)

Enable Turbopack to merge modules together (when possible), getting rid of the `"module id...": (__turbopack_context__) => { ... __turbopack_esm_export__({foo: () => foo})` overhead:

<img width="1591" alt="Bildschirmfoto 2025-06-14 um 14 13 36" src="https://github.com/user-attachments/assets/b6baccbe-a2c7-4acd-b17b-f7bd91c8f349" />

## Details

There is a lot of complexity here, mostly for the problem of when this is actually safe to do.
Generally, modules can only be merged if they are known to always execute in that exact order (also meaning in every chunk these modules are part of).
This is done by first propagating a bitmap on the whole module graph, to determine which modules are safe to merge together (module type is compatible, only ESM, etc). Then, these potential merged groups are linearized in execution order _per chunk_. These orderings are then split up until every module is just part of a single ordering (together with the modules that are always executed before/after it, if any). Each of these orderings then becomes a merged module.

This information is then used to replace these modules with the merged version in the middle of chunking (after the module graph -> vec of modules step, but before turning vec of modules -> vec of chunks).

For code generation, the runtime helpers are omitted and imports just become straight variable references (i.e. across modules). This (together with preventing local variable naming collisions in the merged module) is achieved by leveraging swc's syntax contexts (each module effectively gets it's own syntax context, so that variables of the same name don't collide, and then imports are rewritten to point to the local variable in the other module). Comments and source maps work by encoding the module into the `BytePos`, so that a given `BytePos` in the merged AST can be decoded back into the module that it originated from, to lookup comments and source locations in the right structs.
parent 925fd916
......@@ -66,6 +66,7 @@ test/e2e/async-modules/amp-validator-wasm.js
/turbopack/crates/next-transform-dynamic/tests
/turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/error/input/broken.js
/turbopack/crates/turbopack-tests/tests/snapshot/import-meta/cjs/input/mod.cjs
/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/*
/turbopack/crates/turbopack-tests/tests/**/output*
/turbopack/crates/turbopack-tests/tests/**/static
......@@ -8198,9 +8198,9 @@ dependencies = [
[[package]]
name = "swc_sourcemap"
version = "9.3.1"
version = "9.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "604e837ad12213d381dea886a184e06b4e09fd502e5d1a4db9d4b7341eb85105"
checksum = "9755c673c6a83c461e98fa018f681adb8394a3f44f89a06f27e80fd4fe4fa1e4"
dependencies = [
"base64-simd 0.8.0",
"bitvec",
......@@ -9674,6 +9674,7 @@ dependencies = [
"either",
"indexmap 2.7.1",
"indoc",
"itertools 0.10.5",
"num-bigint",
"num-traits",
"once_cell",
......
......@@ -411,7 +411,7 @@ smallvec = { version = "1.13.1", features = [
"union",
"const_new",
] }
swc_sourcemap = "9.2.2"
swc_sourcemap = "9.3.2"
strsim = "0.11.1"
shrink-to-fit = "0.2.10"
syn = "2.0.100"
......
......@@ -1018,6 +1018,7 @@ impl Project {
self.next_config().turbo_minify(self.next_mode()),
self.next_config().client_source_maps(self.next_mode()),
self.no_mangling(),
self.next_config().turbo_scope_hoisting(self.next_mode()),
))
}
......@@ -1039,6 +1040,7 @@ impl Project {
self.next_config().turbo_minify(self.next_mode()),
self.next_config().server_source_maps(),
self.no_mangling(),
self.next_config().turbo_scope_hoisting(self.next_mode()),
)
} else {
get_server_chunking_context(
......@@ -1051,6 +1053,7 @@ impl Project {
self.next_config().turbo_minify(self.next_mode()),
self.next_config().server_source_maps(),
self.no_mangling(),
self.next_config().turbo_scope_hoisting(self.next_mode()),
)
})
}
......@@ -1073,6 +1076,7 @@ impl Project {
self.next_config().turbo_minify(self.next_mode()),
self.next_config().server_source_maps(),
self.no_mangling(),
self.next_config().turbo_scope_hoisting(self.next_mode()),
)
} else {
get_edge_chunking_context(
......@@ -1085,6 +1089,7 @@ impl Project {
self.next_config().turbo_minify(self.next_mode()),
self.next_config().server_source_maps(),
self.no_mangling(),
self.next_config().turbo_scope_hoisting(self.next_mode()),
)
})
}
......
......@@ -435,6 +435,7 @@ pub async fn get_client_chunking_context(
minify: Vc<bool>,
source_maps: Vc<bool>,
no_mangling: Vc<bool>,
scope_hoisting: Vc<bool>,
) -> Result<Vc<Box<dyn ChunkingContext>>> {
let next_mode = mode.await?;
let mut builder = BrowserChunkingContext::builder(
......@@ -471,23 +472,25 @@ pub async fn get_client_chunking_context(
if next_mode.is_development() {
builder = builder.hot_module_replacement().use_file_source_map_uris();
} else {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 50_000,
max_chunk_count_per_group: 40,
max_merge_chunk_size: 200_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder.use_content_hashing(ContentHashing::Direct { length: 16 })
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 50_000,
max_chunk_count_per_group: 40,
max_merge_chunk_size: 200_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.use_content_hashing(ContentHashing::Direct { length: 16 })
.module_merging(*scope_hoisting.await?);
}
Ok(Vc::upcast(builder.build()))
......
......@@ -795,6 +795,7 @@ pub struct ExperimentalConfig {
turbopack_persistent_caching: Option<bool>,
turbopack_source_maps: Option<bool>,
turbopack_tree_shaking: Option<bool>,
turbopack_scope_hoisting: Option<bool>,
// Whether to enable the global-not-found convention
global_not_found: Option<bool>,
/// Defaults to false in development mode, true in production mode.
......@@ -1593,6 +1594,15 @@ impl NextConfig {
))
}
#[turbo_tasks::function]
pub async fn turbo_scope_hoisting(&self, mode: Vc<NextMode>) -> Result<Vc<bool>> {
Ok(Vc::cell(match *mode.await? {
// Ignore configuration in development mode to not break HMR
NextMode::Development => false,
NextMode::Build => self.experimental.turbopack_scope_hoisting.unwrap_or(true),
}))
}
#[turbo_tasks::function]
pub async fn client_source_maps(&self, _mode: Vc<NextMode>) -> Result<Vc<bool>> {
// Temporarily always enable client source maps as tests regress.
......
......@@ -231,6 +231,7 @@ pub async fn get_edge_chunking_context_with_client_assets(
turbo_minify: Vc<bool>,
turbo_source_maps: Vc<bool>,
no_mangling: Vc<bool>,
scope_hoisting: Vc<bool>,
) -> Result<Vc<Box<dyn ChunkingContext>>> {
let output_root = node_root.join(rcstr!("server/edge")).to_resolved().await?;
let next_mode = mode.await?;
......@@ -264,20 +265,22 @@ pub async fn get_edge_chunking_context_with_client_assets(
.module_id_strategy(module_id_strategy);
if !next_mode.is_development() {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.module_merging(*scope_hoisting.await?);
}
Ok(Vc::upcast(builder.build()))
......@@ -294,6 +297,7 @@ pub async fn get_edge_chunking_context(
turbo_minify: Vc<bool>,
turbo_source_maps: Vc<bool>,
no_mangling: Vc<bool>,
scope_hoisting: Vc<bool>,
) -> Result<Vc<Box<dyn ChunkingContext>>> {
let output_root = node_root.join(rcstr!("server/edge")).to_resolved().await?;
let next_mode = mode.await?;
......@@ -327,20 +331,22 @@ pub async fn get_edge_chunking_context(
.module_id_strategy(module_id_strategy);
if !next_mode.is_development() {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.module_merging(*scope_hoisting.await?);
}
Ok(Vc::upcast(builder.build()))
......
......@@ -998,6 +998,7 @@ pub async fn get_server_chunking_context_with_client_assets(
turbo_minify: Vc<bool>,
turbo_source_maps: Vc<bool>,
no_mangling: Vc<bool>,
scope_hoisting: Vc<bool>,
) -> Result<Vc<NodeJsChunkingContext>> {
let next_mode = mode.await?;
// TODO(alexkirsz) This should return a trait that can be implemented by the
......@@ -1039,22 +1040,24 @@ pub async fn get_server_chunking_context_with_client_assets(
if next_mode.is_development() {
builder = builder.use_file_source_map_uris();
} else {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
max_chunk_count_per_group: 100,
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
max_chunk_count_per_group: 100,
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.module_merging(*scope_hoisting.await?);
}
Ok(builder.build())
......@@ -1071,6 +1074,7 @@ pub async fn get_server_chunking_context(
turbo_minify: Vc<bool>,
turbo_source_maps: Vc<bool>,
no_mangling: Vc<bool>,
scope_hoisting: Vc<bool>,
) -> Result<Vc<NodeJsChunkingContext>> {
let next_mode = mode.await?;
// TODO(alexkirsz) This should return a trait that can be implemented by the
......@@ -1110,22 +1114,24 @@ pub async fn get_server_chunking_context(
if next_mode.is_development() {
builder = builder.use_file_source_map_uris()
} else {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
max_chunk_count_per_group: 100,
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
max_chunk_count_per_group: 100,
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.module_merging(*scope_hoisting.await?);
}
Ok(builder.build())
......
......@@ -462,6 +462,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
turbopackSourceMaps: z.boolean().optional(),
turbopackTreeShaking: z.boolean().optional(),
turbopackRemoveUnusedExports: z.boolean().optional(),
turbopackScopeHoisting: z.boolean().optional(),
optimizePackageImports: z.array(z.string()).optional(),
optimizeServerReact: z.boolean().optional(),
clientTraceMetadata: z.array(z.string()).optional(),
......
......@@ -452,6 +452,11 @@ export interface ExperimentalConfig {
*/
turbopackMinify?: boolean
/**
* Enable scope hoisting. Defaults to true in build mode. Always disabled in development mode.
*/
turbopackScopeHoisting?: boolean
/**
* Enable persistent caching for the turbopack dev server and build.
*/
......
......@@ -97,6 +97,11 @@ impl BrowserChunkingContextBuilder {
self
}
pub fn module_merging(mut self, enable_module_merging: bool) -> Self {
self.chunking_context.enable_module_merging = enable_module_merging;
self
}
pub fn asset_base_path(mut self, asset_base_path: Option<RcStr>) -> Self {
self.chunking_context.asset_base_path = asset_base_path;
self
......@@ -202,6 +207,8 @@ pub struct BrowserChunkingContext {
enable_hot_module_replacement: bool,
/// Enable tracing for this chunking
enable_tracing: bool,
/// Enable module merging
enable_module_merging: bool,
/// The environment chunks will be evaluated in.
environment: ResolvedVc<Environment>,
/// The kind of runtime to include in the output.
......@@ -248,6 +255,7 @@ impl BrowserChunkingContext {
asset_base_path: None,
enable_hot_module_replacement: false,
enable_tracing: false,
enable_module_merging: false,
environment,
runtime_type,
minify_type: MinifyType::NoMinify,
......@@ -513,6 +521,11 @@ impl ChunkingContext for BrowserChunkingContext {
Vc::cell(self.enable_tracing)
}
#[turbo_tasks::function]
fn is_module_merging_enabled(&self) -> Vc<bool> {
Vc::cell(self.enable_module_merging)
}
#[turbo_tasks::function]
pub fn minify_type(&self) -> Vc<MinifyType> {
self.minify_type.cell()
......
......@@ -342,23 +342,25 @@ async fn build_internal(
match *node_env.await? {
NodeEnv::Development => {}
NodeEnv::Production => {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 50_000,
max_chunk_count_per_group: 40,
max_merge_chunk_size: 200_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder.use_content_hashing(ContentHashing::Direct { length: 16 })
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 50_000,
max_chunk_count_per_group: 40,
max_merge_chunk_size: 200_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.use_content_hashing(ContentHashing::Direct { length: 16 })
.module_merging(true);
}
}
......@@ -386,22 +388,24 @@ async fn build_internal(
match *node_env.await? {
NodeEnv::Development => {}
NodeEnv::Production => {
builder = builder.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
max_chunk_count_per_group: 100,
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
);
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
min_chunk_size: 20_000,
max_chunk_count_per_group: 100,
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.chunking_config(
Vc::<CssChunkType>::default().to_resolved().await?,
ChunkingConfig {
max_merge_chunk_size: 100_000,
..Default::default()
},
)
.module_merging(true);
}
}
......
use std::collections::HashSet;
use std::{collections::HashSet, sync::atomic::AtomicBool};
use anyhow::{Context, Result};
use rustc_hash::FxHashMap;
use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Vc};
use smallvec::{SmallVec, smallvec};
use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
use super::{
Chunk, ChunkGroupContent, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext,
......@@ -45,6 +46,7 @@ pub async fn make_chunk_group(
ChunkLoading::Edge
);
let should_trace = *chunking_context.is_tracing_enabled().await?;
let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
let batching_config = chunking_context.batching_config();
let ChunkGroupContent {
......@@ -52,12 +54,14 @@ pub async fn make_chunk_group(
batch_groups,
async_modules,
traced_modules,
availability_info,
} = chunk_group_content(
module_graph,
chunk_group_entries.clone(),
availability_info,
can_split_async,
should_trace,
should_merge_modules,
batching_config,
)
.await?;
......@@ -95,11 +99,6 @@ pub async fn make_chunk_group(
.try_join()
.await?;
// Compute new [AvailabilityInfo]
let availability_info = availability_info
.with_modules(Vc::cell(chunkable_items))
.await?;
// Insert async chunk loaders for every referenced async module
let async_loaders = async_modules
.into_iter()
......@@ -189,6 +188,7 @@ pub async fn chunk_group_content(
availability_info: AvailabilityInfo,
can_split_async: bool,
should_trace: bool,
should_merge_modules: bool,
batching_config: Vc<BatchingConfig>,
) -> Result<ChunkGroupContent> {
let module_batches_graph = module_graph.module_batches(batching_config).await?;
......@@ -315,6 +315,76 @@ pub async fn chunk_group_content(
},
)?;
// This needs to use the unmerged items
let availability_info = availability_info
.with_modules(Vc::cell(state.chunkable_items.clone()))
.await?;
if should_merge_modules {
let merged_modules = module_graph.merged_modules().await?;
state.chunkable_items = state
.chunkable_items
.into_iter()
.map(async |chunkable_module| match chunkable_module {
ChunkableModuleOrBatch::Module(module) => {
if !merged_modules.should_create_chunk_item_for(ResolvedVc::upcast(module)) {
return Ok(smallvec![]);
}
let module = if let Some(replacement) =
merged_modules.should_replace_module(ResolvedVc::upcast(module))
{
replacement
} else {
module
};
Ok(smallvec![ChunkableModuleOrBatch::Module(module)])
}
ChunkableModuleOrBatch::Batch(batch) => {
let batch_ref = batch.await?;
let modules = &batch_ref.modules;
let modified = AtomicBool::new(false);
let modules = modules
.iter()
.filter(|module| {
if merged_modules
.should_create_chunk_item_for(ResolvedVc::upcast(**module))
{
true
} else {
modified.store(true, std::sync::atomic::Ordering::Relaxed);
false
}
})
.map(|&module| {
if let Some(replacement) =
merged_modules.should_replace_module(ResolvedVc::upcast(module))
{
modified.store(true, std::sync::atomic::Ordering::Relaxed);
replacement
} else {
module
}
})
.map(ChunkableModuleOrBatch::Module)
.collect::<SmallVec<[_; 1]>>();
if modified.load(std::sync::atomic::Ordering::Relaxed) {
Ok(modules)
} else {
Ok(smallvec![ChunkableModuleOrBatch::Batch(batch)])
}
}
ChunkableModuleOrBatch::None(i) => Ok(smallvec![ChunkableModuleOrBatch::None(i)]),
})
.try_flat_join()
.await?
.into_iter()
.collect();
};
let mut batch_groups = FxIndexSet::default();
for &module in &state.chunkable_items {
if let Some(batch_group) = module_batches_graph.get_batch_group(&module.into()) {
......@@ -327,5 +397,6 @@ pub async fn chunk_group_content(
batch_groups,
async_modules: state.async_modules,
traced_modules: state.traced_modules,
availability_info,
})
}
use std::{borrow::Cow, collections::BinaryHeap, hash::BuildHasherDefault, mem::take};
use anyhow::Result;
use anyhow::{Context, Result};
use rustc_hash::FxHasher;
use smallvec::SmallVec;
use tracing::{Instrument, field::Empty};
......@@ -32,6 +32,7 @@ pub async fn make_production_chunks(
let span = span_outer.clone();
async move {
let chunk_group_info = module_graph.chunk_group_info().await?;
let merged_modules = module_graph.merged_modules().await?;
#[derive(Default)]
struct GrouppedChunkItems<'l> {
......@@ -69,9 +70,21 @@ pub async fn make_production_chunks(
..
},
..
} => chunk_group_info
.module_chunk_groups
.get(&ResolvedVc::upcast(module)),
} => Some(
chunk_group_info
.module_chunk_groups
.get(&ResolvedVc::upcast(module))
.or_else(|| {
// Merged modules don't have a chunk group in chunk_group_info, so
// lookup using the original module.
merged_modules
.get_original_module(ResolvedVc::upcast(module))
.and_then(|module| {
chunk_group_info.module_chunk_groups.get(&module)
})
})
.context("every module should have a chunk group")?,
),
&ChunkItemOrBatchWithInfo::ChunkItem {
chunk_item: ChunkItemWithAsyncModuleInfo { module: None, .. },
..
......
......@@ -218,11 +218,19 @@ pub trait ChunkingContext {
})
}
/// Whether `ChunkingType::Traced` are used to create corresponding output assets for each
/// traced module.
#[turbo_tasks::function]
fn is_tracing_enabled(self: Vc<Self>) -> Vc<bool> {
Vc::cell(false)
}
/// Whether to use `MergableModule` to merge modules if possible.
#[turbo_tasks::function]
fn is_module_merging_enabled(self: Vc<Self>) -> Vc<bool> {
Vc::cell(false)
}
#[turbo_tasks::function]
fn minify_type(self: Vc<Self>) -> Vc<MinifyType> {
MinifyType::NoMinify.cell()
......
......@@ -36,6 +36,7 @@ pub use self::{
};
use crate::{
asset::Asset,
chunk::availability_info::AvailabilityInfo,
ident::AssetIdent,
module::Module,
module_graph::{
......@@ -108,6 +109,89 @@ impl ChunkableModules {
}
}
/// A [Module] that can be merged with other [Module]s (to perform scope hoisting)
// TODO currently this is only used for ecmascript modules, and with the current API cannot be used
// with other module types (as a MergeableModule cannot prevent itself from being merged with other
// module types)
#[turbo_tasks::value_trait]
pub trait MergeableModule: Module + Asset {
/// Even though MergeableModule is implemented, this allows a dynamic condition to determine
/// mergeability
#[turbo_tasks::function]
fn is_mergeable(self: Vc<Self>) -> Vc<bool> {
Vc::cell(true)
}
/// Create a new module representing the merged content of the given `modules`.
///
/// Group entry points are not referenced by any other module in the group. This list is needed
/// because the merged module is created by recursively inlining modules when they are imported,
/// but this process has to start somewhere (= with these entry points).
#[turbo_tasks::function]
fn merge(
self: Vc<Self>,
modules: Vc<MergeableModulesExposed>,
entry_points: Vc<MergeableModules>,
) -> Vc<Box<dyn ChunkableModule>>;
}
#[turbo_tasks::value(transparent)]
pub struct MergeableModules(Vec<ResolvedVc<Box<dyn MergeableModule>>>);
#[turbo_tasks::value_impl]
impl MergeableModules {
#[turbo_tasks::function]
pub fn interned(modules: Vec<ResolvedVc<Box<dyn MergeableModule>>>) -> Vc<Self> {
Vc::cell(modules)
}
}
/// Whether a given module needs to be exposed (depending on how it is imported by other modules)
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
Serialize,
Deserialize,
TraceRawVcs,
NonLocalValue,
TaskInput,
Hash,
)]
pub enum MergeableModuleExposure {
// This module is only used from within the current group, and only individual exports are
// used (and no namespace object is required).
None,
// This module is only used from within the current group, and but the namespace object is
// needed.
Internal,
// The exports of this module are read from outside this group (necessitating a namespace
// object anyway).
External,
}
#[turbo_tasks::value(transparent)]
pub struct MergeableModulesExposed(
Vec<(
ResolvedVc<Box<dyn MergeableModule>>,
MergeableModuleExposure,
)>,
);
#[turbo_tasks::value_impl]
impl MergeableModulesExposed {
#[turbo_tasks::function]
pub fn interned(
modules: Vec<(
ResolvedVc<Box<dyn MergeableModule>>,
MergeableModuleExposure,
)>,
) -> Vc<Self> {
Vc::cell(modules)
}
}
#[turbo_tasks::value(transparent)]
pub struct Chunks(Vec<ResolvedVc<Box<dyn Chunk>>>);
......@@ -289,12 +373,12 @@ pub trait ChunkableModuleReference: ModuleReference + ValueToString {
}
}
#[derive(Default)]
pub struct ChunkGroupContent {
pub chunkable_items: FxIndexSet<ChunkableModuleOrBatch>,
pub batch_groups: FxIndexSet<ResolvedVc<ModuleBatchGroup>>,
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
pub availability_info: AvailabilityInfo,
}
#[turbo_tasks::value_trait]
......
......@@ -32,6 +32,7 @@ use crate::{
module_graph::{
async_module_info::{AsyncModulesInfo, compute_async_module_info},
chunk_group_info::{ChunkGroupEntry, ChunkGroupInfo, compute_chunk_group_info},
merged_modules::{MergedModuleInfo, compute_merged_modules},
module_batches::{ModuleBatchesGraph, compute_module_batches},
style_groups::{StyleGroups, StyleGroupsConfig, compute_style_groups},
traced_di_graph::{TracedDiGraph, iter_neighbors_rev},
......@@ -42,6 +43,7 @@ use crate::{
pub mod async_module_info;
pub mod chunk_group_info;
pub mod merged_modules;
pub mod module_batch;
pub(crate) mod module_batches;
pub(crate) mod style_groups;
......@@ -992,6 +994,11 @@ impl ModuleGraph {
compute_chunk_group_info(self).await
}
#[turbo_tasks::function]
pub async fn merged_modules(self: Vc<Self>) -> Result<Vc<MergedModuleInfo>> {
compute_merged_modules(self).await
}
#[turbo_tasks::function]
pub async fn module_batches(
self: Vc<Self>,
......
......@@ -423,7 +423,7 @@ fn generate_minimal_source_map(filename: String, source: String) -> Result<Rope>
}
let sm: Arc<SourceMap> = Default::default();
sm.new_source_file(FileName::Custom(filename).into(), source);
let map = generate_js_source_map(sm, mappings, None, true)?;
let map = generate_js_source_map(&*sm, mappings, None, true, true)?;
Ok(map)
}
......
......@@ -109,10 +109,10 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
t: runtimeRequire,
f: moduleContext,
i: esmImport.bind(null, module),
s: esmExport.bind(null, module, module.exports),
j: dynamicExport.bind(null, module, module.exports),
v: exportValue.bind(null, module),
n: exportNamespace.bind(null, module),
s: esmExport.bind(null, module, module.exports, moduleCache),
j: dynamicExport.bind(null, module, module.exports, moduleCache),
v: exportValue.bind(null, module, moduleCache),
n: exportNamespace.bind(null, module, moduleCache),
m: module,
c: moduleCache,
M: moduleFactories,
......
......@@ -43,7 +43,7 @@ interface TurbopackDevContext extends TurbopackDevBaseContext {}
type ModuleFactory = (
this: Module['exports'],
context: TurbopackDevBaseContext
) => undefined
) => unknown
interface DevRuntimeBackend {
reloadChunk?: (chunkUrl: ChunkUrl) => Promise<void>
......@@ -205,10 +205,10 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
t: runtimeRequire,
f: moduleContext,
i: esmImport.bind(null, module),
s: esmExport.bind(null, module, module.exports),
j: dynamicExport.bind(null, module, module.exports),
v: exportValue.bind(null, module),
n: exportNamespace.bind(null, module),
s: esmExport.bind(null, module, module.exports, devModuleCache),
j: dynamicExport.bind(null, module, module.exports, devModuleCache),
v: exportValue.bind(null, module, devModuleCache),
n: exportNamespace.bind(null, module, devModuleCache),
m: module,
c: devModuleCache,
M: moduleFactories,
......
......@@ -31,7 +31,7 @@ type RuntimeParams = {
type ChunkRegistration = [
chunkPath: ChunkScript,
chunkModules: ModuleFactories,
chunkModules: CompressedModuleFactories,
params: RuntimeParams | undefined,
]
......@@ -361,7 +361,15 @@ function registerChunk([
const chunkPath = getPathFromScript(chunkScript)
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
moduleFactories[moduleId] = moduleFactory
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
}
} else {
moduleFactories[moduleId] = moduleFactory
}
}
addModuleToChunk(moduleId, chunkPath)
}
......
......@@ -56,7 +56,7 @@ interface TurbopackNodeBuildContext extends TurbopackBaseContext<Module> {
type ModuleFactory = (
this: Module['exports'],
context: TurbopackNodeBuildContext
) => undefined
) => unknown
const url = require('url') as typeof import('url')
const fs = require('fs/promises') as typeof import('fs/promises')
......@@ -107,11 +107,19 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void {
try {
const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
const chunkModules: ModuleFactories = require(resolved)
const chunkModules: CompressedModuleFactories = require(resolved)
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
moduleFactories[moduleId] = moduleFactory
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
}
} else {
moduleFactories[moduleId] = moduleFactory
}
}
}
loadedChunks.add(chunkPath)
......@@ -165,10 +173,18 @@ async function loadChunkAsync(
url.pathToFileURL(resolved)
)(module, module.exports, localRequire, path.dirname(resolved), resolved)
const chunkModules: ModuleFactories = module.exports
const chunkModules: CompressedModuleFactories = module.exports
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
moduleFactories[moduleId] = moduleFactory
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
}
} else {
moduleFactories[moduleId] = moduleFactory
}
}
}
loadedChunks.add(chunkPath)
......@@ -235,20 +251,6 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
)
}
let parents: ModuleId[]
switch (source.type) {
case SourceType.Runtime:
parents = []
break
case SourceType.Parent:
// No need to add this module as a child of the parent module here, this
// has already been taken care of in `getOrInstantiateModuleFromParent`.
parents = [source.parentId]
break
default:
invariant(source, (source) => `Unknown source type: ${source?.type}`)
}
const module: Module = {
exports: {},
error: undefined,
......@@ -270,10 +272,10 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
y: externalImport,
f: moduleContext,
i: esmImport.bind(null, module),
s: esmExport.bind(null, module, module.exports),
j: dynamicExport.bind(null, module, module.exports),
v: exportValue.bind(null, module),
n: exportNamespace.bind(null, module),
s: esmExport.bind(null, module, module.exports, moduleCache),
j: dynamicExport.bind(null, module, module.exports, moduleCache),
v: exportValue.bind(null, module, moduleCache),
n: exportNamespace.bind(null, module, moduleCache),
m: module,
c: moduleCache,
M: moduleFactories,
......
......@@ -13,6 +13,7 @@ type ChunkListScript = CurrentScript & { readonly brand: unique symbol }
type ChunkPath = string & { readonly brand: unique symbol }
type ChunkScript = CurrentScript & { readonly brand: unique symbol }
type ChunkUrl = string & { readonly brand: unique symbol }
// TODO this should actually be `string | number`
type ModuleId = string
interface Exports {
......@@ -36,10 +37,16 @@ type EsmImport = (
moduleId: ModuleId,
allowExportDefault: boolean
) => EsmNamespaceObject | Promise<EsmNamespaceObject>
type EsmExport = (exportGetters: Record<string, () => any>) => void
type ExportValue = (value: any) => void
type ExportNamespace = (namespace: any) => void
type DynamicExport = (object: Record<string, any>) => void
type EsmExport = (
exportGetters: Record<string, () => any>,
id: ModuleId | undefined
) => void
type ExportValue = (value: any, id: ModuleId | undefined) => void
type ExportNamespace = (namespace: any, id: ModuleId | undefined) => void
type DynamicExport = (
object: Record<string, any>,
id: ModuleId | undefined
) => void
type LoadChunk = (chunkPath: ChunkPath) => Promise<any> | undefined
type LoadChunkByUrl = (chunkUrl: ChunkUrl) => Promise<any> | undefined
......@@ -54,7 +61,13 @@ type LoadWebAssemblyModule = (
) => WebAssembly.Module
type ModuleCache<M> = Record<ModuleId, M>
type ModuleFactories = Record<ModuleId, unknown>
// TODO properly type values here
type ModuleFactories = Record<ModuleId, Function>
// The value is an array with scope hoisting
type CompressedModuleFactories = Record<
ModuleId,
Function | [Function, ModuleId[]]
>
type RelativeURL = (inputUrl: string) => void
type ResolvePathFromModule = (moduleId: string) => string
......
......@@ -59,6 +59,26 @@ function defineProp(
if (!hasOwnProperty.call(obj, name)) Object.defineProperty(obj, name, options)
}
function getOverwrittenModule(
moduleCache: ModuleCache<Module>,
id: ModuleId
): Module {
let module = moduleCache[id]
if (!module) {
// This is invoked when a module is merged into another module, thus it wasn't invoked via
// instantiateModule and the cache entry wasn't created yet.
module = {
exports: {},
error: undefined,
loaded: false,
id,
namespaceObject: undefined,
}
moduleCache[id] = module
}
return module
}
/**
* Adds the getters to the exports object.
*/
......@@ -89,8 +109,14 @@ function esm(
function esmExport(
module: Module,
exports: Exports,
getters: Record<string, () => any>
moduleCache: ModuleCache<Module>,
getters: Record<string, () => any>,
id: ModuleId | undefined
) {
if (id != null) {
module = getOverwrittenModule(moduleCache, id)
exports = module.exports
}
module.namespaceObject = module.exports
esm(exports, getters)
}
......@@ -134,8 +160,14 @@ function ensureDynamicExports(module: Module, exports: Exports) {
function dynamicExport(
module: Module,
exports: Exports,
object: Record<string, any>
moduleCache: ModuleCache<Module>,
object: Record<string, any>,
id: ModuleId | undefined
) {
if (id != null) {
module = getOverwrittenModule(moduleCache, id)
exports = module.exports
}
ensureDynamicExports(module, exports)
if (typeof object === 'object' && object !== null) {
......@@ -143,11 +175,27 @@ function dynamicExport(
}
}
function exportValue(module: Module, value: any) {
function exportValue(
module: Module,
moduleCache: ModuleCache<Module>,
value: any,
id: ModuleId | undefined
) {
if (id != null) {
module = getOverwrittenModule(moduleCache, id)
}
module.exports = value
}
function exportNamespace(module: Module, namespace: any) {
function exportNamespace(
module: Module,
moduleCache: ModuleCache<Module>,
namespace: any,
id: ModuleId | undefined
) {
if (id != null) {
module = getOverwrittenModule(moduleCache, id)
}
module.exports = module.namespaceObject = namespace
}
......
......@@ -21,6 +21,7 @@ data-encoding = { workspace = true }
either = { workspace = true }
indexmap = { workspace = true }
indoc = { workspace = true }
itertools = { workspace = true }
num-bigint = "0.4"
num-traits = "0.2.15"
once_cell = { workspace = true }
......
......@@ -2,13 +2,15 @@ use std::io::Write;
use anyhow::{Result, bail};
use serde::{Deserialize, Serialize};
use smallvec::SmallVec;
use turbo_rcstr::rcstr;
use turbo_tasks::{
NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc, trace::TraceRawVcs,
NonLocalValue, ResolvedVc, TaskInput, TryJoinIterExt, Upcast, ValueToString, Vc,
trace::TraceRawVcs,
};
use turbo_tasks_fs::{FileSystemPath, rope::Rope};
use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext},
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext, ModuleId},
code_builder::{Code, CodeBuilder},
error::PrettyPrintError,
issue::{IssueExt, IssueSeverity, StyledString, code_gen::CodeGenerationIssue},
......@@ -18,7 +20,7 @@ use turbopack_core::{
use crate::{
EcmascriptModuleContent, EcmascriptOptions,
references::async_module::{AsyncModuleOptions, OptionAsyncModuleOptions},
utils::FormatIter,
utils::{FormatIter, StringifyJs},
};
#[turbo_tasks::value(shared)]
......@@ -26,6 +28,7 @@ use crate::{
pub struct EcmascriptChunkItemContent {
pub inner_code: Rope,
pub source_map: Option<Rope>,
pub additional_ids: SmallVec<[ResolvedVc<ModuleId>; 1]>,
pub options: EcmascriptChunkItemOptions,
pub rewrite_source_path: Option<ResolvedVc<FileSystemPath>>,
pub placeholder_for_future_extensions: (),
......@@ -57,6 +60,7 @@ impl EcmascriptChunkItemContent {
},
inner_code: content.inner_code.clone(),
source_map: content.source_map.clone(),
additional_ids: content.additional_ids.clone(),
options: if content.is_esm {
EcmascriptChunkItemOptions {
strict: true,
......@@ -105,7 +109,12 @@ impl EcmascriptChunkItemContent {
args.push("w: __turbopack_wasm__");
args.push("u: __turbopack_wasm_module__");
}
let mut code = CodeBuilder::default();
let additional_ids = self.additional_ids.iter().try_join().await?;
if !additional_ids.is_empty() {
code += "["
}
if self.options.this {
code += "(function(__turbopack_context__) {\n";
} else {
......@@ -148,6 +157,10 @@ impl EcmascriptChunkItemContent {
}
code += "})";
if !additional_ids.is_empty() {
writeln!(code, ", {}]", StringifyJs(&additional_ids))?;
}
Ok(code.build().cell())
}
}
......@@ -261,11 +274,7 @@ async fn module_factory_with_code_generation_issue(
{
Ok(factory) => factory,
Err(error) => {
let id = chunk_item
.chunking_context()
.chunk_item_id(Vc::upcast(chunk_item))
.to_string()
.await;
let id = chunk_item.asset_ident().to_string().await;
let id = id.as_ref().map_or_else(|_| "unknown", |id| &**id);
let error = error.context(format!(
"An error occurred while generating the chunk item {id}"
......
......@@ -16,26 +16,29 @@ use turbopack_core::{
chunk::ChunkingContext, module_graph::ModuleGraph, reference::ModuleReference,
};
use crate::references::{
AstPath,
amd::AmdDefineWithDependenciesCodeGen,
cjs::{
CjsRequireAssetReferenceCodeGen, CjsRequireCacheAccess,
CjsRequireResolveAssetReferenceCodeGen,
},
constant_condition::ConstantConditionCodeGen,
constant_value::ConstantValueCodeGen,
dynamic_expression::DynamicExpression,
esm::{
EsmBinding, EsmModuleItem, ImportMetaBinding, ImportMetaRef,
dynamic::EsmAsyncAssetReferenceCodeGen, module_id::EsmModuleIdAssetReferenceCodeGen,
url::UrlAssetReferenceCodeGen,
use crate::{
ScopeHoistingContext,
references::{
AstPath,
amd::AmdDefineWithDependenciesCodeGen,
cjs::{
CjsRequireAssetReferenceCodeGen, CjsRequireCacheAccess,
CjsRequireResolveAssetReferenceCodeGen,
},
constant_condition::ConstantConditionCodeGen,
constant_value::ConstantValueCodeGen,
dynamic_expression::DynamicExpression,
esm::{
EsmBinding, EsmModuleItem, ImportMetaBinding, ImportMetaRef,
dynamic::EsmAsyncAssetReferenceCodeGen, module_id::EsmModuleIdAssetReferenceCodeGen,
url::UrlAssetReferenceCodeGen,
},
ident::IdentReplacement,
member::MemberReplacement,
require_context::RequireContextAssetReferenceCodeGen,
unreachable::Unreachable,
worker::WorkerAssetReferenceCodeGen,
},
ident::IdentReplacement,
member::MemberReplacement,
require_context::RequireContextAssetReferenceCodeGen,
unreachable::Unreachable,
worker::WorkerAssetReferenceCodeGen,
};
#[derive(Default)]
......@@ -193,6 +196,7 @@ impl CodeGen {
&self,
g: Vc<ModuleGraph>,
ctx: Vc<Box<dyn ChunkingContext>>,
scope_hoisting_context: ScopeHoistingContext<'_>,
) -> Result<CodeGeneration> {
match self {
Self::AmdDefineWithDependenciesCodeGen(v) => v.code_generation(g, ctx).await,
......@@ -200,7 +204,7 @@ impl CodeGen {
Self::ConstantConditionCodeGen(v) => v.code_generation(g, ctx).await,
Self::ConstantValueCodeGen(v) => v.code_generation(g, ctx).await,
Self::DynamicExpression(v) => v.code_generation(g, ctx).await,
Self::EsmBinding(v) => v.code_generation(g, ctx).await,
Self::EsmBinding(v) => v.code_generation(g, ctx, scope_hoisting_context).await,
Self::EsmModuleItem(v) => v.code_generation(g, ctx).await,
Self::IdentReplacement(v) => v.code_generation(g, ctx).await,
Self::ImportMetaBinding(v) => v.code_generation(g, ctx).await,
......
use anyhow::Result;
use turbo_tasks::{ResolvedVc, Vc};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{
AsyncModuleInfo, ChunkItem, ChunkType, ChunkableModule, ChunkingContext,
MergeableModuleExposure,
},
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
reference::ModuleReferences,
};
use crate::{
EcmascriptAnalyzable, EcmascriptModuleContent, EcmascriptOptions,
chunk::{EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkType},
};
#[turbo_tasks::value(shared)]
pub(crate) struct MergedEcmascriptModule {
modules: Vec<(
ResolvedVc<Box<dyn EcmascriptAnalyzable>>,
MergeableModuleExposure,
)>,
entry_points: Vec<ResolvedVc<Box<dyn EcmascriptAnalyzable>>>,
options: ResolvedVc<EcmascriptOptions>,
}
impl MergedEcmascriptModule {
pub fn new(
modules: Vec<(
ResolvedVc<Box<dyn EcmascriptAnalyzable>>,
MergeableModuleExposure,
)>,
entry_points: Vec<ResolvedVc<Box<dyn EcmascriptAnalyzable>>>,
options: ResolvedVc<EcmascriptOptions>,
) -> ResolvedVc<Self> {
MergedEcmascriptModule {
modules,
entry_points,
options,
}
.resolved_cell()
}
}
#[turbo_tasks::value_impl]
impl Asset for MergedEcmascriptModule {
#[turbo_tasks::function]
fn content(&self) -> Vc<AssetContent> {
panic!("content() should not be called");
}
}
#[turbo_tasks::value_impl]
impl Module for MergedEcmascriptModule {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
// This purposely reuses the module's ident as it has replaced the original module, thus
// there can never be a collision.
self.entry_points.first().unwrap().ident()
}
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> {
panic!("references() should not be called");
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
panic!("is_self_async() should not be called");
}
}
#[turbo_tasks::value_impl]
impl ChunkableModule for MergedEcmascriptModule {
#[turbo_tasks::function]
fn as_chunk_item(
self: ResolvedVc<Self>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
) -> Vc<Box<dyn ChunkItem>> {
Vc::upcast(
MergedEcmascriptModuleChunkItem {
module: self,
module_graph,
chunking_context,
}
.cell(),
)
}
}
#[turbo_tasks::value]
struct MergedEcmascriptModuleChunkItem {
module: ResolvedVc<MergedEcmascriptModule>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl ChunkItem for MergedEcmascriptModuleChunkItem {
#[turbo_tasks::function]
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*ResolvedVc::upcast(self.chunking_context)
}
#[turbo_tasks::function]
async fn ty(&self) -> Result<Vc<Box<dyn ChunkType>>> {
Ok(Vc::upcast(
Vc::<EcmascriptChunkType>::default().resolve().await?,
))
}
#[turbo_tasks::function]
fn module(&self) -> Vc<Box<dyn Module>> {
*ResolvedVc::upcast(self.module)
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for MergedEcmascriptModuleChunkItem {
#[turbo_tasks::function]
fn content(self: Vc<Self>) -> Vc<EcmascriptChunkItemContent> {
panic!("content() should not be called");
}
#[turbo_tasks::function]
async fn content_with_async_module_info(
&self,
async_module_info: Option<Vc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptChunkItemContent>> {
let module = self.module.await?;
let modules = &module.modules;
let entry_points = &module.entry_points;
let options = modules
.iter()
.map(|(m, _)| {
let Some(m) = ResolvedVc::try_downcast::<Box<dyn EcmascriptAnalyzable>>(*m) else {
anyhow::bail!("Expected EcmascriptAnalyzable in scope hoisting group");
};
Ok(m.module_content_options(
*self.module_graph,
*self.chunking_context,
async_module_info,
))
})
.collect::<Result<Vec<_>>>()?;
let content = EcmascriptModuleContent::new_merged(
modules.clone(),
options,
ResolvedVc::deref_vec(entry_points.clone()),
);
// Currently, merged modules never include async modules.
let async_module_options = Vc::cell(None);
Ok(EcmascriptChunkItemContent::new(
content,
*self.chunking_context,
*module.options,
async_module_options,
))
}
}
......@@ -132,9 +132,10 @@ pub fn minify(code: Code, source_maps: bool, mangle: Option<MangleType>) -> Resu
builder.push_source(
&src.into(),
Some(generate_js_source_map(
cm,
&*cm,
src_map_buf,
Some(original_map),
true,
// We do not inline source contents.
// We provide a synthesized value to `cm.new_source_file` above, so it cannot be
// the value user expect anyway.
......
......@@ -9,7 +9,7 @@ use swc_core::{
BytePos, FileName, GLOBALS, Globals, LineCol, Mark, SyntaxContext,
errors::{HANDLER, Handler},
input::StringInput,
source_map::SourceMapGenConfig,
source_map::{Files, SourceMapGenConfig, build_source_map},
util::take::Take,
},
ecma::{
......@@ -99,56 +99,70 @@ impl ParseResult {
}
}
/// `original_source_maps_complete` indicates whether the `original_source_maps` cover the whole
/// map, i.e. whether every module that ended up in `mappings` had an original sourcemap.
#[instrument(level = Level::INFO, skip_all)]
pub fn generate_js_source_map(
files_map: Arc<swc_core::common::SourceMap>,
pub fn generate_js_source_map<'a>(
files_map: &impl Files,
mappings: Vec<(BytePos, LineCol)>,
original_source_map: Option<&Rope>,
original_source_maps: impl IntoIterator<Item = &'a Rope>,
original_source_maps_complete: bool,
inline_sources_content: bool,
) -> Result<Rope> {
let original_source_map = original_source_map.map(|x| x.to_bytes());
let input_map = if let Some(original_source_map) = &original_source_map {
Some(swc_sourcemap::lazy::decode(original_source_map)?.into_source_map()?)
} else {
None
};
let new_mappings = files_map.build_source_map(
let original_source_maps = original_source_maps
.into_iter()
.map(|map| map.to_bytes())
.collect::<Vec<_>>();
let original_source_maps = original_source_maps
.iter()
.map(|map| Ok(swc_sourcemap::lazy::decode(map)?.into_source_map()?))
.collect::<Result<Vec<_>>>()?;
let fast_path_single_original_source_map =
original_source_maps.len() == 1 && original_source_maps_complete;
let mut new_mappings = build_source_map(
files_map,
&mappings,
None,
InlineSourcesContentConfig {
&InlineSourcesContentConfig {
// If we are going to adjust the source map, we are going to throw the source contents
// of this source map away regardless.
//
// In other words, we don't need the content of `B` in source map chain of A -> B -> C.
// We only need the source content of `A`, and a way to map the content of `B` back to
// `A`, while constructing the final source map, `C`.
inline_sources_content: inline_sources_content && input_map.is_none(),
inline_sources_content: inline_sources_content && !fast_path_single_original_source_map,
},
);
match input_map {
Some(mut map) => {
// TODO: Make this more efficient
map.adjust_mappings(new_mappings);
// TODO: Enable this when we have a way to handle the ignore list
// add_default_ignore_list(&mut map);
let map = map.into_raw_sourcemap();
let result = serde_json::to_vec(&map)?;
Ok(Rope::from(result))
}
None => {
// We don't convert sourcemap::SourceMap into raw_sourcemap::SourceMap because we don't
// need to adjust mappings
let mut map = new_mappings;
if original_source_maps.is_empty() {
// We don't convert sourcemap::SourceMap into raw_sourcemap::SourceMap because we don't
// need to adjust mappings
add_default_ignore_list(&mut new_mappings);
let mut result = vec![];
new_mappings.to_writer(&mut result)?;
Ok(Rope::from(result))
} else if fast_path_single_original_source_map {
let mut map = original_source_maps.into_iter().next().unwrap();
// TODO: Make this more efficient
map.adjust_mappings(new_mappings);
// TODO: Enable this when we have a way to handle the ignore list
// add_default_ignore_list(&mut map);
let map = map.into_raw_sourcemap();
let result = serde_json::to_vec(&map)?;
Ok(Rope::from(result))
} else {
let mut map = new_mappings.adjust_mappings_from_multiple(original_source_maps);
add_default_ignore_list(&mut map);
add_default_ignore_list(&mut map);
let mut result = vec![];
map.to_writer(&mut result)?;
Ok(Rope::from(result))
}
let mut result = vec![];
map.to_writer(&mut result)?;
Ok(Rope::from(result))
}
}
......
......@@ -17,7 +17,11 @@ use turbopack_core::{
};
use super::esm::base::ReferencedAsset;
use crate::code_gen::{CodeGeneration, CodeGenerationHoistedStmt};
use crate::{
ScopeHoistingContext,
code_gen::{CodeGeneration, CodeGenerationHoistedStmt},
utils::AstSyntaxContext,
};
/// Information needed for generating the async module wrapper for
/// [EcmascriptChunkItem](crate::chunk::EcmascriptChunkItem)s.
......@@ -76,8 +80,10 @@ impl OptionAsyncModule {
}
}
/// The identifiers (and their corresponding syntax context) of all async modules referenced by the
/// current module.
#[turbo_tasks::value(transparent)]
struct AsyncModuleIdents(FxIndexSet<String>);
struct AsyncModuleIdents(FxIndexSet<(String, AstSyntaxContext)>);
async fn get_inherit_async_referenced_asset(
r: Vc<Box<dyn ModuleReference>>,
......@@ -123,7 +129,11 @@ impl AsyncModule {
Ok(match &*referenced_asset {
ReferencedAsset::External(_, ExternalType::EcmaScriptModule) => {
if self.import_externals {
referenced_asset.get_ident(chunking_context).await?
referenced_asset
.get_ident(chunking_context, None, ScopeHoistingContext::None)
.await?
.map(|i| i.into_module_namespace_ident().unwrap())
.map(|(i, ctx)| (i, ctx.unwrap_or_default().into()))
} else {
None
}
......@@ -133,7 +143,11 @@ impl AsyncModule {
.referenced_async_modules
.contains(&ResolvedVc::upcast(*placeable))
{
referenced_asset.get_ident(chunking_context).await?
referenced_asset
.get_ident(chunking_context, None, ScopeHoistingContext::None)
.await?
.map(|i| i.into_module_namespace_ident().unwrap())
.map(|(i, ctx)| (i, ctx.unwrap_or_default().into()))
} else {
None
}
......@@ -208,9 +222,7 @@ impl AsyncModule {
if !async_idents.is_empty() {
let idents = async_idents
.iter()
.map(|ident: &String| {
Ident::new(ident.clone().into(), DUMMY_SP, Default::default())
})
.map(|(ident, ctxt)| Ident::new(ident.clone().into(), DUMMY_SP, **ctxt))
.collect::<Vec<_>>();
return Ok(CodeGeneration::hoisted_stmts([
......
use anyhow::Result;
use serde::{Deserialize, Serialize};
use swc_core::{
common::Span,
ecma::{
ast::{
ComputedPropName, Expr, Ident, KeyValueProp, Lit, MemberExpr, MemberProp, Number, Prop,
PropName, SeqExpr, SimpleAssignTarget, Str,
},
visit::fields::{CalleeField, PropField},
},
use swc_core::ecma::{
ast::{Expr, KeyValueProp, Prop, PropName, SimpleAssignTarget},
visit::fields::{CalleeField, PropField},
};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, Vc, trace::TraceRawVcs};
......@@ -16,9 +10,13 @@ use turbopack_core::{chunk::ChunkingContext, module_graph::ModuleGraph};
use super::EsmAssetReference;
use crate::{
ScopeHoistingContext,
code_gen::{CodeGen, CodeGeneration},
create_visitor,
references::{AstPath, esm::base::ReferencedAsset},
references::{
AstPath,
esm::base::{ReferencedAsset, ReferencedAssetIdent},
},
};
#[derive(Hash, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, TraceRawVcs, NonLocalValue)]
......@@ -45,22 +43,23 @@ impl EsmBinding {
&self,
_module_graph: Vc<ModuleGraph>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
scope_hoisting_context: ScopeHoistingContext<'_>,
) -> Result<CodeGeneration> {
let mut visitors = vec![];
let export = self.export.clone();
let imported_module = self.reference.get_referenced_asset();
let imported_module = self.reference.get_referenced_asset().await?;
enum ImportedIdent {
Module(String),
Module(ReferencedAssetIdent),
None,
Unresolvable,
}
let imported_ident = match &*imported_module.await? {
let imported_ident = match &*imported_module {
ReferencedAsset::None => ImportedIdent::None,
imported_module => imported_module
.get_ident(chunking_context)
.get_ident(chunking_context, export, scope_hoisting_context)
.await?
.map_or(ImportedIdent::Unresolvable, ImportedIdent::Module),
};
......@@ -80,18 +79,14 @@ impl EsmBinding {
ImportedIdent::Module(imported_ident) => {
*prop = Prop::KeyValue(KeyValueProp {
key: PropName::Ident(ident.clone().into()),
value: Box::new(make_expr(
imported_ident,
export.as_deref(),
ident.span,
false,
)),
value: Box::new(imported_ident.as_expr(ident.span, false))
});
}
ImportedIdent::None => {
*prop = Prop::KeyValue(KeyValueProp {
key: PropName::Ident(ident.clone().into()),
value: Expr::undefined(ident.span),
value:
Expr::undefined(ident.span),
});
}
ImportedIdent::Unresolvable => {
......@@ -118,7 +113,7 @@ impl EsmBinding {
use swc_core::common::Spanned;
match &imported_ident {
ImportedIdent::Module(imported_ident) => {
*expr = make_expr(imported_ident, export.as_deref(), expr.span(), in_call);
*expr = imported_ident.as_expr(expr.span(), in_call);
}
ImportedIdent::None => {
*expr = *Expr::undefined(expr.span());
......@@ -145,24 +140,27 @@ impl EsmBinding {
ast_path.pop();
visitors.push(
create_visitor!(exact ast_path, visit_mut_simple_assign_target(l: &mut SimpleAssignTarget) {
use swc_core::common::Spanned;
match &imported_ident {
ImportedIdent::Module(imported_ident) => {
*l = match make_expr(imported_ident, export.as_deref(), l.span(), false) {
Expr::Ident(ident) => SimpleAssignTarget::Ident(ident.into()),
Expr::Member(member) => SimpleAssignTarget::Member(member),
_ => unreachable!(),
};
}
ImportedIdent::None => {
// Do nothing, cannot assign to `undefined`
}
ImportedIdent::Unresolvable => {
// Do nothing, the reference will insert a throw
create_visitor!(exact ast_path, visit_mut_simple_assign_target(l: &mut SimpleAssignTarget) {
use swc_core::common::Spanned;
match &imported_ident {
ImportedIdent::Module(imported_ident) => {
*l = imported_ident
.as_expr_individual(l.span())
.map_either(
|i| SimpleAssignTarget::Ident(i.into()),
SimpleAssignTarget::Member,
)
.into_inner();
}
ImportedIdent::None => {
// Do nothing, cannot assign to `undefined`
}
ImportedIdent::Unresolvable => {
// Do nothing, the reference will insert a throw
}
}
}
}));
})
);
break;
}
}
......@@ -182,40 +180,3 @@ impl From<EsmBinding> for CodeGen {
CodeGen::EsmBinding(val)
}
}
fn make_expr(imported_module: &str, export: Option<&str>, span: Span, in_call: bool) -> Expr {
if let Some(export) = export {
let mut expr = Expr::Member(MemberExpr {
span,
obj: Box::new(Expr::Ident(Ident::new(
imported_module.into(),
span,
Default::default(),
))),
prop: MemberProp::Computed(ComputedPropName {
span,
expr: Box::new(Expr::Lit(Lit::Str(Str {
span,
value: export.into(),
raw: None,
}))),
}),
});
if in_call {
expr = Expr::Seq(SeqExpr {
exprs: vec![
Box::new(Expr::Lit(Lit::Num(Number {
span,
value: 0.0,
raw: None,
}))),
Box::new(expr),
],
span,
});
}
expr
} else {
Expr::Ident(Ident::new(imported_module.into(), span, Default::default()))
}
}
......@@ -125,6 +125,7 @@ impl CachedExternalModule {
inner_code: code.build(),
source_map: None,
is_esm: self.external_type != CachedExternalType::CommonJs,
additional_ids: Default::default(),
}
.cell())
}
......
......@@ -761,7 +761,7 @@ pub(crate) async fn analyse_ecmascript_module_internal(
ImportedSymbol::Part(part_id) => Some(ModulePart::internal(*part_id)),
ImportedSymbol::Exports => Some(ModulePart::exports()),
},
Some(TreeShakingMode::ReexportsOnly) => match &r.imported_symbol {
_ => match &r.imported_symbol {
ImportedSymbol::ModuleEvaluation => {
should_add_evaluation = true;
Some(ModulePart::evaluation())
......@@ -777,10 +777,6 @@ pub(crate) async fn analyse_ecmascript_module_internal(
}
ImportedSymbol::Exports => None,
},
None => {
should_add_evaluation = true;
None
}
},
import_externals,
)
......
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use swc_core::{common::DUMMY_SP, ecma::ast::Ident, quote};
use swc_core::{common::DUMMY_SP, quote};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, ValueToString, Vc, trace::TraceRawVcs};
use turbopack_core::{
......@@ -17,7 +17,7 @@ use super::{
facade::module::EcmascriptModuleFacadeModule, locals::module::EcmascriptModuleLocalsModule,
};
use crate::{
chunk::EcmascriptChunkPlaceable, code_gen::CodeGeneration,
ScopeHoistingContext, chunk::EcmascriptChunkPlaceable, code_gen::CodeGeneration,
references::esm::base::ReferencedAsset, runtime_functions::TURBOPACK_IMPORT,
utils::module_id_to_lit,
};
......@@ -156,9 +156,11 @@ impl EcmascriptModulePartReference {
let referenced_asset = ReferencedAsset::from_resolve_result(self.resolve_reference());
let referenced_asset = referenced_asset.await?;
let ident = referenced_asset
.get_ident(chunking_context)
.get_ident(chunking_context, None, ScopeHoistingContext::None)
.await?
.context("part module reference should have an ident")?;
.context("part module reference should have an ident")?
.as_expr_individual(DUMMY_SP)
.unwrap_left();
let ReferencedAsset::Some(module) = *referenced_asset else {
bail!("part module reference should have an module reference");
......@@ -166,10 +168,10 @@ impl EcmascriptModulePartReference {
let id = module.chunk_item_id(Vc::upcast(chunking_context)).await?;
Ok(CodeGeneration::hoisted_stmt(
ident.clone().into(),
ident.sym.as_str().into(),
quote!(
"var $name = $turbopack_import($id);" as Stmt,
name = Ident::new(ident.clone().into(), DUMMY_SP, Default::default()),
name = ident,
turbopack_import: Expr = TURBOPACK_IMPORT.into(),
id: Expr = module_id_to_lit(&id),
),
......
......@@ -183,6 +183,7 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
},
..Default::default()
},
additional_ids: Default::default(),
placeholder_for_future_extensions: (),
}
.cell())
......
......@@ -242,6 +242,7 @@ async fn run_inner_operation(
struct TestOptions {
tree_shaking_mode: Option<TreeShakingMode>,
remove_unused_exports: Option<bool>,
scope_hoisting: Option<bool>,
}
#[turbo_tasks::value]
......@@ -435,6 +436,7 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
..Default::default()
},
)
.module_merging(options.scope_hoisting.unwrap_or(true))
.build();
let jest_entry_source = FileSource::new(jest_entry_path);
......
import { a } from './reexport.js'
it('should work', () => {
expect(a).toBe(123)
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment