Unverified Commit 1d0ed883 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Add Turbopack constant export inlining

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent c8e811ee
......@@ -379,6 +379,7 @@ pub async fn get_client_module_options_context(
.await?,
cjs_scope_hoisting: *next_config.turbopack_cjs_scope_hoisting().await?,
cross_module_constants: *next_config.turbopack_cross_module_constants().await?,
inline_constant_exports: *next_config.turbopack_inline_constant_exports().await?,
lazy_compilation: *next_config
.turbopack_lazy_dynamic_imports(*next_mode)
.await?,
......
......@@ -1243,8 +1243,10 @@ pub struct ExperimentalConfig {
turbopack_mangle_via_materialized_namespace_object: Option<bool>,
/// Enable scope hoisting of static CommonJS modules. Defaults to false.
turbopack_cjs_scope_hoisting: Option<bool>,
/// Enable cross-module constant inlining. Defaults to false.
/// Enable analyzer-aware cross-module constant inlining. Defaults to false.
turbopack_cross_module_constants: Option<bool>,
/// Inline short constant exports during code generation. Defaults to false.
turbopack_inline_constant_exports: Option<bool>,
/// Devtool option for the segment explorer.
devtool_segment_explorer: Option<bool>,
/// Whether to report inlined system environment variables as warnings or errors.
......@@ -2403,6 +2405,15 @@ impl NextConfig {
)
}
#[turbo_tasks::function]
pub fn turbopack_inline_constant_exports(&self) -> Vc<bool> {
Vc::cell(
self.experimental
.turbopack_inline_constant_exports
.unwrap_or(false),
)
}
#[turbo_tasks::function]
pub fn turbopack_plugin_runtime_strategy(&self) -> Vc<TurbopackPluginRuntimeStrategy> {
// Child processes cannot be spawned from inside wasm, so worker threads are the only
......
......@@ -552,6 +552,7 @@ pub async fn get_server_module_options_context(
.await?,
cjs_scope_hoisting: *next_config.turbopack_cjs_scope_hoisting().await?,
cross_module_constants: *next_config.turbopack_cross_module_constants().await?,
inline_constant_exports: *next_config.turbopack_inline_constant_exports().await?,
..Default::default()
},
execution_context: Some(execution_context),
......
......@@ -461,6 +461,7 @@ export const experimentalSchema = {
turbopackMangleViaMaterializedNamespaceObject: z.boolean().optional(),
turbopackCjsScopeHoisting: z.boolean().optional(),
turbopackCrossModuleConstants: z.boolean().optional(),
turbopackInlineConstantExports: z.boolean().optional(),
turbopackServerFastRefresh: z.boolean().optional(),
optimizePackageImports: z.array(z.string()).optional(),
optimizeServerReact: z.boolean().optional(),
......
......@@ -1127,6 +1127,14 @@ export interface ExperimentalConfig {
*/
turbopackCrossModuleConstants?: boolean
/**
* Inline short, identity-free constant exports during Turbopack code generation without using
* their values for compile-time analysis.
*
* Defaults to `false`
*/
turbopackInlineConstantExports?: boolean
/**
* Set this to `false` to disable the automatic configuration of the babel loader when a Babel
* configuration file is present. This option is enabled by default.
......
......@@ -257,8 +257,12 @@ pub struct EcmascriptOptions {
pub mangle_via_materialized_namespace_object: bool,
/// Whether to scope hoist static CommonJS modules. Defaults to false.
pub cjs_scope_hoisting: bool,
/// Whether to enable cross-module constant inlining. Defaults to false.
/// Whether to use cross-module constants during analysis and code generation. Defaults to
/// false.
pub cross_module_constants: bool,
/// Whether to inline short, identity-free constant exports only during code generation.
/// Defaults to false.
pub inline_constant_exports: bool,
/// Whether dynamic import targets are compiled after their runtime proxy is activated.
pub lazy_compilation: bool,
}
......
......@@ -6,14 +6,17 @@ use bincode::{Decode, Encode};
use num_bigint::BigInt;
use parking_lot::Mutex;
use rustc_hash::FxHashMap;
use swc_core::common::{GLOBALS, source_map::SmallPos};
use swc_core::{
common::{GLOBALS, source_map::SmallPos},
ecma::ast::{ModuleDecl, ModuleItem, Program},
};
use thread_local::ThreadLocal;
use tracing::instrument;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{NonLocalValue, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
compile_time_info::CompileTimeInfo,
compile_time_info::{CompileTimeDefineValue, CompileTimeInfo},
issue::{Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, StyledString},
module::Module,
reference::ModuleReference,
......@@ -35,6 +38,38 @@ const STRING_INLINE_THRESHOLD: usize = 6;
const NUMBER_INLINE_THRESHOLD: f64 = 1_000_000.0;
const BIGINT_INLINE_THRESHOLD: i64 = 1_000_000;
#[derive(Clone, Copy)]
enum AutomaticInlineStatus {
Inline,
TooLong,
HasIdentity,
}
fn automatic_inline_status(value: &ConstantValue) -> AutomaticInlineStatus {
match value {
ConstantValue::Str(s) if s.as_str().len() > STRING_INLINE_THRESHOLD => {
AutomaticInlineStatus::TooLong
}
ConstantValue::Num(n) if n.0.abs() > NUMBER_INLINE_THRESHOLD => {
AutomaticInlineStatus::TooLong
}
ConstantValue::BigInt(n)
if **n > BigInt::from(BIGINT_INLINE_THRESHOLD)
|| **n < BigInt::from(-BIGINT_INLINE_THRESHOLD) =>
{
AutomaticInlineStatus::TooLong
}
ConstantValue::Regex(_) => AutomaticInlineStatus::HasIdentity,
ConstantValue::Undefined
| ConstantValue::Null
| ConstantValue::True
| ConstantValue::False
| ConstantValue::Num(_)
| ConstantValue::BigInt(_)
| ConstantValue::Str(_) => AutomaticInlineStatus::Inline,
}
}
/// Import names that are all-uppercase and contain at least one letter are eligible for automatic
/// constant inlining, even without an import attribute.
pub fn is_import_name_eligible_for_exports(name: &str) -> bool {
......@@ -126,42 +161,21 @@ impl ConstantsModule {
match value {
ConstantsModuleExport::Constant(value) => {
if !has_opt_in {
// when not having opt in, only inline short literals
match &value.0 {
ConstantValue::Str(s)
if s.as_str().len() > STRING_INLINE_THRESHOLD =>
{
JsValue::unknown_empty(
false,
rcstr!("constant too long"),
)
// when not having opt in, only inline short primitive values
match automatic_inline_status(&value.0) {
AutomaticInlineStatus::Inline => {
JsValue::Constant(value.0.clone())
}
ConstantValue::Num(n)
if n.0.abs() > NUMBER_INLINE_THRESHOLD =>
{
JsValue::unknown_empty(
false,
rcstr!("constant too long"),
)
}
ConstantValue::BigInt(n)
if **n > BigInt::from(BIGINT_INLINE_THRESHOLD)
|| **n < BigInt::from(-BIGINT_INLINE_THRESHOLD) =>
{
JsValue::unknown_empty(
false,
rcstr!("constant too long"),
)
}
ConstantValue::Regex(_) => {
// Regexes are literals, but they are also objects, so
// have identity and aren't inlined without opt in.
AutomaticInlineStatus::TooLong => JsValue::unknown_empty(
false,
rcstr!("constant too long"),
),
AutomaticInlineStatus::HasIdentity => {
JsValue::unknown_empty(
false,
rcstr!("regex not inlined"),
)
}
v => JsValue::Constant(v.clone()),
}
} else {
JsValue::Constant(value.0.clone())
......@@ -189,10 +203,53 @@ impl ConstantsModule {
}
}
pub async fn get_inline_export(
import_reference: ResolvedVc<EsmAssetReference>,
export: &str,
compile_time_info: Vc<CompileTimeInfo>,
) -> Result<Option<CompileTimeDefineValue>> {
let resolved = import_reference.resolve_reference().await?;
let Some(module) = resolved.first_module().await? else {
return Ok(None);
};
let constants = get_inline_constants(*module, compile_time_info).await?;
let Some(constants) = constants.as_ref() else {
return Ok(None);
};
let Some((_, ConstantsModuleExport::Constant(value))) =
constants.exports.iter().find(|(name, _)| name == export)
else {
return Ok(None);
};
if !matches!(
automatic_inline_status(&value.0),
AutomaticInlineStatus::Inline
) {
return Ok(None);
}
Ok(CompileTimeDefineValue::try_from(&value.0).ok())
}
#[turbo_tasks::function]
pub async fn get_constants(
module: ResolvedVc<Box<dyn Module>>,
compile_time_info: Vc<CompileTimeInfo>,
) -> Result<Vc<OptionConstantsModule>> {
get_constants_internal(module, compile_time_info, false).await
}
#[turbo_tasks::function]
async fn get_inline_constants(
module: ResolvedVc<Box<dyn Module>>,
compile_time_info: Vc<CompileTimeInfo>,
) -> Result<Vc<OptionConstantsModule>> {
get_constants_internal(module, compile_time_info, true).await
}
async fn get_constants_internal(
module: ResolvedVc<Box<dyn Module>>,
compile_time_info: Vc<CompileTimeInfo>,
include_default_expression: bool,
) -> Result<Vc<OptionConstantsModule>> {
let Some(parseable) = ResolvedVc::try_sidecast::<Box<dyn EcmascriptParsable>>(module) else {
// should never actually happen, there should be a "imported module is not chunkable" error
......@@ -216,6 +273,20 @@ pub async fn get_constants(
let arena = ThreadLocal::new();
let default_export_value =
if include_default_expression && let Program::Module(module) = program {
module.body.iter().find_map(|item| match item {
ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) => {
Some(GLOBALS.set(globals, || {
eval_context.eval(arena.get_or_default(), &export.expr)
}))
}
_ => None,
})
} else {
None
};
let var_graph = {
let supports_block_scoping = *compile_time_info
.environment()
......@@ -251,7 +322,13 @@ pub async fn get_constants(
.iter()
.map(async |(export_name, (binding, span))| {
let value = GLOBALS.set(globals, || {
eval_context.eval_id(arena.get_or_default(), binding.clone())
if export_name.as_str() == "default"
&& let Some(value) = &default_export_value
{
value.clone_in(arena.get_or_default())
} else {
eval_context.eval_id(arena.get_or_default(), binding.clone())
}
});
let linked_value = link(
......
......@@ -126,7 +126,8 @@ use crate::{
async_module::{AsyncModule, OptionAsyncModule},
cjs::{CjsAssetReference, CjsRequireAssetReference, CjsRequireResolveAssetReference},
cross_module_constants::{
is_import_name_eligible_for_exports, module_value_to_constants_module,
get_inline_export, is_import_name_eligible_for_exports,
module_value_to_constants_module,
},
dynamic_expression::DynamicExpression,
emit_collect::{CollectReference, EmitReference},
......@@ -552,6 +553,8 @@ struct AnalysisState<'a> {
var_cache: Mutex<FxHashMap<Id, JsValue<'a>>>,
/// A cache for the linked value of imported constants.
constants_cache: Mutex<FxHashMap<ModuleValue, Option<JsValue<'a>>>>,
/// A cache for constants used only during export inlining code generation.
inline_exports_cache: Mutex<FxHashMap<(usize, RcStr), Option<CompileTimeDefineValue>>>,
// There can be many references to import.meta, but only the first should hoist
// the object allocation.
first_import_meta: bool,
......@@ -582,6 +585,24 @@ struct AnalysisState<'a> {
}
impl<'a> AnalysisState<'a> {
/// Gets a short constant export for code-generation-only inlining.
async fn get_inline_export(
&self,
reference_idx: usize,
export: RcStr,
) -> Result<Option<CompileTimeDefineValue>> {
let key = (reference_idx, export.clone());
if let Some(value) = self.inline_exports_cache.lock().get(&key) {
return Ok(value.clone());
}
let Some(reference) = self.import_references.get(reference_idx) else {
bail!("couldn't find import reference at index {reference_idx}");
};
let value = get_inline_export(*reference, &export, *self.compile_time_info).await?;
self.inline_exports_cache.lock().insert(key, value.clone());
Ok(value)
}
/// Links a value to the graph, returning the linked value.
async fn link_value(
&self,
......@@ -929,6 +950,7 @@ async fn analyze_ecmascript_module_internal(
fun_args_values: Default::default(),
var_cache: Default::default(),
constants_cache: Default::default(),
inline_exports_cache: Default::default(),
first_import_meta: true,
first_webpack_exports_info: true,
module_fragments_enabled: options.module_fragments_enabled,
......@@ -1457,6 +1479,16 @@ async fn analyze_ecmascript_module_internal(
c,
analysis.intern_path(&ast_path),
));
} else if options.inline_constant_exports
&& let Some(export) = export.as_ref()
&& let Some(c) = analysis_state
.get_inline_export(esm_reference_index, export.clone())
.await?
{
analysis.add_code_gen(ConstantValueCodeGen::new_inline_export(
c,
analysis.intern_path(&ast_path),
));
} else if let Some("__turbopack_module_id__") = export.as_deref() {
let chunking_type = r.await?.chunking_type();
analysis.add_reference_code_gen(
......
......@@ -292,6 +292,8 @@ struct TestOptions {
#[serde(default = "default_true")]
cross_module_constants: bool,
#[serde(default)]
inline_constant_exports: bool,
#[serde(default)]
cjs_scope_hoisting: bool,
#[serde(default)]
minify: bool,
......@@ -331,6 +333,7 @@ impl Default for TestOptions {
mangle_via_materialized_namespace_object: default_true(),
cjs_scope_hoisting: false,
cross_module_constants: true,
inline_constant_exports: false,
infer_module_side_effects: default_true(),
minify: false,
production_chunking: false,
......@@ -513,6 +516,7 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
.mangle_via_materialized_namespace_object,
cjs_scope_hoisting: options.cjs_scope_hoisting,
cross_module_constants: options.cross_module_constants,
inline_constant_exports: options.inline_constant_exports,
infer_module_side_effects: options.infer_module_side_effects,
..Default::default()
},
......
import { readA } from './b'
export const value = 'a'
export function completeCycle() {
return readA()
}
import { readA } from './b'
it('inlines across an import cycle without retaining the constants module', () => {
expect(readA()).toBe('a')
const source = readA.toString()
expect(source).toContain('"a"')
expect(source).not.toContain('TURBOPACK compile-time value')
expect(source).not.toContain('value')
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).toContainEqual(expect.stringMatching(/input\/b\.js/))
expect(modules).not.toContainEqual(expect.stringMatching(/input\/a\.js/))
})
{
"scopeHoisting": false,
"mangleExportNames": false,
"crossModuleConstants": false,
"inlineConstantExports": true
}
import { value } from 'disabled-constants'
function readValue() {
return value
}
it('keeps runtime bindings when both constant features are disabled', () => {
expect(readValue()).toBe('dev')
expect(readValue.toString()).toContain('["value"]')
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/disabled-constants\/index\.js/)
)
})
{
"scopeHoisting": false,
"mangleExportNames": false,
"crossModuleConstants": false,
"inlineConstantExports": false
}
import { ANALYZED, late } from 'inline-values'
import { forced } from 'inline-values' with { turbopackConstants: 'true' }
import { annotationOff } from 'inline-values' with { turbopackConstants: 'false' }
if (ANALYZED) require('./analysis-marker')
if (forced) require('./forced-marker')
if (late !== 'dev') require('./late-marker')
if (annotationOff !== 'off') require('./annotation-off-marker')
function readAnalyzed() {
return ANALYZED
}
function readLate() {
return late
}
function readAnnotationOff() {
return annotationOff
}
it('preserves analyzer-aware precedence and falls back to codegen-only inlining', () => {
expect(readAnalyzed()).toBe(false)
expect(readLate()).toBe('dev')
expect(readAnnotationOff()).toBe('off')
expect(readAnalyzed.toString()).toContain('TURBOPACK compile-time value')
expect(readLate.toString()).not.toContain('TURBOPACK compile-time value')
expect(readLate.toString()).not.toContain('late')
expect(readAnnotationOff.toString()).not.toContain('annotationOff')
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).not.toContainEqual(
expect.stringMatching(/input\/analysis-marker\.js/)
)
expect(modules).not.toContainEqual(
expect.stringMatching(/input\/forced-marker\.js/)
)
expect(modules).toContainEqual(expect.stringMatching(/input\/late-marker\.js/))
expect(modules).toContainEqual(
expect.stringMatching(/input\/annotation-off-marker\.js/)
)
})
export const ANALYZED = false
export const forced = false
export const late = 'dev'
export const annotationOff = 'off'
{
"scopeHoisting": false,
"mangleExportNames": false,
"crossModuleConstants": true,
"inlineConstantExports": true
}
import { bValue } from './b'
globalThis.mutualAEvaluations = (globalThis.mutualAEvaluations ?? 0) + 1
export const aValue = 'a'
export function readB() {
return bValue
}
import { aValue } from './a'
globalThis.mutualBEvaluations = (globalThis.mutualBEvaluations ?? 0) + 1
export const bValue = 'b'
export function readA() {
return aValue
}
import { readB } from './a'
import { readA } from './b'
it('supports two retained modules importing constants from each other', () => {
expect(readB()).toBe('b')
expect(readA()).toBe('a')
expect(globalThis.mutualAEvaluations).toBe(1)
expect(globalThis.mutualBEvaluations).toBe(1)
expect(readB.toString()).toContain('"b"')
expect(readB.toString()).not.toContain('bValue')
expect(readA.toString()).toContain('"a"')
expect(readA.toString()).not.toContain('aValue')
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).toContainEqual(expect.stringMatching(/input\/a\.js/))
expect(modules).toContainEqual(expect.stringMatching(/input\/b\.js/))
})
{
"scopeHoisting": false,
"mangleExportNames": false,
"crossModuleConstants": false,
"inlineConstantExports": true
}
import {
shortString,
smallNumber,
smallBigInt,
booleanFalse,
nil,
undef,
stableLet,
stableVar,
} from 'short-constants'
import { regex } from 'identity-values'
import {
longString,
nan,
positiveInfinity,
negativeInfinity,
} from 'long-values'
import shortDefault from 'short-default'
import longDefault from 'long-default'
import { mode } from 'analysis-constant'
import { effectValue } from 'effect-constant'
import { reassigned } from 'reassigned-binding'
import * as staticNamespace from 'namespace-static'
import * as escapingNamespace from 'namespace-escape'
const escapedNamespace = escapingNamespace
if (mode !== 'dev') {
require('./marker')
}
function readShortValues() {
return [
shortString,
smallNumber,
smallBigInt,
booleanFalse,
nil,
undef,
stableLet,
stableVar,
]
}
function readMode() {
return mode
}
function readStaticNamespace() {
return staticNamespace.short
}
function readShortDefault() {
return shortDefault
}
it('inlines short primitive exports during code generation', () => {
expect(readShortValues()).toEqual([
'dev',
1,
1n,
false,
null,
undefined,
'let',
'var',
])
expect(readMode()).toBe('dev')
expect(readStaticNamespace()).toBe('ns')
expect(readShortDefault()).toBe('def')
const source = readShortValues.toString()
expect(source).toContain('"dev"')
expect(source).toContain('1n')
expect(source).toContain('false')
expect(source).toContain('null')
expect(source).toContain('void 0')
expect(source).not.toContain('TURBOPACK compile-time value')
expect(source).not.toContain('shortString')
expect(source).not.toContain('smallNumber')
expect(readStaticNamespace.toString()).not.toContain('staticNamespace')
expect(readShortDefault.toString()).toContain('"def"')
expect(readShortDefault.toString()).not.toContain('shortDefault')
})
it('omits inlined export modules without using values for analysis', () => {
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).not.toContainEqual(
expect.stringMatching(/node_modules\/short-constants\/index\.js/)
)
expect(modules).not.toContainEqual(
expect.stringMatching(/node_modules\/analysis-constant\/index\.js/)
)
expect(modules).not.toContainEqual(
expect.stringMatching(/node_modules\/namespace-static\/index\.js/)
)
expect(modules).not.toContainEqual(
expect.stringMatching(/node_modules\/short-default\/index\.js/)
)
expect(modules).toContainEqual(expect.stringMatching(/input\/marker\.js/))
})
it('preserves evaluation references for side-effectful modules', () => {
expect(effectValue).toBe('ok')
expect(globalThis.inlineExportSideEffect).toBe(1)
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/effect-constant\/index\.js/)
)
})
it('does not inline reassigned bindings, identity values, long values, or escaping namespaces', () => {
expect(reassigned).toBe('b')
expect(escapedNamespace.short).toBe('ns')
expect(regex).toBe(regex)
expect(longString).toBe('this value is deliberately too long')
expect(nan).toBeNaN()
expect(positiveInfinity).toBe(Infinity)
expect(negativeInfinity).toBe(-Infinity)
expect(longDefault).toBe('this default value is too long')
const modules = Array.from(__turbopack_modules__.keys())
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/reassigned-binding\/index\.js/)
)
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/namespace-escape\/index\.js/)
)
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/identity-values\/index\.js/)
)
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/long-values\/index\.js/)
)
expect(modules).toContainEqual(
expect.stringMatching(/node_modules\/long-default\/index\.js/)
)
})
globalThis.inlineExportSideEffect = (globalThis.inlineExportSideEffect ?? 0) + 1
export const effectValue = 'ok'
export const longString = 'this value is deliberately too long'
export const nan = NaN
export const positiveInfinity = Infinity
export const negativeInfinity = -Infinity
export const shortString = 'dev'
export const smallNumber = 1
export const smallBigInt = 1n
export const booleanFalse = false
export const nil = null
export const undef = undefined
export let stableLet = 'let'
export var stableVar = 'var'
{
"scopeHoisting": false,
"mangleExportNames": false,
"crossModuleConstants": false,
"inlineConstantExports": true
}
......@@ -250,6 +250,7 @@ impl ModuleOptions {
mangle_via_materialized_namespace_object,
cjs_scope_hoisting,
cross_module_constants,
inline_constant_exports,
lazy_compilation,
ref preset_env_config,
..
......@@ -349,6 +350,7 @@ impl ModuleOptions {
mangle_via_materialized_namespace_object,
cjs_scope_hoisting,
cross_module_constants,
inline_constant_exports,
lazy_compilation,
..Default::default()
};
......
......@@ -298,11 +298,16 @@ pub struct EcmascriptOptionsContext {
/// Whether to scope-hoist static CommonJS modules. Defaults to false.
pub cjs_scope_hoisting: bool,
/// Whether to enable cross-module constant inlining. Defaults to false.
/// Whether to use cross-module constants during analysis and code generation. Defaults to
/// false.
pub cross_module_constants: bool,
/// Whether dynamic import targets are compiled after their runtime proxy is activated.
pub lazy_compilation: bool,
/// Whether to inline short, identity-free constant exports only during code generation.
/// Defaults to false.
pub inline_constant_exports: bool,
/// Additional SWC preset-env options (mode, coreJs, include, exclude, etc.).
pub preset_env_config: Option<ResolvedVc<PresetEnvConfig>>,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment