Unverified Commit c54eda4b authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Mark packages as side effect free when local analysis determines…

[turbopack] Mark packages as side effect free when local analysis determines that they are. (#86398)

This implements some simple static analysis to determine if modules are side effect free based on their top level module evaluation items.

The rules are simple
 * declarations ✅ 
 * literals ✅ 
 * property access ✅  (potentially controversal but consistent with other bundlers)
 * function calls ❌  (unless part of an builtin allowlist, e.g. `Object.keys`)

imports are obviously the difficult part, for that reason we can evaluate a module to `SideEffectful` `SideEffectFree` or `ModuleEvaluationIsSideEffectFree`.  A later step will leverage a graph analysis step to determine of transitive dependencies of ModuleEvaluationIsSideEffectFree modules are also side effect free and upgrade it.

So for right now the impact is minor.  There is also an experimental option to disable it in case it causes trouble this can help us debug

I tested the performance on vercel site and as expected it isn't really observable

with side effect analysis disabled
```
~/projects/front/apps/vercel-site (main *)$ hyperfine -p 'rm -rf .next' -w 2 -r 20  'pnpm next build --turbopack --experimental-build-mode=compile'
Benchmark 1: pnpm next build --turbopack --experimental-build-mode=compile
  Time (mean ± σ):     48.017 s ±  1.696 s    [User: 319.651 s, System: 44.273 s]
  Range (min … max):   44.855 s … 50.600 s    20 runs
```

With it enabled.
```
~/projects/front/apps/vercel-site (main *)$ hyperfine -p 'rm -rf .next' -w 2 -r 20  'pnpm next build --turbopack --experimental-build-mode=compile'
Benchmark 1: pnpm next build --turbopack --experimental-build-mode=compile
  Time (mean ± σ):     48.038 s ±  2.337 s    [User: 320.261 s, System: 45.434 s]
  Range (min … max):   44.156 s … 52.774 s    20 runs
```

so if there is a regression it is well within the noise.
parent b17b31f1
......@@ -9775,6 +9775,7 @@ dependencies = [
"once_cell",
"parking_lot",
"petgraph 0.8.3",
"phf",
"regex",
"rustc-hash 2.1.1",
"serde",
......
......@@ -326,6 +326,7 @@ pub async fn get_client_module_options_context(
ecmascript: EcmascriptOptionsContext {
enable_typeof_window_inlining: Some(TypeofWindow::Object),
source_maps,
infer_module_side_effects: *next_config.turbopack_infer_module_side_effects().await?,
..Default::default()
},
css: CssOptionsContext {
......
......@@ -1055,6 +1055,8 @@ pub struct ExperimentalConfig {
turbopack_remove_unused_imports: Option<bool>,
/// Defaults to false in development mode, true in production mode.
turbopack_remove_unused_exports: Option<bool>,
/// Enable local analysis to infer side effect free modules. Defaults to true.
turbopack_infer_module_side_effects: Option<bool>,
/// Devtool option for the segment explorer.
devtool_segment_explorer: Option<bool>,
}
......@@ -2028,6 +2030,15 @@ impl NextConfig {
))
}
#[turbo_tasks::function]
pub fn turbopack_infer_module_side_effects(&self) -> Vc<bool> {
Vc::cell(
self.experimental
.turbopack_infer_module_side_effects
.unwrap_or(true),
)
}
#[turbo_tasks::function]
pub async fn module_ids(&self, mode: Vc<NextMode>) -> Result<Vc<ModuleIds>> {
Ok(match *mode.await? {
......
......@@ -582,6 +582,7 @@ pub async fn get_server_module_options_context(
import_externals: *next_config.import_externals().await?,
ignore_dynamic_requests: true,
source_maps,
infer_module_side_effects: *next_config.turbopack_infer_module_side_effects().await?,
..Default::default()
},
execution_context: Some(execution_context),
......
......@@ -310,6 +310,7 @@ export const experimentalSchema = {
turbopackUseBuiltinBabel: z.boolean().optional(),
turbopackUseBuiltinSass: z.boolean().optional(),
turbopackModuleIds: z.enum(['named', 'deterministic']).optional(),
turbopackInferModuleSideEffects: z.boolean().optional(),
optimizePackageImports: z.array(z.string()).optional(),
optimizeServerReact: z.boolean().optional(),
clientTraceMetadata: z.array(z.string()).optional(),
......
......@@ -464,6 +464,15 @@ export interface ExperimentalConfig {
*/
turbopackRemoveUnusedExports?: boolean
/**
* Enable local analysis to infer side effect free modules. When enabled, Turbopack will
* analyze module code to determine if it has side effects. This can improve tree shaking
* and bundle size at the cost of some additional analysis.
*
* Defaults to `true`.
*/
turbopackInferModuleSideEffects?: boolean
/**
* Use the system-provided CA roots instead of bundled CA roots for external HTTPS requests
* made by Turbopack. Currently this is only used for fetching data from Google Fonts.
......
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, TaskInput, ValueToString, Vc};
use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, ValueToString, Vc, trace::TraceRawVcs};
use turbo_tasks_fs::glob::Glob;
use crate::{asset::Asset, ident::AssetIdent, reference::ModuleReferences, source::OptionSource};
......@@ -11,6 +13,22 @@ pub enum StyleType {
GlobalStyle,
}
#[derive(
Serialize, Deserialize, Hash, Eq, PartialEq, Debug, NonLocalValue, TraceRawVcs, Encode, Decode,
)]
pub enum ModuleSideEffects {
/// Analysis determined that the module evaluation is side effect free
/// the module may still be side effectful based on its imports.
ModuleEvaluationIsSideEffectFree,
/// Is known to be side effect free either due to static analysis or some kind of configuration.
/// ```js
/// "use turbopack no side effects"
/// ```
SideEffectFree,
// Neither of the above, so we should assume it has side effects.
SideEffectful,
}
/// A module. This usually represents parsed source code, which has references
/// to other modules.
#[turbo_tasks::value_trait]
......
use anyhow::Result;
use rustc_hash::{FxHashMap, FxHashSet};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use crate::{
module::{Module, ModuleSideEffects},
module_graph::{GraphTraversalAction, ModuleGraph, SingleModuleGraphWithBindingUsage},
};
/// This lists all the modules that are side effect free
/// This means they are either declared side effect free by some configuration or they have been
/// determined to be side effect free via static analysis of the module evaluation and dependencies.
#[turbo_tasks::value(transparent)]
pub struct SideEffectFreeModules(FxHashSet<ResolvedVc<Box<dyn Module>>>);
/// Computes the set of side effect free modules in the module graph.
///
/// This leverages the module graph to compute if modules with `ModuleEvalutationIsSideEffectFree`
/// status are actually side effectful or not.
///
/// A current limitation is that the module graph doesn't contain information on how 'late' imports
/// are used. e.g. if there is a dynamic import with side effects in an event handler, we will mark
/// the module as side effectful even though it isn't. To fix this the module graph would need to
/// record more information about the nature of the edges.
#[turbo_tasks::function]
pub async fn compute_side_effect_free_module_info(
graphs: ResolvedVc<ModuleGraph>,
) -> Result<Vc<SideEffectFreeModules>> {
// Layout segment optimization, we can individually compute the side effect free modules for
// each graph.
let mut result: Vc<SideEffectFreeModules> = Vc::cell(Default::default());
let graphs = graphs.await?;
for graph in graphs.iter_graphs() {
result = compute_side_effect_free_module_info_single(graph, result);
}
Ok(result)
}
#[turbo_tasks::function]
async fn compute_side_effect_free_module_info_single(
graph: SingleModuleGraphWithBindingUsage,
parent_side_effect_free_modules: Vc<SideEffectFreeModules>,
) -> Result<Vc<SideEffectFreeModules>> {
let parent_side_effect_free_modules = parent_side_effect_free_modules.await?;
let graph = graph.read().await?;
let module_side_effects = graph
.enumerate_nodes()
.map(async |(_, node)| {
Ok(match node {
super::SingleModuleGraphNode::Module(module) => {
// This turbo task always has a cache hit since it is called when building the
// module graph. we could consider moving this information
// into to the module graph, but then changes would invalidate the whole graph.
(*module, *module.side_effects().await?)
}
super::SingleModuleGraphNode::VisitedModule { idx: _, module } => (
*module,
if parent_side_effect_free_modules.contains(module) {
ModuleSideEffects::SideEffectFree
} else {
ModuleSideEffects::SideEffectful
},
),
})
})
.try_join()
.await?
.into_iter()
.collect::<FxHashMap<_, _>>();
// Modules are categorized as side-effectful, locally side effect free and side effect free.
// So we are really just interested in determining what modules that are locally side effect
// free. logically we want to start at all such modules are determine if their transitive
// dependencies are side effect free.
let mut locally_side_effect_free_modules_that_have_side_effects = FxHashSet::default();
graph.traverse_edges_from_entries_dfs_reversed(
// Start from all the side effectful nodes
module_side_effects.iter().filter_map(|(m, e)| {
if *e == ModuleSideEffects::SideEffectful {
Some(*m)
} else {
None
}
}),
&mut (),
// child is a previously visited module that we know is side effectful
// parent is a module that depends on it.
|child, parent, _s| {
Ok(if child.is_some() {
match module_side_effects.get(&parent).unwrap() {
ModuleSideEffects::SideEffectful | ModuleSideEffects::SideEffectFree => {
// We have either already seen this or don't want to follow it
GraphTraversalAction::Exclude
}
ModuleSideEffects::ModuleEvaluationIsSideEffectFree => {
// this module is side effect free locally but depends on `child` which is
// effectful so it too is effectful
locally_side_effect_free_modules_that_have_side_effects.insert(parent);
GraphTraversalAction::Continue
}
}
} else {
// entry point, we already determined it was effectful, keep going
GraphTraversalAction::Continue
})
},
|_, _, _| Ok(()),
)?;
let side_effect_free_modules = module_side_effects
.into_iter()
.filter_map(|(m, e)| match e {
ModuleSideEffects::SideEffectful => None,
ModuleSideEffects::SideEffectFree => Some(m),
ModuleSideEffects::ModuleEvaluationIsSideEffectFree => {
if locally_side_effect_free_modules_that_have_side_effects.contains(&m) {
None
} else {
Some(m)
}
}
})
.chain(parent_side_effect_free_modules.iter().copied())
.collect::<FxHashSet<_>>();
Ok(Vc::cell(side_effect_free_modules))
}
......@@ -35,6 +35,7 @@ num-bigint = "0.4"
num-traits = "0.2.15"
once_cell = { workspace = true }
parking_lot = { workspace = true }
phf = { version = "0.11", features = ["macros"] }
petgraph = { workspace = true }
dashmap = { workspace = true }
regex = { workspace = true }
......
......@@ -41,6 +41,7 @@ pub mod builtin;
pub mod graph;
pub mod imports;
pub mod linker;
pub mod side_effects;
pub mod top_level_await;
pub mod well_known;
......
......@@ -100,7 +100,7 @@ use turbopack_core::{
compile_time_info::CompileTimeInfo,
context::AssetContext,
ident::AssetIdent,
module::{Module, OptionModule},
module::{Module, ModuleSideEffects, OptionModule},
module_graph::ModuleGraph,
output::OutputAssetsReference,
reference::ModuleReferences,
......@@ -285,6 +285,8 @@ pub struct EcmascriptOptions {
pub enable_exports_info_inlining: bool,
pub inline_helpers: bool,
/// Whether to infer side effect free modules via local analysis. Defaults to true.
pub infer_module_side_effects: bool,
}
#[turbo_tasks::value]
......@@ -778,7 +780,7 @@ impl Module for EcmascriptModuleAsset {
Ok(if *pkg_side_effect_free.await? {
pkg_side_effect_free
} else {
Vc::cell(self.analyze().await?.has_side_effect_free_directive)
Vc::cell(self.analyze().await?.side_effects == ModuleSideEffects::SideEffectFree)
})
}
}
......
......@@ -77,7 +77,7 @@ use turbopack_core::{
environment::Rendering,
error::PrettyPrintError,
issue::{IssueExt, IssueSeverity, IssueSource, StyledString, analyze::AnalyzeIssue},
module::Module,
module::{Module, ModuleSideEffects},
reference::{ModuleReference, ModuleReferences},
reference_type::{CommonJsReferenceSubType, ReferenceType},
resolve::{
......@@ -111,7 +111,7 @@ use crate::{
},
imports::{ImportAnnotations, ImportAttributes, ImportedSymbol, Reexport},
linker::link,
parse_require_context,
parse_require_context, side_effects,
top_level_await::has_top_level_await,
well_known::replace_well_known,
},
......@@ -172,7 +172,7 @@ pub struct AnalyzeEcmascriptModuleResult {
pub code_generation: ResolvedVc<CodeGens>,
pub exports: ResolvedVc<EcmascriptExports>,
pub async_module: ResolvedVc<OptionAsyncModule>,
pub has_side_effect_free_directive: bool,
pub side_effects: ModuleSideEffects,
/// `true` when the analysis was successful.
pub successful: bool,
pub source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
......@@ -233,7 +233,7 @@ struct AnalyzeEcmascriptModuleResultBuilder {
async_module: ResolvedVc<OptionAsyncModule>,
successful: bool,
source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
has_side_effect_free_directive: bool,
side_effects: ModuleSideEffects,
#[cfg(debug_assertions)]
ident: RcStr,
}
......@@ -253,7 +253,7 @@ impl AnalyzeEcmascriptModuleResultBuilder {
async_module: ResolvedVc::cell(None),
successful: false,
source_map: None,
has_side_effect_free_directive: false,
side_effects: ModuleSideEffects::SideEffectful,
#[cfg(debug_assertions)]
ident: Default::default(),
}
......@@ -331,8 +331,8 @@ impl AnalyzeEcmascriptModuleResultBuilder {
}
/// Set whether this module is side-effect free according to a user-provided directive.
pub fn set_has_side_effect_free_directive(&mut self, value: bool) {
self.has_side_effect_free_directive = value;
pub fn set_side_effects_mode(&mut self, value: ModuleSideEffects) {
self.side_effects = value;
}
/// Sets whether the analysis was successful.
......@@ -443,7 +443,7 @@ impl AnalyzeEcmascriptModuleResultBuilder {
code_generation: ResolvedVc::cell(code_generation),
exports: self.exports.resolved_cell(),
async_module: self.async_module,
has_side_effect_free_directive: self.has_side_effect_free_directive,
side_effects: self.side_effects,
successful: self.successful,
source_map: self.source_map,
},
......@@ -658,7 +658,21 @@ async fn analyze_ecmascript_module_internal(
},
_ => false,
});
analysis.set_has_side_effect_free_directive(has_side_effect_free_directive);
analysis.set_side_effects_mode(if has_side_effect_free_directive {
ModuleSideEffects::SideEffectFree
} else if options.infer_module_side_effects {
// Analyze the AST to infer side effects
GLOBALS.set(globals, || {
side_effects::compute_module_evaluation_side_effects(
program,
comments,
eval_context.unresolved_mark,
)
})
} else {
// If inference is disabled, assume side effects
ModuleSideEffects::SideEffectful
});
let is_esm = eval_context.is_esm(specified_type);
let compile_time_info = compile_time_info_for_module_options(
......
......@@ -1699,7 +1699,7 @@ pub(crate) fn find_turbopack_part_id_in_asserts(asserts: &ObjectLit) -> Option<P
_ => None,
})
}
/// givin a number, return a base54 encoded string
/// given a number, return a base54 encoded string
/// `usize -> [a-zA-Z$_][a-zA-Z$_0-9]*`
pub(crate) fn encode_base54(init: &mut usize, skip_reserved: bool) -> Atom {
static BASE54_CHARS: &[u8; 64] =
......
// Combined test for issue-13063
// Tests that side effects optimization doesn't crash with specific vendor patterns
// Test 1: UiSelectButton (from tst_examples_uiform.js)
it('should not crash with UiSelectButton', () => {
require('./vendors').UiSelectButton()
require('./vendors').UiSelectButton2()
})
// Test 2: UiButton (from tst_examples_uitable.js)
import { UiButton } from './vendors'
it('should not crash with UiButton', () => {
UiButton()
})
'use strict'
module.exports = {
findBundle() {
return ['./vendors.js', './tst_examples_uiform.js']
},
}
it('should not crash', () => {
require('./vendors').UiSelectButton()
require('./vendors').UiSelectButton2()
})
import { UiButton } from './vendors'
it('should not crash', () => {
UiButton()
})
import uuid from './uuid'
import { checkIsNonemptyString } from './types'
export { UiSelectButton } from './select'
export { UiSelectButton2 } from './select2'
export function UiButton() {
checkIsNonemptyString()
uuid()
}
import uuid from './uuid'
import { checkIsNonemptyString } from './types'
export function UiSelectButton() {
checkIsNonemptyString()
uuid()
}
console.log.bind(console)
import uuid from './uuid'
export function UiSelectButton2() {
uuid()
}
console.log.bind(console)
import { b } from 'dep'
b.c()
import { modules } from 'dep/trackModules.js'
it('should not contain side-effect-free modules', () => {
// TODO: webpack can trim 'b'. The issue is that turbopack cannot follow namespace re-exports
// when it is tree shaking exports.
expect(modules).toEqual(['b', 'c'])
})
'use strict'
/** @typedef {import("../../../../").Compiler} Compiler */
/** @typedef {import("../../../../").NormalModule} NormalModule */
class ReorderModulesPlugin {
/**
* @param {Compiler} compiler compiler
*/
apply(compiler) {
compiler.hooks.compilation.tap('ReorderModulesPlugin', (compilation) => {
compilation.hooks.seal.tap('ReorderModulesPlugin', () => {
const sortedModules = [...compilation.modules].sort((a, _b) =>
/** @type {NormalModule} */
(a).request.includes('b.js') ? -1 : 1
)
compilation.modules = new Set(sortedModules)
})
})
}
}
module.exports = {
plugins: [new ReorderModulesPlugin()],
optimization: {
sideEffects: true,
},
}
import { log as booleanValueModuleLog } from 'boolean-value-module/tracker'
import booleanValueModule from 'boolean-value-module'
import { log as globValueModuleLog } from 'glob-value-module/tracker'
import globValueModule from 'glob-value-module'
it('should handle a boolean', function () {
expect(booleanValueModule).toBe('def')
expect(booleanValueModuleLog).toEqual(['index.js'])
})
it('should handle globs', function () {
expect(globValueModule).toBe('def')
expect(globValueModuleLog).toEqual(['./src/a.js', 'a.js', 'index.js'])
})
var a = 'a'
var b = 'b'
var c = 'c'
export { a, b, c }
import { track } from './tracker'
track('a.js')
var x = 'x'
var y = 'y'
export { x, y }
export { z } from './c'
import { track } from './tracker'
track('b.js')
export * from './a'
export { x, y, z } from './b'
import { track } from './tracker'
track('index.js')
export default 'def'
export function track(file) {
log.push(file)
log.sort()
}
export var log = []
export function reset() {
log.length = 0
}
var x = 'x'
var y = 'y'
export { x, y }
export { z } from './a'
import { track } from './tracker'
track('b.js')
import * as a from './a'
import * as a2 from './src/a'
import * as b from './b'
import { track } from './tracker'
track('index.js')
export default 'def'
export function track(file) {
log.push(file)
log.sort()
}
export var log = []
export function reset() {
log.length = 0
}
import { a, b } from './module'
import empty from './empty'
it('should skip over module', () => {
expect(a).toBe('a')
expect(b).toBe('b')
// Check that module.js is optimized away (not loaded directly)
// because it only re-exports from other modules
const modules = Array.from(__turbopack_modules__.keys())
// module.js should be optimized away by tree-shaking
// (it's just a re-export aggregator with no side effects)
// TODO: this is wrong currently since we do not infer the module to be side effect free
expect(modules).toContainEqual(expect.stringMatching(/module\.js/))
// But a.js and b.js should be loaded directly
expect(modules).toContainEqual(expect.stringMatching(/a\.js/))
expect(modules).toContainEqual(expect.stringMatching(/b\.js/))
})
warning - [analysis] /turbopack/crates/turbopack-tests/tests/execution/webpack/side-effects/type-reexports/input/empty.js unexpected export *
export * used with module [project]/turbopack/crates/turbopack-tests/tests/execution/webpack/side-effects/type-reexports/input/empty.js [test] (ecmascript) which has no exports
Typescript only: Did you want to export only types with `export type * from "..."`?
Note: Using `export type` is more efficient than `export *` as it won't emit any runtime code.
Import trace:
test:
./turbopack/crates/turbopack-tests/tests/execution/webpack/side-effects/type-reexports/input/empty.js
./turbopack/crates/turbopack-tests/tests/execution/webpack/side-effects/type-reexports/input/index.js
\ No newline at end of file
import { used } from './module'
it('should not include unused assets', () => {
expect(used.href).toMatch(/png/)
// Check that modules are loaded/not loaded based on side effects
const modules = Array.from(__turbopack_modules__.keys())
// 'used' should be included (not orphaned)
expect(modules).toContainEqual(expect.stringMatching(/file\.png\?used/))
// unused exports should not be included (orphaned by tree-shaking)
// These tree assertions are WRONG, the url modules are included even though they
expect(modules).toContainEqual(expect.stringMatching(/file\.png\?default/))
expect(modules).toContainEqual(expect.stringMatching(/file\.png\?named/))
expect(modules).toContainEqual(expect.stringMatching(/file\.png\?indirect/))
})
export default new URL('file.png?default', import.meta.url)
export const named = new URL('file.png?named', import.meta.url)
export const indirect = fn
export const used = new URL('file.png?used', import.meta.url)
function fn() {
return new URL('file.png?indirect', import.meta.url)
}
......@@ -74,4 +74,4 @@ if (Date.now() > 0) {
}),
]);
//# sourceMappingURL=aaf3a_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_2490dad7._.js.map
//# sourceMappingURL=aaf3a_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_2490dad7._.js.map
\ No newline at end of file
......@@ -210,6 +210,7 @@ impl ModuleOptions {
enable_exports_info_inlining,
source_maps: ecmascript_source_maps,
inline_helpers,
infer_module_side_effects,
..
},
enable_mdx,
......@@ -285,6 +286,7 @@ impl ModuleOptions {
enable_typeof_window_inlining,
enable_exports_info_inlining,
inline_helpers,
infer_module_side_effects,
..Default::default()
};
let ecmascript_options_vc = ecmascript_options.resolved_cell();
......
......@@ -252,6 +252,9 @@ pub struct EcmascriptOptionsContext {
// TODO should this be a part of Environment instead?
pub inline_helpers: bool,
/// Whether to infer side effect free modules via local analysis. Defaults to true.
pub infer_module_side_effects: bool,
pub placeholder_for_future_extensions: (),
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment