Unverified Commit b694eeb5 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: only compute module ids for included modules (#93276)

Currently doesn't have a measureable performance impact.

Previously we assigned module ids for all nodes in the graph, even the subgraphs we detect as unused. They weren't chunked already, but the codegen that references those wasn't gracefully handling `.chunk_item_id(m)` failing (where `m` is a unused skipped module)

So `.iter_nodes()` should basically be deprecated. It doesn't respect `removed_unused_imports` and the side-effect-free inference. 
And this would get much much worse once NFT is on the module graph, because then there are way more modules that should be skipped via the same mechanism

This manifested as the following error after switching from `.iter_nodes` to `.traverse_edges_unordered` in global_module_ids.rs
```
Debug info:
- An error occurred while generating the chunk item [project]/node_modules/.pnpm/next@file+..+next.js+packages+next_@babel+core@7.29.0_@opentelemetry+api@1.7.0_@playwri_9a8ea08672cbf59c2720abbb0c8879be/node_modules/next/dist/esm/server/stream-utils/node-web-streams-helper.js [app-rsc] (ecmascript)
- Execution of <MergedEcmascriptModule as EcmascriptChunkPlaceable>::chunk_item_content failed
- Execution of *EcmascriptChunkItemContent::new failed
- Execution of EcmascriptModuleContent::new_merged failed
- ModuleId not found for ident: [project]/node_modules/.pnpm/next@file+..+next.js+packages+next_@babel+core@7.29.0_@opentelemetry+api@1.7.0_@playwri_9a8ea08672cbf59c2720abbb0c8879be/node_modules/next/dist/esm/shared/lib/hash.js [app-rsc] (ecmascript)

```
parent 908af501
...@@ -292,6 +292,22 @@ pub async fn compute_binding_usage_info( ...@@ -292,6 +292,22 @@ pub async fn compute_binding_usage_info(
.await? .await?
); );
} }
static PRINT_USED_EXPORTS: Lazy<bool> = Lazy::new(|| {
std::env::var_os("TURBOPACK_PRINT_USED_EXPORTS")
.is_some_and(|v| v == "1" || v == "true")
});
if *PRINT_USED_EXPORTS {
use turbo_tasks::TryJoinIterExt;
println!(
"used exports: {:#?}",
used_exports
.iter()
.map(async |(m, v)| Ok((m.ident_string().await?, v,)))
.try_join()
.await?
);
}
} }
Ok(BindingUsageInfo { Ok(BindingUsageInfo {
......
...@@ -139,9 +139,8 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc< ...@@ -139,9 +139,8 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
FxHashSet::with_capacity_and_hasher(module_count, Default::default()); FxHashSet::with_capacity_and_hasher(module_count, Default::default());
let inner_span = tracing::info_span!("collect mergeable modules"); let inner_span = tracing::info_span!("collect mergeable modules");
let mergeable = graphs let mergeable = module_graph
.iter() .iter_nodes()
.flat_map(|g| g.iter_nodes())
.map(async |module| { .map(async |module| {
if let Some(mergeable) = if let Some(mergeable) =
ResolvedVc::try_downcast::<Box<dyn MergeableModule>>(module) ResolvedVc::try_downcast::<Box<dyn MergeableModule>>(module)
......
...@@ -981,9 +981,7 @@ impl ModuleGraphSnapshot { ...@@ -981,9 +981,7 @@ impl ModuleGraphSnapshot {
/// This is primarily useful for debugging. /// This is primarily useful for debugging.
pub async fn get_ids(&self) -> Result<FxHashMap<ResolvedVc<Box<dyn Module>>, ReadRef<RcStr>>> { pub async fn get_ids(&self) -> Result<FxHashMap<ResolvedVc<Box<dyn Module>>, ReadRef<RcStr>>> {
Ok(self Ok(self
.graphs .iter_nodes()
.iter()
.flat_map(|g| g.iter_nodes())
.map(async |n| Ok((n, n.ident().to_string().await?))) .map(async |n| Ok((n, n.ident().to_string().await?)))
.try_join() .try_join()
.await? .await?
......
...@@ -105,6 +105,44 @@ async fn compute_side_effect_free_module_info_single( ...@@ -105,6 +105,44 @@ async fn compute_side_effect_free_module_info_single(
}, },
|_, _, _| Ok(()), |_, _, _| Ok(()),
)?; )?;
#[cfg(debug_assertions)]
{
use once_cell::sync::Lazy;
static PRINT_SIDE_EFFECT_INFO: Lazy<bool> = Lazy::new(|| {
std::env::var_os("TURBOPACK_PRINT_SIDE_EFFECT_INFO")
.is_some_and(|v| v == "1" || v == "true")
});
if *PRINT_SIDE_EFFECT_INFO {
use turbo_tasks::TryJoinIterExt;
println!(
"side effect free modules: {:#?}",
module_side_effects
.iter()
.filter_map(|(m, e)| match e {
ModuleSideEffects::SideEffectful => None,
ModuleSideEffects::SideEffectFree => Some((m, false)),
ModuleSideEffects::ModuleEvaluationIsSideEffectFree => {
if locally_side_effect_free_modules_that_have_side_effects.contains(m) {
None
} else {
Some((m, true))
}
}
})
.map(async |(m, locally)| Ok((
m.ident_string().await?,
if locally {
"inferred locally"
} else {
"declared"
}
)))
.try_join()
.await?
);
}
}
let side_effect_free_modules = module_side_effects let side_effect_free_modules = module_side_effects
.into_iter() .into_iter()
.filter_map(|(m, e)| match e { .filter_map(|(m, e)| match e {
......
...@@ -62,6 +62,14 @@ impl EsmBinding { ...@@ -62,6 +62,14 @@ impl EsmBinding {
chunking_context: Vc<Box<dyn ChunkingContext>>, chunking_context: Vc<Box<dyn ChunkingContext>>,
scope_hoisting_context: ScopeHoistingContext<'_>, scope_hoisting_context: ScopeHoistingContext<'_>,
) -> Result<CodeGeneration> { ) -> Result<CodeGeneration> {
if chunking_context
.unused_references()
.contains_key(&ResolvedVc::upcast(self.reference))
.await?
{
return Ok(CodeGeneration::empty());
}
let mut visitors = vec![]; let mut visitors = vec![];
let export = self.export.clone(); let export = self.export.clone();
......
...@@ -16,7 +16,7 @@ __turbopack_context__.s([ ...@@ -16,7 +16,7 @@ __turbopack_context__.s([
var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$leaf$2e$js__$5b$test$5d$__$28$ecmascript$29$__ = __turbopack_context__.i("[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/leaf.js [test] (ecmascript)"); var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$leaf$2e$js__$5b$test$5d$__$28$ecmascript$29$__ = __turbopack_context__.i("[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/leaf.js [test] (ecmascript)");
; ;
function sharedX() { function sharedX() {
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$leaf$2e$js__$5b$test$5d$__$28$ecmascript$29$__["leafX"])(); leafX();
} }
function sharedY() { function sharedY() {
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$leaf$2e$js__$5b$test$5d$__$28$ecmascript$29$__["leafY"])(); (0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$leaf$2e$js__$5b$test$5d$__$28$ecmascript$29$__["leafY"])();
...@@ -50,7 +50,7 @@ var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbo ...@@ -50,7 +50,7 @@ var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbo
; ;
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$y$2e$js__$5b$test$5d$__$28$ecmascript$29$__["y"])(); (0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$y$2e$js__$5b$test$5d$__$28$ecmascript$29$__["y"])();
function helper() { function helper() {
return (0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$remove$2d$unused$2d$imports$2f$exports$2f$input$2f$library$2f$x$2e$js__$5b$test$5d$__$28$ecmascript$29$__["x"])(); return x();
} }
function unused() { function unused() {
return helper(); return helper();
......
...@@ -3,8 +3,8 @@ ...@@ -3,8 +3,8 @@
"sources": [], "sources": [],
"sections": [ "sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/leaf.js"],"sourcesContent":["export function leafX() {}\nexport function leafY() {}\n"],"names":["leafX","leafY"],"mappings":"AAAO,SAASA,SAAS;AAClB,SAASC,SAAS"}}, {"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/leaf.js"],"sourcesContent":["export function leafX() {}\nexport function leafY() {}\n"],"names":["leafX","leafY"],"mappings":"AAAO,SAASA,SAAS;AAClB,SAASC,SAAS"}},
{"offset": {"line": 15, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/shared.js"],"sourcesContent":["import { leafX, leafY } from './leaf'\n\nexport function sharedX() {\n leafX()\n}\nexport function sharedY() {\n leafY()\n}\n"],"names":["sharedX","sharedY"],"mappings":"AAAA;;AAEO,SAASA;IACd,IAAA,uOAAK;AACP;AACO,SAASC;IACd,IAAA,uOAAK;AACP"}}, {"offset": {"line": 15, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/shared.js"],"sourcesContent":["import { leafX, leafY } from './leaf'\n\nexport function sharedX() {\n leafX()\n}\nexport function sharedY() {\n leafY()\n}\n"],"names":["sharedX","leafX","sharedY"],"mappings":"AAAA;;AAEO,SAASA;IACdC;AACF;AACO,SAASC;IACd,IAAA,uOAAK;AACP"}},
{"offset": {"line": 32, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/y.js"],"sourcesContent":["import { sharedY } from './shared'\n\nglobalThis.yBundled = true\n\nexport function y() {\n sharedY()\n}\n"],"names":["globalThis","yBundled","y"],"mappings":"AAAA;;AAEAA,WAAWC,QAAQ,GAAG;AAEf,SAASC;IACd,IAAA,2OAAO;AACT"}}, {"offset": {"line": 32, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/library/y.js"],"sourcesContent":["import { sharedY } from './shared'\n\nglobalThis.yBundled = true\n\nexport function y() {\n sharedY()\n}\n"],"names":["globalThis","yBundled","y"],"mappings":"AAAA;;AAEAA,WAAWC,QAAQ,GAAG;AAEf,SAASC;IACd,IAAA,2OAAO;AACT"}},
{"offset": {"line": 47, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/a.js"],"sourcesContent":["import { x } from './library/x.js'\nimport { y } from './library/y.js'\n\ny()\n\nfunction helper() {\n return x()\n}\n\nexport function unused() {\n return helper()\n}\n\nexport function used() {\n return 1234\n}\n"],"names":["helper","unused","used"],"mappings":"AACA;;;AAEA,IAAA,gOAAC;AAED,SAASA;IACP,OAAO,IAAA,gOAAC;AACV;AAEO,SAASC;IACd,OAAOD;AACT;AAEO,SAASE;IACd,OAAO;AACT"}}, {"offset": {"line": 47, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/a.js"],"sourcesContent":["import { x } from './library/x.js'\nimport { y } from './library/y.js'\n\ny()\n\nfunction helper() {\n return x()\n}\n\nexport function unused() {\n return helper()\n}\n\nexport function used() {\n return 1234\n}\n"],"names":["helper","x","unused","used"],"mappings":"AACA;;;AAEA,IAAA,gOAAC;AAED,SAASA;IACP,OAAOC;AACT;AAEO,SAASC;IACd,OAAOF;AACT;AAEO,SAASG;IACd,OAAO;AACT"}},
{"offset": {"line": 69, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/index.js"],"sourcesContent":["import { used } from './a.js'\n\nconsole.log(used)\n"],"names":["console","log"],"mappings":"AAAA;;AAEAA,QAAQC,GAAG,CAAC,wNAAI"}}] {"offset": {"line": 69, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/remove-unused-imports/exports/input/index.js"],"sourcesContent":["import { used } from './a.js'\n\nconsole.log(used)\n"],"names":["console","log"],"mappings":"AAAA;;AAEAA,QAAQC,GAAG,CAAC,wNAAI"}}]
} }
\ No newline at end of file
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use rustc_hash::FxHashMap; use rustc_hash::{FxHashMap, FxHashSet};
use smallvec::SmallVec; use smallvec::SmallVec;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
...@@ -23,17 +23,13 @@ pub async fn get_global_module_id_strategy( ...@@ -23,17 +23,13 @@ pub async fn get_global_module_id_strategy(
let span = tracing::info_span!("compute module id map"); let span = tracing::info_span!("compute module id map");
async move { async move {
let module_graph = module_graph.await?; let module_graph = module_graph.await?;
let graphs = &module_graph.graphs;
// All modules in the graph // All modules in the graph and additionally, all the modules that are inserted by chunking
let module_idents = graphs // (i.e. async loaders)
.iter() let mut modules = FxHashSet::default();
.flat_map(|graph| graph.iter_nodes())
.map(|m| m.ident());
// And additionally, all the modules that are inserted by chunking (i.e. async loaders)
let mut async_idents = vec![]; let mut async_idents = vec![];
module_graph.traverse_edges_unordered(|parent, current| { module_graph.traverse_edges_unordered(|parent, current| {
modules.insert(current);
if let Some(( if let Some((
_, _,
&RefData { &RefData {
...@@ -49,7 +45,9 @@ pub async fn get_global_module_id_strategy( ...@@ -49,7 +45,9 @@ pub async fn get_global_module_id_strategy(
Ok(()) Ok(())
})?; })?;
let mut module_id_map = module_idents let mut module_id_map = modules
.into_iter()
.map(|m| m.ident())
.chain(async_idents.into_iter()) .chain(async_idents.into_iter())
.map(|ident| async move { .map(|ident| async move {
let ident = ident.to_resolved().await?; let ident = ident.to_resolved().await?;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment