Unverified Commit ba182aa1 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Cache bundle analysis per output file (#100475)

## Summary

Analyze each emitted and traced file once in a cached task that returns
its filename, chunk membership, and chunk parts with normalized sources
and compressed sizes. Routes previously made 3–5 task calls per shared
file, and task-graph bookkeeping for those calls dominated analysis.
Splitting, compression, and membership enumeration run as plain helpers
under that task, since each is read once.

Measured on v0/chat with `next build --analyze` (#99962), analysis phase
only: new tasks went from 269k to 242k, calls into existing tasks from
8.49M to 5.07M, and median CPU time from 150s to 124s. All 455 data
artifacts were byte-identical across 12 runs.

Supersedes #100486.

## Verification

- `cargo test -p turbopack-analyze --test split_chunk` and `cargo test
-p next-api --lib analyze` passed.
- Rebuilt the native binding with `release-with-debug`, installed it in
v0/chat, and compared `next build --analyze` output against the previous
binary across alternating runs.

<!-- NEXT_JS_LLM -->
parent 62e889a5
mod groups;
mod membership;
mod output_file;
use std::{borrow::Cow, io::Write};
use std::io::Write;
use anyhow::Result;
use bincode::{Decode, Encode};
use byteorder::{BE, WriteBytesExt};
use either::Either;
use membership::output_chunk_modules;
use next_core::app_structure::FileSystemPathVec;
use output_file::{AnalyzedChunkPart, analyze_output_file, analyze_traced_file};
use rustc_hash::{FxHashMap, FxHashSet};
use serde::Serialize;
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexSet, JoinIterExt, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, ValueToString,
ValueToStringRef, Vc,
FxIndexSet, JoinIterExt, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt,
ValueToString, ValueToStringRef, Vc,
};
use turbo_tasks_fs::{
File, FileContent, FileSystemPath,
rope::{Rope, RopeBuilder},
};
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_analyze::split_chunk::{split_output_asset_into_parts, split_traced_file_into_parts};
use turbopack_browser::ecmascript::EcmascriptBrowserChunk;
use turbopack_core::{
SOURCE_URL_PROTOCOL,
asset::{Asset, AssetContent},
chunk::{ChunkingType, TracedMode},
module::Module,
......@@ -254,11 +251,12 @@ struct ModulesDataHeader {
pub traced_module_dependencies: EdgesDataReference,
}
#[derive(Clone, Copy, Serialize)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, NonLocalValue, Encode, Decode, Serialize)]
#[serde(rename_all = "snake_case")]
enum AnalyzeOutputFileCoverage {
Exact,
Unsupported,
#[default]
NotAChunk,
}
......@@ -687,7 +685,8 @@ pub async fn analyze_output_assets(
},
};
let route_entries = route_entries.await?.iter().cloned().collect();
let module_index = analyze_module_index(module_graph).await?;
let module_index_vc = analyze_module_index(module_graph);
let module_index = module_index_vc.await?;
let mut builder =
AnalyzeDataBuilder::new(route_entries, module_index.module_index_hash.clone());
......@@ -696,78 +695,63 @@ pub async fn analyze_output_assets(
let mut candidates = Vec::new();
let mut browser_chunks = FxHashSet::default();
let prefix = format!("{SOURCE_URL_PROTOCOL}///");
// Process the output assets and extract chunk parts.
// Also creates sources for the chunk parts.
for asset in output_assets
.await?
let output_assets = output_assets.await?;
let traced_files = traced_files.await?;
// One call per file: per-route fan-out over shared assets dominates analysis
// through task-graph bookkeeping, not the work behind each call.
let files = output_assets
.iter()
.copied()
.map(Either::Left)
.chain(traced_files.await?.iter().cloned().map(Either::Right))
{
let file_system_path = match &asset {
Either::Left(asset) => Either::Left(asset.path().await?),
Either::Right(path) => Either::Right(path),
};
let path = match &file_system_path {
Either::Left(path) => &path.path,
Either::Right(path) => &path.path,
};
if path.ends_with(".map") || path.ends_with(".nft.json") {
// Skip source maps.
continue;
}
let filename = match &file_system_path {
Either::Left(path) => path.to_string_ref().await?,
Either::Right(path) => path.to_string_ref().await?,
};
.map(|&asset| analyze_output_file(*asset, module_index_vc))
.chain(
traced_files
.iter()
.map(|path| analyze_traced_file(path.clone())),
)
.try_join()
.await?;
let is_client = filename.starts_with("[client-fs]/");
for file in files.iter().filter_map(|file| (**file).as_ref()) {
let is_client = file.filename.starts_with("[client-fs]/");
let output_file_index = builder.add_output_file(AnalyzeOutputFile {
filename: filename.clone(),
filename: file.filename.clone(),
});
if let Either::Left(asset) = &asset {
if let Some(emitted) = &file.emitted {
let membership = &emitted.membership;
asset_indices
.entry(*asset)
.entry(emitted.asset)
.or_default()
.push(output_file_index);
if ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(*asset).is_some() {
if emitted.is_browser_chunk {
browser_chunks.insert(output_file_index);
}
let (indices, coverage, loads, async_loaders) =
output_chunk_modules(*asset, &filename, &module_index, output_file_index).await?;
candidates.extend(loads);
let file = &mut builder.output_files[output_file_index as usize];
file.module_indices.extend(indices);
file.module_indices.sort_unstable();
file.module_indices.dedup();
file.async_loader_indices.extend(async_loaders);
file.async_loader_indices.sort_unstable();
file.async_loader_indices.dedup();
file.module_coverage = coverage;
candidates.extend(
membership
.load_targets
.iter()
.map(|load| ChunkLoadCandidate {
source: output_file_index,
target: load.target,
trigger_module_index: load.trigger_module_index,
}),
);
let output = &mut builder.output_files[output_file_index as usize];
output.module_indices.extend(&membership.module_indices);
output.module_indices.sort_unstable();
output.module_indices.dedup();
output
.async_loader_indices
.extend(&membership.async_loader_indices);
output.async_loader_indices.sort_unstable();
output.async_loader_indices.dedup();
output.module_coverage = membership.coverage;
}
let chunk_parts = match asset {
Either::Left(asset) => split_output_asset_into_parts(*asset).await?,
Either::Right(path) => split_traced_file_into_parts(path).await?,
};
for chunk_part in &chunk_parts {
let decoded_source = urlencoding::decode(&chunk_part.source)?;
let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) {
Cow::Borrowed(stripped)
} else if decoded_source.starts_with('[') && decoded_source.contains("]/") {
decoded_source
} else {
Cow::Owned(format!(
"[project]/{}",
decoded_source.trim_start_matches("../")
))
};
let source_index = builder.ensure_source(&source).1;
let size = chunk_part.real_size + chunk_part.unaccounted_size;
let compressed_size = chunk_part.get_compressed_size().await?.unwrap_or(size);
for &AnalyzedChunkPart {
ref source,
size,
compressed_size,
} in &file.chunk_parts
{
let source_index = builder.ensure_source(source).1;
summary.size += u64::from(size);
summary.compressed_size += u64::from(compressed_size);
if is_client {
......
......@@ -2,9 +2,10 @@
//! snapshot's ordered module index. Unknown wrappers remain unsupported.
use anyhow::{Context, Result, anyhow};
use bincode::{Decode, Encode};
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, ValueToString, Vc};
use turbo_tasks::{FxIndexSet, NonLocalValue, ResolvedVc, ValueToString, Vc};
use turbopack_browser::ecmascript::{
EcmascriptBrowserChunk, EcmascriptBrowserRuntimeChunk, EcmascriptBrowserSingleEntryChunk,
EcmascriptBrowserWorkerEntrypoint,
......@@ -23,7 +24,22 @@ use turbopack_ecmascript::{
};
use turbopack_nodejs::{EcmascriptBuildNodeChunk, EcmascriptBuildNodeRuntimeChunk};
use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage, ChunkLoadCandidate};
use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage};
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct ChunkLoadTarget {
pub target: ResolvedVc<Box<dyn OutputAsset>>,
/// Index of the dynamically imported target module in the analyzer's shared module index.
pub trigger_module_index: u32,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct OutputChunkModules {
pub module_indices: Vec<u32>,
pub coverage: AnalyzeOutputFileCoverage,
pub load_targets: Vec<ChunkLoadTarget>,
pub async_loader_indices: Vec<u32>,
}
async fn insert_chunk_item(
module: Vc<Box<dyn Module>>,
......@@ -78,8 +94,7 @@ fn insert_chunk_item_ident(
async fn chunk_item_load_candidates(
item: ResolvedVc<Box<dyn turbopack_ecmascript::chunk::EcmascriptChunkItem>>,
index: &AnalyzeModuleIndex,
source: u32,
) -> Result<Vec<ChunkLoadCandidate>> {
) -> Result<Vec<ChunkLoadTarget>> {
let module = item.module().to_resolved().await?;
let trigger = if let Some(loader) = ResolvedVc::try_downcast_type::<AsyncLoaderModule>(module) {
loader.await?.inner
......@@ -101,8 +116,7 @@ async fn chunk_item_load_candidates(
Ok(assets
.iter()
.copied()
.map(|target| ChunkLoadCandidate {
source,
.map(|target| ChunkLoadTarget {
target,
trigger_module_index,
})
......@@ -112,22 +126,16 @@ async fn chunk_item_load_candidates(
/// Enumerate known browser/Node.js JS and CSS chunk items; unknown wrappers remain unsupported.
pub(super) async fn output_chunk_modules(
asset: ResolvedVc<Box<dyn OutputAsset>>,
filename: &str,
filename: &RcStr,
module_index: &AnalyzeModuleIndex,
output_file_index: u32,
) -> Result<(
Vec<u32>,
AnalyzeOutputFileCoverage,
Vec<ChunkLoadCandidate>,
Vec<u32>,
)> {
) -> Result<OutputChunkModules> {
if ResolvedVc::try_downcast_type::<EcmascriptBrowserRuntimeChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBuildNodeRuntimeChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<SingleFileEcmascriptOutput>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBrowserSingleEntryChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBrowserWorkerEntrypoint>(asset).is_some()
{
return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![], vec![]));
return Ok(OutputChunkModules::default());
}
let mut indices = FxIndexSet::default();
let mut async_loaders = FxIndexSet::default();
......@@ -157,14 +165,8 @@ pub(super) async fn output_chunk_modules(
)
.await
.with_context(|| format!("Analyzing output {filename}"))?;
candidates.extend(
chunk_item_load_candidates(
item.chunk_item,
module_index,
output_file_index,
)
.await?,
);
candidates
.extend(chunk_item_load_candidates(item.chunk_item, module_index).await?);
}
EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => {
for item in &batch.await?.chunk_items {
......@@ -177,12 +179,7 @@ pub(super) async fn output_chunk_modules(
.await
.with_context(|| format!("Analyzing output {filename}"))?;
candidates.extend(
chunk_item_load_candidates(
item.chunk_item,
module_index,
output_file_index,
)
.await?,
chunk_item_load_candidates(item.chunk_item, module_index).await?,
);
}
}
......@@ -204,19 +201,19 @@ pub(super) async fn output_chunk_modules(
// Other emitted JS/CSS wrappers cannot enumerate their module members.
enumerated = false;
} else {
return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![], vec![]));
return Ok(OutputChunkModules::default());
}
let coverage = if enumerated {
AnalyzeOutputFileCoverage::Exact
} else {
AnalyzeOutputFileCoverage::Unsupported
};
Ok((
indices.into_iter().collect(),
Ok(OutputChunkModules {
module_indices: indices.into_iter().collect(),
coverage,
candidates,
async_loaders.into_iter().collect(),
))
load_targets: candidates,
async_loader_indices: async_loaders.into_iter().collect(),
})
}
#[cfg(test)]
......
//! One emitted or traced file's analysis, cached per file and shared by every
//! route that includes it.
use anyhow::Result;
use bincode::{Decode, Encode};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, ValueToStringRef, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_analyze::split_chunk::{
ChunkPart, split_output_asset_into_parts, split_traced_file_into_parts,
};
use turbopack_browser::ecmascript::EcmascriptBrowserChunk;
use turbopack_core::{SOURCE_URL_PROTOCOL, output::OutputAsset};
use crate::analyze::{
AnalyzeModuleIndex,
membership::{OutputChunkModules, output_chunk_modules},
};
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct AnalyzedChunkPart {
/// Normalized source path, ready for the route's source tree.
pub source: RcStr,
pub size: u32,
pub compressed_size: u32,
}
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct EmittedChunk {
pub asset: ResolvedVc<Box<dyn OutputAsset>>,
pub is_browser_chunk: bool,
pub membership: OutputChunkModules,
}
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct AnalyzedOutputFile {
pub filename: RcStr,
pub chunk_parts: Vec<AnalyzedChunkPart>,
/// `None` for traced files.
pub emitted: Option<EmittedChunk>,
}
/// `None` for source maps and trace manifests, which are not reported.
#[turbo_tasks::value(transparent)]
pub(super) struct OptionAnalyzedOutputFile(Option<AnalyzedOutputFile>);
#[turbo_tasks::function]
pub(super) async fn analyze_output_file(
asset: ResolvedVc<Box<dyn OutputAsset>>,
module_index: Vc<AnalyzeModuleIndex>,
) -> Result<Vc<OptionAnalyzedOutputFile>> {
let path = asset.path().await?;
if !is_reported_file(&path) {
return Ok(Vc::cell(None));
}
let filename = path.to_string_ref().await?;
let membership = output_chunk_modules(asset, &filename, &*module_index.await?).await?;
let chunk_parts = analyzed_chunk_parts(&split_output_asset_into_parts(*asset).await?).await?;
Ok(Vc::cell(Some(AnalyzedOutputFile {
filename,
chunk_parts,
emitted: Some(EmittedChunk {
asset,
is_browser_chunk: ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(asset)
.is_some(),
membership,
}),
})))
}
#[turbo_tasks::function]
pub(super) async fn analyze_traced_file(
path: FileSystemPath,
) -> Result<Vc<OptionAnalyzedOutputFile>> {
if !is_reported_file(&path) {
return Ok(Vc::cell(None));
}
Ok(Vc::cell(Some(AnalyzedOutputFile {
filename: path.to_string_ref().await?,
chunk_parts: analyzed_chunk_parts(&split_traced_file_into_parts(path).await?).await?,
emitted: None,
})))
}
fn is_reported_file(path: &FileSystemPath) -> bool {
!(path.path.ends_with(".map") || path.path.ends_with(".nft.json"))
}
async fn analyzed_chunk_parts(chunk_parts: &[ChunkPart]) -> Result<Vec<AnalyzedChunkPart>> {
let mut analyzed = Vec::with_capacity(chunk_parts.len());
for part in chunk_parts {
let size = part.real_size + part.unaccounted_size;
analyzed.push(AnalyzedChunkPart {
source: normalize_source(&part.source)?,
size,
compressed_size: part.get_compressed_size().await?.unwrap_or(size),
});
}
Ok(analyzed)
}
fn normalize_source(raw: &str) -> Result<RcStr> {
let decoded = urlencoding::decode(raw)?;
Ok(
if let Some(path) = decoded
.strip_prefix(SOURCE_URL_PROTOCOL.as_str())
.and_then(|rest| rest.strip_prefix("///"))
{
path.into()
} else if decoded.starts_with('[') && decoded.contains("]/") {
decoded.as_ref().into()
} else {
format!("[project]/{}", decoded.trim_start_matches("../")).into()
},
)
}
......@@ -60,19 +60,14 @@ impl ChunkPart {
}
}
#[turbo_tasks::value(transparent)]
#[derive(Debug)]
pub struct ChunkParts(Vec<ChunkPart>);
#[turbo_tasks::function]
pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vc<ChunkParts>> {
pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vec<ChunkPart>> {
let source = FileSource::new(path.clone());
let content = source.content().await?;
let AssetContent::File(file_content) = &*content else {
return Ok(Vc::cell(vec![]));
return Ok(vec![]);
};
let FileContent::Content(content) = &*file_content.await? else {
return Ok(Vc::cell(vec![]));
return Ok(vec![]);
};
let content = content.content();
let lines_vc = file_content.lines().to_resolved().await?;
......@@ -80,16 +75,15 @@ pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vc<Chu
self_mapped(path.to_string_ref().await?, content, lines_vc).await
}
#[turbo_tasks::function]
pub async fn split_output_asset_into_parts(
asset: Vc<Box<dyn OutputAsset>>,
) -> Result<Vc<ChunkParts>> {
) -> Result<Vec<ChunkPart>> {
let content = asset.content().await?;
let AssetContent::File(file_content) = &*content else {
return Ok(Vc::cell(vec![]));
return Ok(vec![]);
};
let FileContent::Content(content) = &*file_content.await? else {
return Ok(Vc::cell(vec![]));
return Ok(vec![]);
};
let content = content.content();
let lines_vc = file_content.lines().to_resolved().await?;
......@@ -397,37 +391,37 @@ pub async fn split_output_asset_into_parts(
}
}
Ok(Vc::cell(chunk_parts.into_values().collect()))
Ok(chunk_parts.into_values().collect())
}
pub async fn self_mapped(
path: RcStr,
content: &Rope,
lines: ResolvedVc<FileLinesContent>,
) -> Result<Vc<ChunkParts>> {
) -> Result<Vec<ChunkPart>> {
let len = content.len().try_into().unwrap_or(u32::MAX);
Ok(Vc::cell(vec![ChunkPart {
Ok(vec![ChunkPart {
source: path,
real_size: len,
unaccounted_size: 0,
ranges: vec![],
lines,
}]))
}])
}
async fn unaccounted(
asset: Vc<Box<dyn OutputAsset>>,
content: &Rope,
lines: ResolvedVc<FileLinesContent>,
) -> Result<Vc<ChunkParts>> {
) -> Result<Vec<ChunkPart>> {
let len = content.len().try_into().unwrap_or(u32::MAX);
Ok(Vc::cell(vec![ChunkPart {
Ok(vec![ChunkPart {
source: asset.path().to_string().owned().await?,
real_size: 0,
unaccounted_size: len,
ranges: vec![],
lines,
}]))
}])
}
fn append_content_between(
......
......@@ -5,12 +5,12 @@
use anyhow::Result;
use serde_json::json;
use turbo_rcstr::rcstr;
use turbo_tasks::{OperationVc, ResolvedVc, Vc};
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{
File, FileContent, FileSystem, FileSystemPath, VirtualFileSystem, rope::Rope,
};
use turbo_tasks_testing::{Registration, register, run_once};
use turbopack_analyze::split_chunk::{ChunkPartRange, ChunkParts, split_output_asset_into_parts};
use turbopack_analyze::split_chunk::{ChunkPart, ChunkPartRange, split_output_asset_into_parts};
use turbopack_core::{
asset::{Asset, AssetContent},
code_builder::{Code, CodeBuilder},
......@@ -55,13 +55,10 @@ async fn split_chunk() {
}
.resolved_cell();
#[turbo_tasks::function(operation, root)]
fn split_parts_operation(asset: ResolvedVc<TestAsset>) -> Vc<ChunkParts> {
split_output_asset_into_parts(Vc::upcast(*asset))
}
let parts_op = split_parts_operation(asset);
let parts = parts_op.read_strongly_consistent().await?;
let result = split_parts_operation(asset)
.read_strongly_consistent()
.await?;
let parts = &result.parts;
assert_eq!(parts.len(), 2);
......@@ -96,27 +93,7 @@ async fn split_chunk() {
}]
);
#[turbo_tasks::function(operation, root)]
async fn compressed_size_operation(
parts: OperationVc<ChunkParts>,
index: usize,
) -> Result<Vc<u32>> {
Ok(Vc::cell(
parts.connect().await?[index]
.get_compressed_size()
.await?
.unwrap(),
))
}
let compressed_size_0 = compressed_size_operation(parts_op, 0)
.read_strongly_consistent()
.await?;
let compressed_size_1 = compressed_size_operation(parts_op, 1)
.read_strongly_consistent()
.await?;
assert_eq!(*compressed_size_0, 43);
assert_eq!(*compressed_size_1, 28);
assert_eq!(result.compressed_sizes, [Some(43), Some(28)]);
anyhow::Ok(())
})
......@@ -124,6 +101,26 @@ async fn split_chunk() {
.unwrap()
}
#[turbo_tasks::value]
struct SplitResult {
parts: Vec<ChunkPart>,
compressed_sizes: Vec<Option<u32>>,
}
#[turbo_tasks::function(operation, root)]
async fn split_parts_operation(asset: ResolvedVc<TestAsset>) -> Result<Vc<SplitResult>> {
let parts = split_output_asset_into_parts(Vc::upcast(*asset)).await?;
let mut compressed_sizes = Vec::with_capacity(parts.len());
for part in &parts {
compressed_sizes.push(part.get_compressed_size().await?);
}
Ok(SplitResult {
parts,
compressed_sizes,
}
.cell())
}
#[turbo_tasks::value]
struct TestAsset {
code: ResolvedVc<Code>,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment