Unverified Commit 8eabe35e authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Bundle Analyzer Agent Skill: Show modules in generated JS/CSS files (#99173)

## Summary

Before this PR, an agent could see a route's generated files and follow
imports to find modules relevant to that route. It could not reliably
tell **which exact modules landed in a particular JavaScript or CSS
file**. This PR adds that per-file mapping, so an agent can trace a
large generated file back to the modules it contains.

Files that cannot be mapped are marked as such rather than shown as
empty. The graph checks the module information before reporting a match.

## Verification

- The broad graph-dump converter unit suite is removed in the root and
not reintroduced here; existing e2e and producer code are unchanged. The
snapshot-selection unit suite is removed too, without replacement tests;
its targeted mocked-history/path coverage is intentionally dropped.
Neither analyzer unit suite remains; Next package no-emit types pass.
Unique removed edge-case coverage was explicitly accepted.
- Earlier stack verification included Rust analyzer tests, analyzer e2e
and a full build; these are historical results, not validation of the
new signed heads. **New-head CI is pending.**

<!-- NEXT_JS_LLM -->

<!-- fleet d7bdbc2b-b0bc-4591-99eb-dccb0fa49c73 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent 8d3606c1
...@@ -32,7 +32,11 @@ import { ...@@ -32,7 +32,11 @@ import {
TooltipProvider, TooltipProvider,
TooltipTrigger, TooltipTrigger,
} from '@/components/ui/tooltip' } from '@/components/ui/tooltip'
import { AnalyzeData, ModulesData } from '@/lib/analyze-data' import {
AnalyzeData,
ModulesData,
assertMatchingModuleIndex,
} from '@/lib/analyze-data'
import { import {
analyzeDataUrl, analyzeDataUrl,
fetchAnalyzeData, fetchAnalyzeData,
...@@ -245,6 +249,7 @@ function useAnalyzerModel(compare: boolean) { ...@@ -245,6 +249,7 @@ function useAnalyzerModel(compare: boolean) {
if (!analyzeData) { if (!analyzeData) {
return { moduleDepthMap: new Map(), sourceLoadScopes: new Map() } return { moduleDepthMap: new Map(), sourceLoadScopes: new Map() }
} }
assertMatchingModuleIndex(modulesData, analyzeData, 'Current snapshot')
const activeEntries = computeActiveEntries(modulesData, analyzeData) const activeEntries = computeActiveEntries(modulesData, analyzeData)
return { return {
...@@ -614,6 +619,13 @@ function ComparisonContent({ ...@@ -614,6 +619,13 @@ function ComparisonContent({
baselineAnalyzeData: AnalyzeData | null baselineAnalyzeData: AnalyzeData | null
layoutProps: ComparisonLayoutProps layoutProps: ComparisonLayoutProps
}) { }) {
if (baselineAnalyzeData) {
assertMatchingModuleIndex(
layoutProps.baselineModulesData,
baselineAnalyzeData,
'Baseline snapshot'
)
}
const baselineModuleDepthMap = baselineAnalyzeData const baselineModuleDepthMap = baselineAnalyzeData
? computeModuleDepthMap( ? computeModuleDepthMap(
layoutProps.baselineModulesData, layoutProps.baselineModulesData,
......
...@@ -35,6 +35,7 @@ interface EdgesDataReference { ...@@ -35,6 +35,7 @@ interface EdgesDataReference {
} }
interface AnalyzeDataHeader { interface AnalyzeDataHeader {
module_index_hash: string
sources: AnalyzeSource[] sources: AnalyzeSource[]
chunk_parts: AnalyzeChunkPart[] chunk_parts: AnalyzeChunkPart[]
output_files: AnalyzeOutputFile[] output_files: AnalyzeOutputFile[]
...@@ -45,6 +46,7 @@ interface AnalyzeDataHeader { ...@@ -45,6 +46,7 @@ interface AnalyzeDataHeader {
} }
interface ModulesDataHeader { interface ModulesDataHeader {
module_index_hash: string
modules: AnalyzeModule[] modules: AnalyzeModule[]
module_dependents: EdgesDataReference module_dependents: EdgesDataReference
async_module_dependents: EdgesDataReference async_module_dependents: EdgesDataReference
...@@ -92,6 +94,14 @@ export class ModulesData { ...@@ -92,6 +94,14 @@ export class ModulesData {
} }
} }
moduleIndexHash(): string {
const hash = this.modulesHeader.module_index_hash
if (typeof hash !== 'string' || !hash) {
throw new Error('Missing analyzer module-index fingerprint')
}
return hash
}
module(index: ModuleIndex): AnalyzeModule | undefined { module(index: ModuleIndex): AnalyzeModule | undefined {
return this.modulesHeader.modules[index] return this.modulesHeader.modules[index]
} }
...@@ -202,6 +212,19 @@ export class ModulesData { ...@@ -202,6 +212,19 @@ export class ModulesData {
} }
} }
/** Only join a route to the module table from the same snapshot. */
export function assertMatchingModuleIndex(
modules: ModulesData,
route: AnalyzeData,
side: string
): void {
const moduleHash = modules.moduleIndexHash()
const routeHash = route.moduleIndexHash()
if (moduleHash !== routeHash) {
throw new Error(`${side}: analyzer module-index fingerprint mismatch`)
}
}
/** /**
* Represents route-specific analyze data * Represents route-specific analyze data
*/ */
...@@ -237,6 +260,14 @@ export class AnalyzeData { ...@@ -237,6 +260,14 @@ export class AnalyzeData {
// Accessor methods for header data // Accessor methods for header data
moduleIndexHash(): string {
const hash = this.analyzeHeader.module_index_hash
if (typeof hash !== 'string' || !hash) {
throw new Error('Missing analyzer module-index fingerprint')
}
return hash
}
source(index: SourceIndex): AnalyzeSource | undefined { source(index: SourceIndex): AnalyzeSource | undefined {
return this.analyzeHeader.sources[index] return this.analyzeHeader.sources[index]
} }
......
This diff is collapsed.
//! Enumerate emitted chunk modules against this snapshot's ordered module index.
use anyhow::{Context, Result, anyhow};
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, ValueToString, Vc};
use turbopack_browser::ecmascript::{
EcmascriptBrowserChunk, EcmascriptBrowserRuntimeChunk, EcmascriptBrowserSingleEntryChunk,
EcmascriptBrowserWorkerEntrypoint,
};
use turbopack_core::{
chunk::{Chunk, ChunkItem},
module::Module,
output::OutputAsset,
};
use turbopack_css::chunk::CssChunk;
use turbopack_ecmascript::{
async_chunk::module::AsyncLoaderModule,
chunk::{EcmascriptChunk, EcmascriptChunkItemOrBatchWithAsyncInfo},
single_file_ecmascript_output::SingleFileEcmascriptOutput,
};
use turbopack_nodejs::{EcmascriptBuildNodeChunk, EcmascriptBuildNodeRuntimeChunk};
use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage};
async fn insert_chunk_item(
module: Vc<Box<dyn Module>>,
module_index: &AnalyzeModuleIndex,
indices: &mut FxIndexSet<u32>,
async_loaders: &mut FxIndexSet<u32>,
) -> Result<()> {
let module = module.to_resolved().await?;
let loader_target_ident =
if let Some(loader) = ResolvedVc::try_downcast_type::<AsyncLoaderModule>(module) {
Some(loader.await?.inner.ident().to_string().owned().await?)
} else {
None
};
insert_chunk_item_ident(
module.ident().to_string().owned().await?,
loader_target_ident,
&module_index.by_ident,
indices,
async_loaders,
)
}
fn insert_chunk_item_ident(
ident: RcStr,
loader_target_ident: Option<RcStr>,
by_ident: &FxHashMap<RcStr, u32>,
indices: &mut FxIndexSet<u32>,
async_loaders: &mut FxIndexSet<u32>,
) -> Result<()> {
let lookup_ident = loader_target_ident.as_ref().unwrap_or(&ident);
let &index = by_ident.get(lookup_ident).ok_or_else(|| {
if loader_target_ident.is_some() {
anyhow!(
"Async loader {ident} target {lookup_ident} is missing from the analyzer module \
index"
)
} else {
anyhow!("Chunk module {ident} is missing from the analyzer module index")
}
})?;
if loader_target_ident.is_some() {
async_loaders.insert(index);
} else {
indices.insert(index);
}
Ok(())
}
/// Get exact constituent module identities only for chunk types whose items we
/// can enumerate. Empty rows marked unsupported must not be treated as empty chunks.
pub(super) async fn output_chunk_modules(
asset: ResolvedVc<Box<dyn OutputAsset>>,
filename: &str,
module_index: &AnalyzeModuleIndex,
) -> Result<(Vec<u32>, AnalyzeOutputFileCoverage, Vec<u32>)> {
if ResolvedVc::try_downcast_type::<EcmascriptBrowserRuntimeChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBuildNodeRuntimeChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<SingleFileEcmascriptOutput>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBrowserSingleEntryChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBrowserWorkerEntrypoint>(asset).is_some()
{
return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![]));
}
let mut indices = FxIndexSet::default();
let mut async_loaders = FxIndexSet::default();
let mut enumerated = true;
let ecmascript_content = if let Some(browser_chunk) =
ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(asset)
{
let chunk: ResolvedVc<Box<dyn Chunk>> = browser_chunk.chunk().to_resolved().await?;
let chunk = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
.ok_or_else(|| anyhow::anyhow!("browser output did not contain an ecmascript chunk"))?;
Some(chunk.chunk_content())
} else {
ResolvedVc::try_downcast_type::<EcmascriptBuildNodeChunk>(asset)
.map(|node_chunk| node_chunk.chunk_content())
};
if let Some(content) = ecmascript_content {
let content = content.await?;
for chunk_item in &content.chunk_items {
match chunk_item {
EcmascriptChunkItemOrBatchWithAsyncInfo::ChunkItem(item) => {
insert_chunk_item(
item.chunk_item.module(),
module_index,
&mut indices,
&mut async_loaders,
)
.await
.with_context(|| format!("Analyzing output {filename}"))?;
}
EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => {
for item in &batch.await?.chunk_items {
insert_chunk_item(
item.chunk_item.module(),
module_index,
&mut indices,
&mut async_loaders,
)
.await
.with_context(|| format!("Analyzing output {filename}"))?;
}
}
}
}
} else if let Some(chunk) = ResolvedVc::try_downcast_type::<CssChunk>(asset) {
let content = chunk.await?.content.await?;
for chunk_item in &content.chunk_items {
insert_chunk_item(
chunk_item.module(),
module_index,
&mut indices,
&mut async_loaders,
)
.await
.with_context(|| format!("Analyzing output {filename}"))?;
}
} else if filename.ends_with(".js") || filename.ends_with(".css") {
// Other emitted JS/CSS wrappers cannot enumerate their module members.
enumerated = false;
} else {
return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![]));
}
let coverage = if enumerated {
AnalyzeOutputFileCoverage::Exact
} else {
AnalyzeOutputFileCoverage::Unsupported
};
Ok((
indices.into_iter().collect(),
coverage,
async_loaders.into_iter().collect(),
))
}
#[cfg(test)]
mod tests {
use anyhow::Result;
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
use turbo_tasks::FxIndexSet;
use crate::analyze::membership::insert_chunk_item_ident;
fn insert(ident: &str, loader_target: Option<&str>) -> Result<(Vec<u32>, Vec<u32>)> {
let by_ident = FxHashMap::from_iter([
(RcStr::from("target"), 7),
(RcStr::from("ordinary (async loader)"), 9),
]);
let mut indices = FxIndexSet::default();
let mut async_loaders = FxIndexSet::default();
insert_chunk_item_ident(
ident.into(),
loader_target.map(RcStr::from),
&by_ident,
&mut indices,
&mut async_loaders,
)?;
Ok((
indices.into_iter().collect(),
async_loaders.into_iter().collect(),
))
}
#[test]
fn confirmed_async_loader_is_not_a_member() {
let (indices, async_loaders) = insert("target (async loader)", Some("target")).unwrap();
assert!(indices.is_empty());
assert_eq!(async_loaders, [7]);
}
#[test]
fn missing_async_loader_target_is_an_error() {
let error = insert("missing (async loader)", Some("missing")).unwrap_err();
assert_eq!(
error.to_string(),
"Async loader missing (async loader) target missing is missing from the analyzer \
module index"
);
}
#[test]
fn ordinary_members_are_not_classified_by_ident_suffix() {
let (indices, async_loaders) = insert("ordinary (async loader)", None).unwrap();
assert_eq!(indices, [9]);
assert!(async_loaders.is_empty());
let error = insert("unknown (async loader)", None).unwrap_err();
assert!(
error
.to_string()
.contains("Chunk module unknown (async loader)")
);
}
}
...@@ -251,6 +251,7 @@ async fn get_analyze_data_operation( ...@@ -251,6 +251,7 @@ async fn get_analyze_data_operation(
output_assets, output_assets,
traced_files, traced_files,
route_entries, route_entries,
*whole_app_module_graphs.await?.full,
) )
.to_resolved() .to_resolved()
.await?; .await?;
...@@ -258,9 +259,14 @@ async fn get_analyze_data_operation( ...@@ -258,9 +259,14 @@ async fn get_analyze_data_operation(
let summary = if let EndpointGroupKey::Route(route) = key { let summary = if let EndpointGroupKey::Route(route) = key {
Some(RouteSummary { Some(RouteSummary {
route: route.to_string(), route: route.to_string(),
bundles: analyze_output_assets(output_assets, traced_files, route_entries) bundles: analyze_output_assets(
.await? output_assets,
.summary, traced_files,
route_entries,
*whole_app_module_graphs.await?.full,
)
.await?
.summary,
}) })
} else { } else {
None None
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
"type", "type",
"schema_version", "schema_version",
"snapshot_name", "snapshot_name",
"module_index_hash",
"route_count", "route_count",
"selected_routes" "selected_routes"
], ],
...@@ -41,6 +42,11 @@ ...@@ -41,6 +42,11 @@
"minLength": 1, "minLength": 1,
"description": "Generated or provided snapshot name. Reusing an explicit name replaces its earlier capture." "description": "Generated or provided snapshot name. Reusing an explicit name replaces its earlier capture."
}, },
"module_index_hash": {
"type": "string",
"minLength": 1,
"description": "Fingerprint of the ordered, snapshot-scoped modules.data indices; not proof of identical graphs or builds."
},
"route_count": { "route_count": {
"$ref": "#/$defs/index", "$ref": "#/$defs/index",
"description": "Total routes in this snapshot." "description": "Total routes in this snapshot."
...@@ -175,13 +181,32 @@ ...@@ -175,13 +181,32 @@
}, },
"output": { "output": {
"type": "object", "type": "object",
"required": ["type", "route", "filename"], "required": [
"type",
"route",
"filename",
"modules",
"async_loaders",
"coverage"
],
"properties": { "properties": {
"type": { "const": "output" }, "type": { "const": "output" },
"route": { "type": "string", "description": "Owning route path." }, "route": { "type": "string", "description": "Owning route path." },
"filename": { "filename": {
"type": "string", "type": "string",
"description": "Snapshot-scoped emitted path, not stable across builds or proof of a browser request." "description": "Snapshot-scoped emitted path, not stable across builds or proof of a browser request."
},
"modules": {
"$ref": "#/$defs/identities",
"description": "Sorted identities of the output chunk's real members, joined to this snapshot's module index. Async loader stubs and their targets are represented separately in async_loaders. Unsupported coverage may have partial membership."
},
"async_loaders": {
"$ref": "#/$defs/identities",
"description": "Sorted target module identities of typed async loader stubs contained in this chunk. Each target is recorded only after joining the snapshot module index through the loader's inner module."
},
"coverage": {
"description": "Membership evidence: exact enumerates and joins every real chunk member and async loader target; unsupported means the output type cannot enumerate its members; not_a_chunk identifies non-chunk outputs, including runtime, precompiled polyfill and separately compiled worker outputs by concrete type. Attribution part records remain available.",
"enum": ["exact", "unsupported", "not_a_chunk"]
} }
}, },
"additionalProperties": true, "additionalProperties": true,
......
...@@ -14,6 +14,7 @@ type Part = { ...@@ -14,6 +14,7 @@ type Part = {
} }
type ModuleHeader = { type ModuleHeader = {
schema_version: number schema_version: number
module_index_hash: string
modules: Module[] modules: Module[]
module_dependencies: EdgeRef module_dependencies: EdgeRef
async_module_dependencies: EdgeRef async_module_dependencies: EdgeRef
...@@ -21,6 +22,7 @@ type ModuleHeader = { ...@@ -21,6 +22,7 @@ type ModuleHeader = {
} }
type RouteHeader = { type RouteHeader = {
schema_version: number schema_version: number
module_index_hash: string
sources: Source[] sources: Source[]
chunk_parts: Part[] chunk_parts: Part[]
output_files: Array<{ filename: string }> output_files: Array<{ filename: string }>
...@@ -36,6 +38,9 @@ type RouteHeader = { ...@@ -36,6 +38,9 @@ type RouteHeader = {
reference_kind: 'ecmascript' | 'css' reference_kind: 'ecmascript' | 'css'
}> }>
}> }>
output_file_modules: EdgeRef
output_file_async_loaders: EdgeRef
output_file_module_coverage: Array<'exact' | 'unsupported' | 'not_a_chunk'>
} }
type Data<H> = { header: H; binary: Buffer } type Data<H> = { header: H; binary: Buffer }
...@@ -183,11 +188,13 @@ function parseModules(file: string) { ...@@ -183,11 +188,13 @@ function parseModules(file: string) {
field field
) )
} }
return { modules, edges } if (typeof header.module_index_hash !== 'string' || !header.module_index_hash)
throw new Error('Missing analyzer module-index fingerprint')
return { modules, edges, hash: header.module_index_hash }
} }
function parseRoutes(file: string) { function parseRoutes(file: string, modules: ReturnType<typeof parseModules>) {
const { header } = readData<RouteHeader>(file) const { header, binary } = readData<RouteHeader>(file)
if ( if (
!Array.isArray(header.sources) || !Array.isArray(header.sources) ||
!Array.isArray(header.output_files) || !Array.isArray(header.output_files) ||
...@@ -208,10 +215,49 @@ function parseRoutes(file: string) { ...@@ -208,10 +215,49 @@ function parseRoutes(file: string) {
throw new Error('Invalid analyzer part size') throw new Error('Invalid analyzer part size')
} }
} }
if (header.module_index_hash !== modules.hash) {
throw new Error('Analyzer module-index fingerprint mismatch')
}
const membership = validateEdges(
binary,
header.output_file_modules,
outputs.length,
modules.modules.length,
'output modules'
)
const asyncLoaders = validateEdges(
binary,
header.output_file_async_loaders,
outputs.length,
modules.modules.length,
'output async loaders'
)
if (
!Array.isArray(header.output_file_module_coverage) ||
header.output_file_module_coverage.length !== outputs.length
) {
throw new Error('Missing analyzer output coverage')
}
if (
header.output_file_module_coverage.some(
(coverage) =>
coverage !== 'exact' &&
coverage !== 'unsupported' &&
coverage !== 'not_a_chunk'
)
) {
throw new Error('Invalid analyzer output coverage')
}
for (const output of outputs) for (const output of outputs)
if (typeof output.filename !== 'string') if (typeof output.filename !== 'string')
throw new Error('Invalid output filename') throw new Error('Invalid output filename')
return { paths, header, entries: header.route_entries ?? null } return {
paths,
header,
entries: header.route_entries ?? null,
membership,
asyncLoaders,
}
} }
function writeRecord(record: object): void { function writeRecord(record: object): void {
...@@ -250,11 +296,13 @@ export function dumpAnalyzeGraph( ...@@ -250,11 +296,13 @@ export function dumpAnalyzeGraph(
) )
if (routeFilter !== undefined && selected.length === 0) if (routeFilter !== undefined && selected.length === 0)
throw new Error(`Unknown analyzer route: ${routeFilter}`) throw new Error(`Unknown analyzer route: ${routeFilter}`)
const { modules, edges } = parseModules(join(directory, 'modules.data')) const moduleData = parseModules(join(directory, 'modules.data'))
const { modules, edges, hash } = moduleData
writeRecord({ writeRecord({
type: 'meta', type: 'meta',
schema_version: 1, schema_version: 1,
snapshot_name: snapshotName, snapshot_name: snapshotName,
module_index_hash: hash,
route_count: routes.length, route_count: routes.length,
selected_routes: selected.length, selected_routes: selected.length,
}) })
...@@ -274,7 +322,10 @@ export function dumpAnalyzeGraph( ...@@ -274,7 +322,10 @@ export function dumpAnalyzeGraph(
// Retain only the current route's data. A later validation error may leave // Retain only the current route's data. A later validation error may leave
// partial output; callers must check the exit status before using it. // partial output; callers must check the exit status before using it.
for (const route of selected) { for (const route of selected) {
const { paths, header, entries } = parseRoutes(routeFile(directory, route)) const { paths, header, entries, membership, asyncLoaders } = parseRoutes(
routeFile(directory, route),
moduleData
)
const prefix = { route } const prefix = { route }
writeRecord({ writeRecord({
type: 'route', type: 'route',
...@@ -287,6 +338,15 @@ export function dumpAnalyzeGraph( ...@@ -287,6 +338,15 @@ export function dumpAnalyzeGraph(
type: 'output', type: 'output',
...prefix, ...prefix,
filename: header.output_files[i].filename, filename: header.output_files[i].filename,
modules: membership
.row(i)
.map((id) => modules[id].ident)
.sort(),
async_loaders: asyncLoaders
.row(i)
.map((id) => modules[id].ident)
.sort(),
coverage: header.output_file_module_coverage[i],
}) })
} }
for (const part of header.chunk_parts) { for (const part of header.chunk_parts) {
......
'use client' 'use client'
import { useState } from 'react' import { useEffect, useState } from 'react'
import dynamic from 'next/dynamic'
const DynamicTarget = dynamic(() => import('./dynamic-target'))
export default function ClientEntry() { export default function ClientEntry() {
const [count, setCount] = useState(0) const [count, setCount] = useState(0)
return <button onClick={() => setCount(count + 1)}>Count: {count}</button> useEffect(() => {
if ('serviceWorker' in navigator) {
navigator.serviceWorker.register(new URL('./pwa', import.meta.url))
}
}, [])
return (
<>
<button onClick={() => setCount(count + 1)}>Count: {count}</button>
<button
onClick={async () => {
const { asyncValue } = await import('./async-target')
setCount(asyncValue)
}}
>
Load async target
</button>
<DynamicTarget />
</>
)
} }
export default function DynamicTarget() {
return <p>Dynamic target</p>
}
/// <reference lib="webworker" />
declare const self: ServiceWorkerGlobalScope
self.addEventListener('install', () => {
self.skipWaiting()
})
export {}
...@@ -22,10 +22,51 @@ type RouteEntry = { ...@@ -22,10 +22,51 @@ type RouteEntry = {
}> }>
} }
function readAnalyzeHeader<T>(filename: string): T { type EdgesReference = { offset: number; length: number }
type ChunkGraphHeader = {
schema_version: number
module_index_hash: string
output_files: Array<{ filename: string }>
output_file_modules: EdgesReference
output_file_async_loaders: EdgesReference
output_file_module_coverage: Array<'exact' | 'unsupported' | 'not_a_chunk'>
}
function readAnalyzeFile<T>(filename: string) {
const buffer = readFileSync(filename) const buffer = readFileSync(filename)
const jsonLength = buffer.readUInt32BE(0) const jsonLength = buffer.readUInt32BE(0)
return JSON.parse(buffer.subarray(4, 4 + jsonLength).toString('utf8')) as T const binaryStart = 4 + jsonLength
expect(binaryStart).toBeLessThanOrEqual(buffer.length)
return {
header: JSON.parse(buffer.subarray(4, binaryStart).toString('utf8')) as T,
binary: buffer.subarray(binaryStart),
}
}
function readAnalyzeHeader<T>(filename: string): T {
return readAnalyzeFile<T>(filename).header
}
function readRows(binary: Buffer, reference: EdgesReference): number[][] {
const { offset, length } = reference
expect(offset + length).toBeLessThanOrEqual(binary.length)
const section = binary.subarray(offset, offset + length)
const count = section.readUInt32BE(0)
const offsets = Array.from({ length: count }, (_, i) =>
section.readUInt32BE(4 + 4 * i)
)
const total = offsets.at(-1) ?? 0
expect(section.length).toBe(4 * (1 + count + total))
let start = 0
return offsets.map((end) => {
expect(end).toBeGreaterThanOrEqual(start)
const row = Array.from({ length: end - start }, (_, i) =>
section.readUInt32BE(4 * (1 + count + start + i))
)
start = end
return row
})
} }
// TODO(deploy-test-completion): Re-enable this suite in deploy mode. // TODO(deploy-test-completion): Re-enable this suite in deploy mode.
...@@ -200,6 +241,87 @@ describe('next analyze', () => { ...@@ -200,6 +241,87 @@ describe('next analyze', () => {
]) ])
expect(alias.exitCode).toBe(0) expect(alias.exitCode).toBe(0)
expect(alias.stdout).toBe(named.stdout) expect(alias.stdout).toBe(named.stdout)
const outputRecords = records.filter(
(record) => record.type === 'output' && record.route === '/'
)
for (const output of records.filter((record) => record.type === 'output')) {
expect(output.modules).toEqual([...output.modules].sort())
expect(output.async_loaders).toEqual([...output.async_loaders].sort())
}
const clientEntries = records.filter(
(record) =>
record.type === 'module' &&
record.path.endsWith('/app/client-entry.tsx') &&
record.ident.includes('[app-client]')
)
const asyncTargets = records.filter(
(record) =>
record.type === 'module' &&
record.ident.includes('[app-client]') &&
(record.path.endsWith('/app/async-target.ts') ||
record.path.endsWith('/app/dynamic-target.tsx'))
)
expect(clientEntries.length).toBeGreaterThan(0)
expect(
asyncTargets.some((record) =>
record.path.endsWith('/app/async-target.ts')
)
).toBe(true)
expect(
asyncTargets.some(
(record) =>
record.path.endsWith('/app/dynamic-target.tsx') &&
record.ident.includes('next/dynamic entry')
)
).toBe(true)
const asyncDependencies = clientEntries.flatMap(
(record) => record.dependencies.async
)
for (const target of asyncTargets) {
// The import graph records the target, not a synthetic loader module.
if (
target.ident.includes('next/dynamic entry') ||
target.path.endsWith('/app/async-target.ts')
) {
expect(asyncDependencies).toContain(target.ident)
}
}
const importerOutputs = outputRecords.filter((record) =>
clientEntries.some((entry) => record.modules.includes(entry.ident))
)
expect(importerOutputs.length).toBeGreaterThan(0)
for (const target of asyncTargets.filter(
(target) =>
target.ident.includes('next/dynamic entry') ||
target.path.endsWith('/app/async-target.ts')
)) {
expect(
importerOutputs.some((output) =>
output.async_loaders.includes(target.ident)
)
).toBe(true)
}
for (const target of asyncTargets) {
expect(
outputRecords.some((output) => output.modules.includes(target.ident))
).toBe(true)
}
for (const output of importerOutputs) {
expect(output.coverage).toBe('exact')
// Joining a loader's target must not attribute that target to the importer.
for (const target of asyncTargets) {
expect(output.modules).not.toContain(target.ident)
}
}
expect(
outputRecords
.flatMap((record) => record.modules)
.some((ident: string) => ident.endsWith('async loader)'))
).toBe(false)
const filtered = await next.runCommand([ const filtered = await next.runCommand([
'analyze', 'analyze',
'export', 'export',
...@@ -285,6 +407,45 @@ describe('next analyze', () => { ...@@ -285,6 +407,45 @@ describe('next analyze', () => {
} finally { } finally {
writeFileSync(moduleFile, originalModules) writeFileSync(moduleFile, originalModules)
} }
const routeFile = path.join(snapshotDir, 'analyze.data')
const routeOriginal = readFileSync(routeFile)
try {
const oldHeaderLength = routeOriginal.readUInt32BE(0)
const header = JSON.parse(
routeOriginal.toString('utf8', 4, 4 + oldHeaderLength)
)
const altered = Buffer.from(
JSON.stringify({ ...header, module_index_hash: 'wrong' })
)
const length = Buffer.alloc(4)
length.writeUInt32BE(altered.length)
writeFileSync(
routeFile,
Buffer.concat([
length,
altered,
routeOriginal.subarray(4 + oldHeaderLength),
])
)
const mismatch = await next.runCommand([
'analyze',
'export',
'--snapshot',
name,
])
expect(mismatch.exitCode).not.toBe(0)
const partial = mismatch.stdout
.trim()
.split('\n')
.map((line) => JSON.parse(line))
expect(partial[0]).toMatchObject({ type: 'meta', snapshot_name: name })
expect(partial.some((record) => record.type === 'module')).toBe(true)
expect(partial.some((record) => record.route === '/')).toBe(false)
expect(mismatch.stderr).toContain('module-index fingerprint mismatch')
} finally {
writeFileSync(routeFile, routeOriginal)
}
}, 120_000) }, 120_000)
it('replays a snapshot created by next build --analyze', async () => { it('replays a snapshot created by next build --analyze', async () => {
...@@ -490,6 +651,74 @@ describe('next analyze', () => { ...@@ -490,6 +651,74 @@ describe('next analyze', () => {
expect(entry).not.toHaveProperty('initial') expect(entry).not.toHaveProperty('initial')
expect(entry).not.toHaveProperty('load_scope') expect(entry).not.toHaveProperty('load_scope')
} }
const routeGraphs = [
'analyze.data',
'legacy/analyze.data',
'api/ping/analyze.data',
].map((route) =>
readAnalyzeFile<ChunkGraphHeader>(path.join(dataDir, route))
)
for (const { header, binary } of routeGraphs) {
const rows = readRows(binary, header.output_file_modules)
for (const [i, row] of rows.entries()) {
if (header.output_file_module_coverage[i] === 'not_a_chunk') {
expect(row).toEqual([])
}
}
}
expect(
routeGraphs.some(({ header, binary }) => {
const rows = readRows(binary, header.output_file_modules)
return header.output_files.some(
(output, i) =>
output.filename.includes('/server/chunks/') &&
output.filename.endsWith('.js') &&
header.output_file_module_coverage[i] === 'exact' &&
rows[i].length > 0
)
})
).toBe(true)
const runtimeOutputs = routeGraphs.flatMap(({ header }) =>
header.output_files.flatMap((output, i) =>
output.filename.includes('[turbopack]_runtime')
? [header.output_file_module_coverage[i]]
: []
)
)
expect(runtimeOutputs.length).toBeGreaterThan(0)
expect(
runtimeOutputs.every((coverage) => coverage === 'not_a_chunk')
).toBe(true)
const workers = routeGraphs.flatMap(({ header }) =>
header.output_files.flatMap((output, index) =>
output.filename.includes('/service-worker/')
? [header.output_file_module_coverage[index]]
: []
)
)
expect(workers.length).toBeGreaterThan(0)
expect(workers.every((coverage) => coverage === 'not_a_chunk')).toBe(
true
)
const appGraph = routeGraphs[0].header
const appRows = readRows(
routeGraphs[0].binary,
appGraph.output_file_modules
)
expect(
appRows.some((row) =>
row.some((index) => modules[index].ident.includes('client-entry'))
)
).toBe(true)
expect(appGraph.output_file_module_coverage).toContain('exact')
expect(
appRows.some(
(row, index) =>
appGraph.output_files[index].filename.endsWith('.css') &&
row.some((module) => modules[module].ident.includes('page.css'))
)
).toBe(true)
}) })
}) })
}) })
......
...@@ -50,9 +50,13 @@ export function validateGraphDump(output: string): void { ...@@ -50,9 +50,13 @@ export function validateGraphDump(output: string): void {
throw new Error(`Unknown route for graph record ${index}`) throw new Error(`Unknown route for graph record ${index}`)
} }
if (record.type === 'output') { if (record.type === 'output') {
outputs.add(JSON.stringify([key, record.filename])) const outputKey = JSON.stringify([key, record.filename])
if (outputs.has(outputKey)) {
throw new Error(`Duplicate analyzer output: ${record.filename}`)
}
outputs.add(outputKey)
} else if ( } else if (
(record.type === 'part' || record.type === 'unjoined') && record.type === 'part' &&
!outputs.has(JSON.stringify([key, record.filename])) !outputs.has(JSON.stringify([key, record.filename]))
) { ) {
throw new Error(`Unknown output for graph record ${index}`) throw new Error(`Unknown output for graph record ${index}`)
......
...@@ -24,7 +24,7 @@ use crate::BrowserChunkingContext; ...@@ -24,7 +24,7 @@ use crate::BrowserChunkingContext;
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(ValueToString)] #[derive(ValueToString)]
#[value_to_string("Ecmascript Browser Runtime Chunk")] #[value_to_string("Ecmascript Browser Runtime Chunk")]
pub(crate) struct EcmascriptBrowserRuntimeChunk { pub struct EcmascriptBrowserRuntimeChunk {
chunking_context: ResolvedVc<BrowserChunkingContext>, chunking_context: ResolvedVc<BrowserChunkingContext>,
include_async_module_runtime: bool, include_async_module_runtime: bool,
} }
......
...@@ -27,7 +27,7 @@ use crate::{ ...@@ -27,7 +27,7 @@ use crate::{
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(ValueToString)] #[derive(ValueToString)]
#[value_to_string("Ecmascript Browser Single Entry Chunk")] #[value_to_string("Ecmascript Browser Single Entry Chunk")]
pub(crate) struct EcmascriptBrowserSingleEntryChunk { pub struct EcmascriptBrowserSingleEntryChunk {
chunking_context: ResolvedVc<BrowserChunkingContext>, chunking_context: ResolvedVc<BrowserChunkingContext>,
path: FileSystemPath, path: FileSystemPath,
chunk: ResolvedVc<EcmascriptChunk>, chunk: ResolvedVc<EcmascriptChunk>,
......
...@@ -6,4 +6,7 @@ pub(crate) mod worker; ...@@ -6,4 +6,7 @@ pub(crate) mod worker;
pub use chunk::EcmascriptBrowserChunk; pub use chunk::EcmascriptBrowserChunk;
pub use content::EcmascriptBrowserChunkContent; pub use content::EcmascriptBrowserChunkContent;
pub use evaluate::{
runtime::EcmascriptBrowserRuntimeChunk, single_entry_chunk::EcmascriptBrowserSingleEntryChunk,
};
pub use worker::EcmascriptBrowserWorkerEntrypoint; pub use worker::EcmascriptBrowserWorkerEntrypoint;
...@@ -10,7 +10,7 @@ use turbopack_core::{ ...@@ -10,7 +10,7 @@ use turbopack_core::{
source_map::{GenerateSourceMap, SourceMapAsset}, source_map::{GenerateSourceMap, SourceMapAsset},
version::VersionedContent, version::VersionedContent,
}; };
use turbopack_ecmascript::chunk::EcmascriptChunk; use turbopack_ecmascript::chunk::{EcmascriptChunk, EcmascriptChunkContent};
use super::content::EcmascriptNodeChunkContent; use super::content::EcmascriptNodeChunkContent;
use crate::NodeJsChunkingContext; use crate::NodeJsChunkingContext;
...@@ -19,7 +19,7 @@ use crate::NodeJsChunkingContext; ...@@ -19,7 +19,7 @@ use crate::NodeJsChunkingContext;
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(ValueToString)] #[derive(ValueToString)]
#[value_to_string("Ecmascript Build Node Chunk")] #[value_to_string("Ecmascript Build Node Chunk")]
pub(crate) struct EcmascriptBuildNodeChunk { pub struct EcmascriptBuildNodeChunk {
chunking_context: ResolvedVc<NodeJsChunkingContext>, chunking_context: ResolvedVc<NodeJsChunkingContext>,
chunk: ResolvedVc<EcmascriptChunk>, chunk: ResolvedVc<EcmascriptChunk>,
} }
...@@ -39,6 +39,12 @@ impl EcmascriptBuildNodeChunk { ...@@ -39,6 +39,12 @@ impl EcmascriptBuildNodeChunk {
.cell() .cell()
} }
/// Constituent chunk items and batches, before the Node.js output wrapper.
#[turbo_tasks::function]
pub fn chunk_content(&self) -> Vc<EcmascriptChunkContent> {
self.chunk.chunk_content()
}
#[turbo_tasks::function] #[turbo_tasks::function]
async fn source_map(self: Vc<Self>) -> Result<Vc<SourceMapAsset>> { async fn source_map(self: Vc<Self>) -> Result<Vc<SourceMapAsset>> {
let this = self.await?; let this = self.await?;
......
...@@ -22,7 +22,7 @@ use crate::NodeJsChunkingContext; ...@@ -22,7 +22,7 @@ use crate::NodeJsChunkingContext;
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(ValueToString)] #[derive(ValueToString)]
#[value_to_string("Ecmascript Build Node Runtime Chunk")] #[value_to_string("Ecmascript Build Node Runtime Chunk")]
pub(crate) struct EcmascriptBuildNodeRuntimeChunk { pub struct EcmascriptBuildNodeRuntimeChunk {
chunking_context: ResolvedVc<NodeJsChunkingContext>, chunking_context: ResolvedVc<NodeJsChunkingContext>,
include_async_module_runtime: bool, include_async_module_runtime: bool,
} }
......
...@@ -6,3 +6,6 @@ pub mod ecmascript; ...@@ -6,3 +6,6 @@ pub mod ecmascript;
pub mod fs; pub mod fs;
pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder}; pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder};
pub use ecmascript::node::{
chunk::EcmascriptBuildNodeChunk, entry::runtime::EcmascriptBuildNodeRuntimeChunk,
};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment