Unverified Commit 2d70605d authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

[lts-active] fix(analyze): use route summaries as the analyzer route index (#100974)

Backports #100081 to `releases/lts/active`.

Uses route summaries as the single analyzer route index so generated
metadata routes export correctly. Clean cherry-pick with the original
regression coverage; no compatibility adaptations required.

<!-- NEXT_JS_LLM -->

<!-- fleet 16332d2e-fbbc-4958-bbc6-c9983f2dace6 -->
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent ae7105a2
......@@ -41,10 +41,12 @@ import {
analyzeDataUrl,
fetchAnalyzeData,
fetchModulesData,
routesFromSummaries,
useHistoryIndex,
useSuspenseJsonData,
useSuspenseRouteSummaries,
} from '@/lib/analyzer-data'
import { diffRoutesWithSizes, diffSources, type RouteSummary } from '@/lib/diff'
import { diffRoutesWithSizes, diffSources } from '@/lib/diff'
import { useSidebarResize } from '@/lib/use-sidebar-resize'
import { useAnalyzerRoute } from '@/lib/use-analyzer-route'
import {
......@@ -169,12 +171,10 @@ function useAnalyzerModel(compare: boolean) {
)
// Routes for the active build, which is also comparison side B.
const currentRoutes = useSuspenseJsonData<string[]>(
`${comparisonBaseDir}/routes.json`,
{ revalidateOnFocus: false, revalidateOnReconnect: false }
)
const routeSummaries = useSuspenseRouteSummaries(comparisonBaseDir)
const currentRoutes = routesFromSummaries(routeSummaries)
// Whether the selected route exists on comparison side B. routes.json is
// Whether the selected route exists on comparison side B. The summaries are
// the source of truth because output directories can retain stale routes.
const currentAnalyzeRouteExists =
selectedRoute != null && currentRoutes.includes(selectedRoute)
......@@ -310,10 +310,6 @@ function useAnalyzerModel(compare: boolean) {
})
}, [analyzeData, baselineSnapshot, filterSource])
const routeSummaries = useSuspenseJsonData<RouteSummary[]>(
`${comparisonBaseDir}/route-summaries.json`,
{ revalidateOnFocus: false, revalidateOnReconnect: false }
)
const clientRouteTotals = new Map(
routeSummaries.map(({ route, client }) => [
route,
......@@ -517,14 +513,8 @@ function ValidComparisonContent({
suspense: true,
}
)
const baselineRoutes = useSuspenseJsonData<string[]>(
`${baselineBaseDir}/routes.json`,
{ revalidateOnFocus: false, revalidateOnReconnect: false }
)
const baselineRouteSummaries = useSuspenseJsonData<RouteSummary[]>(
`${baselineBaseDir}/route-summaries.json`,
{ revalidateOnFocus: false, revalidateOnReconnect: false }
)
const baselineRouteSummaries = useSuspenseRouteSummaries(baselineBaseDir)
const baselineRoutes = routesFromSummaries(baselineRouteSummaries)
const baselineRouteTotals = new Map(
baselineRouteSummaries.map(({ route, size, compressed_size }) => [
route,
......@@ -541,6 +531,7 @@ function ValidComparisonContent({
baselineSnapshot,
baselineIsLatest: model.baselineIsLatest,
comparisonSnapshot: model.comparisonSnapshot,
baselineRouteCount: baselineRoutes.length,
comparisonRouteCount: model.currentRoutes.length,
routeDiff,
baselineModulesData,
......@@ -606,6 +597,7 @@ type ComparisonLayoutProps = Pick<
| 'baselineSnapshot'
| 'baselineIsLatest'
| 'comparisonSnapshot'
| 'baselineRouteCount'
| 'comparisonRouteCount'
| 'routeDiff'
> & { baselineModulesData: ModulesData }
......
......@@ -30,6 +30,8 @@ export interface CompareLayoutModel {
baselineSnapshot: SnapshotMetadata
baselineIsLatest: boolean
comparisonSnapshot: SnapshotMetadata | null
/** Route counts from each side's route summaries, not saved metadata. */
baselineRouteCount: number
comparisonRouteCount: number | null
routeDiff: ReturnType<typeof diffRoutesWithSizes> | null
selectedRoute: string | null
......@@ -71,7 +73,7 @@ export function CompareLayout({ model, onResizeSidebar }: CompareLayoutProps) {
className="xl:min-w-0 xl:flex-1 xl:basis-0 xl:border-r xl:border-border"
/>
<CompareContextStrip
baselineSnapshot={model.baselineSnapshot}
baselineRouteCount={model.baselineRouteCount}
comparisonRouteCount={model.comparisonRouteCount}
routeDiff={model.routeDiff}
compressed
......@@ -144,19 +146,20 @@ export function CompareLayout({ model, onResizeSidebar }: CompareLayoutProps) {
* A/B label pills below identify which snapshot is which.
*/
export function CompareContextStrip({
baselineSnapshot,
baselineRouteCount,
comparisonRouteCount,
routeDiff,
compressed,
className,
}: {
baselineSnapshot: SnapshotMetadata
baselineRouteCount: number
comparisonRouteCount: number | null
routeDiff: ReturnType<typeof diffRoutesWithSizes> | null
compressed: boolean
className?: string
}) {
const baselineRoutes = baselineSnapshot.routeCount
// Counted from the summaries, not `routeCount` saved in older metadata.
const baselineRoutes = baselineRouteCount
const comparisonRoutes =
comparisonRouteCount ??
routeDiff?.rows.filter((r) => r.status !== 'removed').length ??
......
......@@ -3,7 +3,10 @@
import { Check, ChevronsUpDown, Route } from 'lucide-react'
import { useEffect, useState, type Ref } from 'react'
import { Button } from '@/components/ui/button'
import { useSuspenseJsonData } from '@/lib/analyzer-data'
import {
routesFromSummaries,
useSuspenseRouteSummaries,
} from '@/lib/analyzer-data'
import {
Command,
CommandEmpty,
......@@ -171,8 +174,8 @@ export function RouteTypeaheadContent(
}
) {
const { selectedRoute, routeDiff, useCompressed = true, routeTotals } = props
const routes = useSuspenseJsonData<string[]>(
`${props.routesBaseDir ?? '/data'}/routes.json`
const routes = routesFromSummaries(
useSuspenseRouteSummaries(props.routesBaseDir ?? '/data')
)
// When a route diff is provided, sort routes by largest absolute impact so
......
import useSWR, { type SWRConfiguration } from 'swr'
import { AnalyzeData, ModulesData } from './analyze-data'
import type { RouteSummary } from './diff'
import type { HistoryIndex } from './snapshot'
import { fetchStrict, jsonFetcher } from './utils'
......@@ -32,6 +33,22 @@ export function useSuspenseJsonData<Data>(
return data
}
/**
* Per-route summaries written by Turbopack for a build. This is the analyzer's
* only route index: each `route` names the directory its `analyze.data` was
* written to.
*/
export function useSuspenseRouteSummaries(baseDir: string): RouteSummary[] {
return useSuspenseJsonData<RouteSummary[]>(
`${baseDir}/route-summaries.json`,
staticDataOptions
)
}
export function routesFromSummaries(summaries: RouteSummary[]): string[] {
return summaries.map(({ route }) => route)
}
export async function fetchAnalyzeData(url: string): Promise<AnalyzeData> {
const response = await fetchStrict(url)
return new AnalyzeData(await response.arrayBuffer())
......
import { readFileSync, writeSync } from 'node:fs'
import { join, resolve, sep } from 'node:path'
import { readAnalyzeRoutes } from './snapshot'
const UINT32_LIMIT = 0x1_0000_0000
......@@ -333,15 +334,7 @@ export function dumpAnalyzeGraph(
// @ts-ignore Node's internal handle, also used by the native bindings loader.
process.stdout._handle.setBlocking?.(true)
}
const routes = JSON.parse(
readFileSync(join(directory, 'routes.json'), 'utf8')
) as string[]
if (
!Array.isArray(routes) ||
routes.some((route) => typeof route !== 'string')
) {
throw new Error('Invalid analyzer routes')
}
const routes = readAnalyzeRoutes(directory)
const selected = routes.filter(
(route) => routeFilter === undefined || route === routeFilter
)
......
......@@ -10,13 +10,9 @@ import { turbopackAnalyze, type AnalyzeContext } from '../turbopack-analyze'
import { durationToString } from '../duration-to-string'
import { Lockfile } from '../lockfile'
import { installBindings } from '../swc/install-bindings'
import { cpSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'
import { discoverRoutes } from '../route-discovery'
import { findPagesDir } from '../../lib/find-pages-dir'
import loadCustomRoutes from '../../lib/load-custom-routes'
import { generateRoutesManifest } from '../generate-routes-manifest'
import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths'
import { cpSync, mkdirSync, rmSync } from 'node:fs'
import {
readAnalyzeRoutes,
writeAnalyzeSnapshot,
snapshotNameSchema,
type SnapshotMetadata,
......@@ -28,7 +24,6 @@ import serveHandler from 'next/dist/compiled/serve-handler'
import { Telemetry } from '../../telemetry/storage'
import { eventAnalyzeCompleted } from '../../telemetry/events'
import { traceGlobals } from '../../trace/shared'
import type { RoutesManifest } from '..'
import { Bundler } from '../../lib/bundler'
export type AnalyzeOptions = {
......@@ -108,16 +103,11 @@ export default async function analyze({
await shutdownPromise
const routes = await collectRoutesForAnalyze(dir, config, appDirOnly)
cpSync(path.join(__dirname, '../../bundle-analyzer'), analyzeDir, {
recursive: true,
})
mkdirSync(path.join(analyzeDir, 'data'), { recursive: true })
writeFileSync(
path.join(analyzeDir, 'data', 'routes.json'),
JSON.stringify(routes, null, 2)
)
const routes = readAnalyzeRoutes(path.join(analyzeDir, 'data'))
// Capture this build alongside any prior builds so the analyzer UI can
// offer it as a comparison baseline in the future.
......@@ -164,80 +154,6 @@ export default async function analyze({
}
}
/**
* Collects all routes from the project for the bundle analyzer.
* Returns a list of route paths (both static and dynamic).
*/
async function collectRoutesForAnalyze(
dir: string,
config: NextConfigComplete,
appDirOnly: boolean
): Promise<string[]> {
const { pagesDir, appDir } = findPagesDir(dir)
let appType: RoutesManifest['appType']
if (pagesDir && appDir) {
appType = 'hybrid'
} else if (pagesDir) {
appType = 'pages'
} else if (appDir) {
appType = 'app'
} else {
throw new Error('No pages or app directory found.')
}
const discovery = await discoverRoutes({
appDir,
pagesDir,
pageExtensions: config.pageExtensions,
isDev: false,
baseDir: dir,
isSrcDir: path.relative(dir, pagesDir || appDir || '').startsWith('src'),
appDirOnly,
})
const pageKeys = {
pages: Object.keys(discovery.mappedPages || {}),
app: discovery.mappedAppPages
? Object.keys(discovery.mappedAppPages).map((key) =>
normalizeAppPath(key)
)
: [],
}
// Load custom routes
const { redirects, headers, onMatchHeaders, rewrites } =
await loadCustomRoutes(config)
// Compute restricted redirect paths
const restrictedRedirectPaths = ['/_next'].map((pathPrefix) =>
config.basePath ? `${config.basePath}${pathPrefix}` : pathPrefix
)
const isAppPPREnabled = Boolean(config.cacheComponents)
// Generate routes manifest
const { routesManifest } = generateRoutesManifest({
appType,
pageKeys,
config,
redirects,
headers,
onMatchHeaders,
rewrites,
restrictedRedirectPaths,
isAppPPREnabled,
})
return Array.from(
new Set(
routesManifest.dynamicRoutes
.map((r) => r.page)
.concat(routesManifest.staticRoutes.map((r) => r.page))
)
)
}
function startServer(dir: string, port: number): Promise<void> {
const server = http.createServer((req, res) => {
return serveHandler(req, res, {
......
......@@ -64,6 +64,32 @@ const DATA_DIRNAME = 'data'
const HISTORY_DIRNAME = 'history'
const METADATA_FILENAME = 'metadata.json'
const HISTORY_INDEX_FILENAME = 'history.json'
const ROUTE_SUMMARIES_FILENAME = 'route-summaries.json'
const routeSummariesSchema = z.array(z.object({ route: z.string() }))
/**
* Read the analyzer's routes from `route-summaries.json` in a live `data/`
* directory or a saved snapshot.
*
* Turbopack writes one summary per route it wrote `analyze.data` for, keyed by
* the directory that data lives in. It is the only route index: the routes
* manifest names differ for metadata routes using `generateImageMetadata` or
* `generateSitemaps` (`/icon/[__metadata_id__]` vs. the stored `/icon`).
*/
export function readAnalyzeRoutes(directory: string): string[] {
const summariesFile = path.join(directory, ROUTE_SUMMARIES_FILENAME)
try {
return routeSummariesSchema
.parse(JSON.parse(readFileSync(summariesFile, 'utf8')))
.map(({ route }) => route)
} catch (error) {
throw new Error(
`Failed to read analyzer route summaries from ${summariesFile}. The analysis may be from an older Next.js version or incomplete; re-run \`next analyze\` to capture it again.`,
{ cause: error }
)
}
}
interface BuildSnapshotInputs {
projectDir: string
......
......@@ -132,7 +132,7 @@ import { isWriteable } from './is-writeable'
import * as Log from './output/log'
import createSpinner, { finishSpinner } from './spinner'
import { trace, flushAllTraces, setGlobal, type Span } from '../trace'
import { writeAnalyzeSnapshot } from './analyze/snapshot'
import { readAnalyzeRoutes, writeAnalyzeSnapshot } from './analyze/snapshot'
import { writeRouteBundleStats } from './route-bundle-stats'
import {
detectConflictingPaths,
......@@ -236,7 +236,7 @@ import {
sortPagesObject,
sortSortableRouteObjects,
} from '../shared/lib/router/utils/sortable-routes'
import { cp, mkdir, writeFile } from 'fs/promises'
import { cp, mkdir } from 'fs/promises'
import {
createRouteTypesManifest,
writeRouteTypesManifest,
......@@ -4858,18 +4858,7 @@ export default async function build(
recursive: true,
})
// Write an index of routes for the route picker
const routes = Array.from(
new Set(
routesManifest.dynamicRoutes
.map((r) => r.page)
.concat(routesManifest.staticRoutes.map((r) => r.page))
)
)
await writeFile(
path.join(analyzeDir, 'data/routes.json'),
JSON.stringify(routes, null, 2)
)
const routes = readAnalyzeRoutes(path.join(analyzeDir, 'data'))
// Capture this build alongside any prior builds so the analyzer UI
// can offer it as a comparison baseline in the future.
......
import { ImageResponse } from 'next/og'
// `generateImageMetadata` makes this a `/meta/icon/[__metadata_id__]` route in
// the routes manifest, while the analyzer writes its data under `/meta/icon`.
export function generateImageMetadata() {
return [
{ id: 'small', size: { width: 32, height: 32 }, contentType: 'image/png' },
{ id: 'large', size: { width: 64, height: 64 }, contentType: 'image/png' },
]
}
export default async function Icon({ id }: { id: Promise<string> }) {
const iconId = await id
return new ImageResponse(<div>{iconId}</div>, { width: 32, height: 32 })
}
import type { MetadataRoute } from 'next'
// `generateSitemaps` makes this a `/sitemap/[__metadata_id__]` route in the
// routes manifest, while the analyzer writes its data under `/sitemap.xml`.
export async function generateSitemaps() {
return [{ id: 0 }, { id: 1 }]
}
export default async function sitemap({
id,
}: {
id: Promise<string>
}): Promise<MetadataRoute.Sitemap> {
return [{ url: `https://example.com/${await id}` }]
}
......@@ -8,7 +8,7 @@ import { validateGraphDump } from '../../lib/analyze-graph-schema'
import { shouldUseTurbopack } from 'next-test-utils'
import path from 'node:path'
import type { ChildProcess } from 'node:child_process'
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
type RouteEntry = {
route_entry_id: string
......@@ -75,6 +75,36 @@ function readRows(binary: Buffer, reference: EdgesReference): number[][] {
})
}
// Routes the analyzer writes data for, as listed in `route-summaries.json`.
// Metadata routes that use `generateImageMetadata` / `generateSitemaps` are
// listed under the key their data is stored at, not as
// `.../[__metadata_id__]` routes-manifest pages.
const EXPECTED_ROUTES = [
'/',
'/_global-error',
'/_not-found',
'/api/ping',
'/legacy',
'/meta/icon',
'/sitemap.xml',
].sort()
function readRoutes(directory: string): string[] {
return JSON.parse(
readFileSync(path.join(directory, 'route-summaries.json'), 'utf8')
).map((summary: { route: string }) => summary.route)
}
function routeDataFile(directory: string, route: string): string {
return path.join(directory, route.slice(1), 'analyze.data')
}
function exportedRoutes(records: Array<{ type: string; route?: string }>) {
return records
.filter((record) => record.type === 'route')
.map((record) => record.route)
}
// TODO(deploy-test-completion): Re-enable this suite in deploy mode.
// It likely inspects local build artifacts that deploy tests do not expose.
// @force-gate !deploy
......@@ -236,6 +266,54 @@ describe('next analyze', () => {
}
for (const record of records)
expect(record).not.toHaveProperty('route_index')
// Every summarized route, including metadata routes, is exported.
expect(exportedRoutes(records)).toEqual(readRoutes(snapshotDir))
expect([...readRoutes(snapshotDir)].sort()).toEqual(EXPECTED_ROUTES)
expect(
JSON.parse(readFileSync(path.join(snapshotDir, 'metadata.json'), 'utf8'))
.routeCount
).toBe(readRoutes(snapshotDir).length)
// route-summaries.json is the only route index.
expect(existsSync(path.join(snapshotDir, 'routes.json'))).toBe(false)
expect(existsSync(path.join(analyzeDir, 'data/routes.json'))).toBe(false)
// A stale routes.json left by older captures, with routes-manifest names
// that have no data, is ignored.
const legacyRoutesFile = path.join(snapshotDir, 'routes.json')
try {
writeFileSync(
legacyRoutesFile,
JSON.stringify(['/', '/meta/icon/[__metadata_id__]'])
)
const legacy = await next.runCommand([
'analyze',
'export',
'--snapshot',
name,
])
expect(legacy.exitCode).toBe(0)
expect(legacy.stdout).toBe(named.stdout)
} finally {
rmSync(legacyRoutesFile, { force: true })
}
// Snapshots without route summaries cannot be exported.
const summariesFile = path.join(snapshotDir, 'route-summaries.json')
const originalSummaries = readFileSync(summariesFile)
try {
rmSync(summariesFile)
const missing = await next.runCommand([
'analyze',
'export',
'--snapshot',
name,
])
expect(missing.exitCode).not.toBe(0)
expect(missing.stdout).toBe('')
expect(missing.stderr).toMatch(/re-run `next analyze`/i)
} finally {
writeFileSync(summariesFile, originalSummaries)
}
const latest = await next.runCommand(['analyze', 'export'])
expect(latest.exitCode).toBe(0)
expect(latest.stdout).toBe(named.stdout)
......@@ -541,6 +619,29 @@ describe('next analyze', () => {
])
expect(replay).toMatchObject({ exitCode: 0 })
validateGraphDump(replay.stdout)
const snapshotDir = path.join(
analyzeDir,
'history',
`snapshot-${encodeURIComponent(name)}`
)
const routes = readRoutes(snapshotDir)
expect([...routes].sort()).toEqual(EXPECTED_ROUTES)
expect(existsSync(path.join(snapshotDir, 'routes.json'))).toBe(false)
expect(
JSON.parse(readFileSync(path.join(snapshotDir, 'metadata.json'), 'utf8'))
.routeCount
).toBe(routes.length)
for (const route of routes) {
expect(existsSync(routeDataFile(snapshotDir, route))).toBe(true)
}
expect(
exportedRoutes(
replay.stdout
.trim()
.split('\n')
.map((line) => JSON.parse(line))
)
).toEqual(routes)
expect(JSON.parse(replay.stdout.split('\n')[0])).toMatchObject({
type: 'meta',
snapshot_name: name,
......@@ -564,19 +665,41 @@ describe('next analyze', () => {
expect(stdout).toContain('.next/diagnostics/analyze')
for (const file of [
'index.html',
'data/routes.json',
'data/route-summaries.json',
'data/modules.data',
'data/analyze.data',
]) {
expect(existsSync(path.join(defaultOutputPath, file))).toBe(true)
}
const routes = JSON.parse(
readFileSync(path.join(defaultOutputPath, 'data/routes.json'), 'utf8')
const routes = readRoutes(path.join(defaultOutputPath, 'data'))
expect([...routes].sort()).toEqual(EXPECTED_ROUTES)
const latestSnapshotDir = path.join(
defaultOutputPath,
'history',
`snapshot-${encodeURIComponent(
JSON.parse(
readFileSync(
path.join(defaultOutputPath, 'history/history.json'),
'utf8'
)
).snapshots[0].name
)}`
)
expect([...routes].sort()).toEqual(
['/', '/_not-found', '/api/ping', '/legacy'].sort()
expect(readRoutes(latestSnapshotDir)).toEqual(routes)
expect(
existsSync(path.join(defaultOutputPath, 'data/routes.json'))
).toBe(false)
expect(existsSync(path.join(latestSnapshotDir, 'routes.json'))).toBe(
false
)
for (const route of routes) {
expect(
existsSync(
routeDataFile(path.join(defaultOutputPath, 'data'), route)
)
).toBe(true)
expect(existsSync(routeDataFile(latestSnapshotDir, route))).toBe(true)
}
const routeSummaries = JSON.parse(
readFileSync(
path.join(defaultOutputPath, 'data/route-summaries.json'),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment