Unverified Commit be66341c authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Build: Add .next/trace-build with high level trace (#83949)

## What?

- Adds a new file `.next/trace-build` with a high level overview of the
build process. This is an extra file next to `.next/trace`.
`.next/trace` includes a lower level trace when using webpack.
`.next/trace-build` is the same between webpack/Turbopack in terms of
spans emitted (though `run-turbopack` vs `run-webpack` span).

Included trace spans are:

```
  'next-build',
  'run-turbopack',
  'run-webpack',
  'run-typescript',
  'run-eslint',
  'static-check',
  'static-generation',
  'output-export-full-static-export',
```

`run-turbopack` is only included when Turbopack is used. `run-webpack`
when webpack is used.

`run-typescript` and `run-eslint` are concurrent.

`output-export-full-static-export` is only included when using `output:
'export'`.
parent 8c8e9a50
......@@ -4098,14 +4098,18 @@ export default async function build(
}
if (config.output === 'export') {
await writeFullyStaticExport(
config,
dir,
enabledDirectories,
configOutDir,
nextBuildSpan,
appDirOnly
)
await nextBuildSpan
.traceChild('output-export-full-static-export')
.traceAsyncFn(async () => {
await writeFullyStaticExport(
config,
dir,
enabledDirectories,
configOutDir,
nextBuildSpan,
appDirOnly
)
})
}
if (config.experimental.adapterPath) {
......
......@@ -52,15 +52,13 @@ export function turbopackBuild(
withWorker: boolean
): ReturnType<typeof import('./impl').turbopackBuild> {
const nextBuildSpan = NextBuildContext.nextBuildSpan!
return nextBuildSpan
.traceChild('run-turbopack-compiler')
.traceAsyncFn(async () => {
if (withWorker) {
return await turbopackBuildWithWorker()
} else {
const build = (require('./impl') as typeof import('./impl'))
.turbopackBuild
return await build()
}
})
return nextBuildSpan.traceChild('run-turbopack').traceAsyncFn(async () => {
if (withWorker) {
return await turbopackBuildWithWorker()
} else {
const build = (require('./impl') as typeof import('./impl'))
.turbopackBuild
return await build()
}
})
}
......@@ -128,7 +128,7 @@ export async function startTypeChecking({
try {
const [[verifyResult, typeCheckEnd]] = await Promise.all([
nextBuildSpan.traceChild('verify-typescript-setup').traceAsyncFn(() =>
nextBuildSpan.traceChild('run-typescript').traceAsyncFn(() =>
verifyTypeScriptSetup(
dir,
config.distDir,
......@@ -146,7 +146,7 @@ export async function startTypeChecking({
})
),
shouldLint &&
nextBuildSpan.traceChild('verify-and-lint').traceAsyncFn(async () => {
nextBuildSpan.traceChild('run-eslint').traceAsyncFn(async () => {
await verifyAndLint(
dir,
eslintCacheDir,
......
......@@ -128,26 +128,29 @@ export async function webpackBuild(
| Awaited<ReturnType<typeof webpackBuildWithWorker>>
| Awaited<ReturnType<typeof import('./impl').webpackBuildImpl>>
> {
if (withWorker) {
debug('using separate compiler workers')
return await webpackBuildWithWorker(compilerNames)
} else {
debug('building all compilers in same process')
const webpackBuildImpl = (require('./impl') as typeof import('./impl'))
.webpackBuildImpl
const curResult = await webpackBuildImpl(null)
// Mirror what happens in webpackBuildWithWorker
if (curResult.telemetryState) {
NextBuildContext.telemetryState = {
...curResult.telemetryState,
useCacheTracker: mergeUseCacheTrackers(
NextBuildContext.telemetryState?.useCacheTracker,
curResult.telemetryState.useCacheTracker
),
const nextBuildSpan = NextBuildContext.nextBuildSpan!
return nextBuildSpan.traceChild('run-webpack').traceAsyncFn(async () => {
if (withWorker) {
debug('using separate compiler workers')
return await webpackBuildWithWorker(compilerNames)
} else {
debug('building all compilers in same process')
const webpackBuildImpl = (require('./impl') as typeof import('./impl'))
.webpackBuildImpl
const curResult = await webpackBuildImpl(null)
// Mirror what happens in webpackBuildWithWorker
if (curResult.telemetryState) {
NextBuildContext.telemetryState = {
...curResult.telemetryState,
useCacheTracker: mergeUseCacheTrackers(
NextBuildContext.telemetryState?.useCacheTracker,
curResult.telemetryState.useCacheTracker
),
}
}
}
return curResult
}
return curResult
}
})
}
import type { TraceEvent } from '../types'
import reportToTelemetry from './to-telemetry'
import reportToJson from './to-json'
import reportToJsonBuild from './to-json-build'
import type { Reporter } from './types'
class MultiReporter implements Reporter {
......@@ -20,4 +21,8 @@ class MultiReporter implements Reporter {
}
// JSON is always reported to allow for diagnostics
export const reporter = new MultiReporter([reportToJson, reportToTelemetry])
export const reporter = new MultiReporter([
reportToJson,
reportToJsonBuild,
reportToTelemetry,
])
import { traceGlobals, traceId } from '../shared'
import fs from 'fs'
import path from 'path'
import {
PHASE_DEVELOPMENT_SERVER,
PHASE_PRODUCTION_BUILD,
} from '../../shared/lib/constants'
import type { TraceEvent } from '../types'
import { batcher } from './to-json'
let writeStream: RotatingWriteStream
let batch: ReturnType<typeof batcher> | undefined
const writeStreamOptions = {
flags: 'a',
encoding: 'utf8' as const,
}
class RotatingWriteStream {
file: string
writeStream!: fs.WriteStream
size: number
sizeLimit: number
private rotatePromise: Promise<void> | undefined
private drainPromise: Promise<void> | undefined
constructor(file: string, sizeLimit: number) {
this.file = file
this.size = 0
this.sizeLimit = sizeLimit
this.createWriteStream()
}
private createWriteStream() {
this.writeStream = fs.createWriteStream(this.file, writeStreamOptions)
}
// Recreate the file
private async rotate() {
await this.end()
try {
fs.unlinkSync(this.file)
} catch (err: any) {
// It's fine if the file does not exist yet
if (err.code !== 'ENOENT') {
throw err
}
}
this.size = 0
this.createWriteStream()
this.rotatePromise = undefined
}
async write(data: string): Promise<void> {
if (this.rotatePromise) await this.rotatePromise
this.size += data.length
if (this.size > this.sizeLimit) {
await (this.rotatePromise = this.rotate())
}
if (!this.writeStream.write(data, 'utf8')) {
if (this.drainPromise === undefined) {
this.drainPromise = new Promise<void>((resolve, _reject) => {
this.writeStream.once('drain', () => {
this.drainPromise = undefined
resolve()
})
})
}
await this.drainPromise
}
}
end(): Promise<void> {
return new Promise((resolve) => {
this.writeStream.end(resolve)
})
}
}
const allowlistedEvents = new Set([
'next-build',
'run-turbopack',
'run-webpack',
'run-typescript',
'run-eslint',
'static-check',
'static-generation',
'output-export-full-static-export',
])
function reportToJsonBuild(event: TraceEvent) {
if (!allowlistedEvents.has(event.name)) {
return
}
const distDir = traceGlobals.get('distDir')
const phase = traceGlobals.get('phase')
if (!distDir || !phase) {
return
}
// Only report in production builds
if (phase !== PHASE_PRODUCTION_BUILD) {
return
}
if (!batch) {
batch = batcher(async (events: TraceEvent[]) => {
if (!writeStream) {
await fs.promises.mkdir(distDir, { recursive: true })
const file = path.join(distDir, 'trace-build')
writeStream = new RotatingWriteStream(
file,
// Development is limited to 50MB, production is unlimited
phase === PHASE_DEVELOPMENT_SERVER ? 52428800 : Infinity
)
}
const eventsJson = JSON.stringify(events)
try {
await writeStream.write(eventsJson + '\n')
} catch (err) {
console.log(err)
}
})
}
batch.report({
...event,
traceId,
})
}
export default {
flushAll: (opts?: { end: boolean }) =>
batch
? batch.flushAll().then(() => {
const phase = traceGlobals.get('phase')
// Only end writeStream when manually flushing in production
if (opts?.end || phase !== PHASE_DEVELOPMENT_SERVER) {
return writeStream.end()
}
})
: undefined,
report: reportToJsonBuild,
}
......@@ -4,20 +4,9 @@ import path from 'path'
import { PHASE_DEVELOPMENT_SERVER } from '../../shared/lib/constants'
import type { TraceEvent } from '../types'
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const localEndpoint = {
serviceName: 'nextjs',
ipv4: '127.0.0.1',
port: 9411,
}
type Event = TraceEvent & {
localEndpoint?: typeof localEndpoint
}
// Batch events as zipkin allows for multiple events to be sent in one go
export function batcher(reportEvents: (evts: Event[]) => Promise<void>) {
const events: Event[] = []
export function batcher(reportEvents: (evts: TraceEvent[]) => Promise<void>) {
const events: TraceEvent[] = []
// Promise queue to ensure events are always sent on flushAll
const queue = new Set()
return {
......@@ -28,7 +17,7 @@ export function batcher(reportEvents: (evts: Event[]) => Promise<void>) {
events.length = 0
}
},
report: (event: Event) => {
report: (event: TraceEvent) => {
events.push(event)
if (events.length > 100) {
......@@ -108,7 +97,7 @@ class RotatingWriteStream {
}
}
const reportToLocalHost = (event: TraceEvent) => {
function reportToJson(event: TraceEvent) {
const distDir = traceGlobals.get('distDir')
const phase = traceGlobals.get('phase')
if (!distDir || !phase) {
......@@ -116,7 +105,7 @@ const reportToLocalHost = (event: TraceEvent) => {
}
if (!batch) {
batch = batcher(async (events: Event[]) => {
batch = batcher(async (events: TraceEvent[]) => {
if (!writeStream) {
await fs.promises.mkdir(distDir, { recursive: true })
const file = path.join(distDir, 'trace')
......@@ -152,5 +141,5 @@ export default {
}
})
: undefined,
report: reportToLocalHost,
report: reportToJson,
}
......@@ -28,7 +28,7 @@ const DEV_ALLOWED_EVENTS = new Set([
const BUILD_ALLOWED_EVENTS = new Set([
...COMMON_ALLOWED_EVENTS,
'next-build',
'run-turbopack-compiler',
'run-turbopack',
'webpack-compilation',
'run-webpack-compiler',
'create-entrypoints',
......@@ -50,14 +50,14 @@ const BUILD_ALLOWED_EVENTS = new Set([
'node-file-trace-build',
'static-generation',
'next-export',
'verify-typescript-setup',
'verify-and-lint',
'run-typescript',
'run-eslint',
])
const {
NEXT_TRACE_UPLOAD_DEBUG,
// An external env to allow to upload full trace without picking up the relavant spans.
// This is mainly for the debugging purpose, to allwo manual audit for full trace for the given build.
// This is mainly for the debugging purpose, to allow manual audit for full trace for the given build.
// [NOTE] This may fail if build is large and generated trace is excessively large.
NEXT_TRACE_UPLOAD_FULL,
} = process.env
......
import { ReactNode } from 'react'
export default function Root({ children }: { children: ReactNode }) {
return (
<html>
<body>{children}</body>
</html>
)
}
export default function Page() {
return <p>hello world</p>
}
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {}
module.exports = nextConfig
import { nextTestSetup, isNextDev, isNextStart } from 'e2e-utils'
import { join } from 'path'
import { existsSync, readFileSync } from 'fs'
type SpanId = number
type TraceEvent = {
traceId?: string
parentId?: SpanId
name: string
id: SpanId
timestamp: number
duration: number
tags?: Object
startTime?: number
}
interface TraceStructure {
events: TraceEvent[]
eventsByName: Map<string, TraceEvent[]>
eventsById: Map<string, TraceEvent>
rootEvents: TraceEvent[]
orphanedEvents: TraceEvent[]
}
function parseTraceFile(traceBuildPath: string): TraceStructure {
const traceContent = readFileSync(traceBuildPath, 'utf8')
const traceLines = traceContent
.trim()
.split('\n')
.filter((line) => line.trim())
const allEvents: TraceEvent[] = []
for (const line of traceLines) {
const events = JSON.parse(line) as TraceEvent[]
allEvents.push(...events)
}
const eventsByName = new Map<string, TraceEvent[]>()
const eventsById = new Map<string, TraceEvent>()
const rootEvents: TraceEvent[] = []
const orphanedEvents: TraceEvent[] = []
// Index all events
for (const event of allEvents) {
if (!eventsByName.has(event.name)) {
eventsByName.set(event.name, [])
}
eventsByName.get(event.name).push(event)
eventsById.set(event.id.toString(), event)
}
// Categorize events as root or orphaned
for (const event of allEvents) {
if (!event.parentId) {
rootEvents.push(event)
} else if (!eventsById.has(event.parentId.toString())) {
orphanedEvents.push(event)
}
}
return {
events: allEvents,
eventsByName,
eventsById,
rootEvents,
orphanedEvents,
}
}
describe('trace-build-file', () => {
const { next } = nextTestSetup({
files: __dirname,
skipStart: !isNextDev,
skipDeployment: true,
})
if (isNextStart) {
it('should create .next/trace-build file during production build', async () => {
// Build the app to trigger trace generation
await next.build()
// Check that trace-build file exists
const traceBuildPath = join(next.testDir, '.next/trace-build')
expect(existsSync(traceBuildPath)).toBe(true)
})
it('should contain high-level build trace events', async () => {
// Ensure we have a fresh build
await next.build()
const traceBuildPath = join(next.testDir, '.next/trace-build')
expect(existsSync(traceBuildPath)).toBe(true)
const traceStructure = parseTraceFile(traceBuildPath)
// Should have events
expect(traceStructure.events.length).toBeGreaterThan(0)
// Should contain the main next-build event
const nextBuildEvents = traceStructure.eventsByName.get('next-build')
expect(nextBuildEvents).toBeDefined()
expect(nextBuildEvents.length).toBe(1)
const nextBuildEvent = nextBuildEvents[0]
expect(nextBuildEvent).toHaveProperty('name', 'next-build')
expect(nextBuildEvent).toHaveProperty('traceId')
expect(nextBuildEvent).toHaveProperty('id')
expect(nextBuildEvent).toHaveProperty('duration')
expect(typeof nextBuildEvent.duration).toBe('number')
expect(typeof nextBuildEvent.traceId).toBe('string')
expect(typeof nextBuildEvent.id).toBe('number')
})
it('should only contain allowlisted events', async () => {
await next.build()
const traceBuildPath = join(next.testDir, '.next/trace-build')
const traceStructure = parseTraceFile(traceBuildPath)
const allowlistedEvents = new Set([
'next-build',
'run-turbopack',
'run-webpack',
'run-typescript',
'run-eslint',
'static-check',
'static-generation',
'output-export-full-static-export',
])
for (const event of traceStructure.events) {
expect(allowlistedEvents.has(event.name)).toBe(true)
}
})
it('should have next-build as root span with proper hierarchy', async () => {
await next.build()
const traceBuildPath = join(next.testDir, '.next/trace-build')
const traceStructure = parseTraceFile(traceBuildPath)
// Should have no orphaned events (all events should have valid parent references)
expect(traceStructure.orphanedEvents).toHaveLength(0)
// Should have at one root event
expect(traceStructure.rootEvents.length).toBe(1)
// next-build should be the main root event
const nextBuildEvents = traceStructure.eventsByName.get('next-build')
expect(nextBuildEvents).toBeDefined()
expect(nextBuildEvents.length).toBe(1)
const nextBuildEvent = nextBuildEvents[0]
expect(nextBuildEvent.parentId).toBeUndefined() // Should be root
expect(traceStructure.rootEvents).toContain(nextBuildEvent)
// Other build events should be children of next-build or have valid parent references
const buildEvents = ['run-webpack', 'run-typescript', 'run-eslint']
for (const eventName of buildEvents) {
const events = traceStructure.eventsByName.get(eventName)
if (events && events.length > 0) {
for (const event of events) {
if (event.parentId) {
// Should have a valid parent
expect(
traceStructure.eventsById.has(event.parentId.toString())
).toBe(true)
const parent = traceStructure.eventsById.get(
event.parentId.toString()
)
// Parent should either be next-build or another valid event
expect(parent).toBeDefined()
expect(parent.traceId).toBe(event.traceId) // Same trace
}
}
}
}
})
it('should have consistent traceId across all events', async () => {
await next.build()
const traceBuildPath = join(next.testDir, '.next/trace-build')
const traceStructure = parseTraceFile(traceBuildPath)
expect(traceStructure.events.length).toBeGreaterThan(0)
const firstEvent = traceStructure.events[0]
expect(firstEvent.traceId).toBeDefined()
expect(typeof firstEvent.traceId).toBe('string')
expect(firstEvent.traceId.length).toBeGreaterThan(0)
// All events should have the same traceId
for (const event of traceStructure.events) {
expect(event.traceId).toBe(firstEvent.traceId)
}
})
}
if (isNextDev) {
it('should not create trace-build file in development mode', async () => {
// Make a request to trigger some activity
await next.render('/')
// Check that trace-build file does not exist
const traceBuildPath = join(next.testDir, '.next/trace-build')
expect(existsSync(traceBuildPath)).toBe(false)
})
}
it('should work with basic page rendering', async () => {
if (isNextStart) {
await next.start()
}
const $ = await next.render$('/')
expect($('p').text()).toBe('hello world')
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment