Unverified Commit 3aa62d1f authored by mknichel's avatar mknichel Committed by GitHub

[.next/trace] Track server requests and memory usage (#58690)

This PR adds more information to `.next/trace` for local development:

- Break down server startup times into setting up the dev bundler and running the instrumentation hook
- Add top level span for handling a request such as a page request, app router request, or API request
- Add span for compilation time tracking the path of the route that triggered the compilation.
- Track memory usage after every request
- Track event when server is restarted due to reaching the memory limit


Platform stats and memory usage on startup:
![Screenshot 2023-11-20 at 8 47 38 AM](https://github.com/vercel/next.js/assets/7355009/870226cc-8caa-41bd-92ae-b0025aecdee7)

Memory usage after each request:
<img width="340" alt="Screenshot 2023-11-20 at 9 16 44 AM" src="https://github.com/vercel/next.js/assets/7355009/d14279d6-35c5-45a6-b431-b485d5d14ac6">

Split server startup times into setting up the dev bundler and running the instrumentation hook:
![Screenshot 2023-11-20 at 8 46 41 AM](https://github.com/vercel/next.js/assets/7355009/eda8b16d-a784-4332-b8b1-5f0dce0611ef)
Co-authored-by: 's avatarTim Neutkens <6324199+timneutkens@users.noreply.github.com>
parent cf102cf1
......@@ -51,7 +51,7 @@ export function runCompiler(
compiler.fsStartTime = Date.now()
compiler.run((err, stats) => {
const webpackCloseSpan = runWebpackSpan.traceChild('webpack-close', {
name: config.name,
name: config.name || 'unknown',
})
webpackCloseSpan
.traceAsyncFn(() => closeCompiler(compiler))
......
......@@ -1605,7 +1605,7 @@ export default async function build(
httpAgentOptions: config.httpAgentOptions,
locales: config.i18n?.locales,
defaultLocale: config.i18n?.defaultLocale,
parentId: isPageStaticSpan.id,
parentId: isPageStaticSpan.getId(),
pageRuntime,
edgeInfo,
pageType,
......
import createStore from 'next/dist/compiled/unistore'
import stripAnsi from 'next/dist/compiled/strip-ansi'
import { flushAllTraces } from '../../trace'
import { type Span, flushAllTraces, trace } from '../../trace'
import {
teardownCrashReporter,
teardownHeapProfiler,
......@@ -52,6 +52,7 @@ function hasStoreChanged(nextStore: OutputState) {
let startTime = 0
let trigger = '' // default, use empty string for trigger
let loadingLogTimer: NodeJS.Timeout | null = null
let traceSpan: Span | null = null
store.subscribe((state) => {
if (!hasStoreChanged(state)) {
......@@ -66,6 +67,9 @@ store.subscribe((state) => {
if (state.trigger) {
trigger = state.trigger
if (trigger !== 'initial') {
traceSpan = trace('compile-path', undefined, {
trigger: trigger,
})
if (!loadingLogTimer) {
// Only log compiling if compiled is not finished in 3 seconds
loadingLogTimer = setTimeout(() => {
......@@ -144,6 +148,10 @@ store.subscribe((state) => {
clearTimeout(loadingLogTimer)
loadingLogTimer = null
}
if (traceSpan) {
traceSpan.stop()
traceSpan = null
}
Log.event(
`Compiled${trigger ? ' ' + trigger : ''}${timeMessage}${modulesMessage}`
)
......
......@@ -86,7 +86,7 @@ async function webpackBuildWithWorker(
compilerName,
traceState: {
...exportTraceState(),
defaultParentSpanId: nextBuildSpan?.id,
defaultParentSpanId: nextBuildSpan?.getId(),
shouldSaveTraceEvents: true,
},
})
......
......@@ -68,7 +68,6 @@ export class CssMinimizerPlugin {
const cssMinimizerSpan = compilationSpan!.traceChild(
'css-minimizer-plugin'
)
cssMinimizerSpan.setAttribute('webpackVersion', 5)
return cssMinimizerSpan.traceAsyncFn(async () => {
const files = Object.keys(assets)
......
......@@ -272,6 +272,19 @@ const nextDev: CliCommand = async (args) => {
return
}
if (code === RESTART_EXIT_CODE) {
// Starting the dev server will overwrite the `.next/trace` file, so we
// must upload the existing contents before restarting the server to
// preserve the metrics.
if (traceUploadUrl) {
uploadTrace({
traceUploadUrl,
mode: 'dev',
isTurboSession,
projectDir: dir,
distDir: config.distDir,
sync: true,
})
}
return startServer(options)
}
await handleSessionStop(signal)
......
......@@ -691,7 +691,7 @@ export async function exportAppImpl(
optimizeCss: nextConfig.experimental.optimizeCss,
disableOptimizedLoading:
nextConfig.experimental.disableOptimizedLoading,
parentSpanId: pageExportSpan.id,
parentSpanId: pageExportSpan.getId(),
httpAgentOptions: nextConfig.httpAgentOptions,
debugOutput: options.debugOutput,
isrMemoryCacheSize: nextConfig.experimental.isrMemoryCacheSize,
......
......@@ -40,7 +40,7 @@ import { normalizePagePath } from '../../shared/lib/page-path/normalize-page-pat
import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix'
import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix'
import { Telemetry } from '../../telemetry/storage'
import { setGlobal } from '../../trace'
import { type Span, setGlobal, trace } from '../../trace'
import { findPageFile } from '../lib/find-page-file'
import { getNodeOptionsWithoutInspect } from '../lib/utils'
import { withCoalescedInvoke } from '../../lib/coalesced-function'
......@@ -84,6 +84,11 @@ export interface Options extends ServerOptions {
* Interface to the development bundler.
*/
bundlerService: DevBundlerService
/**
* Trace span for server startup.
*/
startServerSpan: Span
}
export default class DevServer extends Server {
......@@ -104,6 +109,7 @@ export default class DevServer extends Server {
string,
UnwrapPromise<ReturnType<DevServer['getStaticPaths']>>
>
private startServerSpan: Span
protected staticPathsWorker?: { [key: string]: any } & {
loadStaticPaths: typeof import('./static-paths-worker').loadStaticPaths
......@@ -145,6 +151,8 @@ export default class DevServer extends Server {
} catch {}
super({ ...options, dev: true })
this.bundlerService = options.bundlerService
this.startServerSpan =
options.startServerSpan ?? trace('start-next-dev-server')
this.originalFetch = global.fetch
this.renderOpts.dev = true
this.renderOpts.appDirDevErrorLogger = (err: any) =>
......@@ -268,7 +276,9 @@ export default class DevServer extends Server {
const telemetry = new Telemetry({ distDir: this.distDir })
await super.prepareImpl()
await this.runInstrumentationHookIfAvailable()
await this.startServerSpan
.traceChild('run-instrumentation-hook')
.traceAsyncFn(() => this.runInstrumentationHookIfAvailable())
await this.matchers.reload()
this.ready?.resolve()
......@@ -437,8 +447,21 @@ export default class DevServer extends Server {
res: BaseNextResponse,
parsedUrl?: NextUrlWithParsedQuery
): Promise<void> {
await this.ready?.promise
return await super.handleRequest(req, res, parsedUrl)
const span = trace('handle-request', undefined, { url: req.url })
const result = await span.traceAsyncFn(async () => {
await this.ready?.promise
return await super.handleRequest(req, res, parsedUrl)
})
const memoryUsage = process.memoryUsage()
span
.traceChild('memory-usage', {
url: req.url,
'memory.rss': String(memoryUsage.rss),
'memory.heapUsed': String(memoryUsage.heapUsed),
'memory.heapTotal': String(memoryUsage.heapTotal),
})
.stop()
return result
}
async run(
......
......@@ -3,6 +3,7 @@ import type { DevBundlerService } from './dev-bundler-service'
import type { PropagateToWorkersField } from './router-utils/types'
import next from '../next'
import type { Span } from '../../trace'
let initializations: Record<
string,
......@@ -81,6 +82,7 @@ async function initializeImpl(opts: {
_ipcPort?: string
_ipcKey?: string
bundlerService: DevBundlerService | undefined
startServerSpan: Span | undefined
}) {
const type = process.env.__NEXT_PRIVATE_RENDER_WORKER
if (type) {
......
......@@ -32,6 +32,7 @@ import {
PERMANENT_REDIRECT_STATUS,
} from '../../shared/lib/constants'
import { DevBundlerService } from './dev-bundler-service'
import { type Span, trace } from '../../trace'
const debug = setupDebug('next:router-server:main')
......@@ -62,6 +63,7 @@ export async function initialize(opts: {
customServer?: boolean
experimentalTestProxy?: boolean
experimentalHttpsServer?: boolean
startServerSpan?: Span
}): Promise<[WorkerRequestHandler, WorkerUpgradeHandler]> {
if (!process.env.NODE_ENV) {
// @ts-ignore not readonly
......@@ -102,19 +104,24 @@ export async function initialize(opts: {
const { setupDevBundler } =
require('./router-utils/setup-dev-bundler') as typeof import('./router-utils/setup-dev-bundler')
developmentBundler = await setupDevBundler({
// Passed here but the initialization of this object happens below, doing the initialization before the setupDev call breaks.
renderServer,
appDir,
pagesDir,
telemetry,
fsChecker,
dir: opts.dir,
nextConfig: config,
isCustomServer: opts.customServer,
turbo: !!process.env.TURBOPACK,
port: opts.port,
})
const setupDevBundlerSpan = opts.startServerSpan
? opts.startServerSpan.traceChild('setup-dev-bundler')
: trace('setup-dev-bundler')
developmentBundler = await setupDevBundlerSpan.traceAsyncFn(() =>
setupDevBundler({
// Passed here but the initialization of this object happens below, doing the initialization before the setupDev call breaks.
renderServer,
appDir,
pagesDir,
telemetry,
fsChecker,
dir: opts.dir,
nextConfig: config,
isCustomServer: opts.customServer,
turbo: !!process.env.TURBOPACK,
port: opts.port,
})
)
devBundlerService = new DevBundlerService(
developmentBundler,
......@@ -141,6 +148,7 @@ export async function initialize(opts: {
experimentalTestProxy: !!opts.experimentalTestProxy,
experimentalHttpsServer: !!opts.experimentalHttpsServer,
bundlerService: devBundlerService,
startServerSpan: opts.startServerSpan,
}
// pre-initialize workers
......
......@@ -13,6 +13,7 @@ import v8 from 'v8'
import path from 'path'
import http from 'http'
import https from 'https'
import os from 'os'
import Watchpack from 'watchpack'
import * as Log from '../../build/output/log'
import setupDebug from 'next/dist/compiled/debug'
......@@ -22,10 +23,11 @@ import { initialize } from './router-server'
import { CONFIG_FILES } from '../../shared/lib/constants'
import { getStartServerInfo, logStartInfo } from './app-info-log'
import { validateTurboNextConfig } from '../../lib/turbopack-warning'
import { trace } from '../../trace'
import { type Span, trace, flushAllTraces } from '../../trace'
import { isPostpone } from './router-utils/is-postpone'
const debug = setupDebug('next:start-server')
let startServerSpan: Span | undefined
export interface StartServerOptions {
dir: string
......@@ -75,6 +77,7 @@ export async function getRequestHandlers({
keepAliveTimeout,
experimentalTestProxy,
experimentalHttpsServer,
startServerSpan,
})
}
......@@ -153,6 +156,13 @@ export async function startServer(
Log.warn(
`Server is approaching the used memory threshold, restarting...`
)
trace('server-restart-close-to-memory-threshold', undefined, {
'memory.heapSizeLimit': String(
v8.getHeapStatistics().heap_size_limit
),
'memory.heapUsed': String(v8.getHeapStatistics().used_heap_size),
}).stop()
await flushAllTraces()
process.exit(RESTART_EXIT_CODE)
}
}
......@@ -362,9 +372,26 @@ export async function startServer(
if (process.env.NEXT_PRIVATE_WORKER && process.send) {
process.addListener('message', async (msg: any) => {
if (msg && typeof msg && msg.nextWorkerOptions && process.send) {
await trace('start-dev-server').traceAsyncFn(() =>
startServerSpan = trace('start-dev-server', undefined, {
cpus: String(os.cpus().length),
platform: os.platform(),
'memory.freeMem': String(os.freemem()),
'memory.totalMem': String(os.totalmem()),
'memory.heapSizeLimit': String(v8.getHeapStatistics().heap_size_limit),
})
await startServerSpan.traceAsyncFn(() =>
startServer(msg.nextWorkerOptions)
)
const memoryUsage = process.memoryUsage()
startServerSpan.setAttribute('memory.rss', String(memoryUsage.rss))
startServerSpan.setAttribute(
'memory.heapTotal',
String(memoryUsage.heapTotal)
)
startServerSpan.setAttribute(
'memory.heapUsed',
String(memoryUsage.heapUsed)
)
process.send({ nextServerReady: true })
}
})
......
......@@ -19,6 +19,9 @@ const EVENT_FILTER = new Set([
'webpack-invalidated-server',
'navigation-to-hydration',
'start-dev-server',
'compile-path',
'memory-usage',
'server-restart-close-to-memory-threshold',
])
const {
......
......@@ -86,14 +86,14 @@ describe('Trace', () => {
describe('Worker', () => {
it('exports and initializes trace state', () => {
const root = trace('root-span')
expect(root.id).toEqual(1)
expect(root.getId()).toEqual(1)
const traceState = exportTraceState()
expect(traceState.lastId).toEqual(1)
initializeTraceState({
lastId: 101,
})
const span = trace('another-span')
expect(span.id).toEqual(102)
expect(span.getId()).toEqual(102)
})
it('trace data is serializable to a worker', async () => {
......
......@@ -14,22 +14,24 @@ let savedTraceEvents: TraceEvent[] = []
// eslint typescript has a bug with TS enums
/* eslint-disable no-shadow */
export enum SpanStatus {
Started,
Stopped,
Started = 'started',
Stopped = 'stopped',
}
interface Attributes {
[key: string]: string
}
export class Span {
name: string
id: SpanId
parentId?: SpanId
// Duration of the span in *microseconds*.
duration: number | null
attrs: { [key: string]: any }
status: SpanStatus
now: number
private name: string
private id: SpanId
private parentId?: SpanId
private attrs: { [key: string]: any }
private status: SpanStatus
private now: number
// Number of nanoseconds since epoch.
_start: bigint
private _start: bigint
constructor({
name,
......@@ -40,11 +42,10 @@ export class Span {
name: string
parentId?: SpanId
startTime?: bigint
attrs?: Object
attrs?: Attributes
}) {
this.name = name
this.parentId = parentId ?? defaultParentSpanId
this.duration = null
this.attrs = attrs ? { ...attrs } : {}
this.status = SpanStatus.Started
this.id = getId()
......@@ -62,6 +63,11 @@ export class Span {
// Additionally, ~285 years can be safely represented as microseconds as
// a float64 in both JSON and JavaScript.
stop(stopTime?: bigint) {
if (this.status === SpanStatus.Stopped) {
// Don't report the same span twice.
// TODO: In the future this should throw as `.stop()` shouldn't be called multiple times.
return
}
const end: bigint = stopTime || process.hrtime.bigint()
const duration = (end - this._start) / NUM_OF_MICROSEC_IN_NANOSEC
this.status = SpanStatus.Stopped
......@@ -84,7 +90,7 @@ export class Span {
}
}
traceChild(name: string, attrs?: Object) {
traceChild(name: string, attrs?: Attributes) {
return new Span({ name, parentId: this.id, attrs })
}
......@@ -94,14 +100,18 @@ export class Span {
startTime: bigint,
// Stop time in nanoseconds since epoch.
stopTime: bigint,
attrs?: Object
attrs?: Attributes
) {
const span = new Span({ name, parentId: this.id, attrs, startTime })
span.stop(stopTime)
}
setAttribute(key: string, value: any) {
this.attrs[key] = String(value)
getId() {
return this.id
}
setAttribute(key: string, value: string) {
this.attrs[key] = value
}
traceFn<T>(fn: (span: Span) => T): T {
......
......@@ -4,12 +4,14 @@ export default function uploadTrace({
isTurboSession,
projectDir,
distDir,
sync,
}: {
traceUploadUrl: string
mode: 'dev'
isTurboSession: boolean
projectDir: string
distDir: string
sync?: boolean
}) {
const { NEXT_TRACE_UPLOAD_DEBUG } = process.env
......@@ -20,9 +22,10 @@ export default function uploadTrace({
// we use spawnSync when debugging to ensure logs are piped
// correctly to stdout/stderr
const spawn = NEXT_TRACE_UPLOAD_DEBUG
? child_process.spawnSync
: child_process.spawn
const spawn =
NEXT_TRACE_UPLOAD_DEBUG || sync
? child_process.spawnSync
: child_process.spawn
spawn(
process.execPath,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment