Unverified Commit a01f8255 authored by mknichel's avatar mknichel Committed by GitHub

Add a mode to next build to make it easier to debug memory issues (#63869)

This PR adds a `--experimental-debug-memory-usage` flag to `next build`
to make it easier to debug memory performance. This mode does the
following things:

- Periodically prints the current memory usage of the process
- Records garbage collection events and warns about long running GC
events
- Kills the process if it detects GC thrashing near heap limit
- Automatically takes a heap snapshot if heap usage rises above 70% of
the total heap
- Automatically takes a heap snapshot if the process is close to running
out of memory
- Prints a report at the end of the build with information about peak
memory usage and time spent in garbage collection

---------
Co-authored-by: 's avatarJJ Kasper <jj@jjsweb.site>
parent 5e2ac098
...@@ -105,6 +105,10 @@ program ...@@ -105,6 +105,10 @@ program
)}` )}`
) )
.option('-d, --debug', 'Enables a more verbose build output.') .option('-d, --debug', 'Enables a more verbose build output.')
.option(
'--experimental-debug-memory-usage',
'Enables memory profiling features to debug memory consumption.'
)
.option('--profile', 'Enables production profiling for React.') .option('--profile', 'Enables production profiling for React.')
.option('--no-lint', 'Disables linting.') .option('--no-lint', 'Disables linting.')
.option('--no-mangling', 'Disables mangling.') .option('--no-mangling', 'Disables mangling.')
......
...@@ -182,6 +182,7 @@ import { TurbopackManifestLoader } from '../server/dev/turbopack/manifest-loader ...@@ -182,6 +182,7 @@ import { TurbopackManifestLoader } from '../server/dev/turbopack/manifest-loader
import type { Entrypoints } from '../server/dev/turbopack/types' import type { Entrypoints } from '../server/dev/turbopack/types'
import { buildCustomRoute } from '../lib/build-custom-route' import { buildCustomRoute } from '../lib/build-custom-route'
import { createProgress } from './progress' import { createProgress } from './progress'
import { traceMemoryUsage } from '../lib/memory/trace'
import { generateEncryptionKeyBase64 } from '../server/app-render/encryption-utils' import { generateEncryptionKeyBase64 } from '../server/app-render/encryption-utils'
interface ExperimentalBypassForInfo { interface ExperimentalBypassForInfo {
...@@ -1580,6 +1581,7 @@ export default async function build( ...@@ -1580,6 +1581,7 @@ export default async function build(
} }
Log.info('Creating an optimized production build ...') Log.info('Creating an optimized production build ...')
traceMemoryUsage('Starting build', nextBuildSpan)
if (!isGenerateMode) { if (!isGenerateMode) {
if (runServerAndEdgeInParallel || collectServerBuildTracesInParallel) { if (runServerAndEdgeInParallel || collectServerBuildTracesInParallel) {
...@@ -1588,6 +1590,7 @@ export default async function build( ...@@ -1588,6 +1590,7 @@ export default async function build(
const serverBuildPromise = webpackBuild(useBuildWorker, [ const serverBuildPromise = webpackBuild(useBuildWorker, [
'server', 'server',
]).then((res) => { ]).then((res) => {
traceMemoryUsage('Finished server compilation', nextBuildSpan)
buildTraceContext = res.buildTraceContext buildTraceContext = res.buildTraceContext
durationInSeconds += res.duration durationInSeconds += res.duration
...@@ -1626,6 +1629,7 @@ export default async function build( ...@@ -1626,6 +1629,7 @@ export default async function build(
'edge-server', 'edge-server',
]).then((res) => { ]).then((res) => {
durationInSeconds += res.duration durationInSeconds += res.duration
traceMemoryUsage('Finished edge-server compilation', nextBuildSpan)
}) })
if (runServerAndEdgeInParallel) { if (runServerAndEdgeInParallel) {
await serverBuildPromise await serverBuildPromise
...@@ -1634,6 +1638,7 @@ export default async function build( ...@@ -1634,6 +1638,7 @@ export default async function build(
await webpackBuild(useBuildWorker, ['client']).then((res) => { await webpackBuild(useBuildWorker, ['client']).then((res) => {
durationInSeconds += res.duration durationInSeconds += res.duration
traceMemoryUsage('Finished client compilation', nextBuildSpan)
}) })
Log.event('Compiled successfully') Log.event('Compiled successfully')
...@@ -1648,6 +1653,7 @@ export default async function build( ...@@ -1648,6 +1653,7 @@ export default async function build(
const { duration: compilerDuration, ...rest } = turboNextBuild const { duration: compilerDuration, ...rest } = turboNextBuild
? await turbopackBuild() ? await turbopackBuild()
: await webpackBuild(useBuildWorker, null) : await webpackBuild(useBuildWorker, null)
traceMemoryUsage('Finished build', nextBuildSpan)
buildTraceContext = rest.buildTraceContext buildTraceContext = rest.buildTraceContext
...@@ -1663,6 +1669,7 @@ export default async function build( ...@@ -1663,6 +1669,7 @@ export default async function build(
// For app directory, we run type checking after build. // For app directory, we run type checking after build.
if (appDir && !isCompileMode && !isGenerateMode) { if (appDir && !isCompileMode && !isGenerateMode) {
await startTypeChecking(typeCheckingOptions) await startTypeChecking(typeCheckingOptions)
traceMemoryUsage('Finished type checking', nextBuildSpan)
} }
const postCompileSpinner = createSpinner('Collecting page data') const postCompileSpinner = createSpinner('Collecting page data')
...@@ -2302,6 +2309,7 @@ export default async function build( ...@@ -2302,6 +2309,7 @@ export default async function build(
}) })
if (postCompileSpinner) postCompileSpinner.stopAndPersist() if (postCompileSpinner) postCompileSpinner.stopAndPersist()
traceMemoryUsage('Finished collecting page data', nextBuildSpan)
if (customAppGetInitialProps) { if (customAppGetInitialProps) {
console.warn( console.warn(
......
...@@ -8,12 +8,15 @@ import { warn } from '../build/output/log' ...@@ -8,12 +8,15 @@ import { warn } from '../build/output/log'
import { printAndExit } from '../server/lib/utils' import { printAndExit } from '../server/lib/utils'
import isError from '../lib/is-error' import isError from '../lib/is-error'
import { getProjectDir } from '../lib/get-project-dir' import { getProjectDir } from '../lib/get-project-dir'
import { enableMemoryDebuggingMode } from '../lib/memory/startup'
import { disableMemoryDebuggingMode } from '../lib/memory/shutdown'
type NextBuildOptions = { type NextBuildOptions = {
debug?: boolean debug?: boolean
profile?: boolean profile?: boolean
lint: boolean lint: boolean
mangling: boolean mangling: boolean
experimentalDebugMemoryUsage: boolean
experimentalAppOnly?: boolean experimentalAppOnly?: boolean
experimentalTurbo?: boolean experimentalTurbo?: boolean
experimentalBuildMode: 'default' | 'compile' | 'generate' experimentalBuildMode: 'default' | 'compile' | 'generate'
...@@ -25,6 +28,7 @@ const nextBuild = (options: NextBuildOptions, directory?: string) => { ...@@ -25,6 +28,7 @@ const nextBuild = (options: NextBuildOptions, directory?: string) => {
const { const {
debug, debug,
experimentalDebugMemoryUsage,
profile, profile,
lint, lint,
mangling, mangling,
...@@ -49,6 +53,11 @@ const nextBuild = (options: NextBuildOptions, directory?: string) => { ...@@ -49,6 +53,11 @@ const nextBuild = (options: NextBuildOptions, directory?: string) => {
) )
} }
if (experimentalDebugMemoryUsage) {
process.env.EXPERIMENTAL_DEBUG_MEMORY_USAGE = '1'
enableMemoryDebuggingMode()
}
const dir = getProjectDir(directory) const dir = getProjectDir(directory)
// Check if the provided directory exists // Check if the provided directory exists
...@@ -69,23 +78,32 @@ const nextBuild = (options: NextBuildOptions, directory?: string) => { ...@@ -69,23 +78,32 @@ const nextBuild = (options: NextBuildOptions, directory?: string) => {
experimentalAppOnly, experimentalAppOnly,
!!process.env.TURBOPACK, !!process.env.TURBOPACK,
experimentalBuildMode experimentalBuildMode
).catch((err) => { )
console.error('') .catch((err) => {
if ( if (experimentalDebugMemoryUsage) {
isError(err) && disableMemoryDebuggingMode()
(err.code === 'INVALID_RESOLVE_ALIAS' || }
err.code === 'WEBPACK_ERRORS' || console.error('')
err.code === 'BUILD_OPTIMIZATION_FAILED' || if (
err.code === 'NEXT_EXPORT_ERROR' || isError(err) &&
err.code === 'NEXT_STATIC_GEN_BAILOUT' || (err.code === 'INVALID_RESOLVE_ALIAS' ||
err.code === 'EDGE_RUNTIME_UNSUPPORTED_API') err.code === 'WEBPACK_ERRORS' ||
) { err.code === 'BUILD_OPTIMIZATION_FAILED' ||
printAndExit(`> ${err.message}`) err.code === 'NEXT_EXPORT_ERROR' ||
} else { err.code === 'NEXT_STATIC_GEN_BAILOUT' ||
console.error('> Build error occurred') err.code === 'EDGE_RUNTIME_UNSUPPORTED_API')
printAndExit(err) ) {
} printAndExit(`> ${err.message}`)
}) } else {
console.error('> Build error occurred')
printAndExit(err)
}
})
.finally(() => {
if (experimentalDebugMemoryUsage) {
disableMemoryDebuggingMode()
}
})
} }
export { nextBuild } export { nextBuild }
# Debug Memory Usage
This directory contains a number of utilities to help in debugging
memory usage.
When enabled, code in this directory will:
- Periodically print out memory usage statistics
- Print a report with a summary of memory usage and suggestions for how to
improve.
- Generate heap snapshots automatically when too much memory is being consumed
- Monitor garbage collection events for long running GCs
import { PerformanceObserver } from 'perf_hooks'
import { warn } from '../../build/output/log'
import { bold } from '../picocolors'
const LONG_RUNNING_GC_THRESHOLD_MS = 15
const gcEvents: PerformanceEntry[] = []
const obs = new PerformanceObserver((list) => {
const entry = list.getEntries()[0]
gcEvents.push(entry)
if (entry.duration > LONG_RUNNING_GC_THRESHOLD_MS) {
warn(bold(`Long running GC detected: ${entry.duration.toFixed(2)}ms`))
}
})
/**
* Starts recording garbage collection events in the process and warn on long
* running GCs. To disable, call `stopObservingGc`.
*/
export function startObservingGc() {
obs.observe({ entryTypes: ['gc'] })
}
export function stopObservingGc() {
obs.disconnect()
}
/**
* Returns all recorded garbage collection events. This function will only
* return information from when `startObservingGc` was enabled and before
* `stopObservingGc` was called.
*/
export function getGcEvents() {
return gcEvents
}
import { info } from '../../build/output/log'
import { bold } from '../picocolors'
import { getGcEvents, stopObservingGc } from './gc-observer'
import { getAllMemoryUsageSpans, stopPeriodicMemoryUsageTracing } from './trace'
export function disableMemoryDebuggingMode(): void {
stopPeriodicMemoryUsageTracing()
stopObservingGc()
info(bold('Memory usage report:'))
const gcEvents = getGcEvents()
const totalTimeInGcMs = gcEvents.reduce(
(acc, event) => acc + event.duration,
0
)
info(` - Total time spent in GC: ${totalTimeInGcMs.toFixed(2)}ms`)
const allMemoryUsage = getAllMemoryUsageSpans()
const peakHeapUsage = Math.max(
...allMemoryUsage.map((usage) => usage['memory.heapUsed'])
)
const peakRssUsage = Math.max(
...allMemoryUsage.map((usage) => usage['memory.rss'])
)
info(` - Peak heap usage: ${(peakHeapUsage / 1024 / 1024).toFixed(2)} MB`)
info(` - Peak RSS usage: ${(peakRssUsage / 1024 / 1024).toFixed(2)} MB`)
}
import v8 from 'v8'
import { info, warn } from '../../build/output/log'
import { italic } from '../picocolors'
import { startObservingGc } from './gc-observer'
import { startPeriodicMemoryUsageTracing } from './trace'
export function enableMemoryDebuggingMode(): void {
// This will generate a heap snapshot when the program is close to the
// memory limit. It does not give any warning to the user though which
// can be jarring. If memory is large, this may take a long time.
if ('setHeapSnapshotNearHeapLimit' in v8) {
// @ts-expect-error - this method exists since Node 16.
v8.setHeapSnapshotNearHeapLimit(1)
}
// This flag will kill the process when it starts to GC thrash when it's
// close to the memory limit rather than continuing to try to collect
// memory ineffectively.
v8.setFlagsFromString('--detect-ineffective-gcs-near-heap-limit')
// This allows users to generate a heap snapshot on demand just by sending
// a signal to the process.
process.on('SIGUSR2', () => {
warn(
`Received SIGUSR2 signal. Generating heap snapshot. ${italic(
'Note: this will take some time.'
)}`
)
v8.writeHeapSnapshot()
})
startObservingGc()
startPeriodicMemoryUsageTracing()
warn(
`Memory debugging mode is enabled. ${italic(
'Note: This will affect performance.'
)}`
)
info(
' - Heap snapshots will be automatically generated when the process reaches more than 70% of the memory limit and again when the process is just about to run out of memory.'
)
info(
` - To manually generate a heap snapshot, send the process a SIGUSR2 signal: \`kill -SIGUSR2 ${process.pid}\``
)
info(
' - Heap snapshots when there is high memory will take a very long time to complete and may be difficult to analyze in tools.'
)
info(
' - See https://nextjs.org/docs/app/building-your-application/optimizing/memory-usage for more information.'
)
}
import v8 from 'v8'
import { info, warn } from '../../build/output/log'
import { type Span, trace } from '../../trace'
import { bold, italic } from '../picocolors'
import { join } from 'path'
import { traceGlobals } from '../../trace/shared'
const HEAP_SNAPSHOT_THRESHOLD_PERCENT = 70
let alreadyGeneratedHeapSnapshot = false
const TRACE_MEMORY_USAGE_TIMER_MS = 20000
let traceMemoryUsageTimer: NodeJS.Timeout | undefined
interface MemoryUsage {
'memory.rss': number
'memory.heapUsed': number
'memory.heapTotal': number
'memory.heapMax': number
}
const allMemoryUsage: MemoryUsage[] = []
/**
* Begins a timer that will record memory usage periodically to understand
* memory usage across the lifetime of the process.
*/
export function startPeriodicMemoryUsageTracing(): void {
traceMemoryUsageTimer = setTimeout(() => {
traceMemoryUsage('periodic memory snapshot')
startPeriodicMemoryUsageTracing()
}, TRACE_MEMORY_USAGE_TIMER_MS)
}
export function stopPeriodicMemoryUsageTracing(): void {
if (traceMemoryUsageTimer) {
clearTimeout(traceMemoryUsageTimer)
}
}
/**
* Returns the list of all recorded memory usage snapshots from the process.
*/
export function getAllMemoryUsageSpans(): MemoryUsage[] {
return allMemoryUsage
}
/**
* Records a snapshot of memory usage at this moment in time to the .next/trace
* file.
*/
export function traceMemoryUsage(
description: string,
parentSpan?: Span | undefined
): void {
const memoryUsage = process.memoryUsage()
const v8HeapStatistics = v8.getHeapStatistics()
const heapUsed = v8HeapStatistics.used_heap_size
const heapMax = v8HeapStatistics.heap_size_limit
const tracedMemoryUsage: MemoryUsage = {
'memory.rss': memoryUsage.rss,
'memory.heapUsed': heapUsed,
'memory.heapTotal': memoryUsage.heapTotal,
'memory.heapMax': heapMax,
}
allMemoryUsage.push(tracedMemoryUsage)
const tracedMemoryUsageAsStrings = Object.fromEntries(
Object.entries(tracedMemoryUsage).map(([key, value]) => [
key,
String(value),
])
)
if (parentSpan) {
parentSpan.traceChild('memory-usage', tracedMemoryUsageAsStrings)
} else {
trace('memory-usage', undefined, tracedMemoryUsageAsStrings)
}
if (process.env.EXPERIMENTAL_DEBUG_MEMORY_USAGE) {
const percentageHeapUsed = (100 * heapUsed) / heapMax
info('')
info('***************************************')
info(`Memory usage report at "${description}":`)
info(` - RSS: ${(memoryUsage.rss / 1024 / 1024).toFixed(2)} MB`)
info(` - Heap Used: ${(heapUsed / 1024 / 1024).toFixed(2)} MB`)
info(
` - Heap Total Allocated: ${(memoryUsage.heapTotal / 1024 / 1024).toFixed(
2
)} MB`
)
info(` - Heap Max: ${(heapMax / 1024 / 1024).toFixed(2)} MB`)
info(` - Percentage Heap Used: ${percentageHeapUsed.toFixed(2)}%`)
info('***************************************')
info('')
if (percentageHeapUsed > HEAP_SNAPSHOT_THRESHOLD_PERCENT) {
const distDir = traceGlobals.get('distDir')
const heapFilename = join(
distDir,
`${description.replace(' ', '-')}.heapsnapshot`
)
warn(
bold(
`Heap usage is close to the limit. ${percentageHeapUsed.toFixed(
2
)}% of heap has been used.`
)
)
if (!alreadyGeneratedHeapSnapshot) {
warn(
bold(
`Saving heap snapshot to ${heapFilename}. ${italic(
'Note: this will take some time.'
)}`
)
)
v8.writeHeapSnapshot(heapFilename)
alreadyGeneratedHeapSnapshot = true
} else {
warn(
'Skipping heap snapshot generation since heap snapshot has already been generated.'
)
}
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment