Commit 05f71f96 authored by Jude Gao's avatar Jude Gao Committed by Jiachi Liu

Fix telemetry event loss on build failures and server shutdown (#85867)

## Problem

Telemetry events were not being captured in three scenarios:

### 1. MCP Telemetry Lost on Dev Server Shutdown

Two telemetry instances were created during dev server startup. MCP
events were recorded to one instance but shutdown flushed a different
instance.

**Fix:** Reuse the existing telemetry instance from `traceGlobals`.

### 2. Process Exit Timeout Killing Async Telemetry

The CLI force-kills child processes after 100ms
(`NEXT_EXIT_TIMEOUT_MS`). Async telemetry operations were interrupted
before completion.

**Fix:** Use `flushDetached()` to write events to disk and spawn a
detached process for async submission, allowing immediate parent exit.

### 3. Build Errors Calling `process.exit()` Before Telemetry Flush

`process.exit()` calls in error handlers (SWC failures, missing
dependencies, route conflicts) bypassed finally blocks and killed
processes before telemetry completed.

**Fix:** Throw errors instead of calling `process.exit()`, add finally
blocks with `await telemetry.flush()` in critical paths.

## Changes

**Core:**
- `server/dev/next-dev-server.ts`: Reuse telemetry instance
- `server/lib/start-server.ts`: Use `flushDetached()` for shutdown
- `build/swc/index.ts`: Remove `process.exit()` calls
- `build/turbopack-build/impl.ts`: Add telemetry flush in worker finally
block
- `build/index.ts`: Add telemetry flush in finally block, throw instead
of exit
- `lib/verify-partytown-setup.ts`: Throw instead of exit

**Supporting:**
- `telemetry/storage.ts`: Generate unique event files per PID
- `telemetry/detached-flush.ts`: Accept optional eventsFile parameter
- `errors.json`: Update detached-flush error message

## Testing

All telemetry tests now pass:
- MCP telemetry on dev server shutdown
- SWC load failures in worker threads
- Build configuration errors

✅ Non-blocking shutdown  
✅ Timeout-independent  
✅ No race conditions  
✅ Works in worker threads
parent 78e7f43f
......@@ -905,5 +905,7 @@
"904": "The file \"%s\" must export a function, either as a default export or as a named \"%s\" export.",
"905": "Page \"%s\" cannot use \\`export const unstable_prefetch = ...\\` without enabling \\`cacheComponents\\`.",
"906": "Bindings not loaded yet, but they are being loaded, did you forget to await?",
"907": "bindings not loaded yet. Either call `loadBindings` to wait for them to be available or ensure that `installBindings` has already been called."
"907": "bindings not loaded yet. Either call `loadBindings` to wait for them to be available or ensure that `installBindings` has already been called.",
"908": "Invalid flags should be run as node detached-flush dev ./path-to/project [eventsFile]",
"909": "Failed to load SWC binary for %s/%s, see more info here: https://nextjs.org/docs/messages/failed-loading-swc"
}
......@@ -1151,11 +1151,11 @@ export default async function build(
}
if (appDir && 'exportPathMap' in config) {
Log.error(
const errorMessage =
'The "exportPathMap" configuration cannot be used with the "app" directory. Please use generateStaticParams() instead.'
)
Log.error(errorMessage)
await telemetry.flush()
process.exit(1)
throw new Error(errorMessage)
}
const validFileMatcher = createValidFileMatcher(
......@@ -1495,16 +1495,15 @@ export default async function build(
if (bundler !== Bundler.Turbopack) {
const numConflictingAppPaths = conflictingAppPagePaths.length
if (mappedAppPages && numConflictingAppPaths > 0) {
Log.error(
`Conflicting app and page file${
numConflictingAppPaths === 1 ? ' was' : 's were'
} found, please remove the conflicting files to continue:`
)
const errorMessage = `Conflicting app and page file${
numConflictingAppPaths === 1 ? ' was' : 's were'
} found, please remove the conflicting files to continue:`
Log.error(errorMessage)
for (const [pagePath, appPath] of conflictingAppPagePaths) {
Log.error(` "${pagePath}" - "${appPath}"`)
}
await telemetry.flush()
process.exit(1)
throw new Error(errorMessage)
}
}
......@@ -1835,7 +1834,7 @@ export default async function build(
})
.catch((err) => {
console.error(err)
process.exit(1)
throw err
})
})
}
......@@ -2702,7 +2701,7 @@ export default async function build(
outputFileTracingRoot,
}).catch((err) => {
console.error(err)
process.exit(1)
throw err
})
})
}
......@@ -4304,6 +4303,12 @@ export default async function build(
// Ensure we wait for lockfile patching if present
await lockfilePatchPromise.cur
// Flush telemetry before finishing (waits for async operations like setTimeout in debug mode)
const telemetry: Telemetry | undefined = traceGlobals.get('telemetry')
if (telemetry) {
await telemetry.flush()
}
// Ensure all traces are flushed before finishing the command
await flushAllTraces()
teardownTraceSubscriber()
......
......@@ -224,7 +224,7 @@ export async function loadBindings(
process.stderr._handle.setBlocking?.(true)
}
pendingBindings = new Promise(async (resolve, _reject) => {
pendingBindings = new Promise(async (resolve, reject) => {
if (!lockfilePatchPromise.cur) {
// always run lockfile check once so that it gets patched
// even if it doesn't fail to load locally
......@@ -294,7 +294,13 @@ export async function loadBindings(
}
}
logLoadFailure(attempts, true)
await logLoadFailure(attempts, true)
// Reject the promise to propagate the error (process.exit was removed to allow telemetry flush)
reject(
new Error(
`Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
)
)
})
loadedBindings = await pendingBindings
pendingBindings = undefined
......@@ -380,13 +386,22 @@ function loadBindingsSync() {
attempts = attempts.concat(a)
}
// Fire-and-forget telemetry logging (loadBindingsSync must remain synchronous)
// Worker error handler will await telemetry.flush() before exit
logLoadFailure(attempts)
throw new Error('Failed to load bindings', { cause: attempts })
}
let loggingLoadFailure = false
function logLoadFailure(attempts: any, triedWasm = false) {
/**
* Logs SWC load failure telemetry and error messages.
*
* Note: Does NOT call process.exit() - errors must propagate to caller's error handler
* which will await telemetry.flush() before exit (critical for worker threads with async telemetry).
*/
async function logLoadFailure(attempts: any, triedWasm = false) {
// make sure we only emit the event and log the failure once
if (loggingLoadFailure) return
loggingLoadFailure = true
......@@ -396,17 +411,15 @@ function logLoadFailure(attempts: any, triedWasm = false) {
}
// @ts-expect-error TODO: this event has a wrong type.
eventSwcLoadFailure({
await eventSwcLoadFailure({
wasm: triedWasm ? 'failed' : undefined,
nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode,
})
.then(() => lockfilePatchPromise.cur || Promise.resolve())
.finally(() => {
Log.error(
`Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
)
process.exit(1)
})
await (lockfilePatchPromise.cur || Promise.resolve())
Log.error(
`Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
)
}
type RustifiedEnv = { name: string; value: string }[]
......
......@@ -245,15 +245,20 @@ export async function workerMain(workerData: {
// Install bindings early so we can access synchronously later
await installBindings(config.experimental?.useWasmBinary)
const {
shutdownPromise: resultShutdownPromise,
buildTraceContext,
duration,
} = await turbopackBuild()
shutdownPromise = resultShutdownPromise
return {
buildTraceContext,
duration,
try {
const {
shutdownPromise: resultShutdownPromise,
buildTraceContext,
duration,
} = await turbopackBuild()
shutdownPromise = resultShutdownPromise
return {
buildTraceContext,
duration,
}
} finally {
// Always flush telemetry before worker exits (waits for async operations like setTimeout in debug mode)
await telemetry.flush()
}
}
......
......@@ -91,7 +91,8 @@ export async function verifyPartytownSetup(
// Don't show a stack trace when there is an error due to missing dependencies
if (err instanceof FatalError) {
console.error(err.message)
process.exit(1)
// Throw to allow finally blocks to run (e.g., telemetry flush)
throw err
}
throw err
}
......
......@@ -40,6 +40,7 @@ import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix'
import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix'
import { Telemetry } from '../../telemetry/storage'
import { type Span, setGlobal, trace } from '../../trace'
import { traceGlobals } from '../../trace/shared'
import { findPageFile } from '../lib/find-page-file'
import { getFormattedNodeOptionsWithoutInspect } from '../lib/utils'
import { withCoalescedInvoke } from '../../lib/coalesced-function'
......@@ -289,7 +290,12 @@ export default class DevServer extends Server {
setGlobal('distDir', this.distDir)
setGlobal('phase', PHASE_DEVELOPMENT_SERVER)
const telemetry = new Telemetry({ distDir: this.distDir })
// Use existing telemetry instance from traceGlobals instead of creating a new one.
// Creating a new instance would overwrite the existing one, causing any telemetry
// events recorded to the original instance to be lost during cleanup/flush.
const existingTelemetry = traceGlobals.get('telemetry')
const telemetry =
existingTelemetry || new Telemetry({ distDir: this.distDir })
await super.prepareImpl()
await this.matchers.reload()
......@@ -303,7 +309,10 @@ export default class DevServer extends Server {
// This is required by the tracing subsystem.
setGlobal('appDir', this.appDir)
setGlobal('pagesDir', this.pagesDir)
setGlobal('telemetry', telemetry)
// Only set telemetry if it wasn't already set
if (!existingTelemetry) {
setGlobal('telemetry', telemetry)
}
process.on('unhandledRejection', (reason) => {
if (isPostpone(reason)) {
......
......@@ -417,7 +417,10 @@ export async function startServer(
>
| undefined
if (telemetry) {
await telemetry.flush()
// Use flushDetached to avoid blocking process exit
// Each process writes to a unique file (_events_${pid}.json)
// to avoid race conditions with the parent process
telemetry.flushDetached('dev', dir)
}
} catch (_) {
// Ignore telemetry errors during cleanup
......
......@@ -9,21 +9,27 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
// this process should be started with following arg order
// 1. mode e.g. dev, export, start
// 2. project dir
// 3. events filename (optional, defaults to _events.json)
;(async () => {
const args = [...process.argv]
const eventsFile = args.pop()
let dir = args.pop()
const mode = args.pop()
if (!dir || mode !== 'dev') {
throw new Error(
`Invalid flags should be run as node detached-flush dev ./path-to/project`
`Invalid flags should be run as node detached-flush dev ./path-to/project [eventsFile]`
)
}
dir = getProjectDir(dir)
const config = await loadConfig(PHASE_DEVELOPMENT_SERVER, dir)
const distDir = path.join(dir, config.distDir || '.next')
const eventsPath = path.join(distDir, '_events.json')
// Support both old format (no eventsFile arg) and new format (with eventsFile arg)
const eventsPath = path.join(
distDir,
eventsFile && !eventsFile.includes('/') ? eventsFile : '_events.json'
)
let events: TelemetryEvent[]
try {
......@@ -40,7 +46,8 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
await telemetry.record(events)
await telemetry.flush()
// finished flushing events clean-up/exit
// finished flushing events clean-up
fs.unlinkSync(eventsPath)
process.exit(0)
// Don't call process.exit() here - let Node.js exit naturally after
// all pending work completes (e.g., setTimeout in debug telemetry)
})()
......@@ -216,7 +216,9 @@ export class Telemetry {
return prom
}
flush = async () => Promise.all(this.queue).catch(() => null)
flush = async () => {
return Promise.all(this.queue).catch(() => null)
}
// writes current events to disk and spawns separate
// detached process to submit the records without blocking
......@@ -232,9 +234,17 @@ export class Telemetry {
// if we fail to abort ignore this event
}
})
if (allEvents.length === 0) {
// No events to flush
return
}
fs.mkdirSync(this.distDir, { recursive: true })
// Use unique filename per process to avoid race conditions between parent/child
const eventsFile = `_events_${process.pid}.json`
fs.writeFileSync(
path.join(this.distDir, '_events.json'),
path.join(this.distDir, eventsFile),
JSON.stringify(allEvents)
)
......@@ -249,16 +259,20 @@ export class Telemetry {
? child_process.spawnSync
: child_process.spawn
spawn(process.execPath, [require.resolve('./detached-flush'), mode, dir], {
detached: !this.NEXT_TELEMETRY_DEBUG,
windowsHide: true,
shell: false,
...(this.NEXT_TELEMETRY_DEBUG
? {
stdio: 'inherit',
}
: {}),
})
spawn(
process.execPath,
[require.resolve('./detached-flush'), mode, dir, eventsFile],
{
detached: !this.NEXT_TELEMETRY_DEBUG,
windowsHide: true,
shell: false,
...(this.NEXT_TELEMETRY_DEBUG
? {
stdio: 'inherit',
}
: {}),
}
)
}
private submitRecord = async (
......@@ -276,15 +290,19 @@ export class Telemetry {
}
if (this.NEXT_TELEMETRY_DEBUG) {
// Print to standard error to simplify selecting the output
events.forEach(({ eventName, payload }) =>
console.error(
`[telemetry] ` + JSON.stringify({ eventName, payload }, null, 2)
)
)
// Do not send the telemetry data if debugging. Users may use this feature
// to preview what data would be sent.
return Promise.resolve()
// Return a promise that resolves after logging to ensure the output
// is captured before the process exits (e.g., during flushDetached)
return new Promise((resolve) => {
setTimeout(() => {
// Print to standard error to simplify selecting the output
events.forEach(({ eventName, payload }) =>
console.error(
`[telemetry] ` + JSON.stringify({ eventName, payload }, null, 2)
)
)
resolve(undefined)
}, 100)
})
}
// Skip recording telemetry if the feature is disabled
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment