Commit 05f71f96 authored by Jude Gao's avatar Jude Gao Committed by Jiachi Liu

Fix telemetry event loss on build failures and server shutdown (#85867)

## Problem

Telemetry events were not being captured in three scenarios:

### 1. MCP Telemetry Lost on Dev Server Shutdown

Two telemetry instances were created during dev server startup. MCP
events were recorded to one instance but shutdown flushed a different
instance.

**Fix:** Reuse the existing telemetry instance from `traceGlobals`.

### 2. Process Exit Timeout Killing Async Telemetry

The CLI force-kills child processes after 100ms
(`NEXT_EXIT_TIMEOUT_MS`). Async telemetry operations were interrupted
before completion.

**Fix:** Use `flushDetached()` to write events to disk and spawn a
detached process for async submission, allowing immediate parent exit.

### 3. Build Errors Calling `process.exit()` Before Telemetry Flush

`process.exit()` calls in error handlers (SWC failures, missing
dependencies, route conflicts) bypassed finally blocks and killed
processes before telemetry completed.

**Fix:** Throw errors instead of calling `process.exit()`, add finally
blocks with `await telemetry.flush()` in critical paths.

## Changes

**Core:**
- `server/dev/next-dev-server.ts`: Reuse telemetry instance
- `server/lib/start-server.ts`: Use `flushDetached()` for shutdown
- `build/swc/index.ts`: Remove `process.exit()` calls
- `build/turbopack-build/impl.ts`: Add telemetry flush in worker finally
block
- `build/index.ts`: Add telemetry flush in finally block, throw instead
of exit
- `lib/verify-partytown-setup.ts`: Throw instead of exit

**Supporting:**
- `telemetry/storage.ts`: Generate unique event files per PID
- `telemetry/detached-flush.ts`: Accept optional eventsFile parameter
- `errors.json`: Update detached-flush error message

## Testing

All telemetry tests now pass:
- MCP telemetry on dev server shutdown
- SWC load failures in worker threads
- Build configuration errors

✅ Non-blocking shutdown  
✅ Timeout-independent  
✅ No race conditions  
✅ Works in worker threads
parent 78e7f43f
...@@ -905,5 +905,7 @@ ...@@ -905,5 +905,7 @@
"904": "The file \"%s\" must export a function, either as a default export or as a named \"%s\" export.", "904": "The file \"%s\" must export a function, either as a default export or as a named \"%s\" export.",
"905": "Page \"%s\" cannot use \\`export const unstable_prefetch = ...\\` without enabling \\`cacheComponents\\`.", "905": "Page \"%s\" cannot use \\`export const unstable_prefetch = ...\\` without enabling \\`cacheComponents\\`.",
"906": "Bindings not loaded yet, but they are being loaded, did you forget to await?", "906": "Bindings not loaded yet, but they are being loaded, did you forget to await?",
"907": "bindings not loaded yet. Either call `loadBindings` to wait for them to be available or ensure that `installBindings` has already been called." "907": "bindings not loaded yet. Either call `loadBindings` to wait for them to be available or ensure that `installBindings` has already been called.",
"908": "Invalid flags should be run as node detached-flush dev ./path-to/project [eventsFile]",
"909": "Failed to load SWC binary for %s/%s, see more info here: https://nextjs.org/docs/messages/failed-loading-swc"
} }
...@@ -1151,11 +1151,11 @@ export default async function build( ...@@ -1151,11 +1151,11 @@ export default async function build(
} }
if (appDir && 'exportPathMap' in config) { if (appDir && 'exportPathMap' in config) {
Log.error( const errorMessage =
'The "exportPathMap" configuration cannot be used with the "app" directory. Please use generateStaticParams() instead.' 'The "exportPathMap" configuration cannot be used with the "app" directory. Please use generateStaticParams() instead.'
) Log.error(errorMessage)
await telemetry.flush() await telemetry.flush()
process.exit(1) throw new Error(errorMessage)
} }
const validFileMatcher = createValidFileMatcher( const validFileMatcher = createValidFileMatcher(
...@@ -1495,16 +1495,15 @@ export default async function build( ...@@ -1495,16 +1495,15 @@ export default async function build(
if (bundler !== Bundler.Turbopack) { if (bundler !== Bundler.Turbopack) {
const numConflictingAppPaths = conflictingAppPagePaths.length const numConflictingAppPaths = conflictingAppPagePaths.length
if (mappedAppPages && numConflictingAppPaths > 0) { if (mappedAppPages && numConflictingAppPaths > 0) {
Log.error( const errorMessage = `Conflicting app and page file${
`Conflicting app and page file${ numConflictingAppPaths === 1 ? ' was' : 's were'
numConflictingAppPaths === 1 ? ' was' : 's were' } found, please remove the conflicting files to continue:`
} found, please remove the conflicting files to continue:` Log.error(errorMessage)
)
for (const [pagePath, appPath] of conflictingAppPagePaths) { for (const [pagePath, appPath] of conflictingAppPagePaths) {
Log.error(` "${pagePath}" - "${appPath}"`) Log.error(` "${pagePath}" - "${appPath}"`)
} }
await telemetry.flush() await telemetry.flush()
process.exit(1) throw new Error(errorMessage)
} }
} }
...@@ -1835,7 +1834,7 @@ export default async function build( ...@@ -1835,7 +1834,7 @@ export default async function build(
}) })
.catch((err) => { .catch((err) => {
console.error(err) console.error(err)
process.exit(1) throw err
}) })
}) })
} }
...@@ -2702,7 +2701,7 @@ export default async function build( ...@@ -2702,7 +2701,7 @@ export default async function build(
outputFileTracingRoot, outputFileTracingRoot,
}).catch((err) => { }).catch((err) => {
console.error(err) console.error(err)
process.exit(1) throw err
}) })
}) })
} }
...@@ -4304,6 +4303,12 @@ export default async function build( ...@@ -4304,6 +4303,12 @@ export default async function build(
// Ensure we wait for lockfile patching if present // Ensure we wait for lockfile patching if present
await lockfilePatchPromise.cur await lockfilePatchPromise.cur
// Flush telemetry before finishing (waits for async operations like setTimeout in debug mode)
const telemetry: Telemetry | undefined = traceGlobals.get('telemetry')
if (telemetry) {
await telemetry.flush()
}
// Ensure all traces are flushed before finishing the command // Ensure all traces are flushed before finishing the command
await flushAllTraces() await flushAllTraces()
teardownTraceSubscriber() teardownTraceSubscriber()
......
...@@ -224,7 +224,7 @@ export async function loadBindings( ...@@ -224,7 +224,7 @@ export async function loadBindings(
process.stderr._handle.setBlocking?.(true) process.stderr._handle.setBlocking?.(true)
} }
pendingBindings = new Promise(async (resolve, _reject) => { pendingBindings = new Promise(async (resolve, reject) => {
if (!lockfilePatchPromise.cur) { if (!lockfilePatchPromise.cur) {
// always run lockfile check once so that it gets patched // always run lockfile check once so that it gets patched
// even if it doesn't fail to load locally // even if it doesn't fail to load locally
...@@ -294,7 +294,13 @@ export async function loadBindings( ...@@ -294,7 +294,13 @@ export async function loadBindings(
} }
} }
logLoadFailure(attempts, true) await logLoadFailure(attempts, true)
// Reject the promise to propagate the error (process.exit was removed to allow telemetry flush)
reject(
new Error(
`Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
)
)
}) })
loadedBindings = await pendingBindings loadedBindings = await pendingBindings
pendingBindings = undefined pendingBindings = undefined
...@@ -380,13 +386,22 @@ function loadBindingsSync() { ...@@ -380,13 +386,22 @@ function loadBindingsSync() {
attempts = attempts.concat(a) attempts = attempts.concat(a)
} }
// Fire-and-forget telemetry logging (loadBindingsSync must remain synchronous)
// Worker error handler will await telemetry.flush() before exit
logLoadFailure(attempts) logLoadFailure(attempts)
throw new Error('Failed to load bindings', { cause: attempts }) throw new Error('Failed to load bindings', { cause: attempts })
} }
let loggingLoadFailure = false let loggingLoadFailure = false
function logLoadFailure(attempts: any, triedWasm = false) { /**
* Logs SWC load failure telemetry and error messages.
*
* Note: Does NOT call process.exit() - errors must propagate to caller's error handler
* which will await telemetry.flush() before exit (critical for worker threads with async telemetry).
*/
async function logLoadFailure(attempts: any, triedWasm = false) {
// make sure we only emit the event and log the failure once // make sure we only emit the event and log the failure once
if (loggingLoadFailure) return if (loggingLoadFailure) return
loggingLoadFailure = true loggingLoadFailure = true
...@@ -396,17 +411,15 @@ function logLoadFailure(attempts: any, triedWasm = false) { ...@@ -396,17 +411,15 @@ function logLoadFailure(attempts: any, triedWasm = false) {
} }
// @ts-expect-error TODO: this event has a wrong type. // @ts-expect-error TODO: this event has a wrong type.
eventSwcLoadFailure({ await eventSwcLoadFailure({
wasm: triedWasm ? 'failed' : undefined, wasm: triedWasm ? 'failed' : undefined,
nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode, nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode,
}) })
.then(() => lockfilePatchPromise.cur || Promise.resolve()) await (lockfilePatchPromise.cur || Promise.resolve())
.finally(() => {
Log.error( Log.error(
`Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc` `Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
) )
process.exit(1)
})
} }
type RustifiedEnv = { name: string; value: string }[] type RustifiedEnv = { name: string; value: string }[]
......
...@@ -245,15 +245,20 @@ export async function workerMain(workerData: { ...@@ -245,15 +245,20 @@ export async function workerMain(workerData: {
// Install bindings early so we can access synchronously later // Install bindings early so we can access synchronously later
await installBindings(config.experimental?.useWasmBinary) await installBindings(config.experimental?.useWasmBinary)
const { try {
shutdownPromise: resultShutdownPromise, const {
buildTraceContext, shutdownPromise: resultShutdownPromise,
duration, buildTraceContext,
} = await turbopackBuild() duration,
shutdownPromise = resultShutdownPromise } = await turbopackBuild()
return { shutdownPromise = resultShutdownPromise
buildTraceContext, return {
duration, buildTraceContext,
duration,
}
} finally {
// Always flush telemetry before worker exits (waits for async operations like setTimeout in debug mode)
await telemetry.flush()
} }
} }
......
...@@ -91,7 +91,8 @@ export async function verifyPartytownSetup( ...@@ -91,7 +91,8 @@ export async function verifyPartytownSetup(
// Don't show a stack trace when there is an error due to missing dependencies // Don't show a stack trace when there is an error due to missing dependencies
if (err instanceof FatalError) { if (err instanceof FatalError) {
console.error(err.message) console.error(err.message)
process.exit(1) // Throw to allow finally blocks to run (e.g., telemetry flush)
throw err
} }
throw err throw err
} }
......
...@@ -40,6 +40,7 @@ import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix' ...@@ -40,6 +40,7 @@ import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix'
import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix' import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix'
import { Telemetry } from '../../telemetry/storage' import { Telemetry } from '../../telemetry/storage'
import { type Span, setGlobal, trace } from '../../trace' import { type Span, setGlobal, trace } from '../../trace'
import { traceGlobals } from '../../trace/shared'
import { findPageFile } from '../lib/find-page-file' import { findPageFile } from '../lib/find-page-file'
import { getFormattedNodeOptionsWithoutInspect } from '../lib/utils' import { getFormattedNodeOptionsWithoutInspect } from '../lib/utils'
import { withCoalescedInvoke } from '../../lib/coalesced-function' import { withCoalescedInvoke } from '../../lib/coalesced-function'
...@@ -289,7 +290,12 @@ export default class DevServer extends Server { ...@@ -289,7 +290,12 @@ export default class DevServer extends Server {
setGlobal('distDir', this.distDir) setGlobal('distDir', this.distDir)
setGlobal('phase', PHASE_DEVELOPMENT_SERVER) setGlobal('phase', PHASE_DEVELOPMENT_SERVER)
const telemetry = new Telemetry({ distDir: this.distDir }) // Use existing telemetry instance from traceGlobals instead of creating a new one.
// Creating a new instance would overwrite the existing one, causing any telemetry
// events recorded to the original instance to be lost during cleanup/flush.
const existingTelemetry = traceGlobals.get('telemetry')
const telemetry =
existingTelemetry || new Telemetry({ distDir: this.distDir })
await super.prepareImpl() await super.prepareImpl()
await this.matchers.reload() await this.matchers.reload()
...@@ -303,7 +309,10 @@ export default class DevServer extends Server { ...@@ -303,7 +309,10 @@ export default class DevServer extends Server {
// This is required by the tracing subsystem. // This is required by the tracing subsystem.
setGlobal('appDir', this.appDir) setGlobal('appDir', this.appDir)
setGlobal('pagesDir', this.pagesDir) setGlobal('pagesDir', this.pagesDir)
setGlobal('telemetry', telemetry) // Only set telemetry if it wasn't already set
if (!existingTelemetry) {
setGlobal('telemetry', telemetry)
}
process.on('unhandledRejection', (reason) => { process.on('unhandledRejection', (reason) => {
if (isPostpone(reason)) { if (isPostpone(reason)) {
......
...@@ -417,7 +417,10 @@ export async function startServer( ...@@ -417,7 +417,10 @@ export async function startServer(
> >
| undefined | undefined
if (telemetry) { if (telemetry) {
await telemetry.flush() // Use flushDetached to avoid blocking process exit
// Each process writes to a unique file (_events_${pid}.json)
// to avoid race conditions with the parent process
telemetry.flushDetached('dev', dir)
} }
} catch (_) { } catch (_) {
// Ignore telemetry errors during cleanup // Ignore telemetry errors during cleanup
......
...@@ -9,21 +9,27 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' ...@@ -9,21 +9,27 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
// this process should be started with following arg order // this process should be started with following arg order
// 1. mode e.g. dev, export, start // 1. mode e.g. dev, export, start
// 2. project dir // 2. project dir
// 3. events filename (optional, defaults to _events.json)
;(async () => { ;(async () => {
const args = [...process.argv] const args = [...process.argv]
const eventsFile = args.pop()
let dir = args.pop() let dir = args.pop()
const mode = args.pop() const mode = args.pop()
if (!dir || mode !== 'dev') { if (!dir || mode !== 'dev') {
throw new Error( throw new Error(
`Invalid flags should be run as node detached-flush dev ./path-to/project` `Invalid flags should be run as node detached-flush dev ./path-to/project [eventsFile]`
) )
} }
dir = getProjectDir(dir) dir = getProjectDir(dir)
const config = await loadConfig(PHASE_DEVELOPMENT_SERVER, dir) const config = await loadConfig(PHASE_DEVELOPMENT_SERVER, dir)
const distDir = path.join(dir, config.distDir || '.next') const distDir = path.join(dir, config.distDir || '.next')
const eventsPath = path.join(distDir, '_events.json') // Support both old format (no eventsFile arg) and new format (with eventsFile arg)
const eventsPath = path.join(
distDir,
eventsFile && !eventsFile.includes('/') ? eventsFile : '_events.json'
)
let events: TelemetryEvent[] let events: TelemetryEvent[]
try { try {
...@@ -40,7 +46,8 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' ...@@ -40,7 +46,8 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
await telemetry.record(events) await telemetry.record(events)
await telemetry.flush() await telemetry.flush()
// finished flushing events clean-up/exit // finished flushing events clean-up
fs.unlinkSync(eventsPath) fs.unlinkSync(eventsPath)
process.exit(0) // Don't call process.exit() here - let Node.js exit naturally after
// all pending work completes (e.g., setTimeout in debug telemetry)
})() })()
...@@ -216,7 +216,9 @@ export class Telemetry { ...@@ -216,7 +216,9 @@ export class Telemetry {
return prom return prom
} }
flush = async () => Promise.all(this.queue).catch(() => null) flush = async () => {
return Promise.all(this.queue).catch(() => null)
}
// writes current events to disk and spawns separate // writes current events to disk and spawns separate
// detached process to submit the records without blocking // detached process to submit the records without blocking
...@@ -232,9 +234,17 @@ export class Telemetry { ...@@ -232,9 +234,17 @@ export class Telemetry {
// if we fail to abort ignore this event // if we fail to abort ignore this event
} }
}) })
if (allEvents.length === 0) {
// No events to flush
return
}
fs.mkdirSync(this.distDir, { recursive: true }) fs.mkdirSync(this.distDir, { recursive: true })
// Use unique filename per process to avoid race conditions between parent/child
const eventsFile = `_events_${process.pid}.json`
fs.writeFileSync( fs.writeFileSync(
path.join(this.distDir, '_events.json'), path.join(this.distDir, eventsFile),
JSON.stringify(allEvents) JSON.stringify(allEvents)
) )
...@@ -249,16 +259,20 @@ export class Telemetry { ...@@ -249,16 +259,20 @@ export class Telemetry {
? child_process.spawnSync ? child_process.spawnSync
: child_process.spawn : child_process.spawn
spawn(process.execPath, [require.resolve('./detached-flush'), mode, dir], { spawn(
detached: !this.NEXT_TELEMETRY_DEBUG, process.execPath,
windowsHide: true, [require.resolve('./detached-flush'), mode, dir, eventsFile],
shell: false, {
...(this.NEXT_TELEMETRY_DEBUG detached: !this.NEXT_TELEMETRY_DEBUG,
? { windowsHide: true,
stdio: 'inherit', shell: false,
} ...(this.NEXT_TELEMETRY_DEBUG
: {}), ? {
}) stdio: 'inherit',
}
: {}),
}
)
} }
private submitRecord = async ( private submitRecord = async (
...@@ -276,15 +290,19 @@ export class Telemetry { ...@@ -276,15 +290,19 @@ export class Telemetry {
} }
if (this.NEXT_TELEMETRY_DEBUG) { if (this.NEXT_TELEMETRY_DEBUG) {
// Print to standard error to simplify selecting the output // Return a promise that resolves after logging to ensure the output
events.forEach(({ eventName, payload }) => // is captured before the process exits (e.g., during flushDetached)
console.error( return new Promise((resolve) => {
`[telemetry] ` + JSON.stringify({ eventName, payload }, null, 2) setTimeout(() => {
) // Print to standard error to simplify selecting the output
) events.forEach(({ eventName, payload }) =>
// Do not send the telemetry data if debugging. Users may use this feature console.error(
// to preview what data would be sent. `[telemetry] ` + JSON.stringify({ eventName, payload }, null, 2)
return Promise.resolve() )
)
resolve(undefined)
}, 100)
})
} }
// Skip recording telemetry if the feature is disabled // Skip recording telemetry if the feature is disabled
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment