Unverified Commit 2e8108e6 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Replace jest-worker with custom worker pool

Replace the vendored jest-worker with a purpose-built pool in
lib/worker/: lazy spawning, boot throttling, per-worker concurrency,
crash recovery and graceful shutdown, for both child processes and
worker threads. Build, export, type-check, the dev server, the
dev-validation worker and the 'use cache' probe pool use it.

Prerender early exit now throws back to the build instead of exiting
inside the worker; the CLI prints the message unchanged and deferred
Turbopack warnings still follow it. Calls whose arguments fail to
serialize, and workers whose module fails to load, reject their calls
with the real error instead of wedging or crashing the pool. Leftover
jest-worker references are removed and the Turbopack server NFT prunes
the worker child entrypoints outside standalone builds.
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent ce9c8008
......@@ -298,7 +298,6 @@ fn next_owned_ignores(
rcstr!("**/next/dist/compiled/@mswjs/interceptors/ClientRequest/LICENSE"),
rcstr!("**/next/dist/compiled/@mswjs/interceptors/ClientRequest/llhttp/**"),
rcstr!("**/next/dist/compiled/semver/semver/**/*.js"),
rcstr!("**/next/dist/compiled/jest-worker/**/*"),
// -- The following were added for Turbopack specifically --
// client/components/use-action-queue.ts has a process.env.NODE_ENV guard, but we can't set that due to React: https://github.com/vercel/next.js/pull/75254
rcstr!("**/next/dist/next-devtools/userspace/use-app-dev-rendering-indicator.js"),
......@@ -328,6 +327,9 @@ fn next_owned_ignores(
globs.extend([
rcstr!("**/*/next/dist/server/next.js"),
rcstr!("**/*/next/dist/bin/next"),
// The worker pool's child entrypoints are only needed when the
// standalone server spawns workers (matches `collect-build-traces.ts`).
rcstr!("**/next/dist/lib/worker/worker-*-child*"),
]);
}
......
......@@ -137,7 +137,7 @@ module.exports = ({ dev, turbo, bundleType, experimental, ...rest }) => {
;(async () => {
if (
request.match(
/next[/\\]dist[/\\]compiled[/\\](babel|webpack|source-map|semver|jest-worker|stacktrace-parser|@ampproject\/toolbox-optimizer)/
/next[/\\]dist[/\\]compiled[/\\](babel|webpack|source-map|semver|stacktrace-parser|@ampproject\/toolbox-optimizer)/
)
) {
callback(null, 'commonjs ' + request)
......
......@@ -281,7 +281,6 @@
"ipaddr.js": "2.2.0",
"is-docker": "2.0.0",
"is-wsl": "2.2.0",
"jest-worker": "27.5.1",
"json5": "2.2.3",
"jsonwebtoken": "9.0.0",
"loader-runner": "4.3.0",
......
......@@ -222,7 +222,7 @@ export async function collectBuildTraces({
const sharedIgnores = [
'**/next/dist/compiled/next-server/**/*.dev.js',
...(isStandalone ? [] : ['**/next/dist/compiled/jest-worker/**/*']),
...(isStandalone ? [] : ['**/next/dist/lib/worker/worker-*-child*']),
'**/next/dist/compiled/webpack/*',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/route-resolver*',
......@@ -295,12 +295,12 @@ export async function collectBuildTraces({
if (isStandalone) {
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/processChild'),
require.resolve('../lib/worker/worker-process-child'),
serverTracedFiles
)
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/threadChild'),
require.resolve('../lib/worker/worker-thread-child'),
serverTracedFiles
)
}
......
......@@ -22,7 +22,7 @@ import { bold, yellow } from '../lib/picocolors'
import { makeRe } from 'next/dist/compiled/picomatch'
import { existsSync, promises as fs } from 'fs'
import os from 'os'
import { getNextBuildDebuggerPortOffset, Worker } from '../lib/worker'
import { Worker } from '../lib/worker/index'
import { defaultConfig, getNextConfigRuntime } from '../server/config-shared'
import devalue from 'next/dist/compiled/devalue'
import findUp from 'next/dist/compiled/find-up'
......@@ -1027,8 +1027,8 @@ export function createStaticWorker(
): StaticWorker {
const { numberOfWorkers, debuggerPortOffset, progress } = options
return new Worker(staticWorkerPath, {
logger: Log,
numWorkers: numberOfWorkers,
workerName: 'Next.js static worker',
maxWorkers: numberOfWorkers,
onActivity: () => {
progress?.run()
},
......@@ -1148,7 +1148,11 @@ export default async function build(
deferredTurbopackWarnings = undefined
}
// A failing static generation worker exits the process directly (`prerenderEarlyExit`), skipping the flush points below.
// A prerender early exit fails the build with a `NEXT_EXPORT_ERROR` that the
// CLI prints after `build()` returns and then exits. For that case the
// warnings are left to this exit hook so they follow the error message, as
// they did when the failing worker printed it and exited the process itself.
let deferWarningsToExit = false
process.once('exit', () => flushTurbopackWarnings())
try {
......@@ -1929,9 +1933,8 @@ export default async function build(
const buildTraceWorker = new Worker(
require.resolve('./collect-build-traces'),
{
debuggerPortOffset: -1,
isolatedMemory: false,
numWorkers: 1,
workerName: 'Next.js build trace worker',
maxWorkers: 1,
exposedMethods: ['collectBuildTraces'],
forkOptions: process.env.NEXT_CPU_PROF
? {
......@@ -2336,9 +2339,9 @@ export default async function build(
staticWorker = createStaticWorker(config, {
numberOfWorkers,
debuggerPortOffset: getNextBuildDebuggerPortOffset({
kind: 'export-page',
}),
// Port offset 0: export-page workers get the first debugger port after
// the parent process (base + 1 + 0).
debuggerPortOffset: 0,
})
const analysisBegin = process.hrtime()
......@@ -2432,6 +2435,21 @@ export default async function build(
})
: Promise.resolve([])
// These checks are started early and only awaited after the per-page
// checks below. If those throw first, the build shuts the static worker
// down, which rejects any of these calls that are still queued. Mark
// them as handled so that rejection can't surface as an
// `unhandledRejection` and hide the error that actually failed the
// build. Awaiting them later still observes the original outcome.
for (const promise of [
errorPageHasCustomGetInitialProps,
errorPageStaticResult,
customAppGetInitialPropsPromise,
namedExportsPromise,
]) {
promise.catch(() => {})
}
// eslint-disable-next-line @typescript-eslint/no-shadow
let isNextImageImported: boolean | undefined
......@@ -4479,7 +4497,7 @@ export default async function build(
// When output: export we want to end the worker later as it's still used for writeFullyStaticExport
if (config.output !== 'export') {
// ensure the worker is not left hanging
staticWorker?.end()
await staticWorker?.shutdown()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
}
......@@ -4711,7 +4729,7 @@ export default async function build(
if (config.output === 'export') {
// TODO: When writeFullyStaticExport doesn't fail when staticWorker is passed moved this after writeFullyStaticExport.
// End the worker here when it's output: export.
staticWorker.end()
await staticWorker.shutdown()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
await nextBuildSpan
......@@ -4859,6 +4877,14 @@ export default async function build(
await pendingUpgradeNudge
})
} catch (e) {
if (
isError(e) &&
e.code === 'NEXT_EXPORT_ERROR' &&
(e as { isPrerenderEarlyExit?: boolean }).isPrerenderEarlyExit
) {
deferWarningsToExit = true
}
// A build can fail before the success path awaits this check. Surface an
// independent nudge failure so its retry receipt never hides the full
// reminder on the next build.
......@@ -4890,13 +4916,15 @@ export default async function build(
} finally {
// @ts-expect-error Existence of staticWorker is checked here intentionally.
if (staticWorker) {
staticWorker.end()
await staticWorker.shutdown()
}
// Ensure we wait for lockfile patching if present
await lockfilePatchPromise.cur
// Backstop for builds that never reach the post-static-generation flush.
flushTurbopackWarnings()
if (!deferWarningsToExit) {
flushTurbopackWarnings()
}
// Flush telemetry before finishing (waits for async operations like setTimeout in debug mode)
const telemetry: Telemetry | undefined = traceGlobals.get('telemetry')
......
......@@ -3,10 +3,9 @@ import type { Telemetry } from '../telemetry/storage'
import type { Span } from '../trace'
import * as Log from './output/log'
import { Worker } from '../lib/worker'
import { Worker } from '../lib/worker/index'
import createSpinner from './spinner'
import { eventTypeCheckCompleted } from '../telemetry/events'
import isError from '../lib/is-error'
import { hrtimeDurationToString } from './duration-to-string'
import { getBuildDistDir } from '../export/utils'
......@@ -46,10 +45,9 @@ function verifyAndRunTypeScript(
typeCheckWorker = new Worker(
require.resolve('../lib/verify-typescript-setup'),
{
workerName: 'Next.js type check worker',
exposedMethods: ['verifyAndRunTypeScriptInWorker'],
debuggerPortOffset: -1,
isolatedMemory: false,
numWorkers: 1,
maxWorkers: 1,
enableWorkerThreads,
maxRetries: 0,
}
......@@ -80,15 +78,14 @@ function verifyAndRunTypeScript(
useTypeScriptCli,
onFirstCliOutput,
})
.then((result) => {
typeCheckWorker?.end()
.then(async (result) => {
await typeCheckWorker?.shutdown()
return result
})
.catch(() => {
// The error is already logged (in the worker for the API checker, or
// directly for the in-process CLI checker); we simply exit to prevent the
// `Jest worker encountered 1 child process exceptions, exceeding retry
// limit` message from showing up.
// directly for the in-process CLI checker); we simply exit to avoid
// reporting the same failure a second time.
process.exit(1)
})
}
......@@ -132,63 +129,53 @@ export async function startTypeChecking({
const typeCheckAndLintStart = process.hrtime()
try {
const [verifyResult, typeCheckEnd] = await nextBuildSpan
.traceChild('run-typescript')
.traceAsyncFn(() =>
verifyAndRunTypeScript(
dir,
getBuildDistDir(config),
Boolean(config.experimental.strictRouteTypes),
!ignoreTypeScriptErrors,
config.typescript.tsconfigPath,
Boolean(config.typedRoutes),
config.images.disableStaticImages,
cacheDir,
config.experimental.workerThreads,
!!appDir,
!!pagesDir,
appDir,
pagesDir,
debugBuildPaths,
useTypeScriptCli,
// Stop the spinner before as soon as the subprocess reports output.
useTypeScriptCli && typeCheckingSpinner
? () => typeCheckingSpinner.stop()
: undefined
).then((resolved) => {
const checkEnd = process.hrtime(typeCheckAndLintStart)
return [resolved, checkEnd] as const
})
)
const [verifyResult, typeCheckEnd] = await nextBuildSpan
.traceChild('run-typescript')
.traceAsyncFn(() =>
verifyAndRunTypeScript(
dir,
getBuildDistDir(config),
Boolean(config.experimental.strictRouteTypes),
!ignoreTypeScriptErrors,
config.typescript.tsconfigPath,
Boolean(config.typedRoutes),
config.images.disableStaticImages,
cacheDir,
config.experimental.workerThreads,
!!appDir,
!!pagesDir,
appDir,
pagesDir,
debugBuildPaths,
useTypeScriptCli,
// Stop the spinner before as soon as the subprocess reports output.
useTypeScriptCli && typeCheckingSpinner
? () => typeCheckingSpinner.stop()
: undefined
).then((resolved) => {
const checkEnd = process.hrtime(typeCheckAndLintStart)
return [resolved, checkEnd] as const
})
)
if (typeCheckingSpinner) {
typeCheckingSpinner.stop()
}
if (typeCheckingSpinner) {
typeCheckingSpinner.stop()
}
createSpinner(
`Finished TypeScript${ignoreTypeScriptErrors ? ' config validation' : ''} in ${hrtimeDurationToString(typeCheckEnd)}`
)?.stopAndPersist()
createSpinner(
`Finished TypeScript${ignoreTypeScriptErrors ? ' config validation' : ''} in ${hrtimeDurationToString(typeCheckEnd)}`
)?.stopAndPersist()
if (!ignoreTypeScriptErrors && verifyResult) {
telemetry.record(
eventTypeCheckCompleted({
durationInSeconds: typeCheckEnd[0],
typescriptVersion: verifyResult.version,
inputFilesCount: verifyResult.result?.inputFilesCount,
totalFilesCount: verifyResult.result?.totalFilesCount,
incremental: verifyResult.result?.incremental,
typeCheckMode: verifyResult.typeCheckMode,
})
)
}
} catch (err) {
// prevent showing jest-worker internal error as it
// isn't helpful for users and clutters output
if (isError(err) && err.message === 'Call retries were exceeded') {
await telemetry.flush()
process.exit(1)
}
throw err
if (!ignoreTypeScriptErrors && verifyResult) {
telemetry.record(
eventTypeCheckCompleted({
durationInSeconds: typeCheckEnd[0],
typescriptVersion: verifyResult.version,
inputFilesCount: verifyResult.result?.inputFilesCount,
totalFilesCount: verifyResult.result?.totalFilesCount,
incremental: verifyResult.result?.incremental,
typeCheckMode: verifyResult.typeCheckMode,
})
)
}
}
......@@ -2,7 +2,7 @@ import type { COMPILER_INDEXES } from '../../shared/lib/constants'
import * as Log from '../output/log'
import { NextBuildContext } from '../build-context'
import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { Worker } from '../../lib/worker'
import { Worker } from '../../lib/worker/index'
import origDebug from 'next/dist/compiled/debug'
import path from 'path'
import { exportTraceState, recordTraceEvents } from '../../trace'
......@@ -46,10 +46,9 @@ async function webpackBuildWithWorker(
for (const compilerName of compilerNames) {
const worker = new Worker(path.join(__dirname, 'impl.js'), {
workerName: `Next.js webpack worker (${compilerName})`,
exposedMethods: ['workerMain'],
debuggerPortOffset: -1,
isolatedMemory: false,
numWorkers: 1,
maxWorkers: 1,
maxRetries: 0,
forkOptions: {
env: {
......@@ -78,7 +77,7 @@ async function webpackBuildWithWorker(
recordTraceEvents(curResult.debugTraceEvents)
}
// destroy worker so it's not sticking around using memory
await worker.end()
await worker.shutdown()
// Update plugin state
pluginState = deepMerge(pluginState, curResult.pluginState)
......
......@@ -223,7 +223,7 @@ export class MinifyPlugin {
await Promise.all(scheduledTasks)
if (initializedWorker) {
await initializedWorker.end()
await initializedWorker.shutdown()
}
})
}
......
......@@ -170,6 +170,16 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
if (experimentalDebugMemoryUsage) {
disableMemoryDebuggingMode()
}
// A prerender early exit already printed the page's error. Print the
// exit message as-is, right after it, the way the export worker did
// when it exited the process itself.
if (
isError(err) &&
err.code === 'NEXT_EXPORT_ERROR' &&
(err as { isPrerenderEarlyExit?: boolean }).isPrerenderEarlyExit
) {
printAndExit(err.message)
}
console.error('')
if (
isError(err) &&
......
MIT License
Copyright (c) Facebook, Inc. and its affiliates.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
This diff is collapsed.
{"name":"jest-worker","main":"index.js","license":"MIT"}
(()=>{"use strict";var __webpack_modules__={994:(e,r)=>{Object.defineProperty(r,"__esModule",{value:true});r.PARENT_MESSAGE_SETUP_ERROR=r.PARENT_MESSAGE_OK=r.PARENT_MESSAGE_CUSTOM=r.PARENT_MESSAGE_CLIENT_ERROR=r.CHILD_MESSAGE_INITIALIZE=r.CHILD_MESSAGE_END=r.CHILD_MESSAGE_CALL=void 0;const _=0;r.CHILD_MESSAGE_INITIALIZE=_;const t=1;r.CHILD_MESSAGE_CALL=t;const n=2;r.CHILD_MESSAGE_END=n;const o=0;r.PARENT_MESSAGE_OK=o;const s=1;r.PARENT_MESSAGE_CLIENT_ERROR=s;const c=2;r.PARENT_MESSAGE_SETUP_ERROR=c;const i=3;r.PARENT_MESSAGE_CUSTOM=i}};var __webpack_module_cache__={};function __nccwpck_require__(e){var r=__webpack_module_cache__[e];if(r!==undefined){return r.exports}var _=__webpack_module_cache__[e]={exports:{}};var t=true;try{__webpack_modules__[e](_,_.exports,__nccwpck_require__);t=false}finally{if(t)delete __webpack_module_cache__[e]}return _.exports}if(typeof __nccwpck_require__!=="undefined")__nccwpck_require__.ab=__dirname+"/";var __webpack_exports__={};var _types=__nccwpck_require__(994);let file=null;let setupArgs=[];let initialized=false;const messageListener=e=>{switch(e[0]){case _types.CHILD_MESSAGE_INITIALIZE:const r=e;file=r[2];setupArgs=e[3];break;case _types.CHILD_MESSAGE_CALL:const _=e;execMethod(_[2],_[3]);break;case _types.CHILD_MESSAGE_END:end();break;default:throw new TypeError("Unexpected request from parent process: "+e[0])}};process.on("message",messageListener);function reportSuccess(e){if(!process||!process.send){throw new Error("Child can only be used on a forked process")}process.send([_types.PARENT_MESSAGE_OK,e])}function reportClientError(e){return reportError(e,_types.PARENT_MESSAGE_CLIENT_ERROR)}function reportInitializeError(e){return reportError(e,_types.PARENT_MESSAGE_SETUP_ERROR)}function reportError(e,r){if(!process||!process.send){throw new Error("Child can only be used on a forked process")}if(e==null){e=new Error('"null" or "undefined" thrown')}process.send([r,e.constructor&&e.constructor.name,e.message,e.stack,typeof e==="object"?{...e}:e])}function end(){const main=eval("require")(file);if(!main.teardown){exitProcess();return}execFunction(main.teardown,main,[],exitProcess,exitProcess)}function exitProcess(){process.removeListener("message",messageListener)}function execMethod(method,args){const main=eval("require")(file);let fn;if(method==="default"){fn=main.__esModule?main["default"]:main}else{fn=main[method]}function execHelper(){execFunction(fn,main,args,reportSuccess,reportClientError)}if(initialized||!main.setup){execHelper();return}initialized=true;execFunction(main.setup,main,setupArgs,execHelper,reportInitializeError)}const isPromise=e=>!!e&&(typeof e==="object"||typeof e==="function")&&typeof e.then==="function";function execFunction(e,r,_,t,n){let o;try{o=e.apply(r,_)}catch(e){n(e);return}if(isPromise(o)){o.then(t,n)}else{t(o)}}module.exports=__webpack_exports__})();
\ No newline at end of file
(()=>{"use strict";var __webpack_modules__={994:(e,r)=>{Object.defineProperty(r,"__esModule",{value:true});r.PARENT_MESSAGE_SETUP_ERROR=r.PARENT_MESSAGE_OK=r.PARENT_MESSAGE_CUSTOM=r.PARENT_MESSAGE_CLIENT_ERROR=r.CHILD_MESSAGE_INITIALIZE=r.CHILD_MESSAGE_END=r.CHILD_MESSAGE_CALL=void 0;const t=0;r.CHILD_MESSAGE_INITIALIZE=t;const _=1;r.CHILD_MESSAGE_CALL=_;const n=2;r.CHILD_MESSAGE_END=n;const o=0;r.PARENT_MESSAGE_OK=o;const s=1;r.PARENT_MESSAGE_CLIENT_ERROR=s;const i=2;r.PARENT_MESSAGE_SETUP_ERROR=i;const a=3;r.PARENT_MESSAGE_CUSTOM=a},167:e=>{e.exports=require("worker_threads")}};var __webpack_module_cache__={};function __nccwpck_require__(e){var r=__webpack_module_cache__[e];if(r!==undefined){return r.exports}var t=__webpack_module_cache__[e]={exports:{}};var _=true;try{__webpack_modules__[e](t,t.exports,__nccwpck_require__);_=false}finally{if(_)delete __webpack_module_cache__[e]}return t.exports}if(typeof __nccwpck_require__!=="undefined")__nccwpck_require__.ab=__dirname+"/";var __webpack_exports__={};function _worker_threads(){const e=__nccwpck_require__(167);_worker_threads=function(){return e};return e}var _types=__nccwpck_require__(994);let file=null;let setupArgs=[];let initialized=false;const messageListener=e=>{switch(e[0]){case _types.CHILD_MESSAGE_INITIALIZE:const r=e;file=r[2];setupArgs=e[3];process.env.JEST_WORKER_ID=e[4];break;case _types.CHILD_MESSAGE_CALL:const t=e;execMethod(t[2],t[3]);break;case _types.CHILD_MESSAGE_END:end();break;default:throw new TypeError("Unexpected request from parent process: "+e[0])}};_worker_threads().parentPort.on("message",messageListener);function reportSuccess(e){if(_worker_threads().isMainThread){throw new Error("Child can only be used on a forked process")}_worker_threads().parentPort.postMessage([_types.PARENT_MESSAGE_OK,e])}function reportClientError(e){return reportError(e,_types.PARENT_MESSAGE_CLIENT_ERROR)}function reportInitializeError(e){return reportError(e,_types.PARENT_MESSAGE_SETUP_ERROR)}function reportError(e,r){if(_worker_threads().isMainThread){throw new Error("Child can only be used on a forked process")}if(e==null){e=new Error('"null" or "undefined" thrown')}_worker_threads().parentPort.postMessage([r,e.constructor&&e.constructor.name,e.message,e.stack,typeof e==="object"?{...e}:e])}function end(){const main=eval("require")(file);if(!main.teardown){exitProcess();return}execFunction(main.teardown,main,[],exitProcess,exitProcess)}function exitProcess(){_worker_threads().parentPort.removeListener("message",messageListener)}function execMethod(method,args){const main=eval("require")(file);let fn;if(method==="default"){fn=main.__esModule?main["default"]:main}else{fn=main[method]}function execHelper(){execFunction(fn,main,args,reportSuccess,reportClientError)}if(initialized||!main.setup){execHelper();return}initialized=true;execFunction(main.setup,main,setupArgs,execHelper,reportInitializeError)}const isPromise=e=>!!e&&(typeof e==="object"||typeof e==="function")&&typeof e.then==="function";function execFunction(e,r,t,_,n){let o;try{o=e.apply(r,t)}catch(e){n(e);return}if(isPromise(o)){o.then(_,n)}else{_(o)}}module.exports=__webpack_exports__})();
\ No newline at end of file
......@@ -70,7 +70,6 @@ import { isInterceptionRouteRewrite } from '../lib/is-interception-route-rewrite
import type { ActionManifest } from '../build/webpack/plugins/flight-client-entry-plugin'
import { extractInfoFromServerReferenceId } from '../shared/lib/server-reference-info'
import { convertSegmentPathToStaticExportFilename } from '../shared/lib/segment-cache/segment-value-encoding'
import { getNextBuildDebuggerPortOffset } from '../lib/worker'
import { getParams } from './helpers/get-params'
import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic'
import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
......@@ -799,9 +798,9 @@ async function exportAppImpl(
worker = staticWorker
} else {
worker = createStaticWorker(nextConfig, {
debuggerPortOffset: getNextBuildDebuggerPortOffset({
kind: 'export-page',
}),
// Port offset 0: export-page workers get the first debugger port after
// the parent process (base + 1 + 0).
debuggerPortOffset: 0,
numberOfWorkers: options.numWorkers,
progress,
})
......@@ -1079,7 +1078,7 @@ async function exportAppImpl(
}
if (!staticWorker && worker) {
await worker.end()
await worker.shutdown()
}
return collector
......
......@@ -72,6 +72,26 @@ class ExportPageError extends Error {
code = 'NEXT_EXPORT_PAGE_ERROR'
}
/**
* Set to true when prerenderEarlyExit fires. Suppresses logging from other
* pages that are concurrently in-flight so the user only sees the first error.
*/
let isEarlyExiting = false
/**
* Saved reference to the real console.error. When isEarlyExiting is set we
* replace console.error with a no-op so that render-time logging from
* concurrent pages (e.g. logDisallowedDynamicError) is silenced.
*/
let realConsoleError: typeof console.error = console.error
function setEarlyExiting(): void {
isEarlyExiting = true
// Silence all further console.error output so concurrent pages don't
// produce noise after the first failure has been reported.
console.error = () => {}
}
async function exportPageImpl(
input: ExportPageInput,
fileWriter: MultiFileWriter
......@@ -392,6 +412,13 @@ export async function exportPages(
await installBindings()
installCodeFrameSupport()
// Reset per-call so the flag doesn't leak across sequential batches.
// Restore console.error first (realConsoleError is still the original captured
// at module load; reassigning it here would overwrite it with the no-op set by
// setEarlyExiting() if a previous batch triggered an early exit).
console.error = realConsoleError
isEarlyExiting = false
const {
exportPaths,
dir,
......@@ -530,10 +557,23 @@ export async function exportPages(
}
// If prerenderEarlyExit is enabled, we'll exit the build immediately.
if (nextConfig.experimental.prerenderEarlyExit) {
console.error(
// If another page already triggered an early exit, this page's
// promise was abandoned by the outer Promise.all. Throwing here
// would produce an unhandled rejection, so return silently instead.
if (isEarlyExiting) {
return { result: undefined, path, page, pageKey }
}
// Suppress output from any concurrently-running pages so the user
// only sees the error from the first failing page.
setEarlyExiting()
const exportError = new Error(
`Export encountered an error on ${pageKey}, exiting the build.`
)
process.exit(1)
;(exportError as any).code = 'NEXT_EXPORT_ERROR'
// Lets the build tell an early exit apart from the end-of-build
// export failure summary (which uses the same code).
;(exportError as any).isPrerenderEarlyExit = true
throw exportError
} else {
// Otherwise, this is a no-op. The build will continue, and a summary of failed pages will be displayed at the end.
}
......@@ -627,31 +667,39 @@ async function exportPage(
return { error: result.error, duration: Date.now() - start }
}
} catch (err) {
console.error(
`Error occurred prerendering page "${input.exportPath.path}". Read more: https://nextjs.org/docs/messages/prerender-error`
)
if (!isEarlyExiting) {
console.error(
`Error occurred prerendering page "${input.exportPath.path}". Read more: https://nextjs.org/docs/messages/prerender-error`
)
// bailoutToCSRError errors should not leak to the user as they are not actionable; they're
// a framework signal
if (!isBailoutToCSRError(err)) {
// A static generation bailout error is a framework signal to fail static generation but
// and will encode a reason in the error message. If there is a message, we'll print it.
// Otherwise there's nothing to show as we don't want to leak an error internal error stack to the user.
// TODO: Always log the full error. ignore-listing will take care of hiding internal stacks.
if (isStaticGenBailoutError(err)) {
if (err.message) {
console.error(`Error: ${err.message}`)
// bailoutToCSRError errors should not leak to the user as they are not actionable; they're
// a framework signal
if (!isBailoutToCSRError(err)) {
// A static generation bailout error is a framework signal to fail static generation but
// and will encode a reason in the error message. If there is a message, we'll print it.
// Otherwise there's nothing to show as we don't want to leak an error internal error stack to the user.
// TODO: Always log the full error. ignore-listing will take care of hiding internal stacks.
if (isStaticGenBailoutError(err)) {
if (err.message) {
console.error(`Error: ${err.message}`)
}
} else {
console.error(err)
}
} else {
console.error(err)
}
}
return { error: true, duration: Date.now() - start }
}
// Notify the parent process that we processed a page (used by the progress activity indicator)
process.send?.([3, { type: 'activity' }])
// Notify the parent process that we processed a page (used by the progress activity indicator).
// Wrap in try/catch: if the IPC channel closed because the pool already propagated a
// NEXT_EXPORT_ERROR, the send throws ERR_IPC_CHANNEL_CLOSED — treat it as a no-op.
try {
process.send?.([3, { type: 'activity' }])
} catch {
// Channel already closed; nothing to do.
}
// Otherwise we can return the result.
return {
......@@ -667,7 +715,12 @@ process.on('unhandledRejection', (err: unknown) => {
return
}
console.error(err)
// Suppress errors from concurrent pages when an early exit is in progress.
if (isEarlyExiting) {
return
}
realConsoleError(err)
})
process.on('rejectionHandled', () => {
......@@ -680,12 +733,15 @@ const FATAL_UNHANDLED_NEXT_API_EXIT_CODE = 78
process.on('uncaughtException', (err) => {
if (isDynamicUsageError(err)) {
console.error(
realConsoleError(
'A Next.js API that uses exceptions to signal framework behavior was uncaught. This suggests improper usage of a Next.js API. The original error is printed below and the build will now exit.'
)
console.error(err)
realConsoleError(err)
process.exit(FATAL_UNHANDLED_NEXT_API_EXIT_CODE)
} else if (isEarlyExiting) {
// Suppress uncaught exceptions (e.g. ERR_IPC_CHANNEL_CLOSED) from
// concurrent pages when an early exit is already in progress.
} else {
console.error(err)
realConsoleError(err)
}
})
......@@ -297,9 +297,9 @@ export async function verifyAndRunTypeScript({
} catch (err) {
// Print the user-facing message here and rethrow. This function runs both
// in-process (next dev / next test / next typegen, and the CLI type-check
// during build) and inside a jest worker (the TypeScript-API type-check
// during build). A thrown error does not survive the worker boundary — the
// parent only sees `Call retries were exceeded` — so the message must be
// during build) and inside a worker (the TypeScript-API type-check during
// build). The worker entrypoint exits instead of rethrowing, so the error
// never reaches the parent — so the message must be
// printed on this side. The caller decides what to do with the throw (exit,
// or tolerate it as `next dev` does on re-verification).
if (err instanceof CompileError) {
......@@ -320,9 +320,8 @@ export async function verifyAndRunTypeScript({
/**
* Worker entrypoint used by `next build` for the TypeScript-API type-check.
* `verifyAndRunTypeScript` has already printed any error, so on failure we exit
* the worker directly rather than rethrowing: a thrown error would be retried by
* jest-worker and surface in the parent as the unhelpful
* `Jest worker encountered 1 child process exceptions, exceeding retry limit`.
* the worker directly rather than rethrowing: a thrown error would be
* serialized back to the parent and reported a second time.
*/
export async function verifyAndRunTypeScriptInWorker(
options: Parameters<typeof verifyAndRunTypeScript>[0]
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
// ---------------------------------------------------------------------------
// Message type constants for parent ↔ child IPC communication.
//
// Child messages (parent → child):
// INITIALIZE Load the worker module and optionally call its setup() function.
// CALL Invoke an exported method in the worker module.
// END Gracefully shut down the worker process.
//
// Parent messages (child → parent):
// OK A CALL completed successfully — carries the return value.
// CLIENT_ERROR A CALL threw an error — carries serialized error details.
// SETUP_ERROR The setup() function threw during initialization.
// CUSTOM An arbitrary message sent by the worker via process.send().
// READY The worker has finished loading its module and running setup().
// ---------------------------------------------------------------------------
/** Load the worker module and optionally call its setup() function */
export const CHILD_MESSAGE_INITIALIZE = 0
/** Invoke an exported method in the worker module */
export const CHILD_MESSAGE_CALL = 1
/** Gracefully shut down the worker process */
export const CHILD_MESSAGE_END = 2
/** A CALL completed successfully — carries the return value */
export const PARENT_MESSAGE_OK = 0
/** A CALL threw an error — carries serialized error details */
export const PARENT_MESSAGE_CLIENT_ERROR = 1
/** The setup() function threw (or isn't a function) during initialization; the module is loaded */
export const PARENT_MESSAGE_SETUP_ERROR = 2
/** An arbitrary message sent by the worker via process.send() */
export const PARENT_MESSAGE_CUSTOM = 3
/** The worker has finished loading its module and running setup() */
export const PARENT_MESSAGE_READY = 4
/** Loading (require) the worker module threw; the worker can't handle CALLs */
export const PARENT_MESSAGE_LOAD_ERROR = 5
// Parent → Child message types
export type ChildMessageInitialize = [
type: typeof CHILD_MESSAGE_INITIALIZE,
workerPath: string,
setupArgs: unknown[],
]
export type ChildMessageCall = [
type: typeof CHILD_MESSAGE_CALL,
requestId: number,
methodName: string,
args: unknown[],
]
export type ChildMessageEnd = [type: typeof CHILD_MESSAGE_END]
export type ChildMessage =
| ChildMessageInitialize
| ChildMessageCall
| ChildMessageEnd
// Child → Parent message types
export type ParentMessageOk = [
type: typeof PARENT_MESSAGE_OK,
requestId: number,
result: unknown,
]
export type ParentMessageClientError = [
type: typeof PARENT_MESSAGE_CLIENT_ERROR,
requestId: number,
errorName: string,
errorMessage: string,
errorStack: string | undefined,
errorProperties: Record<string, unknown> | unknown,
]
export type ParentMessageSetupError = [
type: typeof PARENT_MESSAGE_SETUP_ERROR,
errorName: string,
errorMessage: string,
errorStack: string | undefined,
]
export type ParentMessageCustom = [
type: typeof PARENT_MESSAGE_CUSTOM,
payload: unknown,
]
export type ParentMessageReady = [type: typeof PARENT_MESSAGE_READY]
export type ParentMessageLoadError = [
type: typeof PARENT_MESSAGE_LOAD_ERROR,
errorName: string,
errorMessage: string,
errorStack: string | undefined,
]
export type ParentMessage =
| ParentMessageOk
| ParentMessageClientError
| ParentMessageSetupError
| ParentMessageCustom
| ParentMessageReady
| ParentMessageLoadError
/**
* Shared protocol logic for both child_process and worker_threads child files.
*
* The only difference between the two child variants is how they send/receive
* messages and how they "exit" (remove the listener vs process.exit). This
* module captures all the shared logic: module loading, method execution,
* setup/teardown orchestration, and error serialization.
*/
import {
CHILD_MESSAGE_INITIALIZE,
CHILD_MESSAGE_CALL,
CHILD_MESSAGE_END,
PARENT_MESSAGE_OK,
PARENT_MESSAGE_CLIENT_ERROR,
PARENT_MESSAGE_SETUP_ERROR,
PARENT_MESSAGE_READY,
PARENT_MESSAGE_LOAD_ERROR,
type ChildMessage,
} from './types'
/** Transport abstraction: how the child sends messages back to the parent */
export interface ChildTransport {
send(message: unknown[]): void
/** Called when the child should stop listening (e.g. after END) */
disconnect(): void
}
/**
* Create a message listener that handles the parent→child protocol.
*
* Call the returned function with each incoming `ChildMessage`. It handles
* INITIALIZE, CALL, and END internally, using the provided `transport` to
* reply.
*
* An optional `onInitialize` hook is called after INITIALIZE, before any CALL.
*/
export function createMessageHandler(
transport: ChildTransport,
onInitialize?: (message: ChildMessage) => void
): (message: ChildMessage) => void {
/** The loaded worker module (set during INITIALIZE) */
let main: any = null
function isPromise(value: unknown): value is Promise<unknown> {
return (
!!value &&
(typeof value === 'object' || typeof value === 'function') &&
typeof (value as any).then === 'function'
)
}
function reportSuccess(requestId: number, result: unknown): void {
transport.send([PARENT_MESSAGE_OK, requestId, result])
}
/** Normalize an unknown thrown value into a structured error descriptor */
function normalizeError(error: unknown): {
name: string
message: string
stack: string | undefined
} {
if (error == null) {
error = new Error('"null" or "undefined" thrown')
}
const err = error as Error
return {
name: err.constructor?.name ?? 'Error',
message: err.message,
stack: err.stack,
}
}
function reportClientError(requestId: number, error: unknown): void {
const e = normalizeError(error)
transport.send([
PARENT_MESSAGE_CLIENT_ERROR,
requestId,
e.name,
e.message,
e.stack,
// Spread extra properties from the original error (e.g. `code`, `digest`)
typeof error === 'object' ? { ...error } : error,
])
}
function reportInitializeError(error: unknown): void {
const e = normalizeError(error)
transport.send([PARENT_MESSAGE_SETUP_ERROR, e.name, e.message, e.stack])
}
function reportLoadError(error: unknown): void {
const e = normalizeError(error)
transport.send([PARENT_MESSAGE_LOAD_ERROR, e.name, e.message, e.stack])
}
/**
* Call `fn` with `args` in `ctx`, routing the result (sync or async) to
* `onSuccess` or `onError`.
*/
function execFunction(
fn: Function,
ctx: any,
args: unknown[],
onSuccess: (result: unknown) => void,
onError: (error: unknown) => void
): void {
let result: unknown
try {
result = fn.apply(ctx, args)
} catch (err) {
onError(err)
return
}
if (isPromise(result)) {
result.then(onSuccess, onError)
} else {
onSuccess(result)
}
}
/**
* Load the worker module, run its setup() if present, and send READY.
* Called once during INITIALIZE.
*/
function initialize(workerPath: string, setupArgs: unknown[]): void {
try {
main = require(workerPath)
} catch (err) {
// Module not found, syntax error, etc. Report it as a load error (not a
// setup error): the module isn't available, so the parent must not
// dispatch CALLs to this worker.
reportLoadError(err)
return
}
if (!main.setup) {
transport.send([PARENT_MESSAGE_READY])
return
}
if (typeof main.setup !== 'function') {
reportInitializeError(
new TypeError(
`Worker module exports a non-function "setup": ${typeof main.setup}`
)
)
return
}
execFunction(
main.setup,
main,
setupArgs,
() => transport.send([PARENT_MESSAGE_READY]),
reportInitializeError
)
}
function execMethod(
requestId: number,
method: string,
args: unknown[]
): void {
// The parent stops dispatching after a load error, but never let a stray
// CALL crash the worker by dereferencing the missing module.
if (main == null) {
reportClientError(requestId, new Error('Worker module is not loaded'))
return
}
let fn: unknown
if (method === 'default') {
fn = main.__esModule ? main['default'] : main
if (typeof fn !== 'function') {
reportClientError(
requestId,
new TypeError(`Worker default export is not a function: ${typeof fn}`)
)
return
}
} else {
// Guard against prototype-chain traversal (e.g. "constructor", "__proto__",
// "toString"). Only own properties of the module are callable.
if (!Object.prototype.hasOwnProperty.call(main, method)) {
reportClientError(
requestId,
new TypeError(`Worker has no own export named "${method}"`)
)
return
}
fn = main[method]
if (typeof fn !== 'function') {
reportClientError(
requestId,
new TypeError(
`Worker export "${method}" is not a function: ${typeof fn}`
)
)
return
}
}
execFunction(
fn as Function,
main,
args,
(result) => reportSuccess(requestId, result),
(error) => reportClientError(requestId, error)
)
}
function end(): void {
if (typeof main?.teardown !== 'function') {
transport.disconnect()
return
}
execFunction(
main.teardown,
main,
[],
() => transport.disconnect(),
(error) => {
console.error('Worker teardown() threw:', error)
transport.disconnect()
}
)
}
// The message listener returned to the caller
return function messageListener(message: ChildMessage): void {
switch (message[0]) {
case CHILD_MESSAGE_INITIALIZE:
onInitialize?.(message)
initialize(message[1], message[2])
break
case CHILD_MESSAGE_CALL:
execMethod(message[1], message[2], message[3])
break
case CHILD_MESSAGE_END:
end()
break
default:
throw new TypeError(
'Unexpected request from parent process: ' + (message as any)[0]
)
}
}
}
This diff is collapsed.
This diff is collapsed.
/**
* Child-process entry point for the worker pool.
*
* This file is loaded by `child_process.fork()`. It creates a transport
* that uses `process.send()` / `process.on('message')` and delegates all
* protocol handling to `createMessageHandler` from the shared module.
*/
import type { ChildMessage } from './types'
import {
createMessageHandler,
type ChildTransport,
} from './worker-child-common'
let disconnected = false
const transport: ChildTransport = {
send(message: unknown[]): void {
// Silently drop messages after disconnect. This can happen when an async
// operation in the worker completes after the parent has sent END and the
// IPC channel has been closed — attempting process.send() in that state
// throws ERR_IPC_CHANNEL_CLOSED and pollutes the build output.
if (disconnected) return
if (!process.send) {
throw new Error('Child can only be used on a forked process')
}
// Note: unlike worker_threads (which uses structured clone and throws
// DataCloneError for non-clonable values), IPC uses JSON serialization.
// Non-serializable values are silently coerced (symbols → undefined,
// functions are dropped). No explicit error handling is needed here.
process.send(message)
},
disconnect(): void {
disconnected = true
process.removeListener('message', listener)
// Close the IPC channel so the child can exit naturally.
// This allows process 'exit' handlers (e.g. cpu-profile saving) to fire
// before the parent's force-kill timeout.
process.disconnect()
},
}
const listener = createMessageHandler(transport)
process.on('message', listener as (message: ChildMessage) => void)
This diff is collapsed.
......@@ -12,7 +12,7 @@ import type { NonPartial } from '../../shared/lib/typescript-utils'
* Cross-module handoff for the dev validation worker (client-module warmup,
* static-shell validation, and instant-config validation all run on it). A
* symbol on `globalThis` decouples the dev-server entry point (which installs
* the jest-worker pool) from the read site in `runDevValidationInBackground`
* the worker pool) from the read site in `runDevValidationInBackground`
* inside `app-render.tsx`, avoiding a direct import of dev-only pool code from
* the render module. When the symbol is not set (e.g. because the feature flag
* is disabled), `getDevValidationWorker()` returns undefined and the caller
......
......@@ -2,7 +2,7 @@ import type { Params } from '../request/params'
// Cross-module handoff for the `'use cache'` hang-detection probe. A symbol on
// `globalThis` decouples the dev-server entry point (which installs the
// jest-worker pool) from the read site inside `use-cache-wrapper.ts` — avoiding
// worker pool) from the read site inside `use-cache-wrapper.ts` — avoiding
// a direct import of dev-only code from the use-cache module. In any process
// where the symbol is not set (prod, edge, unit tests, the probe worker itself)
// `getUseCacheProbe()` returns undefined, which doubles as the recursion guard
......
This diff is collapsed.
......@@ -495,10 +495,6 @@ declare module 'next/dist/compiled/@mswjs/interceptors/ClientRequest' {
export * from '@mswjs/interceptors/ClientRequest'
}
declare module 'next/dist/compiled/jest-worker' {
export * from 'jest-worker'
}
// TODO: Use tsconfig#paths instead
declare module 'next/dist/compiled/next-devtools'
......
This diff is collapsed.
......@@ -1718,9 +1718,6 @@ importers:
is-wsl:
specifier: 2.2.0
version: 2.2.0
jest-worker:
specifier: 27.5.1
version: 27.5.1
json5:
specifier: 2.2.3
version: 2.2.3
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment