Unverified Commit a038ddb2 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

fix: drain build worker output before shutdown

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent aad56b54
...@@ -4293,7 +4293,7 @@ export default async function build( ...@@ -4293,7 +4293,7 @@ export default async function build(
// When output: export we want to end the worker later as it's still used for writeFullyStaticExport // When output: export we want to end the worker later as it's still used for writeFullyStaticExport
if (config.output !== 'export') { if (config.output !== 'export') {
// ensure the worker is not left hanging // ensure the worker is not left hanging
staticWorker?.end() await staticWorker?.end()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally` staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
} }
...@@ -4528,7 +4528,7 @@ export default async function build( ...@@ -4528,7 +4528,7 @@ export default async function build(
if (config.output === 'export') { if (config.output === 'export') {
// TODO: When writeFullyStaticExport doesn't fail when staticWorker is passed moved this after writeFullyStaticExport. // TODO: When writeFullyStaticExport doesn't fail when staticWorker is passed moved this after writeFullyStaticExport.
// End the worker here when it's output: export. // End the worker here when it's output: export.
staticWorker.end() await staticWorker.end()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally` staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
await nextBuildSpan await nextBuildSpan
...@@ -4690,7 +4690,7 @@ export default async function build( ...@@ -4690,7 +4690,7 @@ export default async function build(
} finally { } finally {
// @ts-expect-error Existence of staticWorker is checked here intentionally. // @ts-expect-error Existence of staticWorker is checked here intentionally.
if (staticWorker) { if (staticWorker) {
staticWorker.end() await staticWorker.end()
} }
// Ensure we wait for lockfile patching if present // Ensure we wait for lockfile patching if present
await lockfilePatchPromise.cur await lockfilePatchPromise.cur
......
import { PassThrough } from 'stream' import { PassThrough } from 'stream'
let latestForkEnv: NodeJS.ProcessEnv | undefined let latestForkEnv: NodeJS.ProcessEnv | undefined
let latestStdout: PassThrough | undefined
let latestStderr: PassThrough | undefined
let latestEnd: jest.Mock | undefined
let latestKill: jest.Mock | undefined
jest.mock('next/dist/compiled/jest-worker', () => { jest.mock('next/dist/compiled/jest-worker', () => {
const WorkerMock = jest.fn().mockImplementation((_path, options) => { const WorkerMock = jest.fn().mockImplementation((_path, options) => {
latestForkEnv = options?.forkOptions?.env latestForkEnv = options?.forkOptions?.env
latestStdout = new PassThrough()
latestStderr = new PassThrough()
latestEnd = jest.fn().mockResolvedValue({ forceExited: false })
latestKill = jest.fn()
return { return {
_workerPool: { _workers: [] }, _workerPool: {
getStdout: () => new PassThrough(), _workers: [{ _child: { kill: latestKill, on: jest.fn() } }],
getStderr: () => new PassThrough(), },
end: jest.fn().mockResolvedValue(undefined), getStdout: () => latestStdout,
getStderr: () => latestStderr,
end: latestEnd,
close: jest.fn(), close: jest.fn(),
} }
}) })
...@@ -62,8 +73,14 @@ describe('lib/worker color propagation', () => { ...@@ -62,8 +73,14 @@ describe('lib/worker color propagation', () => {
const restore = restoreDescriptors.pop() const restore = restoreDescriptors.pop()
restore?.() restore?.()
} }
latestStdout?.end()
latestStderr?.end()
jest.resetModules() jest.resetModules()
latestForkEnv = undefined latestForkEnv = undefined
latestStdout = undefined
latestStderr = undefined
latestEnd = undefined
latestKill = undefined
}) })
it('enables FORCE_COLOR when the parent supports colors', () => { it('enables FORCE_COLOR when the parent supports colors', () => {
...@@ -125,3 +142,53 @@ describe('lib/worker color propagation', () => { ...@@ -125,3 +142,53 @@ describe('lib/worker color propagation', () => {
expect(latestForkEnv?.FORCE_COLOR).toBeUndefined() expect(latestForkEnv?.FORCE_COLOR).toBeUndefined()
}) })
}) })
describe('lib/worker shutdown', () => {
afterEach(() => {
latestStdout?.end()
latestStderr?.end()
jest.useRealTimers()
jest.resetModules()
latestForkEnv = undefined
latestStdout = undefined
latestStderr = undefined
latestEnd = undefined
latestKill = undefined
})
it('waits for stdout and stderr to drain without interrupting the worker', async () => {
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, noopOptions)
let didEnd = false
const endPromise = worker.end().then((result) => {
didEnd = true
return result
})
await Promise.resolve()
expect(latestEnd).toHaveBeenCalledTimes(1)
expect(latestKill).not.toHaveBeenCalled()
expect(didEnd).toBe(false)
latestStdout!.end()
await Promise.resolve()
expect(didEnd).toBe(false)
latestStderr!.end()
await expect(endPromise).resolves.toEqual({ forceExited: false })
})
it('bounds the output drain wait', async () => {
jest.useFakeTimers()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, noopOptions)
const endPromise = worker.end()
await Promise.resolve()
jest.advanceTimersByTime(1000)
await expect(endPromise).resolves.toEqual({ forceExited: false })
expect(latestKill).toHaveBeenCalledWith('SIGINT')
})
})
import type { ChildProcess } from 'child_process' import type { ChildProcess } from 'child_process'
import { Worker as JestWorker } from 'next/dist/compiled/jest-worker' import { Worker as JestWorker } from 'next/dist/compiled/jest-worker'
import { Transform } from 'stream' import { Transform, type Readable } from 'stream'
import { import {
formatDebugAddress, formatDebugAddress,
formatNodeOptions, formatNodeOptions,
...@@ -22,6 +22,33 @@ const cleanupWorkers = (worker: JestWorker) => { ...@@ -22,6 +22,33 @@ const cleanupWorkers = (worker: JestWorker) => {
} }
} }
const WORKER_STDIO_DRAIN_TIMEOUT_MS = 1000
const waitForStreamEnd = (
stream: NodeJS.ReadableStream,
signal: AbortSignal
): Promise<boolean> => {
const readable = stream as Readable
if (readable.readableEnded || readable.destroyed) {
return Promise.resolve(true)
}
return new Promise((resolve) => {
const finish = (didDrain: boolean) => {
stream.off('end', onEnd)
stream.off('close', onEnd)
signal.removeEventListener('abort', onAbort)
resolve(didDrain)
}
const onEnd = () => finish(true)
const onAbort = () => finish(false)
stream.once('end', onEnd)
stream.once('close', onEnd)
signal.addEventListener('abort', onAbort, { once: true })
})
}
export function getNextBuildDebuggerPortOffset(_: { export function getNextBuildDebuggerPortOffset(_: {
kind: 'export-page' kind: 'export-page'
}): number { }): number {
...@@ -292,14 +319,43 @@ export class Worker { ...@@ -292,14 +319,43 @@ export class Worker {
this._onActivityAbort = onActivityAbort this._onActivityAbort = onActivityAbort
} }
end(): ReturnType<JestWorker['end']> { async end(): ReturnType<JestWorker['end']> {
const worker = this._worker const worker = this._worker
if (!worker) { if (!worker) {
throw new Error('Farm is ended, no more calls can be done to it') throw new Error('Farm is ended, no more calls can be done to it')
} }
cleanupWorkers(worker)
// Worker method results and stdio use separate IPC channels. A method can
// resolve before its final output reaches the parent, so start observing
// both streams before asking jest-worker to shut down.
const outputStreams = [worker.getStdout(), worker.getStderr()]
const drainController = new AbortController()
const outputEnded = Promise.all(
outputStreams.map((stream) =>
waitForStreamEnd(stream, drainController.signal)
)
)
this._worker = undefined this._worker = undefined
return worker.end() let result: Awaited<ReturnType<JestWorker['end']>>
try {
result = await worker.end()
} catch (error) {
drainController.abort()
throw error
}
const drainTimeout = setTimeout(
() => drainController.abort(),
WORKER_STDIO_DRAIN_TIMEOUT_MS
)
const didDrain = await outputEnded
clearTimeout(drainTimeout)
if (didDrain.includes(false)) {
cleanupWorkers(worker)
}
return result
} }
/** /**
......
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
// TODO(lubieowoce): reenable when the cause of flakiness is found and fixed describe('sync IO that blocks the root', () => {
// (seems to have increased sharply around 2026-08-24/25)
describe.skip('sync IO that blocks the root', () => {
const { next } = nextTestSetup({ const { next } = nextTestSetup({
files: __dirname, files: __dirname,
skipStart: true, skipStart: true,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment