Commit f27edfda authored by Tobias Koppers's avatar Tobias Koppers

Fix worker pool bugs: boot timeout, dead code, teardown logging, onWorkerExit on timeout

- #3: BOOTING workers now subject to timeout (prevents hung setup() from deadlocking pool)
- #5: Remove unreachable _rejectActiveRequests in shutdown() (already handled by _handleExit)
- #8: Remove misleading CHILD_MESSAGE_END before forceKill in shutdownNow()/_handleWorkerTimeout
- #9: Call onWorkerExit(null, 'SIGKILL') from _handleWorkerTimeout so timeout kills are observable
- #10: Log teardown() errors in worker-child-common.ts instead of silently swallowing them
- Fix WorkerExitErrorMock to match real error message format
- Add tests: onActivityAbort/setOnActivityAbort, workerName in error messages,
  spawn error queue rejection, booting worker timeout, onWorkerExit on timeout
Co-Authored-By: 's avatarClaude <noreply@anthropic.com>
parent 36d8d585
......@@ -52,9 +52,9 @@ class WorkerPoolMock {
class WorkerExitErrorMock extends Error {
code: number | null
signal: string | null
constructor(code: number | null, signal: string | null) {
constructor(code: number | null, signal: string | null, workerName?: string) {
super(
`Worker exited unexpectedly with code ${code}${signal ? `, signal ${signal}` : ''}`
`${workerName ?? 'Worker'} exited with code: ${code} and signal: ${signal}`
)
this.name = 'WorkerExitError'
this.code = code
......@@ -691,7 +691,7 @@ describe('lib/worker maxRetries', () => {
}
await expect(worker.compute()).rejects.toThrow(
'Worker exited unexpectedly with code 1'
'Worker exited with code: 1 and signal: null'
)
worker.shutdownNow()
......@@ -761,7 +761,7 @@ describe('lib/worker maxRetries', () => {
}
await expect(worker.compute()).rejects.toThrow(
'Worker exited unexpectedly with code 1'
'Worker exited with code: 1 and signal: null'
)
worker.shutdownNow()
......@@ -843,3 +843,167 @@ describe('lib/worker exit handler cleanup', () => {
expect(remaining).toHaveLength(0)
})
})
describe('lib/worker onActivityAbort', () => {
afterEach(() => {
jest.resetModules()
latestPoolOptions = undefined
latestPoolInstance = undefined
})
it('calls onActivityAbort when worker stdout produces output', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
// Write to pool stdout — the Worker pipes it through an abort transform
latestPoolInstance!._stdout.write('some output')
// Wait for the piped data to propagate
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('calls onActivityAbort when worker stderr produces output', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
latestPoolInstance!._stderr.write('error output')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('fires onActivityAbort only once per registration', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
latestPoolInstance!._stdout.write('first')
latestPoolInstance!._stdout.write('second')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('resets the guard when setOnActivityAbort is called', async () => {
const onActivityAbort1 = jest.fn()
const onActivityAbort2 = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort: onActivityAbort1,
})
latestPoolInstance!._stdout.write('first')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort1).toHaveBeenCalledTimes(1)
// Re-register with new callback — resets the dedup guard
worker.setOnActivityAbort(onActivityAbort2)
latestPoolInstance!._stdout.write('second')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort2).toHaveBeenCalledTimes(1)
// Original callback should not fire again
expect(onActivityAbort1).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('does not fire after setOnActivityAbort(undefined)', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
worker.setOnActivityAbort(undefined)
latestPoolInstance!._stdout.write('output')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).not.toHaveBeenCalled()
worker.shutdownNow()
})
})
describe('lib/worker workerName in error messages', () => {
afterEach(() => {
jest.resetModules()
latestPoolOptions = undefined
latestPoolInstance = undefined
})
it('includes workerName in WorkerExitError after exhausting retries', async () => {
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
workerName: 'Next.js export worker',
maxRetries: 1,
exposedMethods: ['render'],
}) as any
latestPoolInstance!.dispatch = () => {
return Promise.reject(new WorkerExitErrorMock(1, null))
}
try {
await worker.render()
throw new Error('should have thrown')
} catch (err: any) {
expect(err.message).toBe(
'Next.js export worker exited with code: 1 and signal: null'
)
}
worker.shutdownNow()
})
it('uses default "Worker" when workerName is not set', async () => {
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
maxRetries: 0,
exposedMethods: ['render'],
}) as any
latestPoolInstance!.dispatch = () => {
return Promise.reject(new WorkerExitErrorMock(1, null))
}
try {
await worker.render()
throw new Error('should have thrown')
} catch (err: any) {
expect(err.message).toBe('Worker exited with code: 1 and signal: null')
}
worker.shutdownNow()
})
})
......@@ -205,7 +205,10 @@ export function createMessageHandler(
main,
[],
() => transport.disconnect(),
() => transport.disconnect()
(error) => {
console.error('Worker teardown() threw:', error)
transport.disconnect()
}
)
}
......
......@@ -1496,8 +1496,11 @@ describe('WorkerPool', () => {
timeout: 1000,
})
pool.dispatch('hung', [])
pool.dispatch('queued', []) // waiting in queue
const p1 = pool.dispatch('hung', [])
const p2 = pool.dispatch('queued', []) // waiting in queue
// Suppress unhandled rejections from shutdownNow()
p1.catch(() => {})
p2.catch(() => {})
const proc1 = latestProcess()
replyReady(proc1)
......@@ -1528,4 +1531,172 @@ describe('WorkerPool', () => {
jest.advanceTimersByTime(5000)
})
})
// -----------------------------------------------------------------------
// Spawn error rejects queued tasks
// -----------------------------------------------------------------------
describe('spawn error rejects queued tasks', () => {
it('rejects both in-flight and queued tasks on spawn error', async () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
concurrencyPerWorker: 1,
})
const p1 = pool.dispatch('first', [])
const p2 = pool.dispatch('second', []) // queued — only 1 worker, at capacity
const proc = latestProcess()
// Simulate a spawn error (e.g. ENOMEM)
proc.emit('error', new Error('spawn ENOMEM'))
await expect(p1).rejects.toThrow('spawn ENOMEM')
await expect(p2).rejects.toThrow('spawn ENOMEM')
pool.shutdownNow()
})
})
// -----------------------------------------------------------------------
// Booting worker timeout
// -----------------------------------------------------------------------
describe('booting worker timeout', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
it('kills a booting worker that never sends READY within timeout', async () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 1000,
})
const promise = pool.dispatch('test', [])
const proc1 = latestProcess()
// Do NOT call replyReady — worker stays in BOOTING state
// Advance past timeout
jest.advanceTimersByTime(1500)
// The booting worker should be killed
expect(proc1.killed).toBe(true)
// A replacement worker should be spawned for the queued task
expect(spawnedProcesses).toHaveLength(2)
const proc2 = latestProcess()
// The replacement worker completes the task
replyReady(proc2)
const calls = proc2.sent.filter((m) => m[0] === CHILD_MESSAGE_CALL)
expect(calls).toHaveLength(1)
replyOk(proc2, calls[0][1] as number, 'done')
await expect(promise).resolves.toBe('done')
pool.shutdownNow()
})
it('spawns replacement for queued tasks after booting worker times out', () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 2,
maxBootingWorkers: 1,
timeout: 1000,
})
pool.dispatch('a', [])
pool.dispatch('b', [])
expect(spawnedProcesses).toHaveLength(1)
const proc1 = spawnedProcesses[0]
// Worker 1 stays booting — timeout fires
jest.advanceTimersByTime(1500)
expect(proc1.killed).toBe(true)
// A replacement should be spawned for the remaining queued task
expect(spawnedProcesses.length).toBeGreaterThanOrEqual(2)
pool.shutdownNow()
})
it('does not kill a booting worker before timeout elapses', () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 2000,
})
pool.dispatch('test', [])
const proc = latestProcess()
// Advance less than timeout
jest.advanceTimersByTime(1000)
expect(proc.killed).toBe(false)
// Now make it ready — should work fine
replyReady(proc)
expect(proc.killed).toBe(false)
pool.shutdownNow()
})
})
// -----------------------------------------------------------------------
// onWorkerExit called on timeout
// -----------------------------------------------------------------------
describe('onWorkerExit on timeout', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
it('calls onWorkerExit when a running worker times out', async () => {
const onWorkerExit = jest.fn()
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 1000,
onWorkerExit,
})
const promise = pool.dispatch('slow', [])
const proc = latestProcess()
replyReady(proc)
// Advance past timeout
jest.advanceTimersByTime(1500)
await expect(promise).rejects.toThrow('Worker timed out')
expect(onWorkerExit).toHaveBeenCalledWith(null, 'SIGKILL')
pool.shutdownNow()
})
it('calls onWorkerExit when a booting worker times out', () => {
const onWorkerExit = jest.fn()
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 1000,
onWorkerExit,
})
pool.dispatch('test', [])
// Worker stays booting — timeout fires
jest.advanceTimersByTime(1500)
expect(onWorkerExit).toHaveBeenCalledWith(null, 'SIGKILL')
pool.shutdownNow()
})
})
})
......@@ -369,13 +369,6 @@ export class WorkerPool {
await exitPromise
clearTimeout(timeout)
// Reject any requests that were still in-flight when the worker
// exited. This prevents Promises from hanging forever (issue #11).
this._rejectActiveRequests(
worker,
new Error('Worker pool ended while requests were in-flight')
)
return forceExited
})
)
......@@ -391,7 +384,6 @@ export class WorkerPool {
/**
* Force-kill all workers immediately.
* All in-flight requests and queued tasks are rejected.
* Sends an END message first to allow graceful cleanup, then force-kills.
*/
shutdownNow(): void {
this._ending = true
......@@ -404,8 +396,6 @@ export class WorkerPool {
for (const worker of this._allWorkers) {
worker.state = WorkerState.SHUTTING_DOWN
this._rejectActiveRequests(worker, new Error('Worker pool closed'))
// Send END message to allow the child to clean up, then force-kill
worker.handle.send([CHILD_MESSAGE_END])
worker.handle.forceKill()
}
this._allWorkers.clear()
......@@ -786,10 +776,12 @@ export class WorkerPool {
}
/**
* Periodic check for workers that have timed out. A worker is considered
* timed out when it has in-flight requests and hasn't reported activity
* within the configured timeout window. Only the hung worker is killed
* and replaced — other workers are unaffected.
* Periodic check for workers that have timed out. A RUNNING worker is
* considered timed out when it has in-flight requests and hasn't reported
* activity within the configured timeout window. A BOOTING worker is
* considered timed out when it hasn't sent PARENT_MESSAGE_READY within
* the same window (e.g. a hung setup()). Only the affected worker is
* killed and replaced — other workers are unaffected.
*/
private _checkTimeouts(): void {
const { timeout } = this._options
......@@ -797,12 +789,15 @@ export class WorkerPool {
const now = Date.now()
for (const worker of this._allWorkers) {
if (now - worker.lastActivity < timeout) continue
if (
worker.state === WorkerState.RUNNING &&
worker.activeRequests.size > 0 &&
now - worker.lastActivity >= timeout
worker.activeRequests.size > 0
) {
this._handleWorkerTimeout(worker)
} else if (worker.state === WorkerState.BOOTING) {
this._handleWorkerTimeout(worker)
}
}
}
......@@ -812,13 +807,17 @@ export class WorkerPool {
* removes it from the pool, and spawns replacements for any queued tasks.
*/
private _handleWorkerTimeout(worker: PoolWorker): void {
if (worker.state === WorkerState.BOOTING) {
this._bootingCount--
}
worker.state = WorkerState.SHUTTING_DOWN
this._rejectActiveRequests(worker, new Error('Worker timed out'))
this._availableWorkers.delete(worker)
this._allWorkers.delete(worker)
// Send END to allow cleanup, then force-kill
worker.handle.send([CHILD_MESSAGE_END])
// Notify the caller about the timeout-induced kill
this._options.onWorkerExit?.(null, 'SIGKILL')
worker.handle.forceKill()
// Spawn replacement workers for any queued tasks
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment