Commit f27edfda authored by Tobias Koppers's avatar Tobias Koppers

Fix worker pool bugs: boot timeout, dead code, teardown logging, onWorkerExit on timeout

- #3: BOOTING workers now subject to timeout (prevents hung setup() from deadlocking pool)
- #5: Remove unreachable _rejectActiveRequests in shutdown() (already handled by _handleExit)
- #8: Remove misleading CHILD_MESSAGE_END before forceKill in shutdownNow()/_handleWorkerTimeout
- #9: Call onWorkerExit(null, 'SIGKILL') from _handleWorkerTimeout so timeout kills are observable
- #10: Log teardown() errors in worker-child-common.ts instead of silently swallowing them
- Fix WorkerExitErrorMock to match real error message format
- Add tests: onActivityAbort/setOnActivityAbort, workerName in error messages,
  spawn error queue rejection, booting worker timeout, onWorkerExit on timeout
Co-Authored-By: 's avatarClaude <noreply@anthropic.com>
parent 36d8d585
...@@ -52,9 +52,9 @@ class WorkerPoolMock { ...@@ -52,9 +52,9 @@ class WorkerPoolMock {
class WorkerExitErrorMock extends Error { class WorkerExitErrorMock extends Error {
code: number | null code: number | null
signal: string | null signal: string | null
constructor(code: number | null, signal: string | null) { constructor(code: number | null, signal: string | null, workerName?: string) {
super( super(
`Worker exited unexpectedly with code ${code}${signal ? `, signal ${signal}` : ''}` `${workerName ?? 'Worker'} exited with code: ${code} and signal: ${signal}`
) )
this.name = 'WorkerExitError' this.name = 'WorkerExitError'
this.code = code this.code = code
...@@ -691,7 +691,7 @@ describe('lib/worker maxRetries', () => { ...@@ -691,7 +691,7 @@ describe('lib/worker maxRetries', () => {
} }
await expect(worker.compute()).rejects.toThrow( await expect(worker.compute()).rejects.toThrow(
'Worker exited unexpectedly with code 1' 'Worker exited with code: 1 and signal: null'
) )
worker.shutdownNow() worker.shutdownNow()
...@@ -761,7 +761,7 @@ describe('lib/worker maxRetries', () => { ...@@ -761,7 +761,7 @@ describe('lib/worker maxRetries', () => {
} }
await expect(worker.compute()).rejects.toThrow( await expect(worker.compute()).rejects.toThrow(
'Worker exited unexpectedly with code 1' 'Worker exited with code: 1 and signal: null'
) )
worker.shutdownNow() worker.shutdownNow()
...@@ -843,3 +843,167 @@ describe('lib/worker exit handler cleanup', () => { ...@@ -843,3 +843,167 @@ describe('lib/worker exit handler cleanup', () => {
expect(remaining).toHaveLength(0) expect(remaining).toHaveLength(0)
}) })
}) })
describe('lib/worker onActivityAbort', () => {
afterEach(() => {
jest.resetModules()
latestPoolOptions = undefined
latestPoolInstance = undefined
})
it('calls onActivityAbort when worker stdout produces output', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
// Write to pool stdout — the Worker pipes it through an abort transform
latestPoolInstance!._stdout.write('some output')
// Wait for the piped data to propagate
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('calls onActivityAbort when worker stderr produces output', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
latestPoolInstance!._stderr.write('error output')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('fires onActivityAbort only once per registration', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
latestPoolInstance!._stdout.write('first')
latestPoolInstance!._stdout.write('second')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('resets the guard when setOnActivityAbort is called', async () => {
const onActivityAbort1 = jest.fn()
const onActivityAbort2 = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort: onActivityAbort1,
})
latestPoolInstance!._stdout.write('first')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort1).toHaveBeenCalledTimes(1)
// Re-register with new callback — resets the dedup guard
worker.setOnActivityAbort(onActivityAbort2)
latestPoolInstance!._stdout.write('second')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort2).toHaveBeenCalledTimes(1)
// Original callback should not fire again
expect(onActivityAbort1).toHaveBeenCalledTimes(1)
worker.shutdownNow()
})
it('does not fire after setOnActivityAbort(undefined)', async () => {
const onActivityAbort = jest.fn()
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
onActivityAbort,
})
worker.setOnActivityAbort(undefined)
latestPoolInstance!._stdout.write('output')
await new Promise((r) => setImmediate(r))
expect(onActivityAbort).not.toHaveBeenCalled()
worker.shutdownNow()
})
})
describe('lib/worker workerName in error messages', () => {
afterEach(() => {
jest.resetModules()
latestPoolOptions = undefined
latestPoolInstance = undefined
})
it('includes workerName in WorkerExitError after exhausting retries', async () => {
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
workerName: 'Next.js export worker',
maxRetries: 1,
exposedMethods: ['render'],
}) as any
latestPoolInstance!.dispatch = () => {
return Promise.reject(new WorkerExitErrorMock(1, null))
}
try {
await worker.render()
throw new Error('should have thrown')
} catch (err: any) {
expect(err.message).toBe(
'Next.js export worker exited with code: 1 and signal: null'
)
}
worker.shutdownNow()
})
it('uses default "Worker" when workerName is not set', async () => {
const { Worker } = require('./worker') as typeof import('./worker')
const worker = new Worker(__filename, {
...noopOptions,
maxRetries: 0,
exposedMethods: ['render'],
}) as any
latestPoolInstance!.dispatch = () => {
return Promise.reject(new WorkerExitErrorMock(1, null))
}
try {
await worker.render()
throw new Error('should have thrown')
} catch (err: any) {
expect(err.message).toBe('Worker exited with code: 1 and signal: null')
}
worker.shutdownNow()
})
})
...@@ -205,7 +205,10 @@ export function createMessageHandler( ...@@ -205,7 +205,10 @@ export function createMessageHandler(
main, main,
[], [],
() => transport.disconnect(), () => transport.disconnect(),
() => transport.disconnect() (error) => {
console.error('Worker teardown() threw:', error)
transport.disconnect()
}
) )
} }
......
...@@ -1496,8 +1496,11 @@ describe('WorkerPool', () => { ...@@ -1496,8 +1496,11 @@ describe('WorkerPool', () => {
timeout: 1000, timeout: 1000,
}) })
pool.dispatch('hung', []) const p1 = pool.dispatch('hung', [])
pool.dispatch('queued', []) // waiting in queue const p2 = pool.dispatch('queued', []) // waiting in queue
// Suppress unhandled rejections from shutdownNow()
p1.catch(() => {})
p2.catch(() => {})
const proc1 = latestProcess() const proc1 = latestProcess()
replyReady(proc1) replyReady(proc1)
...@@ -1528,4 +1531,172 @@ describe('WorkerPool', () => { ...@@ -1528,4 +1531,172 @@ describe('WorkerPool', () => {
jest.advanceTimersByTime(5000) jest.advanceTimersByTime(5000)
}) })
}) })
// -----------------------------------------------------------------------
// Spawn error rejects queued tasks
// -----------------------------------------------------------------------
describe('spawn error rejects queued tasks', () => {
it('rejects both in-flight and queued tasks on spawn error', async () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
concurrencyPerWorker: 1,
})
const p1 = pool.dispatch('first', [])
const p2 = pool.dispatch('second', []) // queued — only 1 worker, at capacity
const proc = latestProcess()
// Simulate a spawn error (e.g. ENOMEM)
proc.emit('error', new Error('spawn ENOMEM'))
await expect(p1).rejects.toThrow('spawn ENOMEM')
await expect(p2).rejects.toThrow('spawn ENOMEM')
pool.shutdownNow()
})
})
// -----------------------------------------------------------------------
// Booting worker timeout
// -----------------------------------------------------------------------
describe('booting worker timeout', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
it('kills a booting worker that never sends READY within timeout', async () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 1000,
})
const promise = pool.dispatch('test', [])
const proc1 = latestProcess()
// Do NOT call replyReady — worker stays in BOOTING state
// Advance past timeout
jest.advanceTimersByTime(1500)
// The booting worker should be killed
expect(proc1.killed).toBe(true)
// A replacement worker should be spawned for the queued task
expect(spawnedProcesses).toHaveLength(2)
const proc2 = latestProcess()
// The replacement worker completes the task
replyReady(proc2)
const calls = proc2.sent.filter((m) => m[0] === CHILD_MESSAGE_CALL)
expect(calls).toHaveLength(1)
replyOk(proc2, calls[0][1] as number, 'done')
await expect(promise).resolves.toBe('done')
pool.shutdownNow()
})
it('spawns replacement for queued tasks after booting worker times out', () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 2,
maxBootingWorkers: 1,
timeout: 1000,
})
pool.dispatch('a', [])
pool.dispatch('b', [])
expect(spawnedProcesses).toHaveLength(1)
const proc1 = spawnedProcesses[0]
// Worker 1 stays booting — timeout fires
jest.advanceTimersByTime(1500)
expect(proc1.killed).toBe(true)
// A replacement should be spawned for the remaining queued task
expect(spawnedProcesses.length).toBeGreaterThanOrEqual(2)
pool.shutdownNow()
})
it('does not kill a booting worker before timeout elapses', () => {
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 2000,
})
pool.dispatch('test', [])
const proc = latestProcess()
// Advance less than timeout
jest.advanceTimersByTime(1000)
expect(proc.killed).toBe(false)
// Now make it ready — should work fine
replyReady(proc)
expect(proc.killed).toBe(false)
pool.shutdownNow()
})
})
// -----------------------------------------------------------------------
// onWorkerExit called on timeout
// -----------------------------------------------------------------------
describe('onWorkerExit on timeout', () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
it('calls onWorkerExit when a running worker times out', async () => {
const onWorkerExit = jest.fn()
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 1000,
onWorkerExit,
})
const promise = pool.dispatch('slow', [])
const proc = latestProcess()
replyReady(proc)
// Advance past timeout
jest.advanceTimersByTime(1500)
await expect(promise).rejects.toThrow('Worker timed out')
expect(onWorkerExit).toHaveBeenCalledWith(null, 'SIGKILL')
pool.shutdownNow()
})
it('calls onWorkerExit when a booting worker times out', () => {
const onWorkerExit = jest.fn()
const pool = new WorkerPool({
workerPath: '/fake/worker.js',
maxWorkers: 1,
timeout: 1000,
onWorkerExit,
})
pool.dispatch('test', [])
// Worker stays booting — timeout fires
jest.advanceTimersByTime(1500)
expect(onWorkerExit).toHaveBeenCalledWith(null, 'SIGKILL')
pool.shutdownNow()
})
})
}) })
...@@ -369,13 +369,6 @@ export class WorkerPool { ...@@ -369,13 +369,6 @@ export class WorkerPool {
await exitPromise await exitPromise
clearTimeout(timeout) clearTimeout(timeout)
// Reject any requests that were still in-flight when the worker
// exited. This prevents Promises from hanging forever (issue #11).
this._rejectActiveRequests(
worker,
new Error('Worker pool ended while requests were in-flight')
)
return forceExited return forceExited
}) })
) )
...@@ -391,7 +384,6 @@ export class WorkerPool { ...@@ -391,7 +384,6 @@ export class WorkerPool {
/** /**
* Force-kill all workers immediately. * Force-kill all workers immediately.
* All in-flight requests and queued tasks are rejected. * All in-flight requests and queued tasks are rejected.
* Sends an END message first to allow graceful cleanup, then force-kills.
*/ */
shutdownNow(): void { shutdownNow(): void {
this._ending = true this._ending = true
...@@ -404,8 +396,6 @@ export class WorkerPool { ...@@ -404,8 +396,6 @@ export class WorkerPool {
for (const worker of this._allWorkers) { for (const worker of this._allWorkers) {
worker.state = WorkerState.SHUTTING_DOWN worker.state = WorkerState.SHUTTING_DOWN
this._rejectActiveRequests(worker, new Error('Worker pool closed')) this._rejectActiveRequests(worker, new Error('Worker pool closed'))
// Send END message to allow the child to clean up, then force-kill
worker.handle.send([CHILD_MESSAGE_END])
worker.handle.forceKill() worker.handle.forceKill()
} }
this._allWorkers.clear() this._allWorkers.clear()
...@@ -786,10 +776,12 @@ export class WorkerPool { ...@@ -786,10 +776,12 @@ export class WorkerPool {
} }
/** /**
* Periodic check for workers that have timed out. A worker is considered * Periodic check for workers that have timed out. A RUNNING worker is
* timed out when it has in-flight requests and hasn't reported activity * considered timed out when it has in-flight requests and hasn't reported
* within the configured timeout window. Only the hung worker is killed * activity within the configured timeout window. A BOOTING worker is
* and replaced — other workers are unaffected. * considered timed out when it hasn't sent PARENT_MESSAGE_READY within
* the same window (e.g. a hung setup()). Only the affected worker is
* killed and replaced — other workers are unaffected.
*/ */
private _checkTimeouts(): void { private _checkTimeouts(): void {
const { timeout } = this._options const { timeout } = this._options
...@@ -797,12 +789,15 @@ export class WorkerPool { ...@@ -797,12 +789,15 @@ export class WorkerPool {
const now = Date.now() const now = Date.now()
for (const worker of this._allWorkers) { for (const worker of this._allWorkers) {
if (now - worker.lastActivity < timeout) continue
if ( if (
worker.state === WorkerState.RUNNING && worker.state === WorkerState.RUNNING &&
worker.activeRequests.size > 0 && worker.activeRequests.size > 0
now - worker.lastActivity >= timeout
) { ) {
this._handleWorkerTimeout(worker) this._handleWorkerTimeout(worker)
} else if (worker.state === WorkerState.BOOTING) {
this._handleWorkerTimeout(worker)
} }
} }
} }
...@@ -812,13 +807,17 @@ export class WorkerPool { ...@@ -812,13 +807,17 @@ export class WorkerPool {
* removes it from the pool, and spawns replacements for any queued tasks. * removes it from the pool, and spawns replacements for any queued tasks.
*/ */
private _handleWorkerTimeout(worker: PoolWorker): void { private _handleWorkerTimeout(worker: PoolWorker): void {
if (worker.state === WorkerState.BOOTING) {
this._bootingCount--
}
worker.state = WorkerState.SHUTTING_DOWN worker.state = WorkerState.SHUTTING_DOWN
this._rejectActiveRequests(worker, new Error('Worker timed out')) this._rejectActiveRequests(worker, new Error('Worker timed out'))
this._availableWorkers.delete(worker) this._availableWorkers.delete(worker)
this._allWorkers.delete(worker) this._allWorkers.delete(worker)
// Send END to allow cleanup, then force-kill // Notify the caller about the timeout-induced kill
worker.handle.send([CHILD_MESSAGE_END]) this._options.onWorkerExit?.(null, 'SIGKILL')
worker.handle.forceKill() worker.handle.forceKill()
// Spawn replacement workers for any queued tasks // Spawn replacement workers for any queued tasks
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment