Unverified Commit 10ba5f77 authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

Revert static worker refactor (#56767)

The changes to this worker from #55841 seem to also cause strange behavior where the process doesn't exit despite sending an exit signal. Reverting until we can spend the time to investigate what the issue is. 

[slack x-ref](https://vercel.slack.com/archives/C03S8ED1DKM/p1697129702263269)
[slack x-ref](https://vercel.slack.com/archives/C056QDZARTM/p1697149427099799?thread_ts=1696351087.736439&cid=C056QDZARTM)
parent 786ef25e
...@@ -4,11 +4,7 @@ import type { PagesManifest } from './webpack/plugins/pages-manifest-plugin' ...@@ -4,11 +4,7 @@ import type { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
import type { ExportPathMap, NextConfigComplete } from '../server/config-shared' import type { ExportPathMap, NextConfigComplete } from '../server/config-shared'
import type { MiddlewareManifest } from './webpack/plugins/middleware-plugin' import type { MiddlewareManifest } from './webpack/plugins/middleware-plugin'
import type { ActionManifest } from './webpack/plugins/flight-client-entry-plugin' import type { ActionManifest } from './webpack/plugins/flight-client-entry-plugin'
import type { import type { ExportAppOptions, ExportAppWorker } from '../export/types'
ExportAppOptions,
ExportAppWorker,
ExportPageInput,
} from '../export/types'
import type { Revalidate } from '../server/lib/revalidate' import type { Revalidate } from '../server/lib/revalidate'
import '../lib/setup-exception-listeners' import '../lib/setup-exception-listeners'
...@@ -1209,16 +1205,7 @@ export default async function build( ...@@ -1209,16 +1205,7 @@ export default async function build(
) { ) {
let infoPrinted = false let infoPrinted = false
return Worker.create< return new Worker(staticWorkerPath, {
Pick<
typeof import('./worker'),
| 'hasCustomGetInitialProps'
| 'isPageStatic'
| 'getDefinedNamedExports'
| 'exportPage'
>,
[ExportPageInput]
>(staticWorkerPath, {
timeout: timeout * 1000, timeout: timeout * 1000,
onRestart: (method, [arg], attempts) => { onRestart: (method, [arg], attempts) => {
if (method === 'exportPage') { if (method === 'exportPage') {
...@@ -1265,7 +1252,14 @@ export default async function build( ...@@ -1265,7 +1252,14 @@ export default async function build(
'getDefinedNamedExports', 'getDefinedNamedExports',
'exportPage', 'exportPage',
], ],
}) }) as Worker &
Pick<
typeof import('./worker'),
| 'hasCustomGetInitialProps'
| 'isPageStatic'
| 'getDefinedNamedExports'
| 'exportPage'
>
} }
let CacheHandler: any let CacheHandler: any
......
...@@ -3,7 +3,6 @@ import type { ...@@ -3,7 +3,6 @@ import type {
ExportAppOptions, ExportAppOptions,
ExportWorker, ExportWorker,
WorkerRenderOptsPartial, WorkerRenderOptsPartial,
ExportPageInput,
} from './types' } from './types'
import type { PrerenderManifest } from '../build' import type { PrerenderManifest } from '../build'
import type { PagesManifest } from '../build/webpack/plugins/pages-manifest-plugin' import type { PagesManifest } from '../build/webpack/plugins/pages-manifest-plugin'
...@@ -174,32 +173,29 @@ function setupWorkers( ...@@ -174,32 +173,29 @@ function setupWorkers(
let infoPrinted = false let infoPrinted = false
const worker = Worker.create<typeof import('./worker'), [ExportPageInput]>( const worker = new Worker(require.resolve('./worker'), {
require.resolve('./worker'), timeout: timeout * 1000,
{ onRestart: (_method, [{ path }], attempts) => {
timeout: timeout * 1000, if (attempts >= 3) {
onRestart: (_method, [{ path }], attempts) => { throw new ExportError(
if (attempts >= 3) { `Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
throw new ExportError( )
`Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout` }
) Log.warn(
} `Restarted static page generation for ${path} because it took more than ${timeout} seconds`
)
if (!infoPrinted) {
Log.warn( Log.warn(
`Restarted static page generation for ${path} because it took more than ${timeout} seconds` 'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
) )
if (!infoPrinted) { infoPrinted = true
Log.warn( }
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout' },
) maxRetries: 0,
infoPrinted = true numWorkers: threads,
} enableWorkerThreads: nextConfig.experimental.workerThreads,
}, exposedMethods: ['default'],
maxRetries: 0, }) as Worker & typeof import('./worker')
numWorkers: threads,
enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'],
}
)
return { return {
pages: worker.default, pages: worker.default,
......
import type { ChildProcess } from 'child_process' import type { ChildProcess } from 'child_process'
import { Worker as JestWorker } from 'next/dist/compiled/jest-worker' import { Worker as JestWorker } from 'next/dist/compiled/jest-worker'
import { getNodeOptionsWithoutInspect } from '../server/lib/utils' import { getNodeOptionsWithoutInspect } from '../server/lib/utils'
// We need this as we're using `Promise.withResolvers` which is not available in the node typings
import '../lib/polyfill-promise-with-resolvers'
type FarmOptions = ConstructorParameters<typeof JestWorker>[1] type FarmOptions = ConstructorParameters<typeof JestWorker>[1]
const RESTARTED = Symbol('restarted') const RESTARTED = Symbol('restarted')
...@@ -17,58 +13,45 @@ const cleanupWorkers = (worker: JestWorker) => { ...@@ -17,58 +13,45 @@ const cleanupWorkers = (worker: JestWorker) => {
} }
} }
type Options< export class Worker {
T extends object = object, private _worker: JestWorker | undefined
Args extends any[] = any[]
> = FarmOptions & {
timeout?: number
onRestart?: (method: string, args: Args, attempts: number) => void
exposedMethods: ReadonlyArray<keyof T>
enableWorkerThreads?: boolean
}
export class Worker<T extends object = object, Args extends any[] = any[]> {
private _worker?: JestWorker
/** constructor(
* Creates a new worker with the correct typings associated with the selected
* methods.
*/
public static create<T extends object, Args extends any[] = any[]>(
workerPath: string, workerPath: string,
options: Options<T, Args> options: FarmOptions & {
): Worker<T, Args> & T { timeout?: number
return new Worker(workerPath, options) as Worker<T, Args> & T onRestart?: (method: string, args: any[], attempts: number) => void
} exposedMethods: ReadonlyArray<string>
enableWorkerThreads?: boolean
constructor(workerPath: string, options: Options<T, Args>) { }
) {
let { timeout, onRestart, ...farmOptions } = options let { timeout, onRestart, ...farmOptions } = options
let restartPromise: Promise<typeof RESTARTED> let restartPromise: Promise<typeof RESTARTED>
let resolveRestartPromise: (arg: typeof RESTARTED) => void let resolveRestartPromise: (arg: typeof RESTARTED) => void
let activeTasks = 0 let activeTasks = 0
this._worker = undefined
const createWorker = () => { const createWorker = () => {
const worker = new JestWorker(workerPath, { this._worker = new JestWorker(workerPath, {
...farmOptions, ...farmOptions,
forkOptions: { forkOptions: {
...farmOptions.forkOptions, ...farmOptions.forkOptions,
env: { env: {
...farmOptions.forkOptions?.env, ...((farmOptions.forkOptions?.env || {}) as any),
...process.env, ...process.env,
// We don't pass down NODE_OPTIONS as it can lead to extra memory // we don't pass down NODE_OPTIONS as it can
// usage, // extra memory usage
NODE_OPTIONS: getNodeOptionsWithoutInspect() NODE_OPTIONS: getNodeOptionsWithoutInspect()
.replace(/--max-old-space-size=[\d]{1,}/, '') .replace(/--max-old-space-size=[\d]{1,}/, '')
.trim(), .trim(),
}, } as any,
stdio: 'inherit',
}, },
}) }) as JestWorker
restartPromise = new Promise(
const { promise, resolve } = Promise.withResolvers<typeof RESTARTED>() (resolve) => (resolveRestartPromise = resolve)
restartPromise = promise )
resolveRestartPromise = resolve
/** /**
* Jest Worker has two worker types, ChildProcessWorker (uses child_process) and NodeThreadWorker (uses worker_threads) * Jest Worker has two worker types, ChildProcessWorker (uses child_process) and NodeThreadWorker (uses worker_threads)
...@@ -80,14 +63,11 @@ export class Worker<T extends object = object, Args extends any[] = any[]> { ...@@ -80,14 +63,11 @@ export class Worker<T extends object = object, Args extends any[] = any[]> {
* But this property is not available in NodeThreadWorker, so we need to check if we are using ChildProcessWorker * But this property is not available in NodeThreadWorker, so we need to check if we are using ChildProcessWorker
*/ */
if (!farmOptions.enableWorkerThreads) { if (!farmOptions.enableWorkerThreads) {
const poolWorkers: { _child?: ChildProcess }[] = for (const worker of ((this._worker as any)._workerPool?._workers ||
// @ts-expect-error - we're accessing a private property []) as {
worker._workerPool?._workers ?? [] _child?: ChildProcess
}[]) {
for (const poolWorker of poolWorkers) { worker._child?.on('exit', (code, signal) => {
if (!poolWorker._child) continue
poolWorker._child.once('exit', (code, signal) => {
// log unexpected exit if .end() wasn't called // log unexpected exit if .end() wasn't called
if ((code || (signal && signal !== 'SIGINT')) && this._worker) { if ((code || (signal && signal !== 'SIGINT')) && this._worker) {
console.error( console.error(
...@@ -98,22 +78,16 @@ export class Worker<T extends object = object, Args extends any[] = any[]> { ...@@ -98,22 +78,16 @@ export class Worker<T extends object = object, Args extends any[] = any[]> {
} }
} }
return worker this._worker.getStdout().pipe(process.stdout)
this._worker.getStderr().pipe(process.stderr)
} }
createWorker()
// Create the first worker.
this._worker = createWorker()
const onHanging = () => { const onHanging = () => {
const worker = this._worker const worker = this._worker
if (!worker) return if (!worker) return
// Grab the current restart promise, and create a new worker.
const resolve = resolveRestartPromise const resolve = resolveRestartPromise
this._worker = createWorker() createWorker()
// Once the old worker is ended, resolve the restart promise to signal to
// any active tasks that the worker had to be restarted.
worker.end().then(() => { worker.end().then(() => {
resolve(RESTARTED) resolve(RESTARTED)
}) })
...@@ -122,62 +96,33 @@ export class Worker<T extends object = object, Args extends any[] = any[]> { ...@@ -122,62 +96,33 @@ export class Worker<T extends object = object, Args extends any[] = any[]> {
let hangingTimer: NodeJS.Timeout | false = false let hangingTimer: NodeJS.Timeout | false = false
const onActivity = () => { const onActivity = () => {
// If there was an active hanging timer, clear it.
if (hangingTimer) clearTimeout(hangingTimer) if (hangingTimer) clearTimeout(hangingTimer)
hangingTimer = activeTasks > 0 && setTimeout(onHanging, timeout)
// If there are no active tasks, we don't need to start a new hanging
// timer.
if (activeTasks === 0) return
hangingTimer = setTimeout(onHanging, timeout)
} }
const wrapMethodWithTimeout = for (const method of farmOptions.exposedMethods) {
(methodName: keyof T) => if (method.startsWith('_')) continue
async (...args: Args) => { ;(this as any)[method] = timeout
activeTasks++ ? // eslint-disable-next-line no-loop-func
async (...args: any[]) => {
try { activeTasks++
let attempts = 0 try {
for (;;) { let attempts = 0
// Mark that we're doing work, we want to ensure that if the worker for (;;) {
// halts for any reason, we restart it. onActivity()
onActivity() const result = await Promise.race([
(this._worker as any)[method](...args),
const result = await Promise.race([ restartPromise,
// Either we'll get the result from the worker, or we'll get the ])
// restart promise to fire. if (result !== RESTARTED) return result
// @ts-expect-error - we're grabbing a dynamic method on the worker if (onRestart) onRestart(method, args, ++attempts)
this._worker[methodName](...args), }
restartPromise, } finally {
]) activeTasks--
onActivity()
// If the result anything besides `RESTARTED`, we can return it, as
// it's the actual result from the worker.
if (result !== RESTARTED) {
return result
} }
// Otherwise, we'll need to restart the worker, and try again.
if (onRestart) onRestart(methodName.toString(), args, ++attempts)
} }
} finally { : (this._worker as any)[method].bind(this._worker)
activeTasks--
onActivity()
}
}
for (const name of farmOptions.exposedMethods) {
if (name.startsWith('_')) continue
// @ts-expect-error - we're grabbing a dynamic method on the worker
let method = this._worker[name].bind(this._worker)
if (timeout) {
method = wrapMethodWithTimeout(name)
}
// @ts-expect-error - we're dynamically creating methods
this[name] = method
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment