Unverified Commit e7ee310f authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add build worker exit tracking and enable tests (#46698)

Ensures we properly log when a compilation worker unexpectedly exits and ensures we enable the flag for some test suites to ensure it's covered. 

x-ref: https://github.com/vercel/next.js/pull/46666
x-ref: [slack thread](https://vercel.slack.com/archives/C04S835KUC9/p1677777414887749?thread_ts=1677775341.172509&cid=C04S835KUC9)
parent 111ef395
...@@ -7,6 +7,7 @@ import { ...@@ -7,6 +7,7 @@ import {
CLIENT_STATIC_FILES_RUNTIME_MAIN_APP, CLIENT_STATIC_FILES_RUNTIME_MAIN_APP,
APP_CLIENT_INTERNALS, APP_CLIENT_INTERNALS,
PHASE_PRODUCTION_BUILD, PHASE_PRODUCTION_BUILD,
COMPILER_INDEXES,
} from '../shared/lib/constants' } from '../shared/lib/constants'
import { runCompiler } from './compiler' import { runCompiler } from './compiler'
import * as Log from './output/log' import * as Log from './output/log'
...@@ -28,6 +29,7 @@ import * as flightManifestPluginModule from './webpack/plugins/flight-manifest-p ...@@ -28,6 +29,7 @@ import * as flightManifestPluginModule from './webpack/plugins/flight-manifest-p
import * as pagesPluginModule from './webpack/plugins/pages-manifest-plugin' import * as pagesPluginModule from './webpack/plugins/pages-manifest-plugin'
import { Worker } from 'next/dist/compiled/jest-worker' import { Worker } from 'next/dist/compiled/jest-worker'
import origDebug from 'next/dist/compiled/debug' import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process'
const debug = origDebug('next:build:webpack-build') const debug = origDebug('next:build:webpack-build')
...@@ -53,7 +55,9 @@ function isTraceEntryPointsPlugin( ...@@ -53,7 +55,9 @@ function isTraceEntryPointsPlugin(
return plugin instanceof TraceEntryPointsPlugin return plugin instanceof TraceEntryPointsPlugin
} }
async function webpackBuildImpl(compilerIdx?: number): Promise<{ async function webpackBuildImpl(
compilerName?: keyof typeof COMPILER_INDEXES
): Promise<{
duration: number duration: number
turbotraceContext?: TurbotraceContext turbotraceContext?: TurbotraceContext
serializedFlightMaps?: typeof NextBuildContext['serializedFlightMaps'] serializedFlightMaps?: typeof NextBuildContext['serializedFlightMaps']
...@@ -157,7 +161,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{ ...@@ -157,7 +161,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{
webpackBuildStart = process.hrtime() webpackBuildStart = process.hrtime()
debug(`starting compiler`, compilerIdx) debug(`starting compiler`, compilerName)
// We run client and server compilation separately to optimize for memory usage // We run client and server compilation separately to optimize for memory usage
await runWebpackSpan.traceAsyncFn(async () => { await runWebpackSpan.traceAsyncFn(async () => {
// Run the server compilers first and then the client // Run the server compilers first and then the client
...@@ -171,14 +175,14 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{ ...@@ -171,14 +175,14 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{
let edgeServerResult: UnwrapPromise<ReturnType<typeof runCompiler>> | null = let edgeServerResult: UnwrapPromise<ReturnType<typeof runCompiler>> | null =
null null
if (!compilerIdx || compilerIdx === 1) { if (!compilerName || compilerName === 'server') {
serverResult = await runCompiler(serverConfig, { serverResult = await runCompiler(serverConfig, {
runWebpackSpan, runWebpackSpan,
}) })
debug('server result', serverResult) debug('server result', serverResult)
} }
if (!compilerIdx || compilerIdx === 2) { if (!compilerName || compilerName === 'edge-server') {
edgeServerResult = configs[2] edgeServerResult = configs[2]
? await runCompiler(configs[2], { runWebpackSpan }) ? await runCompiler(configs[2], { runWebpackSpan })
: null : null
...@@ -208,7 +212,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{ ...@@ -208,7 +212,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{
} }
}) })
if (!compilerIdx || compilerIdx === 3) { if (!compilerName || compilerName === 'client') {
clientResult = await runCompiler(clientConfig, { clientResult = await runCompiler(clientConfig, {
runWebpackSpan, runWebpackSpan,
}) })
...@@ -295,7 +299,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{ ...@@ -295,7 +299,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{
Log.warn('Compiled with warnings\n') Log.warn('Compiled with warnings\n')
console.warn(result.warnings.filter(Boolean).join('\n\n')) console.warn(result.warnings.filter(Boolean).join('\n\n'))
console.warn() console.warn()
} else if (!compilerIdx) { } else if (!compilerName) {
Log.info('Compiled successfully') Log.info('Compiled successfully')
} }
...@@ -331,7 +335,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{ ...@@ -331,7 +335,7 @@ async function webpackBuildImpl(compilerIdx?: number): Promise<{
// the main function when this file is run as a worker // the main function when this file is run as a worker
export async function workerMain(workerData: { export async function workerMain(workerData: {
compilerIdx?: number compilerName: keyof typeof COMPILER_INDEXES
buildContext: typeof NextBuildContext buildContext: typeof NextBuildContext
}) { }) {
// setup new build context from the serialized data passed from the parent // setup new build context from the serialized data passed from the parent
...@@ -385,7 +389,7 @@ export async function workerMain(workerData: { ...@@ -385,7 +389,7 @@ export async function workerMain(workerData: {
) )
NextBuildContext.nextBuildSpan = trace('next-build') NextBuildContext.nextBuildSpan = trace('next-build')
const result = await webpackBuildImpl(workerData.compilerIdx) const result = await webpackBuildImpl(workerData.compilerName)
const { entriesTrace } = result.turbotraceContext ?? {} const { entriesTrace } = result.turbotraceContext ?? {}
if (entriesTrace) { if (entriesTrace) {
const { entryNameMap, depModArray } = entriesTrace const { entryNameMap, depModArray } = entriesTrace
...@@ -410,10 +414,11 @@ async function webpackBuildWithWorker() { ...@@ -410,10 +414,11 @@ async function webpackBuildWithWorker() {
...prunedBuildContext ...prunedBuildContext
} = NextBuildContext } = NextBuildContext
const getWorker = () => { const getWorker = (compilerName: string) => {
const _worker = new Worker(__filename, { const _worker = new Worker(__filename, {
exposedMethods: ['workerMain'], exposedMethods: ['workerMain'],
numWorkers: 1, numWorkers: 1,
maxRetries: 0,
forkOptions: { forkOptions: {
env: { env: {
...process.env, ...process.env,
...@@ -423,6 +428,19 @@ async function webpackBuildWithWorker() { ...@@ -423,6 +428,19 @@ async function webpackBuildWithWorker() {
}) as Worker & { workerMain: typeof workerMain } }) as Worker & { workerMain: typeof workerMain }
_worker.getStderr().pipe(process.stderr) _worker.getStderr().pipe(process.stderr)
_worker.getStdout().pipe(process.stdout) _worker.getStdout().pipe(process.stdout)
for (const worker of ((_worker as any)._workerPool?._workers || []) as {
_child: ChildProcess
}[]) {
worker._child.on('exit', (code, signal) => {
if (code || signal) {
console.error(
`Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}`
)
}
})
}
return _worker return _worker
} }
...@@ -430,12 +448,19 @@ async function webpackBuildWithWorker() { ...@@ -430,12 +448,19 @@ async function webpackBuildWithWorker() {
duration: 0, duration: 0,
turbotraceContext: {} as any, turbotraceContext: {} as any,
} }
// order matters here
const ORDERED_COMPILER_NAMES = [
'server',
'edge-server',
'client',
] as (keyof typeof COMPILER_INDEXES)[]
for (const compilerName of ORDERED_COMPILER_NAMES) {
const worker = getWorker(compilerName)
for (let i = 1; i < 4; i++) {
const worker = getWorker()
const curResult = await worker.workerMain({ const curResult = await worker.workerMain({
buildContext: prunedBuildContext, buildContext: prunedBuildContext,
compilerIdx: i, compilerName,
}) })
// destroy worker so it's not sticking around using memory // destroy worker so it's not sticking around using memory
await worker.end() await worker.end()
......
...@@ -4,6 +4,7 @@ module.exports = { ...@@ -4,6 +4,7 @@ module.exports = {
sri: { sri: {
algorithm: 'sha256', algorithm: 'sha256',
}, },
webpackBuildWorker: true,
}, },
// output: 'standalone', // output: 'standalone',
rewrites: async () => { rewrites: async () => {
......
module.exports = { module.exports = {
experimental: {
webpackBuildWorker: true,
},
i18n: { i18n: {
locales: ['en', 'fr', 'nl'], locales: ['en', 'fr', 'nl'],
defaultLocale: 'en', defaultLocale: 'en',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment