Unverified Commit e5e80ead authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Build: Share StaticWorker between static check and static generation (#85860)

## What?

Shares `static check` and `static generation` workers.

On hello-world:

Before:

```
next build test/e2e/app-dir/hello-world  7.58s user 1.07s system 332% cpu 2.603 total
```

After:

```
next build test/e2e/app-dir/hello-world  7.18s user 0.87s system 308% cpu 2.612 total
```
parent 8d891a26
...@@ -872,6 +872,9 @@ async function writeFullyStaticExport( ...@@ -872,6 +872,9 @@ async function writeFullyStaticExport(
configOutDir: string, configOutDir: string,
nextBuildSpan: Span, nextBuildSpan: Span,
appDirOnly: boolean appDirOnly: boolean
// TODO: Reusing the worker seems to break finding if it's `.html` or a JS page.
// Because writeFullyStaticExport is called after `exportApp` has been called before
// worker: StaticWorker | undefined
): Promise<void> { ): Promise<void> {
const exportApp = (require('../export') as typeof import('../export')) const exportApp = (require('../export') as typeof import('../export'))
.default as typeof import('../export').default .default as typeof import('../export').default
...@@ -888,6 +891,7 @@ async function writeFullyStaticExport( ...@@ -888,6 +891,7 @@ async function writeFullyStaticExport(
appDirOnly, appDirOnly,
}, },
nextBuildSpan nextBuildSpan
// worker
) )
} }
...@@ -927,6 +931,7 @@ export default async function build( ...@@ -927,6 +931,7 @@ export default async function build(
let appType: RoutesManifest['appType'] let appType: RoutesManifest['appType']
let loadedConfig: NextConfigComplete | undefined let loadedConfig: NextConfigComplete | undefined
let staticWorker: StaticWorker
try { try {
const nextBuildSpan = trace('next-build', undefined, { const nextBuildSpan = trace('next-build', undefined, {
buildMode: experimentalBuildMode, buildMode: experimentalBuildMode,
...@@ -1977,7 +1982,7 @@ export default async function build( ...@@ -1977,7 +1982,7 @@ export default async function build(
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const worker = createStaticWorker(config, { staticWorker = createStaticWorker(config, {
numberOfWorkers, numberOfWorkers,
debuggerPortOffset: -1, debuggerPortOffset: -1,
}) })
...@@ -2015,7 +2020,7 @@ export default async function build( ...@@ -2015,7 +2020,7 @@ export default async function build(
nonStaticErrorPageSpan.traceAsyncFn( nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
(await worker.hasCustomGetInitialProps({ (await staticWorker.hasCustomGetInitialProps({
page: '/_error', page: '/_error',
distDir, distDir,
checkingApp: false, checkingApp: false,
...@@ -2026,7 +2031,7 @@ export default async function build( ...@@ -2026,7 +2031,7 @@ export default async function build(
const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn( const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
worker.isPageStatic({ staticWorker.isPageStatic({
dir, dir,
page: '/_error', page: '/_error',
distDir, distDir,
...@@ -2048,7 +2053,7 @@ export default async function build( ...@@ -2048,7 +2053,7 @@ export default async function build(
const appPageToCheck = '/_app' const appPageToCheck = '/_app'
const customAppGetInitialPropsPromise = hasUserPagesRoutes const customAppGetInitialPropsPromise = hasUserPagesRoutes
? worker.hasCustomGetInitialProps({ ? staticWorker.hasCustomGetInitialProps({
page: appPageToCheck, page: appPageToCheck,
distDir, distDir,
checkingApp: true, checkingApp: true,
...@@ -2057,7 +2062,7 @@ export default async function build( ...@@ -2057,7 +2062,7 @@ export default async function build(
: Promise.resolve(false) : Promise.resolve(false)
const namedExportsPromise = hasUserPagesRoutes const namedExportsPromise = hasUserPagesRoutes
? worker.getDefinedNamedExports({ ? staticWorker.getDefinedNamedExports({
page: appPageToCheck, page: appPageToCheck,
distDir, distDir,
sriEnabled, sriEnabled,
...@@ -2237,7 +2242,7 @@ export default async function build( ...@@ -2237,7 +2242,7 @@ export default async function build(
checkPageSpan.traceChild('is-page-static') checkPageSpan.traceChild('is-page-static')
let workerResult = await isPageStaticSpan.traceAsyncFn( let workerResult = await isPageStaticSpan.traceAsyncFn(
() => { () => {
return worker.isPageStatic({ return staticWorker.isPageStatic({
dir, dir,
page, page,
originalAppPath, originalAppPath,
...@@ -3089,7 +3094,8 @@ export default async function build( ...@@ -3089,7 +3094,8 @@ export default async function build(
numWorkers: numberOfWorkers, numWorkers: numberOfWorkers,
appDirOnly, appDirOnly,
}, },
nextBuildSpan nextBuildSpan,
staticWorker
) )
// If there was no result, there's nothing more to do. // If there was no result, there's nothing more to do.
...@@ -4018,8 +4024,12 @@ export default async function build( ...@@ -4018,8 +4024,12 @@ export default async function build(
buildTracesSpinner = createSpinner('Collecting build traces') buildTracesSpinner = createSpinner('Collecting build traces')
} }
// ensure the worker is not left hanging // When output: export we want to end the worker later as it's still used for writeFullyStaticExport
worker.end() if (config.output !== 'export') {
// ensure the worker is not left hanging
staticWorker?.end()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
}
const analysisEnd = process.hrtime(analysisBegin) const analysisEnd = process.hrtime(analysisBegin)
telemetry.record( telemetry.record(
...@@ -4194,6 +4204,11 @@ export default async function build( ...@@ -4194,6 +4204,11 @@ export default async function build(
} }
if (config.output === 'export') { if (config.output === 'export') {
// TODO: When writeFullyStaticExport doesn't fail when staticWorker is passed moved this after writeFullyStaticExport.
// End the worker here when it's output: export.
staticWorker.end()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
await nextBuildSpan await nextBuildSpan
.traceChild('output-export-full-static-export') .traceChild('output-export-full-static-export')
.traceAsyncFn(async () => { .traceAsyncFn(async () => {
...@@ -4204,6 +4219,7 @@ export default async function build( ...@@ -4204,6 +4219,7 @@ export default async function build(
configOutDir, configOutDir,
nextBuildSpan, nextBuildSpan,
appDirOnly appDirOnly
// staticWorker
) )
}) })
} }
...@@ -4311,6 +4327,10 @@ export default async function build( ...@@ -4311,6 +4327,10 @@ export default async function build(
} }
throw e throw e
} finally { } finally {
// @ts-expect-error Existence of staticWorker is checked here intentionally.
if (staticWorker) {
staticWorker.end()
}
// Ensure we wait for lockfile patching if present // Ensure we wait for lockfile patching if present
await lockfilePatchPromise.cur await lockfilePatchPromise.cur
......
...@@ -76,7 +76,8 @@ export class ExportError extends Error { ...@@ -76,7 +76,8 @@ export class ExportError extends Error {
async function exportAppImpl( async function exportAppImpl(
dir: string, dir: string,
options: Readonly<ExportAppOptions>, options: Readonly<ExportAppOptions>,
span: Span span: Span,
staticWorker?: StaticWorker
): Promise<ExportAppResult | null> { ): Promise<ExportAppResult | null> {
dir = resolve(dir) dir = resolve(dir)
...@@ -645,13 +646,20 @@ async function exportAppImpl( ...@@ -645,13 +646,20 @@ async function exportAppImpl(
`Exporting using ${options.numWorkers} worker${options.numWorkers > 1 ? 's' : ''}` `Exporting using ${options.numWorkers} worker${options.numWorkers > 1 ? 's' : ''}`
) )
worker = createStaticWorker(nextConfig, { if (staticWorker) {
debuggerPortOffset: getNextBuildDebuggerPortOffset({ // TODO: progress shouldn't rely on "activity" event sent from `exportPage`.
kind: 'export-page', staticWorker.setOnActivity(progress.run)
}), staticWorker.setOnActivityAbort(progress.clear)
numberOfWorkers: options.numWorkers, worker = staticWorker
progress, } else {
}) worker = createStaticWorker(nextConfig, {
debuggerPortOffset: getNextBuildDebuggerPortOffset({
kind: 'export-page',
}),
numberOfWorkers: options.numWorkers,
progress,
})
}
results = await exportPagesInBatches(worker, initialPhaseExportPaths) results = await exportPagesInBatches(worker, initialPhaseExportPaths)
...@@ -913,7 +921,13 @@ async function exportAppImpl( ...@@ -913,7 +921,13 @@ async function exportAppImpl(
await telemetry.flush() await telemetry.flush()
} }
if (worker) { // Clean up activity listeners for progress.
if (staticWorker) {
staticWorker.setOnActivity(undefined)
staticWorker.setOnActivityAbort(undefined)
}
if (!staticWorker && worker) {
await worker.end() await worker.end()
} }
...@@ -957,11 +971,12 @@ async function collectSegmentPathsImpl( ...@@ -957,11 +971,12 @@ async function collectSegmentPathsImpl(
export default async function exportApp( export default async function exportApp(
dir: string, dir: string,
options: ExportAppOptions, options: ExportAppOptions,
span: Span span: Span,
staticWorker?: StaticWorker
): Promise<ExportAppResult | null> { ): Promise<ExportAppResult | null> {
const nextExportSpan = span.traceChild('next-export') const nextExportSpan = span.traceChild('next-export')
return nextExportSpan.traceAsyncFn(async () => { return nextExportSpan.traceAsyncFn(async () => {
return await exportAppImpl(dir, options, nextExportSpan) return await exportAppImpl(dir, options, nextExportSpan, staticWorker)
}) })
} }
...@@ -12,6 +12,7 @@ import type { ...@@ -12,6 +12,7 @@ import type {
import type { FetchMetrics } from '../server/base-http' import type { FetchMetrics } from '../server/base-http'
import type { RouteMetadata } from './routes/types' import type { RouteMetadata } from './routes/types'
import type { RenderResumeDataCache } from '../server/resume-data-cache/resume-data-cache' import type { RenderResumeDataCache } from '../server/resume-data-cache/resume-data-cache'
import type { StaticWorker } from '../build'
export type ExportPathEntry = ExportPathMap[keyof ExportPathMap] & { export type ExportPathEntry = ExportPathMap[keyof ExportPathMap] & {
path: string path: string
...@@ -89,6 +90,7 @@ export type WorkerRenderOpts = WorkerRenderOptsPartial & ...@@ -89,6 +90,7 @@ export type WorkerRenderOpts = WorkerRenderOptsPartial &
LoadComponentsReturnType LoadComponentsReturnType
export interface ExportAppOptions { export interface ExportAppOptions {
staticWorker?: StaticWorker
outdir: string outdir: string
enabledDirectories: NextEnabledDirectories enabledDirectories: NextEnabledDirectories
silent?: boolean silent?: boolean
......
...@@ -32,6 +32,9 @@ export function getNextBuildDebuggerPortOffset(_: { ...@@ -32,6 +32,9 @@ export function getNextBuildDebuggerPortOffset(_: {
export class Worker { export class Worker {
private _worker: JestWorker | undefined private _worker: JestWorker | undefined
private _onActivity: (() => void) | undefined
private _onActivityAbort: (() => void) | undefined
constructor( constructor(
workerPath: string, workerPath: string,
options: Omit<FarmOptions, 'forkOptions'> & { options: Omit<FarmOptions, 'forkOptions'> & {
...@@ -65,9 +68,14 @@ export class Worker { ...@@ -65,9 +68,14 @@ export class Worker {
logger = console, logger = console,
debuggerPortOffset, debuggerPortOffset,
isolatedMemory, isolatedMemory,
onActivity,
onActivityAbort,
...farmOptions ...farmOptions
} = options } = options
this._onActivity = onActivity
this._onActivityAbort = onActivityAbort
let restartPromise: Promise<typeof RESTARTED> let restartPromise: Promise<typeof RESTARTED>
let resolveRestartPromise: (arg: typeof RESTARTED) => void let resolveRestartPromise: (arg: typeof RESTARTED) => void
let activeTasks = 0 let activeTasks = 0
...@@ -182,16 +190,16 @@ export class Worker { ...@@ -182,16 +190,16 @@ export class Worker {
'type' in data && 'type' in data &&
data.type === 'activity' data.type === 'activity'
) { ) {
onActivity() onActivityImpl()
} }
}) })
} }
} }
let aborted = false let aborted = false
const onActivityAbort = () => { const onActivityAbortImpl = () => {
if (!aborted) { if (!aborted) {
options.onActivityAbort?.() this._onActivityAbort?.()
aborted = true aborted = true
} }
} }
...@@ -199,7 +207,7 @@ export class Worker { ...@@ -199,7 +207,7 @@ export class Worker {
// Listen to the worker's stdout and stderr, if there's any thing logged, abort the activity first // Listen to the worker's stdout and stderr, if there's any thing logged, abort the activity first
const abortActivityStreamOnLog = new Transform({ const abortActivityStreamOnLog = new Transform({
transform(_chunk, _encoding, callback) { transform(_chunk, _encoding, callback) {
onActivityAbort() onActivityAbortImpl()
callback() callback()
}, },
}) })
...@@ -230,9 +238,9 @@ export class Worker { ...@@ -230,9 +238,9 @@ export class Worker {
let hangingTimer: NodeJS.Timeout | false = false let hangingTimer: NodeJS.Timeout | false = false
const onActivity = () => { const onActivityImpl = () => {
if (hangingTimer) clearTimeout(hangingTimer) if (hangingTimer) clearTimeout(hangingTimer)
if (options.onActivity) options.onActivity() if (this._onActivity) this._onActivity()
hangingTimer = activeTasks > 0 && setTimeout(onHanging, timeout) hangingTimer = activeTasks > 0 && setTimeout(onHanging, timeout)
} }
...@@ -246,7 +254,7 @@ export class Worker { ...@@ -246,7 +254,7 @@ export class Worker {
try { try {
let attempts = 0 let attempts = 0
for (;;) { for (;;) {
onActivity() onActivityImpl()
const result = await Promise.race([ const result = await Promise.race([
(this._worker as any)[method](...args), (this._worker as any)[method](...args),
restartPromise, restartPromise,
...@@ -256,13 +264,20 @@ export class Worker { ...@@ -256,13 +264,20 @@ export class Worker {
} }
} finally { } finally {
activeTasks-- activeTasks--
onActivity() onActivityImpl()
} }
} }
: (this._worker as any)[method].bind(this._worker) : (this._worker as any)[method].bind(this._worker)
} }
} }
setOnActivity(onActivity: (() => void) | undefined): void {
this._onActivity = onActivity
}
setOnActivityAbort(onActivityAbort: (() => void) | undefined): void {
this._onActivityAbort = onActivityAbort
}
end(): ReturnType<JestWorker['end']> { end(): ReturnType<JestWorker['end']> {
const worker = this._worker const worker = this._worker
if (!worker) { if (!worker) {
......
import { triggerRefresh } from './actions' import { triggerRefresh } from './actions'
import { connection } from 'next/server'
export default async function Page() {
await connection()
export default function Page() {
const timestamp = performance.now() const timestamp = performance.now()
return ( return (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment