Unverified Commit e5e80ead authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Build: Share StaticWorker between static check and static generation (#85860)

## What?

Shares `static check` and `static generation` workers.

On hello-world:

Before:

```
next build test/e2e/app-dir/hello-world  7.58s user 1.07s system 332% cpu 2.603 total
```

After:

```
next build test/e2e/app-dir/hello-world  7.18s user 0.87s system 308% cpu 2.612 total
```
parent 8d891a26
......@@ -872,6 +872,9 @@ async function writeFullyStaticExport(
configOutDir: string,
nextBuildSpan: Span,
appDirOnly: boolean
// TODO: Reusing the worker seems to break finding if it's `.html` or a JS page.
// Because writeFullyStaticExport is called after `exportApp` has been called before
// worker: StaticWorker | undefined
): Promise<void> {
const exportApp = (require('../export') as typeof import('../export'))
.default as typeof import('../export').default
......@@ -888,6 +891,7 @@ async function writeFullyStaticExport(
appDirOnly,
},
nextBuildSpan
// worker
)
}
......@@ -927,6 +931,7 @@ export default async function build(
let appType: RoutesManifest['appType']
let loadedConfig: NextConfigComplete | undefined
let staticWorker: StaticWorker
try {
const nextBuildSpan = trace('next-build', undefined, {
buildMode: experimentalBuildMode,
......@@ -1977,7 +1982,7 @@ export default async function build(
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const worker = createStaticWorker(config, {
staticWorker = createStaticWorker(config, {
numberOfWorkers,
debuggerPortOffset: -1,
})
......@@ -2015,7 +2020,7 @@ export default async function build(
nonStaticErrorPageSpan.traceAsyncFn(
async () =>
hasCustomErrorPage &&
(await worker.hasCustomGetInitialProps({
(await staticWorker.hasCustomGetInitialProps({
page: '/_error',
distDir,
checkingApp: false,
......@@ -2026,7 +2031,7 @@ export default async function build(
const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
async () =>
hasCustomErrorPage &&
worker.isPageStatic({
staticWorker.isPageStatic({
dir,
page: '/_error',
distDir,
......@@ -2048,7 +2053,7 @@ export default async function build(
const appPageToCheck = '/_app'
const customAppGetInitialPropsPromise = hasUserPagesRoutes
? worker.hasCustomGetInitialProps({
? staticWorker.hasCustomGetInitialProps({
page: appPageToCheck,
distDir,
checkingApp: true,
......@@ -2057,7 +2062,7 @@ export default async function build(
: Promise.resolve(false)
const namedExportsPromise = hasUserPagesRoutes
? worker.getDefinedNamedExports({
? staticWorker.getDefinedNamedExports({
page: appPageToCheck,
distDir,
sriEnabled,
......@@ -2237,7 +2242,7 @@ export default async function build(
checkPageSpan.traceChild('is-page-static')
let workerResult = await isPageStaticSpan.traceAsyncFn(
() => {
return worker.isPageStatic({
return staticWorker.isPageStatic({
dir,
page,
originalAppPath,
......@@ -3089,7 +3094,8 @@ export default async function build(
numWorkers: numberOfWorkers,
appDirOnly,
},
nextBuildSpan
nextBuildSpan,
staticWorker
)
// If there was no result, there's nothing more to do.
......@@ -4018,8 +4024,12 @@ export default async function build(
buildTracesSpinner = createSpinner('Collecting build traces')
}
// ensure the worker is not left hanging
worker.end()
// When output: export we want to end the worker later as it's still used for writeFullyStaticExport
if (config.output !== 'export') {
// ensure the worker is not left hanging
staticWorker?.end()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
}
const analysisEnd = process.hrtime(analysisBegin)
telemetry.record(
......@@ -4194,6 +4204,11 @@ export default async function build(
}
if (config.output === 'export') {
// TODO: When writeFullyStaticExport doesn't fail when staticWorker is passed moved this after writeFullyStaticExport.
// End the worker here when it's output: export.
staticWorker.end()
staticWorker = undefined! // Reset staticWorker to make sure it does not end in `finally`
await nextBuildSpan
.traceChild('output-export-full-static-export')
.traceAsyncFn(async () => {
......@@ -4204,6 +4219,7 @@ export default async function build(
configOutDir,
nextBuildSpan,
appDirOnly
// staticWorker
)
})
}
......@@ -4311,6 +4327,10 @@ export default async function build(
}
throw e
} finally {
// @ts-expect-error Existence of staticWorker is checked here intentionally.
if (staticWorker) {
staticWorker.end()
}
// Ensure we wait for lockfile patching if present
await lockfilePatchPromise.cur
......
......@@ -76,7 +76,8 @@ export class ExportError extends Error {
async function exportAppImpl(
dir: string,
options: Readonly<ExportAppOptions>,
span: Span
span: Span,
staticWorker?: StaticWorker
): Promise<ExportAppResult | null> {
dir = resolve(dir)
......@@ -645,13 +646,20 @@ async function exportAppImpl(
`Exporting using ${options.numWorkers} worker${options.numWorkers > 1 ? 's' : ''}`
)
worker = createStaticWorker(nextConfig, {
debuggerPortOffset: getNextBuildDebuggerPortOffset({
kind: 'export-page',
}),
numberOfWorkers: options.numWorkers,
progress,
})
if (staticWorker) {
// TODO: progress shouldn't rely on "activity" event sent from `exportPage`.
staticWorker.setOnActivity(progress.run)
staticWorker.setOnActivityAbort(progress.clear)
worker = staticWorker
} else {
worker = createStaticWorker(nextConfig, {
debuggerPortOffset: getNextBuildDebuggerPortOffset({
kind: 'export-page',
}),
numberOfWorkers: options.numWorkers,
progress,
})
}
results = await exportPagesInBatches(worker, initialPhaseExportPaths)
......@@ -913,7 +921,13 @@ async function exportAppImpl(
await telemetry.flush()
}
if (worker) {
// Clean up activity listeners for progress.
if (staticWorker) {
staticWorker.setOnActivity(undefined)
staticWorker.setOnActivityAbort(undefined)
}
if (!staticWorker && worker) {
await worker.end()
}
......@@ -957,11 +971,12 @@ async function collectSegmentPathsImpl(
export default async function exportApp(
dir: string,
options: ExportAppOptions,
span: Span
span: Span,
staticWorker?: StaticWorker
): Promise<ExportAppResult | null> {
const nextExportSpan = span.traceChild('next-export')
return nextExportSpan.traceAsyncFn(async () => {
return await exportAppImpl(dir, options, nextExportSpan)
return await exportAppImpl(dir, options, nextExportSpan, staticWorker)
})
}
......@@ -12,6 +12,7 @@ import type {
import type { FetchMetrics } from '../server/base-http'
import type { RouteMetadata } from './routes/types'
import type { RenderResumeDataCache } from '../server/resume-data-cache/resume-data-cache'
import type { StaticWorker } from '../build'
export type ExportPathEntry = ExportPathMap[keyof ExportPathMap] & {
path: string
......@@ -89,6 +90,7 @@ export type WorkerRenderOpts = WorkerRenderOptsPartial &
LoadComponentsReturnType
export interface ExportAppOptions {
staticWorker?: StaticWorker
outdir: string
enabledDirectories: NextEnabledDirectories
silent?: boolean
......
......@@ -32,6 +32,9 @@ export function getNextBuildDebuggerPortOffset(_: {
export class Worker {
private _worker: JestWorker | undefined
private _onActivity: (() => void) | undefined
private _onActivityAbort: (() => void) | undefined
constructor(
workerPath: string,
options: Omit<FarmOptions, 'forkOptions'> & {
......@@ -65,9 +68,14 @@ export class Worker {
logger = console,
debuggerPortOffset,
isolatedMemory,
onActivity,
onActivityAbort,
...farmOptions
} = options
this._onActivity = onActivity
this._onActivityAbort = onActivityAbort
let restartPromise: Promise<typeof RESTARTED>
let resolveRestartPromise: (arg: typeof RESTARTED) => void
let activeTasks = 0
......@@ -182,16 +190,16 @@ export class Worker {
'type' in data &&
data.type === 'activity'
) {
onActivity()
onActivityImpl()
}
})
}
}
let aborted = false
const onActivityAbort = () => {
const onActivityAbortImpl = () => {
if (!aborted) {
options.onActivityAbort?.()
this._onActivityAbort?.()
aborted = true
}
}
......@@ -199,7 +207,7 @@ export class Worker {
// Listen to the worker's stdout and stderr, if there's any thing logged, abort the activity first
const abortActivityStreamOnLog = new Transform({
transform(_chunk, _encoding, callback) {
onActivityAbort()
onActivityAbortImpl()
callback()
},
})
......@@ -230,9 +238,9 @@ export class Worker {
let hangingTimer: NodeJS.Timeout | false = false
const onActivity = () => {
const onActivityImpl = () => {
if (hangingTimer) clearTimeout(hangingTimer)
if (options.onActivity) options.onActivity()
if (this._onActivity) this._onActivity()
hangingTimer = activeTasks > 0 && setTimeout(onHanging, timeout)
}
......@@ -246,7 +254,7 @@ export class Worker {
try {
let attempts = 0
for (;;) {
onActivity()
onActivityImpl()
const result = await Promise.race([
(this._worker as any)[method](...args),
restartPromise,
......@@ -256,13 +264,20 @@ export class Worker {
}
} finally {
activeTasks--
onActivity()
onActivityImpl()
}
}
: (this._worker as any)[method].bind(this._worker)
}
}
setOnActivity(onActivity: (() => void) | undefined): void {
this._onActivity = onActivity
}
setOnActivityAbort(onActivityAbort: (() => void) | undefined): void {
this._onActivityAbort = onActivityAbort
}
end(): ReturnType<JestWorker['end']> {
const worker = this._worker
if (!worker) {
......
import { triggerRefresh } from './actions'
import { connection } from 'next/server'
export default async function Page() {
await connection()
export default function Page() {
const timestamp = performance.now()
return (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment