Unverified Commit 8bbb1cd3 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

use a shared worker pool for collecting page data and static page generation (#27924)

this avoid loading all code twice and hopefully improving performance
parent b6411408
......@@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful
#### Possible Ways to Fix It
- Make sure that there is no infinite loop during execution.
- Make sure all Promises in `getStaticProps` `resolve` or `reject` correctly.
- Make sure all Promises in `getStaticPaths`/`getStaticProps` `resolve` or `reject` correctly.
- Avoid very long timeouts for network requests.
- Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds).
### Useful Links
- [`getStaticPaths`](https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation)
- [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation)
......@@ -91,8 +91,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
import { isWebpack5 } from 'next/dist/compiled/webpack/webpack'
import { NextConfigComplete } from '../server/config-shared'
const staticCheckWorker = require.resolve('./utils')
export type SsgRoute = {
initialRevalidateSeconds: number | false
srcRoute: string | null
......@@ -670,6 +668,62 @@ export default async function build(
await promises.readFile(buildManifestPath, 'utf8')
) as BuildManifest
const timeout = config.experimental.staticPageGenerationTimeout || 0
const sharedPool = config.experimental.sharedPool || false
const staticWorker = sharedPool
? require.resolve('./worker')
: require.resolve('./utils')
let infoPrinted = false
const staticWorkers = new Worker(staticWorker, {
timeout: timeout * 1000,
onRestart: (method, [arg], attempts) => {
if (method === 'exportPage') {
const { path: pagePath } = arg
if (attempts >= 3) {
throw new Error(
`Static page generation for ${pagePath} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
`Restarted static page genertion for ${pagePath} because it took more than ${timeout} seconds`
)
} else {
const pagePath = arg
if (attempts >= 2) {
throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
)
}
Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
)
}
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
)
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: sharedPool
? [
'hasCustomGetInitialProps',
'isPageStatic',
'getNamedExports',
'exportPage',
]
: ['hasCustomGetInitialProps', 'isPageStatic', 'getNamedExports'],
}) as Worker &
Pick<
typeof import('./worker'),
| 'hasCustomGetInitialProps'
| 'isPageStatic'
| 'getNamedExports'
| 'exportPage'
>
const analysisBegin = process.hrtime()
const staticCheckSpan = nextBuildSpan.traceChild('static-check')
......@@ -682,39 +736,6 @@ export default async function build(
} = await staticCheckSpan.traceAsyncFn(async () => {
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const timeout = config.experimental.pageDataCollectionTimeout || 0
let infoPrinted = false
const staticCheckWorkers = new Worker(staticCheckWorker, {
timeout: timeout * 1000,
onRestart: (_method, [pagePath], attempts) => {
if (attempts >= 2) {
throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
)
}
Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
)
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/page-data-collection-timeout'
)
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: [
'hasCustomGetInitialProps',
'isPageStatic',
'getNamedExports',
],
}) as Worker &
Pick<
typeof import('./utils'),
'hasCustomGetInitialProps' | 'isPageStatic' | 'getNamedExports'
>
const runtimeEnvConfig = {
publicRuntimeConfig: config.publicRuntimeConfig,
serverRuntimeConfig: config.serverRuntimeConfig,
......@@ -726,7 +747,7 @@ export default async function build(
const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn(
async () =>
hasCustomErrorPage &&
(await staticCheckWorkers.hasCustomGetInitialProps(
(await staticWorkers.hasCustomGetInitialProps(
'/_error',
distDir,
isLikeServerless,
......@@ -738,7 +759,7 @@ export default async function build(
const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
async () =>
hasCustomErrorPage &&
staticCheckWorkers.isPageStatic(
staticWorkers.isPageStatic(
'/_error',
distDir,
isLikeServerless,
......@@ -753,7 +774,7 @@ export default async function build(
// from _error instead
const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps(
const customAppGetInitialPropsPromise = staticWorkers.hasCustomGetInitialProps(
appPageToCheck,
distDir,
isLikeServerless,
......@@ -761,7 +782,7 @@ export default async function build(
true
)
const namedExportsPromise = staticCheckWorkers.getNamedExports(
const namedExportsPromise = staticWorkers.getNamedExports(
appPageToCheck,
distDir,
isLikeServerless,
......@@ -808,7 +829,7 @@ export default async function build(
'is-page-static'
)
let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
return staticCheckWorkers.isPageStatic(
return staticWorkers.isPageStatic(
page,
distDir,
isLikeServerless,
......@@ -926,7 +947,7 @@ export default async function build(
hasNonStaticErrorPage: nonStaticErrorPage,
}
staticCheckWorkers.end()
if (!sharedPool) staticWorkers.end()
return returnValue
})
......@@ -1082,7 +1103,8 @@ export default async function build(
ssgPages,
additionalSsgPaths
)
const exportApp = require('../export').default
const exportApp: typeof import('../export').default = require('../export')
.default
const exportOptions = {
silent: false,
buildExport: true,
......@@ -1090,6 +1112,14 @@ export default async function build(
pages: combinedPages,
outdir: path.join(distDir, 'export'),
statusMessage: 'Generating static pages',
exportPageWorker: sharedPool
? staticWorkers.exportPage.bind(staticWorkers)
: undefined,
endWorker: sharedPool
? async () => {
await staticWorkers.end()
}
: undefined,
}
const exportConfig: any = {
...config,
......
export * from './utils'
export { default as exportPage } from '../export/worker'
......@@ -137,6 +137,8 @@ interface ExportOptions {
pages?: string[]
buildExport?: boolean
statusMessage?: string
exportPageWorker?: typeof import('./worker').default
endWorker?: () => Promise<void>
}
export default async function exportApp(
......@@ -519,29 +521,40 @@ export default async function exportApp(
const timeout = configuration?.experimental.staticPageGenerationTimeout || 0
let infoPrinted = false
const worker = new Worker(require.resolve('./worker'), {
timeout: timeout * 1000,
onRestart: (_method, [{ path }], attempts) => {
if (attempts >= 3) {
throw new Error(
`Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
`Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
)
if (!infoPrinted) {
let exportPage: typeof import('./worker').default
let endWorker: () => Promise<void>
if (options.exportPageWorker) {
exportPage = options.exportPageWorker
endWorker = options.endWorker || (() => Promise.resolve())
} else {
const worker = new Worker(require.resolve('./worker'), {
timeout: timeout * 1000,
onRestart: (_method, [{ path }], attempts) => {
if (attempts >= 3) {
throw new Error(
`Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
`Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
)
infoPrinted = true
}
},
maxRetries: 0,
numWorkers: threads,
enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'],
}) as Worker & typeof import('./worker')
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
)
infoPrinted = true
}
},
maxRetries: 0,
numWorkers: threads,
enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'],
}) as Worker & typeof import('./worker')
exportPage = worker.default.bind(worker)
endWorker = async () => {
await worker.end()
}
}
let renderError = false
const errorPaths: string[] = []
......@@ -553,7 +566,7 @@ export default async function exportApp(
return pageExportSpan.traceAsyncFn(async () => {
const pathMap = exportPathMap[path]
const result = await worker.default({
const result = await exportPage({
path,
pathMap,
distDir,
......@@ -604,7 +617,7 @@ export default async function exportApp(
})
)
worker.end()
const endWorkerPromise = endWorker()
// copy prerendered routes to outDir
if (!options.buildExport && prerenderManifest) {
......@@ -681,5 +694,7 @@ export default async function exportApp(
if (telemetry) {
await telemetry.flush()
}
await endWorkerPromise
})
}
......@@ -105,6 +105,7 @@ export type NextConfig = { [key: string]: any } & {
swcMinify?: boolean
swcLoader?: boolean
cpus?: number
sharedPool?: boolean
plugins?: boolean
profiling?: boolean
isrFlushToDisk?: boolean
......@@ -128,7 +129,6 @@ export type NextConfig = { [key: string]: any } & {
craCompat?: boolean
esmExternals?: boolean | 'loose'
staticPageGenerationTimeout?: number
pageDataCollectionTimeout?: number
isrMemoryCacheSize?: number
concurrentFeatures?: boolean
}
......@@ -184,6 +184,7 @@ export const defaultConfig: NextConfig = {
(Number(process.env.CIRCLE_NODE_TOTAL) ||
(os.cpus() || { length: 1 }).length) - 1
),
sharedPool: false,
plugins: false,
profiling: false,
isrFlushToDisk: true,
......@@ -200,7 +201,6 @@ export const defaultConfig: NextConfig = {
craCompat: false,
esmExternals: false,
staticPageGenerationTimeout: 60,
pageDataCollectionTimeout: 60,
// default to 50MB limit
isrMemoryCacheSize: 50 * 1024 * 1024,
concurrentFeatures: false,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment