Unverified Commit 8bbb1cd3 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

use a shared worker pool for collecting page data and static page generation (#27924)

this avoid loading all code twice and hopefully improving performance
parent b6411408
...@@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful ...@@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful
#### Possible Ways to Fix It #### Possible Ways to Fix It
- Make sure that there is no infinite loop during execution. - Make sure that there is no infinite loop during execution.
- Make sure all Promises in `getStaticProps` `resolve` or `reject` correctly. - Make sure all Promises in `getStaticPaths`/`getStaticProps` `resolve` or `reject` correctly.
- Avoid very long timeouts for network requests. - Avoid very long timeouts for network requests.
- Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds). - Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds).
### Useful Links ### Useful Links
- [`getStaticPaths`](https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation)
- [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation) - [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation)
...@@ -91,8 +91,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path' ...@@ -91,8 +91,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
import { isWebpack5 } from 'next/dist/compiled/webpack/webpack' import { isWebpack5 } from 'next/dist/compiled/webpack/webpack'
import { NextConfigComplete } from '../server/config-shared' import { NextConfigComplete } from '../server/config-shared'
const staticCheckWorker = require.resolve('./utils')
export type SsgRoute = { export type SsgRoute = {
initialRevalidateSeconds: number | false initialRevalidateSeconds: number | false
srcRoute: string | null srcRoute: string | null
...@@ -670,6 +668,62 @@ export default async function build( ...@@ -670,6 +668,62 @@ export default async function build(
await promises.readFile(buildManifestPath, 'utf8') await promises.readFile(buildManifestPath, 'utf8')
) as BuildManifest ) as BuildManifest
const timeout = config.experimental.staticPageGenerationTimeout || 0
const sharedPool = config.experimental.sharedPool || false
const staticWorker = sharedPool
? require.resolve('./worker')
: require.resolve('./utils')
let infoPrinted = false
const staticWorkers = new Worker(staticWorker, {
timeout: timeout * 1000,
onRestart: (method, [arg], attempts) => {
if (method === 'exportPage') {
const { path: pagePath } = arg
if (attempts >= 3) {
throw new Error(
`Static page generation for ${pagePath} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
`Restarted static page genertion for ${pagePath} because it took more than ${timeout} seconds`
)
} else {
const pagePath = arg
if (attempts >= 2) {
throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
)
}
Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
)
}
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
)
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: sharedPool
? [
'hasCustomGetInitialProps',
'isPageStatic',
'getNamedExports',
'exportPage',
]
: ['hasCustomGetInitialProps', 'isPageStatic', 'getNamedExports'],
}) as Worker &
Pick<
typeof import('./worker'),
| 'hasCustomGetInitialProps'
| 'isPageStatic'
| 'getNamedExports'
| 'exportPage'
>
const analysisBegin = process.hrtime() const analysisBegin = process.hrtime()
const staticCheckSpan = nextBuildSpan.traceChild('static-check') const staticCheckSpan = nextBuildSpan.traceChild('static-check')
...@@ -682,39 +736,6 @@ export default async function build( ...@@ -682,39 +736,6 @@ export default async function build(
} = await staticCheckSpan.traceAsyncFn(async () => { } = await staticCheckSpan.traceAsyncFn(async () => {
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const timeout = config.experimental.pageDataCollectionTimeout || 0
let infoPrinted = false
const staticCheckWorkers = new Worker(staticCheckWorker, {
timeout: timeout * 1000,
onRestart: (_method, [pagePath], attempts) => {
if (attempts >= 2) {
throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
)
}
Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
)
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/page-data-collection-timeout'
)
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: [
'hasCustomGetInitialProps',
'isPageStatic',
'getNamedExports',
],
}) as Worker &
Pick<
typeof import('./utils'),
'hasCustomGetInitialProps' | 'isPageStatic' | 'getNamedExports'
>
const runtimeEnvConfig = { const runtimeEnvConfig = {
publicRuntimeConfig: config.publicRuntimeConfig, publicRuntimeConfig: config.publicRuntimeConfig,
serverRuntimeConfig: config.serverRuntimeConfig, serverRuntimeConfig: config.serverRuntimeConfig,
...@@ -726,7 +747,7 @@ export default async function build( ...@@ -726,7 +747,7 @@ export default async function build(
const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn( const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
(await staticCheckWorkers.hasCustomGetInitialProps( (await staticWorkers.hasCustomGetInitialProps(
'/_error', '/_error',
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -738,7 +759,7 @@ export default async function build( ...@@ -738,7 +759,7 @@ export default async function build(
const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn( const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
staticCheckWorkers.isPageStatic( staticWorkers.isPageStatic(
'/_error', '/_error',
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -753,7 +774,7 @@ export default async function build( ...@@ -753,7 +774,7 @@ export default async function build(
// from _error instead // from _error instead
const appPageToCheck = isLikeServerless ? '/_error' : '/_app' const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps( const customAppGetInitialPropsPromise = staticWorkers.hasCustomGetInitialProps(
appPageToCheck, appPageToCheck,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -761,7 +782,7 @@ export default async function build( ...@@ -761,7 +782,7 @@ export default async function build(
true true
) )
const namedExportsPromise = staticCheckWorkers.getNamedExports( const namedExportsPromise = staticWorkers.getNamedExports(
appPageToCheck, appPageToCheck,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -808,7 +829,7 @@ export default async function build( ...@@ -808,7 +829,7 @@ export default async function build(
'is-page-static' 'is-page-static'
) )
let workerResult = await isPageStaticSpan.traceAsyncFn(() => { let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
return staticCheckWorkers.isPageStatic( return staticWorkers.isPageStatic(
page, page,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -926,7 +947,7 @@ export default async function build( ...@@ -926,7 +947,7 @@ export default async function build(
hasNonStaticErrorPage: nonStaticErrorPage, hasNonStaticErrorPage: nonStaticErrorPage,
} }
staticCheckWorkers.end() if (!sharedPool) staticWorkers.end()
return returnValue return returnValue
}) })
...@@ -1082,7 +1103,8 @@ export default async function build( ...@@ -1082,7 +1103,8 @@ export default async function build(
ssgPages, ssgPages,
additionalSsgPaths additionalSsgPaths
) )
const exportApp = require('../export').default const exportApp: typeof import('../export').default = require('../export')
.default
const exportOptions = { const exportOptions = {
silent: false, silent: false,
buildExport: true, buildExport: true,
...@@ -1090,6 +1112,14 @@ export default async function build( ...@@ -1090,6 +1112,14 @@ export default async function build(
pages: combinedPages, pages: combinedPages,
outdir: path.join(distDir, 'export'), outdir: path.join(distDir, 'export'),
statusMessage: 'Generating static pages', statusMessage: 'Generating static pages',
exportPageWorker: sharedPool
? staticWorkers.exportPage.bind(staticWorkers)
: undefined,
endWorker: sharedPool
? async () => {
await staticWorkers.end()
}
: undefined,
} }
const exportConfig: any = { const exportConfig: any = {
...config, ...config,
......
export * from './utils'
export { default as exportPage } from '../export/worker'
...@@ -137,6 +137,8 @@ interface ExportOptions { ...@@ -137,6 +137,8 @@ interface ExportOptions {
pages?: string[] pages?: string[]
buildExport?: boolean buildExport?: boolean
statusMessage?: string statusMessage?: string
exportPageWorker?: typeof import('./worker').default
endWorker?: () => Promise<void>
} }
export default async function exportApp( export default async function exportApp(
...@@ -519,29 +521,40 @@ export default async function exportApp( ...@@ -519,29 +521,40 @@ export default async function exportApp(
const timeout = configuration?.experimental.staticPageGenerationTimeout || 0 const timeout = configuration?.experimental.staticPageGenerationTimeout || 0
let infoPrinted = false let infoPrinted = false
const worker = new Worker(require.resolve('./worker'), { let exportPage: typeof import('./worker').default
timeout: timeout * 1000, let endWorker: () => Promise<void>
onRestart: (_method, [{ path }], attempts) => { if (options.exportPageWorker) {
if (attempts >= 3) { exportPage = options.exportPageWorker
throw new Error( endWorker = options.endWorker || (() => Promise.resolve())
`Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout` } else {
) const worker = new Worker(require.resolve('./worker'), {
} timeout: timeout * 1000,
Log.warn( onRestart: (_method, [{ path }], attempts) => {
`Restarted static page genertion for ${path} because it took more than ${timeout} seconds` if (attempts >= 3) {
) throw new Error(
if (!infoPrinted) { `Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn( Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout' `Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
) )
infoPrinted = true if (!infoPrinted) {
} Log.warn(
}, 'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
maxRetries: 0, )
numWorkers: threads, infoPrinted = true
enableWorkerThreads: nextConfig.experimental.workerThreads, }
exposedMethods: ['default'], },
}) as Worker & typeof import('./worker') maxRetries: 0,
numWorkers: threads,
enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'],
}) as Worker & typeof import('./worker')
exportPage = worker.default.bind(worker)
endWorker = async () => {
await worker.end()
}
}
let renderError = false let renderError = false
const errorPaths: string[] = [] const errorPaths: string[] = []
...@@ -553,7 +566,7 @@ export default async function exportApp( ...@@ -553,7 +566,7 @@ export default async function exportApp(
return pageExportSpan.traceAsyncFn(async () => { return pageExportSpan.traceAsyncFn(async () => {
const pathMap = exportPathMap[path] const pathMap = exportPathMap[path]
const result = await worker.default({ const result = await exportPage({
path, path,
pathMap, pathMap,
distDir, distDir,
...@@ -604,7 +617,7 @@ export default async function exportApp( ...@@ -604,7 +617,7 @@ export default async function exportApp(
}) })
) )
worker.end() const endWorkerPromise = endWorker()
// copy prerendered routes to outDir // copy prerendered routes to outDir
if (!options.buildExport && prerenderManifest) { if (!options.buildExport && prerenderManifest) {
...@@ -681,5 +694,7 @@ export default async function exportApp( ...@@ -681,5 +694,7 @@ export default async function exportApp(
if (telemetry) { if (telemetry) {
await telemetry.flush() await telemetry.flush()
} }
await endWorkerPromise
}) })
} }
...@@ -105,6 +105,7 @@ export type NextConfig = { [key: string]: any } & { ...@@ -105,6 +105,7 @@ export type NextConfig = { [key: string]: any } & {
swcMinify?: boolean swcMinify?: boolean
swcLoader?: boolean swcLoader?: boolean
cpus?: number cpus?: number
sharedPool?: boolean
plugins?: boolean plugins?: boolean
profiling?: boolean profiling?: boolean
isrFlushToDisk?: boolean isrFlushToDisk?: boolean
...@@ -128,7 +129,6 @@ export type NextConfig = { [key: string]: any } & { ...@@ -128,7 +129,6 @@ export type NextConfig = { [key: string]: any } & {
craCompat?: boolean craCompat?: boolean
esmExternals?: boolean | 'loose' esmExternals?: boolean | 'loose'
staticPageGenerationTimeout?: number staticPageGenerationTimeout?: number
pageDataCollectionTimeout?: number
isrMemoryCacheSize?: number isrMemoryCacheSize?: number
concurrentFeatures?: boolean concurrentFeatures?: boolean
} }
...@@ -184,6 +184,7 @@ export const defaultConfig: NextConfig = { ...@@ -184,6 +184,7 @@ export const defaultConfig: NextConfig = {
(Number(process.env.CIRCLE_NODE_TOTAL) || (Number(process.env.CIRCLE_NODE_TOTAL) ||
(os.cpus() || { length: 1 }).length) - 1 (os.cpus() || { length: 1 }).length) - 1
), ),
sharedPool: false,
plugins: false, plugins: false,
profiling: false, profiling: false,
isrFlushToDisk: true, isrFlushToDisk: true,
...@@ -200,7 +201,6 @@ export const defaultConfig: NextConfig = { ...@@ -200,7 +201,6 @@ export const defaultConfig: NextConfig = {
craCompat: false, craCompat: false,
esmExternals: false, esmExternals: false,
staticPageGenerationTimeout: 60, staticPageGenerationTimeout: 60,
pageDataCollectionTimeout: 60,
// default to 50MB limit // default to 50MB limit
isrMemoryCacheSize: 50 * 1024 * 1024, isrMemoryCacheSize: 50 * 1024 * 1024,
concurrentFeatures: false, concurrentFeatures: false,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment