Unverified Commit 8bbb1cd3 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

use a shared worker pool for collecting page data and static page generation (#27924)

this avoid loading all code twice and hopefully improving performance
parent b6411408
...@@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful ...@@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful
#### Possible Ways to Fix It #### Possible Ways to Fix It
- Make sure that there is no infinite loop during execution. - Make sure that there is no infinite loop during execution.
- Make sure all Promises in `getStaticProps` `resolve` or `reject` correctly. - Make sure all Promises in `getStaticPaths`/`getStaticProps` `resolve` or `reject` correctly.
- Avoid very long timeouts for network requests. - Avoid very long timeouts for network requests.
- Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds). - Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds).
### Useful Links ### Useful Links
- [`getStaticPaths`](https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation)
- [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation) - [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation)
...@@ -91,8 +91,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path' ...@@ -91,8 +91,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
import { isWebpack5 } from 'next/dist/compiled/webpack/webpack' import { isWebpack5 } from 'next/dist/compiled/webpack/webpack'
import { NextConfigComplete } from '../server/config-shared' import { NextConfigComplete } from '../server/config-shared'
const staticCheckWorker = require.resolve('./utils')
export type SsgRoute = { export type SsgRoute = {
initialRevalidateSeconds: number | false initialRevalidateSeconds: number | false
srcRoute: string | null srcRoute: string | null
...@@ -670,23 +668,27 @@ export default async function build( ...@@ -670,23 +668,27 @@ export default async function build(
await promises.readFile(buildManifestPath, 'utf8') await promises.readFile(buildManifestPath, 'utf8')
) as BuildManifest ) as BuildManifest
const analysisBegin = process.hrtime() const timeout = config.experimental.staticPageGenerationTimeout || 0
const sharedPool = config.experimental.sharedPool || false
const staticCheckSpan = nextBuildSpan.traceChild('static-check') const staticWorker = sharedPool
const { ? require.resolve('./worker')
customAppGetInitialProps, : require.resolve('./utils')
namedExports,
isNextImageImported,
hasSsrAmpPages,
hasNonStaticErrorPage,
} = await staticCheckSpan.traceAsyncFn(async () => {
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const timeout = config.experimental.pageDataCollectionTimeout || 0
let infoPrinted = false let infoPrinted = false
const staticCheckWorkers = new Worker(staticCheckWorker, { const staticWorkers = new Worker(staticWorker, {
timeout: timeout * 1000, timeout: timeout * 1000,
onRestart: (_method, [pagePath], attempts) => { onRestart: (method, [arg], attempts) => {
if (method === 'exportPage') {
const { path: pagePath } = arg
if (attempts >= 3) {
throw new Error(
`Static page generation for ${pagePath} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
`Restarted static page genertion for ${pagePath} because it took more than ${timeout} seconds`
)
} else {
const pagePath = arg
if (attempts >= 2) { if (attempts >= 2) {
throw new Error( throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout` `Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
...@@ -695,26 +697,45 @@ export default async function build( ...@@ -695,26 +697,45 @@ export default async function build(
Log.warn( Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds` `Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
) )
}
if (!infoPrinted) { if (!infoPrinted) {
Log.warn( Log.warn(
'See more info here https://nextjs.org/docs/messages/page-data-collection-timeout' 'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
) )
infoPrinted = true infoPrinted = true
} }
}, },
numWorkers: config.experimental.cpus, numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads, enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: [ exposedMethods: sharedPool
? [
'hasCustomGetInitialProps', 'hasCustomGetInitialProps',
'isPageStatic', 'isPageStatic',
'getNamedExports', 'getNamedExports',
], 'exportPage',
]
: ['hasCustomGetInitialProps', 'isPageStatic', 'getNamedExports'],
}) as Worker & }) as Worker &
Pick< Pick<
typeof import('./utils'), typeof import('./worker'),
'hasCustomGetInitialProps' | 'isPageStatic' | 'getNamedExports' | 'hasCustomGetInitialProps'
| 'isPageStatic'
| 'getNamedExports'
| 'exportPage'
> >
const analysisBegin = process.hrtime()
const staticCheckSpan = nextBuildSpan.traceChild('static-check')
const {
customAppGetInitialProps,
namedExports,
isNextImageImported,
hasSsrAmpPages,
hasNonStaticErrorPage,
} = await staticCheckSpan.traceAsyncFn(async () => {
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const runtimeEnvConfig = { const runtimeEnvConfig = {
publicRuntimeConfig: config.publicRuntimeConfig, publicRuntimeConfig: config.publicRuntimeConfig,
serverRuntimeConfig: config.serverRuntimeConfig, serverRuntimeConfig: config.serverRuntimeConfig,
...@@ -726,7 +747,7 @@ export default async function build( ...@@ -726,7 +747,7 @@ export default async function build(
const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn( const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
(await staticCheckWorkers.hasCustomGetInitialProps( (await staticWorkers.hasCustomGetInitialProps(
'/_error', '/_error',
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -738,7 +759,7 @@ export default async function build( ...@@ -738,7 +759,7 @@ export default async function build(
const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn( const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
staticCheckWorkers.isPageStatic( staticWorkers.isPageStatic(
'/_error', '/_error',
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -753,7 +774,7 @@ export default async function build( ...@@ -753,7 +774,7 @@ export default async function build(
// from _error instead // from _error instead
const appPageToCheck = isLikeServerless ? '/_error' : '/_app' const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps( const customAppGetInitialPropsPromise = staticWorkers.hasCustomGetInitialProps(
appPageToCheck, appPageToCheck,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -761,7 +782,7 @@ export default async function build( ...@@ -761,7 +782,7 @@ export default async function build(
true true
) )
const namedExportsPromise = staticCheckWorkers.getNamedExports( const namedExportsPromise = staticWorkers.getNamedExports(
appPageToCheck, appPageToCheck,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -808,7 +829,7 @@ export default async function build( ...@@ -808,7 +829,7 @@ export default async function build(
'is-page-static' 'is-page-static'
) )
let workerResult = await isPageStaticSpan.traceAsyncFn(() => { let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
return staticCheckWorkers.isPageStatic( return staticWorkers.isPageStatic(
page, page,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -926,7 +947,7 @@ export default async function build( ...@@ -926,7 +947,7 @@ export default async function build(
hasNonStaticErrorPage: nonStaticErrorPage, hasNonStaticErrorPage: nonStaticErrorPage,
} }
staticCheckWorkers.end() if (!sharedPool) staticWorkers.end()
return returnValue return returnValue
}) })
...@@ -1082,7 +1103,8 @@ export default async function build( ...@@ -1082,7 +1103,8 @@ export default async function build(
ssgPages, ssgPages,
additionalSsgPaths additionalSsgPaths
) )
const exportApp = require('../export').default const exportApp: typeof import('../export').default = require('../export')
.default
const exportOptions = { const exportOptions = {
silent: false, silent: false,
buildExport: true, buildExport: true,
...@@ -1090,6 +1112,14 @@ export default async function build( ...@@ -1090,6 +1112,14 @@ export default async function build(
pages: combinedPages, pages: combinedPages,
outdir: path.join(distDir, 'export'), outdir: path.join(distDir, 'export'),
statusMessage: 'Generating static pages', statusMessage: 'Generating static pages',
exportPageWorker: sharedPool
? staticWorkers.exportPage.bind(staticWorkers)
: undefined,
endWorker: sharedPool
? async () => {
await staticWorkers.end()
}
: undefined,
} }
const exportConfig: any = { const exportConfig: any = {
...config, ...config,
......
export * from './utils'
export { default as exportPage } from '../export/worker'
...@@ -137,6 +137,8 @@ interface ExportOptions { ...@@ -137,6 +137,8 @@ interface ExportOptions {
pages?: string[] pages?: string[]
buildExport?: boolean buildExport?: boolean
statusMessage?: string statusMessage?: string
exportPageWorker?: typeof import('./worker').default
endWorker?: () => Promise<void>
} }
export default async function exportApp( export default async function exportApp(
...@@ -519,6 +521,12 @@ export default async function exportApp( ...@@ -519,6 +521,12 @@ export default async function exportApp(
const timeout = configuration?.experimental.staticPageGenerationTimeout || 0 const timeout = configuration?.experimental.staticPageGenerationTimeout || 0
let infoPrinted = false let infoPrinted = false
let exportPage: typeof import('./worker').default
let endWorker: () => Promise<void>
if (options.exportPageWorker) {
exportPage = options.exportPageWorker
endWorker = options.endWorker || (() => Promise.resolve())
} else {
const worker = new Worker(require.resolve('./worker'), { const worker = new Worker(require.resolve('./worker'), {
timeout: timeout * 1000, timeout: timeout * 1000,
onRestart: (_method, [{ path }], attempts) => { onRestart: (_method, [{ path }], attempts) => {
...@@ -542,6 +550,11 @@ export default async function exportApp( ...@@ -542,6 +550,11 @@ export default async function exportApp(
enableWorkerThreads: nextConfig.experimental.workerThreads, enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'], exposedMethods: ['default'],
}) as Worker & typeof import('./worker') }) as Worker & typeof import('./worker')
exportPage = worker.default.bind(worker)
endWorker = async () => {
await worker.end()
}
}
let renderError = false let renderError = false
const errorPaths: string[] = [] const errorPaths: string[] = []
...@@ -553,7 +566,7 @@ export default async function exportApp( ...@@ -553,7 +566,7 @@ export default async function exportApp(
return pageExportSpan.traceAsyncFn(async () => { return pageExportSpan.traceAsyncFn(async () => {
const pathMap = exportPathMap[path] const pathMap = exportPathMap[path]
const result = await worker.default({ const result = await exportPage({
path, path,
pathMap, pathMap,
distDir, distDir,
...@@ -604,7 +617,7 @@ export default async function exportApp( ...@@ -604,7 +617,7 @@ export default async function exportApp(
}) })
) )
worker.end() const endWorkerPromise = endWorker()
// copy prerendered routes to outDir // copy prerendered routes to outDir
if (!options.buildExport && prerenderManifest) { if (!options.buildExport && prerenderManifest) {
...@@ -681,5 +694,7 @@ export default async function exportApp( ...@@ -681,5 +694,7 @@ export default async function exportApp(
if (telemetry) { if (telemetry) {
await telemetry.flush() await telemetry.flush()
} }
await endWorkerPromise
}) })
} }
...@@ -105,6 +105,7 @@ export type NextConfig = { [key: string]: any } & { ...@@ -105,6 +105,7 @@ export type NextConfig = { [key: string]: any } & {
swcMinify?: boolean swcMinify?: boolean
swcLoader?: boolean swcLoader?: boolean
cpus?: number cpus?: number
sharedPool?: boolean
plugins?: boolean plugins?: boolean
profiling?: boolean profiling?: boolean
isrFlushToDisk?: boolean isrFlushToDisk?: boolean
...@@ -128,7 +129,6 @@ export type NextConfig = { [key: string]: any } & { ...@@ -128,7 +129,6 @@ export type NextConfig = { [key: string]: any } & {
craCompat?: boolean craCompat?: boolean
esmExternals?: boolean | 'loose' esmExternals?: boolean | 'loose'
staticPageGenerationTimeout?: number staticPageGenerationTimeout?: number
pageDataCollectionTimeout?: number
isrMemoryCacheSize?: number isrMemoryCacheSize?: number
concurrentFeatures?: boolean concurrentFeatures?: boolean
} }
...@@ -184,6 +184,7 @@ export const defaultConfig: NextConfig = { ...@@ -184,6 +184,7 @@ export const defaultConfig: NextConfig = {
(Number(process.env.CIRCLE_NODE_TOTAL) || (Number(process.env.CIRCLE_NODE_TOTAL) ||
(os.cpus() || { length: 1 }).length) - 1 (os.cpus() || { length: 1 }).length) - 1
), ),
sharedPool: false,
plugins: false, plugins: false,
profiling: false, profiling: false,
isrFlushToDisk: true, isrFlushToDisk: true,
...@@ -200,7 +201,6 @@ export const defaultConfig: NextConfig = { ...@@ -200,7 +201,6 @@ export const defaultConfig: NextConfig = {
craCompat: false, craCompat: false,
esmExternals: false, esmExternals: false,
staticPageGenerationTimeout: 60, staticPageGenerationTimeout: 60,
pageDataCollectionTimeout: 60,
// default to 50MB limit // default to 50MB limit
isrMemoryCacheSize: 50 * 1024 * 1024, isrMemoryCacheSize: 50 * 1024 * 1024,
concurrentFeatures: false, concurrentFeatures: false,
......
...@@ -13,26 +13,38 @@ const fixturesDir = join(__dirname, '..', 'fixtures') ...@@ -13,26 +13,38 @@ const fixturesDir = join(__dirname, '..', 'fixtures')
const nextConfig = new File(join(fixturesDir, 'basic-app/next.config.js')) const nextConfig = new File(join(fixturesDir, 'basic-app/next.config.js'))
describe('Build Output', () => { describe('Build Output', () => {
for (const gzipSize of [true, false, undefined]) { const configs = [{}]
describe( for (const gzipSize of [true, false]) {
'Basic Application Output' + configs.push(...configs.map((c) => ({ ...c, gzipSize })))
(gzipSize !== undefined }
? ` (with experimental.gzipSize: ${gzipSize})` for (const sharedPool of [true]) {
: ''), configs.push(...configs.map((c) => ({ ...c, sharedPool })))
() => { }
for (const workerThreads of [true]) {
configs.push(...configs.map((c) => ({ ...c, workerThreads })))
}
for (const experimental of configs) {
describe(`Basic Application Output (experimental: ${JSON.stringify(
experimental
)})`, () => {
let stdout let stdout
const appDir = join(fixturesDir, 'basic-app') const appDir = join(fixturesDir, 'basic-app')
const hasExperimentalConfig = Object.keys(experimental).length > 0
beforeAll(async () => { beforeAll(async () => {
await remove(join(appDir, '.next')) await remove(join(appDir, '.next'))
if (gzipSize !== undefined) { if (hasExperimentalConfig) {
nextConfig.write( nextConfig.write(
`module.exports = { experimental: { gzipSize: ${gzipSize} } };` `module.exports = { experimental: ${JSON.stringify(
experimental
)} };`
) )
} }
}) })
if (gzipSize !== undefined) { if (hasExperimentalConfig) {
afterAll(async () => { afterAll(async () => {
nextConfig.delete() nextConfig.delete()
}) })
...@@ -115,7 +127,7 @@ describe('Build Output', () => { ...@@ -115,7 +127,7 @@ describe('Build Output', () => {
expect(parseFloat(size)).toBeGreaterThan(0) expect(parseFloat(size)).toBeGreaterThan(0)
} }
// const gz = gzipSize !== false // const gz = experimental.gzipSize !== false
// expect(parseFloat(indexSize) / 1000).toBeCloseTo( // expect(parseFloat(indexSize) / 1000).toBeCloseTo(
// gz ? 0.251 : 0.394, // gz ? 0.251 : 0.394,
...@@ -187,8 +199,7 @@ describe('Build Output', () => { ...@@ -187,8 +199,7 @@ describe('Build Output', () => {
) )
expect(files).toEqual([]) expect(files).toEqual([])
}) })
} })
)
} }
describe('Custom App Output', () => { describe('Custom App Output', () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment