Unverified Commit 3bf4ae37 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

performance improvement of static generation (#25035)

### move all access to built pages into worker pool

to allow parallelizing and avoid loading the bundles in the main thread

This improves performance of the static check step a bit and helps reducing memory load in main thread

### enable splitChunks for server build in webpack 5

This improves performance for static generation by loading less code due to reduced duplication
parent a4420532
...@@ -58,7 +58,7 @@ export type WebpackEntrypoints = { ...@@ -58,7 +58,7 @@ export type WebpackEntrypoints = {
| string[] | string[]
| { | {
import: string | string[] import: string | string[]
dependOn: string | string[] dependOn?: string | string[]
} }
} }
......
...@@ -81,9 +81,6 @@ import { ...@@ -81,9 +81,6 @@ import {
detectConflictingPaths, detectConflictingPaths,
computeFromManifest, computeFromManifest,
getJsPageSizeInKb, getJsPageSizeInKb,
getNamedExports,
hasCustomGetInitialProps,
isPageStatic,
PageInfo, PageInfo,
printCustomRoutes, printCustomRoutes,
printTreeView, printTreeView,
...@@ -266,7 +263,7 @@ export default async function build( ...@@ -266,7 +263,7 @@ export default async function build(
) )
const pageKeys = Object.keys(mappedPages) const pageKeys = Object.keys(mappedPages)
const conflictingPublicFiles: string[] = [] const conflictingPublicFiles: string[] = []
const hasCustomErrorPage = mappedPages['/_error'].startsWith( const hasCustomErrorPage: boolean = mappedPages['/_error'].startsWith(
'private-next-pages' 'private-next-pages'
) )
const hasPages404 = Boolean( const hasPages404 = Boolean(
...@@ -656,21 +653,22 @@ export default async function build( ...@@ -656,21 +653,22 @@ export default async function build(
await promises.readFile(buildManifestPath, 'utf8') await promises.readFile(buildManifestPath, 'utf8')
) as BuildManifest ) as BuildManifest
let customAppGetInitialProps: boolean | undefined
let namedExports: Array<string> | undefined
let isNextImageImported: boolean | undefined
const analysisBegin = process.hrtime() const analysisBegin = process.hrtime()
let hasSsrAmpPages = false
const staticCheckSpan = nextBuildSpan.traceChild('static-check') const staticCheckSpan = nextBuildSpan.traceChild('static-check')
const { hasNonStaticErrorPage } = await staticCheckSpan.traceAsyncFn( const {
async () => { customAppGetInitialProps,
namedExports,
isNextImageImported,
hasSsrAmpPages,
hasNonStaticErrorPage,
} = await staticCheckSpan.traceAsyncFn(async () => {
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const staticCheckWorkers = new Worker(staticCheckWorker, { const staticCheckWorkers = new Worker(staticCheckWorker, {
numWorkers: config.experimental.cpus, numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads, enableWorkerThreads: config.experimental.workerThreads,
}) as Worker & { isPageStatic: typeof isPageStatic } }) as Worker & typeof import('./utils')
staticCheckWorkers.getStdout().pipe(process.stdout) staticCheckWorkers.getStdout().pipe(process.stdout)
staticCheckWorkers.getStderr().pipe(process.stderr) staticCheckWorkers.getStderr().pipe(process.stderr)
...@@ -683,10 +681,10 @@ export default async function build( ...@@ -683,10 +681,10 @@ export default async function build(
const nonStaticErrorPageSpan = staticCheckSpan.traceChild( const nonStaticErrorPageSpan = staticCheckSpan.traceChild(
'check-static-error-page' 'check-static-error-page'
) )
const nonStaticErrorPage = await nonStaticErrorPageSpan.traceAsyncFn( const nonStaticErrorPagePromise = nonStaticErrorPageSpan.traceAsyncFn(
async () => async () =>
hasCustomErrorPage && hasCustomErrorPage &&
(await hasCustomGetInitialProps( (await staticCheckWorkers.hasCustomGetInitialProps(
'/_error', '/_error',
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -698,7 +696,7 @@ export default async function build( ...@@ -698,7 +696,7 @@ export default async function build(
// from _error instead // from _error instead
const appPageToCheck = isLikeServerless ? '/_error' : '/_app' const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
customAppGetInitialProps = await hasCustomGetInitialProps( const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps(
appPageToCheck, appPageToCheck,
distDir, distDir,
isLikeServerless, isLikeServerless,
...@@ -706,24 +704,17 @@ export default async function build( ...@@ -706,24 +704,17 @@ export default async function build(
true true
) )
namedExports = await getNamedExports( const namedExportsPromise = staticCheckWorkers.getNamedExports(
appPageToCheck, appPageToCheck,
distDir, distDir,
isLikeServerless, isLikeServerless,
runtimeEnvConfig runtimeEnvConfig
) )
if (customAppGetInitialProps) { // eslint-disable-next-line no-shadow
console.warn( let isNextImageImported: boolean | undefined
chalk.bold.yellow(`Warning: `) + // eslint-disable-next-line no-shadow
chalk.yellow( let hasSsrAmpPages = false
`You have opted-out of Automatic Static Optimization due to \`getInitialProps\` in \`pages/_app\`. This does not opt-out pages with \`getStaticProps\``
)
)
console.warn(
'Read more: https://nextjs.org/docs/messages/opt-out-auto-static-optimization\n'
)
}
const computedManifestData = await computeFromManifest( const computedManifestData = await computeFromManifest(
buildManifest, buildManifest,
...@@ -812,17 +803,14 @@ export default async function build( ...@@ -812,17 +803,14 @@ export default async function build(
serverPropsPages.add(page) serverPropsPages.add(page)
} else if ( } else if (
workerResult.isStatic && workerResult.isStatic &&
customAppGetInitialProps === false (await customAppGetInitialPropsPromise) === false
) { ) {
staticPages.add(page) staticPages.add(page)
isStatic = true isStatic = true
} }
if (hasPages404 && page === '/404') { if (hasPages404 && page === '/404') {
if ( if (!workerResult.isStatic && !workerResult.hasStaticProps) {
!workerResult.isStatic &&
!workerResult.hasStaticProps
) {
throw new Error( throw new Error(
`\`pages/404\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}` `\`pages/404\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}`
) )
...@@ -830,7 +818,7 @@ export default async function build( ...@@ -830,7 +818,7 @@ export default async function build(
// we need to ensure the 404 lambda is present since we use // we need to ensure the 404 lambda is present since we use
// it when _app has getInitialProps // it when _app has getInitialProps
if ( if (
customAppGetInitialProps && (await customAppGetInitialPropsPromise) &&
!workerResult.hasStaticProps !workerResult.hasStaticProps
) { ) {
staticPages.delete(page) staticPages.delete(page)
...@@ -864,11 +852,29 @@ export default async function build( ...@@ -864,11 +852,29 @@ export default async function build(
}) })
}) })
) )
const returnValue = {
customAppGetInitialProps: await customAppGetInitialPropsPromise,
namedExports: await namedExportsPromise,
isNextImageImported,
hasSsrAmpPages,
hasNonStaticErrorPage: await nonStaticErrorPagePromise,
}
staticCheckWorkers.end() staticCheckWorkers.end()
return returnValue
})
return { hasNonStaticErrorPage: nonStaticErrorPage } if (customAppGetInitialProps) {
} console.warn(
chalk.bold.yellow(`Warning: `) +
chalk.yellow(
`You have opted-out of Automatic Static Optimization due to \`getInitialProps\` in \`pages/_app\`. This does not opt-out pages with \`getStaticProps\``
)
) )
console.warn(
'Read more: https://nextjs.org/docs/messages/opt-out-auto-static-optimization\n'
)
}
if (!hasSsrAmpPages) { if (!hasSsrAmpPages) {
requiredServerFiles.ignore.push( requiredServerFiles.ignore.push(
......
...@@ -822,7 +822,17 @@ export default async function getBaseWebpackConfig( ...@@ -822,7 +822,17 @@ export default async function getBaseWebpackConfig(
...(isWebpack5 ? { emitOnErrors: !dev } : { noEmitOnErrors: dev }), ...(isWebpack5 ? { emitOnErrors: !dev } : { noEmitOnErrors: dev }),
checkWasmTypes: false, checkWasmTypes: false,
nodeEnv: false, nodeEnv: false,
splitChunks: isServer ? false : splitChunksConfig, splitChunks: isServer
? isWebpack5
? {
// allow to split entrypoints
chunks: 'all',
// size of files is not so relevant for server build
// we want to prefer deduplication to load less code
minSize: 1000,
}
: false
: splitChunksConfig,
runtimeChunk: isServer runtimeChunk: isServer
? isWebpack5 && !isLikeServerless ? isWebpack5 && !isLikeServerless
? { name: 'webpack-runtime' } ? { name: 'webpack-runtime' }
......
...@@ -38,7 +38,7 @@ export default class PagesManifestPlugin implements webpack.Plugin { ...@@ -38,7 +38,7 @@ export default class PagesManifestPlugin implements webpack.Plugin {
!file.includes('webpack-runtime') && file.endsWith('.js') !file.includes('webpack-runtime') && file.endsWith('.js')
) )
if (files.length > 1) { if (!isWebpack5 && files.length > 1) {
console.log( console.log(
`Found more than one file in server entrypoint ${entrypoint.name}`, `Found more than one file in server entrypoint ${entrypoint.name}`,
files files
...@@ -47,7 +47,7 @@ export default class PagesManifestPlugin implements webpack.Plugin { ...@@ -47,7 +47,7 @@ export default class PagesManifestPlugin implements webpack.Plugin {
} }
// Write filename, replace any backslashes in path (on windows) with forwardslashes for cross-platform consistency. // Write filename, replace any backslashes in path (on windows) with forwardslashes for cross-platform consistency.
pages[pagePath] = files[0] pages[pagePath] = files[files.length - 1]
if (isWebpack5 && !this.dev) { if (isWebpack5 && !this.dev) {
pages[pagePath] = pages[pagePath].slice(3) pages[pagePath] = pages[pagePath].slice(3)
......
...@@ -48,7 +48,7 @@ initNext({ webpackHMR }) ...@@ -48,7 +48,7 @@ initNext({ webpackHMR })
const { pages } = JSON.parse(event.data) const { pages } = JSON.parse(event.data)
const router = window.next.router const router = window.next.router
if (pages.includes(router.pathname)) { if (!router.clc && pages.includes(router.pathname)) {
console.log('Refreshing page data due to server-side change') console.log('Refreshing page data due to server-side change')
buildIndicatorHandler('building') buildIndicatorHandler('building')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment