Unverified Commit eba9fca3 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Optimize build trace handling (#56048)

This consolidates our trace handling between turbotrace and the default
`nodeFileTrace` handling to prevent over-tracing. With node-file-trace
it's extremely more efficient to trace in a single `nodeFileTrace` run
so that all resolving/analysis etc can be efficiently cached.

Furthermore, this reduces the amount of chunks we need to trace by
waiting until post build since automatically statically optimized paths
don't need to be traced as they are rendered to purely HTML and never
re-rendered.

This reduces builds times by upwards of 4 - 5 minutes on larger
projects.
parent 7b48a1e2
This diff is collapsed.
This diff is collapsed.
...@@ -1796,7 +1796,8 @@ export async function copyTracedFiles( ...@@ -1796,7 +1796,8 @@ export async function copyTracedFiles(
tracingRoot: string, tracingRoot: string,
serverConfig: NextConfig, serverConfig: NextConfig,
middlewareManifest: MiddlewareManifest, middlewareManifest: MiddlewareManifest,
hasInstrumentationHook: boolean hasInstrumentationHook: boolean,
staticPages: Set<string>
) { ) {
const outputPath = path.join(distDir, 'standalone') const outputPath = path.join(distDir, 'standalone')
let moduleType = false let moduleType = false
...@@ -1889,6 +1890,12 @@ export async function copyTracedFiles( ...@@ -1889,6 +1890,12 @@ export async function copyTracedFiles(
if (middlewareManifest.functions.hasOwnProperty(page)) { if (middlewareManifest.functions.hasOwnProperty(page)) {
continue continue
} }
const route = normalizePagePath(page)
if (staticPages.has(route)) {
continue
}
const pageFile = path.join( const pageFile = path.join(
distDir, distDir,
'server', 'server',
...@@ -1897,7 +1904,9 @@ export async function copyTracedFiles( ...@@ -1897,7 +1904,9 @@ export async function copyTracedFiles(
) )
const pageTraceFile = `${pageFile}.nft.json` const pageTraceFile = `${pageFile}.nft.json`
await handleTraceFiles(pageTraceFile).catch((err) => { await handleTraceFiles(pageTraceFile).catch((err) => {
Log.warn(`Failed to copy traced files for ${pageFile}`, err) if (err.code !== 'ENOENT' || (page !== '/404' && page !== '/500')) {
Log.warn(`Failed to copy traced files for ${pageFile}`, err)
}
}) })
} }
......
...@@ -24,8 +24,8 @@ import loadConfig from '../../server/config' ...@@ -24,8 +24,8 @@ import loadConfig from '../../server/config'
import { trace } from '../../trace' import { trace } from '../../trace'
import { WEBPACK_LAYERS } from '../../lib/constants' import { WEBPACK_LAYERS } from '../../lib/constants'
import { import {
BuildTraceContext,
TraceEntryPointsPlugin, TraceEntryPointsPlugin,
TurbotraceContext,
} from '../webpack/plugins/next-trace-entrypoints-plugin' } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { UnwrapPromise } from '../../lib/coalesced-function' import { UnwrapPromise } from '../../lib/coalesced-function'
import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin' import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'
...@@ -61,7 +61,7 @@ export async function webpackBuildImpl( ...@@ -61,7 +61,7 @@ export async function webpackBuildImpl(
): Promise<{ ): Promise<{
duration: number duration: number
pluginState: any pluginState: any
turbotraceContext?: TurbotraceContext buildTraceContext?: BuildTraceContext
serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries'] serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries']
}> { }> {
let result: CompilerResult | null = { let result: CompilerResult | null = {
...@@ -183,18 +183,22 @@ export async function webpackBuildImpl( ...@@ -183,18 +183,22 @@ export async function webpackBuildImpl(
let inputFileSystem: any let inputFileSystem: any
if (!compilerName || compilerName === 'server') { if (!compilerName || compilerName === 'server') {
debug('starting server compiler')
const start = Date.now()
;[serverResult, inputFileSystem] = await runCompiler(serverConfig, { ;[serverResult, inputFileSystem] = await runCompiler(serverConfig, {
runWebpackSpan, runWebpackSpan,
inputFileSystem, inputFileSystem,
}) })
debug('server result', serverResult) debug(`server compiler finished ${Date.now() - start}ms`)
} }
if (!compilerName || compilerName === 'edge-server') { if (!compilerName || compilerName === 'edge-server') {
debug('starting edge-server compiler')
const start = Date.now()
;[edgeServerResult, inputFileSystem] = edgeConfig ;[edgeServerResult, inputFileSystem] = edgeConfig
? await runCompiler(edgeConfig, { runWebpackSpan, inputFileSystem }) ? await runCompiler(edgeConfig, { runWebpackSpan, inputFileSystem })
: [null] : [null]
debug('edge server result', edgeServerResult) debug(`edge-server compiler finished ${Date.now() - start}ms`)
} }
// Only continue if there were no errors // Only continue if there were no errors
...@@ -223,11 +227,13 @@ export async function webpackBuildImpl( ...@@ -223,11 +227,13 @@ export async function webpackBuildImpl(
} }
if (!compilerName || compilerName === 'client') { if (!compilerName || compilerName === 'client') {
debug('starting client compiler')
const start = Date.now()
;[clientResult, inputFileSystem] = await runCompiler(clientConfig, { ;[clientResult, inputFileSystem] = await runCompiler(clientConfig, {
runWebpackSpan, runWebpackSpan,
inputFileSystem, inputFileSystem,
}) })
debug('client result', clientResult) debug(`client compiler finished ${Date.now() - start}ms`)
} }
} }
...@@ -319,7 +325,7 @@ export async function webpackBuildImpl( ...@@ -319,7 +325,7 @@ export async function webpackBuildImpl(
return { return {
duration: webpackBuildEnd[0], duration: webpackBuildEnd[0],
turbotraceContext: traceEntryPointsPlugin?.turbotraceContext, buildTraceContext: traceEntryPointsPlugin?.buildTraceContext,
pluginState: getPluginState(), pluginState: getPluginState(),
serializedPagesManifestEntries: { serializedPagesManifestEntries: {
edgeServerPages: pagesPluginModule.edgeServerPages, edgeServerPages: pagesPluginModule.edgeServerPages,
...@@ -360,17 +366,25 @@ export async function workerMain(workerData: { ...@@ -360,17 +366,25 @@ export async function workerMain(workerData: {
NextBuildContext.nextBuildSpan = trace('next-build') NextBuildContext.nextBuildSpan = trace('next-build')
const result = await webpackBuildImpl(workerData.compilerName) const result = await webpackBuildImpl(workerData.compilerName)
const { entriesTrace } = result.turbotraceContext ?? {} const { entriesTrace, chunksTrace } = result.buildTraceContext ?? {}
if (entriesTrace) { if (entriesTrace) {
const { entryNameMap, depModArray } = entriesTrace const { entryNameMap, depModArray } = entriesTrace
if (depModArray) { if (depModArray) {
result.turbotraceContext!.entriesTrace!.depModArray = depModArray result.buildTraceContext!.entriesTrace!.depModArray = depModArray
} }
if (entryNameMap) { if (entryNameMap) {
const entryEntries = Array.from(entryNameMap?.entries() ?? []) const entryEntries = Array.from(entryNameMap?.entries() ?? [])
// @ts-expect-error // @ts-expect-error
result.turbotraceContext.entriesTrace.entryNameMap = entryEntries result.buildTraceContext.entriesTrace.entryNameMap = entryEntries
} }
} }
if (chunksTrace?.entryNameFilesMap) {
const entryNameFilesMap = Array.from(
chunksTrace.entryNameFilesMap.entries() ?? []
)
// @ts-expect-error
result.buildTraceContext!.chunksTrace!.entryNameFilesMap = entryNameFilesMap
}
return result return result
} }
import { COMPILER_INDEXES } from '../../shared/lib/constants' import { COMPILER_INDEXES } from '../../shared/lib/constants'
import * as Log from '../output/log' import * as Log from '../output/log'
import { NextBuildContext } from '../build-context' import { NextBuildContext } from '../build-context'
import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin' import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { Worker } from 'next/dist/compiled/jest-worker' import { Worker } from 'next/dist/compiled/jest-worker'
import origDebug from 'next/dist/compiled/debug' import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process' import { ChildProcess } from 'child_process'
...@@ -50,7 +50,7 @@ async function webpackBuildWithWorker() { ...@@ -50,7 +50,7 @@ async function webpackBuildWithWorker() {
const combinedResult = { const combinedResult = {
duration: 0, duration: 0,
turbotraceContext: {} as TurbotraceContext, buildTraceContext: {} as BuildTraceContext,
} }
// order matters here // order matters here
const ORDERED_COMPILER_NAMES = [ const ORDERED_COMPILER_NAMES = [
...@@ -85,15 +85,28 @@ async function webpackBuildWithWorker() { ...@@ -85,15 +85,28 @@ async function webpackBuildWithWorker() {
combinedResult.duration += curResult.duration combinedResult.duration += curResult.duration
if (curResult.turbotraceContext?.entriesTrace) { if (curResult.buildTraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext const { entryNameMap } = curResult.buildTraceContext.entriesTrace!
const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) { if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map( combinedResult.buildTraceContext.entriesTrace =
curResult.buildTraceContext.entriesTrace
combinedResult.buildTraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap entryNameMap
) )
} }
if (curResult.buildTraceContext?.chunksTrace) {
const { entryNameFilesMap } = curResult.buildTraceContext.chunksTrace!
if (entryNameFilesMap) {
combinedResult.buildTraceContext.chunksTrace =
curResult.buildTraceContext.chunksTrace!
combinedResult.buildTraceContext.chunksTrace!.entryNameFilesMap =
new Map(entryNameFilesMap)
}
}
} }
} }
buildSpinner?.stopAndPersist() buildSpinner?.stopAndPersist()
......
import React from 'react' import React from 'react'
import { World } from 'components/world' import { World } from 'components/world'
// prevent static generation for trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() { export default function HelloPage() {
return ( return (
<div> <div>
......
import React from 'react' import React from 'react'
import { World } from '@c/world' import { World } from '@c/world'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() { export default function HelloPage() {
return ( return (
<div> <div>
......
import React from 'react' import React from 'react'
import api from '@lib/b-only' import api from '@lib/b-only'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() { export default function ResolveOrder() {
return <div>{api()}</div> return <div>{api()}</div>
} }
import React from 'react' import React from 'react'
import api from '@lib/api' import api from '@lib/api'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() { export default function ResolveOrder() {
return <div>{api()}</div> return <div>{api()}</div>
} }
import { Hello } from '@mycomponent' import { Hello } from '@mycomponent'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function SingleAlias() { export default function SingleAlias() {
return ( return (
<div> <div>
......
import myfile from 'mypackage/myfile' import myfile from 'mypackage/myfile'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function WildcardAlias() { export default function WildcardAlias() {
return <div>{myfile.hello}</div> return <div>{myfile.hello}</div>
} }
import Link from 'next/link' import Link from 'next/link'
import Error from 'next/error' import Error from 'next/error'
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page(props) { export default function Page(props) {
return ( return (
<> <>
......
...@@ -8,9 +8,18 @@ if (typeof window === 'undefined') { ...@@ -8,9 +8,18 @@ if (typeof window === 'undefined') {
import('nanoid').then((mod) => console.log(mod.nanoid())) import('nanoid').then((mod) => console.log(mod.nanoid()))
} }
export default () => ( // prevent static generation for build trace test
<div> export function getServerSideProps() {
<Link href="/about">About Page</Link> return {
<p className="index-page">Hello World</p> props: {},
</div> }
) }
export default function Page() {
return (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
}
...@@ -4,9 +4,18 @@ import Link from 'next/link' ...@@ -4,9 +4,18 @@ import Link from 'next/link'
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
import next from 'next' import next from 'next'
export default () => ( // prevent static generation for build trace test
<div> export function getServerSideProps() {
<Link href="/about">About Page</Link> return {
<p className="index-page">Hello World</p> props: {},
</div> }
) }
export default function Page() {
return (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
}
...@@ -106,7 +106,7 @@ createNextDescribe( ...@@ -106,7 +106,7 @@ createNextDescribe(
}) })
it('should respond with 405 for POST to static page', async () => { it('should respond with 405 for POST to static page', async () => {
const res = await fetchViaHTTP(next.appPort, '/', undefined, { const res = await fetchViaHTTP(next.appPort, '/about', undefined, {
method: 'POST', method: 'POST',
}) })
expect(res.status).toBe(405) expect(res.status).toBe(405)
...@@ -114,7 +114,7 @@ createNextDescribe( ...@@ -114,7 +114,7 @@ createNextDescribe(
}) })
it('should contain generated page count in output', async () => { it('should contain generated page count in output', async () => {
const pageCount = 40 const pageCount = 37
expect(next.cliOutput).toContain( expect(next.cliOutput).toContain(
`Generating static pages (0/${pageCount})` `Generating static pages (0/${pageCount})`
) )
...@@ -204,18 +204,6 @@ createNextDescribe( ...@@ -204,18 +204,6 @@ createNextDescribe(
], ],
notTests: [/\0/, /\?/, /!/], notTests: [/\0/, /\?/, /!/],
}, },
{
page: '/dynamic',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{ {
page: '/index', page: '/index',
tests: [ tests: [
...@@ -231,19 +219,6 @@ createNextDescribe( ...@@ -231,19 +219,6 @@ createNextDescribe(
], ],
notTests: [/next\/dist\/pages\/_error\.js/, /\0/, /\?/, /!/], notTests: [/next\/dist\/pages\/_error\.js/, /\0/, /\?/, /!/],
}, },
{
page: '/counter',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/react\/cjs\/react\.development\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{ {
page: '/next-import', page: '/next-import',
tests: [ tests: [
...@@ -298,6 +273,7 @@ createNextDescribe( ...@@ -298,6 +273,7 @@ createNextDescribe(
] ]
for (const check of checks) { for (const check of checks) {
require('console').log('checking', check.page)
const { version, files } = await next.readJSON( const { version, files } = await next.readJSON(
join('.next/server/pages/', check.page + '.js.nft.json') join('.next/server/pages/', check.page + '.js.nft.json')
) )
...@@ -940,7 +916,7 @@ createNextDescribe( ...@@ -940,7 +916,7 @@ createNextDescribe(
}) })
it('should add autoExport for auto pre-rendered pages', async () => { it('should add autoExport for auto pre-rendered pages', async () => {
for (const page of ['/', '/about']) { for (const page of ['/about']) {
const html = await renderViaHTTP(next.appPort, page) const html = await renderViaHTTP(next.appPort, page)
const $ = cheerio.load(html) const $ = cheerio.load(html)
const data = JSON.parse($('#__NEXT_DATA__').html()) const data = JSON.parse($('#__NEXT_DATA__').html())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment