Unverified Commit eba9fca3 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Optimize build trace handling (#56048)

This consolidates our trace handling between turbotrace and the default
`nodeFileTrace` handling to prevent over-tracing. With node-file-trace
it's extremely more efficient to trace in a single `nodeFileTrace` run
so that all resolving/analysis etc can be efficiently cached.

Furthermore, this reduces the amount of chunks we need to trace by
waiting until post build since automatically statically optimized paths
don't need to be traced as they are rendered to purely HTML and never
re-rendered.

This reduces builds times by upwards of 4 - 5 minutes on larger
projects.
parent 7b48a1e2
This diff is collapsed.
This diff is collapsed.
......@@ -1796,7 +1796,8 @@ export async function copyTracedFiles(
tracingRoot: string,
serverConfig: NextConfig,
middlewareManifest: MiddlewareManifest,
hasInstrumentationHook: boolean
hasInstrumentationHook: boolean,
staticPages: Set<string>
) {
const outputPath = path.join(distDir, 'standalone')
let moduleType = false
......@@ -1889,6 +1890,12 @@ export async function copyTracedFiles(
if (middlewareManifest.functions.hasOwnProperty(page)) {
continue
}
const route = normalizePagePath(page)
if (staticPages.has(route)) {
continue
}
const pageFile = path.join(
distDir,
'server',
......@@ -1897,7 +1904,9 @@ export async function copyTracedFiles(
)
const pageTraceFile = `${pageFile}.nft.json`
await handleTraceFiles(pageTraceFile).catch((err) => {
Log.warn(`Failed to copy traced files for ${pageFile}`, err)
if (err.code !== 'ENOENT' || (page !== '/404' && page !== '/500')) {
Log.warn(`Failed to copy traced files for ${pageFile}`, err)
}
})
}
......
......@@ -24,8 +24,8 @@ import loadConfig from '../../server/config'
import { trace } from '../../trace'
import { WEBPACK_LAYERS } from '../../lib/constants'
import {
BuildTraceContext,
TraceEntryPointsPlugin,
TurbotraceContext,
} from '../webpack/plugins/next-trace-entrypoints-plugin'
import { UnwrapPromise } from '../../lib/coalesced-function'
import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'
......@@ -61,7 +61,7 @@ export async function webpackBuildImpl(
): Promise<{
duration: number
pluginState: any
turbotraceContext?: TurbotraceContext
buildTraceContext?: BuildTraceContext
serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries']
}> {
let result: CompilerResult | null = {
......@@ -183,18 +183,22 @@ export async function webpackBuildImpl(
let inputFileSystem: any
if (!compilerName || compilerName === 'server') {
debug('starting server compiler')
const start = Date.now()
;[serverResult, inputFileSystem] = await runCompiler(serverConfig, {
runWebpackSpan,
inputFileSystem,
})
debug('server result', serverResult)
debug(`server compiler finished ${Date.now() - start}ms`)
}
if (!compilerName || compilerName === 'edge-server') {
debug('starting edge-server compiler')
const start = Date.now()
;[edgeServerResult, inputFileSystem] = edgeConfig
? await runCompiler(edgeConfig, { runWebpackSpan, inputFileSystem })
: [null]
debug('edge server result', edgeServerResult)
debug(`edge-server compiler finished ${Date.now() - start}ms`)
}
// Only continue if there were no errors
......@@ -223,11 +227,13 @@ export async function webpackBuildImpl(
}
if (!compilerName || compilerName === 'client') {
debug('starting client compiler')
const start = Date.now()
;[clientResult, inputFileSystem] = await runCompiler(clientConfig, {
runWebpackSpan,
inputFileSystem,
})
debug('client result', clientResult)
debug(`client compiler finished ${Date.now() - start}ms`)
}
}
......@@ -319,7 +325,7 @@ export async function webpackBuildImpl(
return {
duration: webpackBuildEnd[0],
turbotraceContext: traceEntryPointsPlugin?.turbotraceContext,
buildTraceContext: traceEntryPointsPlugin?.buildTraceContext,
pluginState: getPluginState(),
serializedPagesManifestEntries: {
edgeServerPages: pagesPluginModule.edgeServerPages,
......@@ -360,17 +366,25 @@ export async function workerMain(workerData: {
NextBuildContext.nextBuildSpan = trace('next-build')
const result = await webpackBuildImpl(workerData.compilerName)
const { entriesTrace } = result.turbotraceContext ?? {}
const { entriesTrace, chunksTrace } = result.buildTraceContext ?? {}
if (entriesTrace) {
const { entryNameMap, depModArray } = entriesTrace
if (depModArray) {
result.turbotraceContext!.entriesTrace!.depModArray = depModArray
result.buildTraceContext!.entriesTrace!.depModArray = depModArray
}
if (entryNameMap) {
const entryEntries = Array.from(entryNameMap?.entries() ?? [])
// @ts-expect-error
result.turbotraceContext.entriesTrace.entryNameMap = entryEntries
result.buildTraceContext.entriesTrace.entryNameMap = entryEntries
}
}
if (chunksTrace?.entryNameFilesMap) {
const entryNameFilesMap = Array.from(
chunksTrace.entryNameFilesMap.entries() ?? []
)
// @ts-expect-error
result.buildTraceContext!.chunksTrace!.entryNameFilesMap = entryNameFilesMap
}
return result
}
import { COMPILER_INDEXES } from '../../shared/lib/constants'
import * as Log from '../output/log'
import { NextBuildContext } from '../build-context'
import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { Worker } from 'next/dist/compiled/jest-worker'
import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process'
......@@ -50,7 +50,7 @@ async function webpackBuildWithWorker() {
const combinedResult = {
duration: 0,
turbotraceContext: {} as TurbotraceContext,
buildTraceContext: {} as BuildTraceContext,
}
// order matters here
const ORDERED_COMPILER_NAMES = [
......@@ -85,15 +85,28 @@ async function webpackBuildWithWorker() {
combinedResult.duration += curResult.duration
if (curResult.turbotraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext
if (curResult.buildTraceContext?.entriesTrace) {
const { entryNameMap } = curResult.buildTraceContext.entriesTrace!
const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
combinedResult.buildTraceContext.entriesTrace =
curResult.buildTraceContext.entriesTrace
combinedResult.buildTraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap
)
}
if (curResult.buildTraceContext?.chunksTrace) {
const { entryNameFilesMap } = curResult.buildTraceContext.chunksTrace!
if (entryNameFilesMap) {
combinedResult.buildTraceContext.chunksTrace =
curResult.buildTraceContext.chunksTrace!
combinedResult.buildTraceContext.chunksTrace!.entryNameFilesMap =
new Map(entryNameFilesMap)
}
}
}
}
buildSpinner?.stopAndPersist()
......
import React from 'react'
import { World } from 'components/world'
// prevent static generation for trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() {
return (
<div>
......
import React from 'react'
import { World } from '@c/world'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() {
return (
<div>
......
import React from 'react'
import api from '@lib/b-only'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() {
return <div>{api()}</div>
}
import React from 'react'
import api from '@lib/api'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() {
return <div>{api()}</div>
}
import { Hello } from '@mycomponent'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function SingleAlias() {
return (
<div>
......
import myfile from 'mypackage/myfile'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function WildcardAlias() {
return <div>{myfile.hello}</div>
}
import Link from 'next/link'
import Error from 'next/error'
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page(props) {
return (
<>
......
......@@ -8,9 +8,18 @@ if (typeof window === 'undefined') {
import('nanoid').then((mod) => console.log(mod.nanoid()))
}
export default () => (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page() {
return (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
}
......@@ -4,9 +4,18 @@ import Link from 'next/link'
// eslint-disable-next-line no-unused-vars
import next from 'next'
export default () => (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page() {
return (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
}
......@@ -106,7 +106,7 @@ createNextDescribe(
})
it('should respond with 405 for POST to static page', async () => {
const res = await fetchViaHTTP(next.appPort, '/', undefined, {
const res = await fetchViaHTTP(next.appPort, '/about', undefined, {
method: 'POST',
})
expect(res.status).toBe(405)
......@@ -114,7 +114,7 @@ createNextDescribe(
})
it('should contain generated page count in output', async () => {
const pageCount = 40
const pageCount = 37
expect(next.cliOutput).toContain(
`Generating static pages (0/${pageCount})`
)
......@@ -204,18 +204,6 @@ createNextDescribe(
],
notTests: [/\0/, /\?/, /!/],
},
{
page: '/dynamic',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{
page: '/index',
tests: [
......@@ -231,19 +219,6 @@ createNextDescribe(
],
notTests: [/next\/dist\/pages\/_error\.js/, /\0/, /\?/, /!/],
},
{
page: '/counter',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/react\/cjs\/react\.development\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{
page: '/next-import',
tests: [
......@@ -298,6 +273,7 @@ createNextDescribe(
]
for (const check of checks) {
require('console').log('checking', check.page)
const { version, files } = await next.readJSON(
join('.next/server/pages/', check.page + '.js.nft.json')
)
......@@ -940,7 +916,7 @@ createNextDescribe(
})
it('should add autoExport for auto pre-rendered pages', async () => {
for (const page of ['/', '/about']) {
for (const page of ['/about']) {
const html = await renderViaHTTP(next.appPort, page)
const $ = cheerio.load(html)
const data = JSON.parse($('#__NEXT_DATA__').html())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment