Unverified Commit eba9fca3 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Optimize build trace handling (#56048)

This consolidates our trace handling between turbotrace and the default
`nodeFileTrace` handling to prevent over-tracing. With node-file-trace
it's extremely more efficient to trace in a single `nodeFileTrace` run
so that all resolving/analysis etc can be efficiently cached.

Furthermore, this reduces the amount of chunks we need to trace by
waiting until post build since automatically statically optimized paths
don't need to be traced as they are rendered to purely HTML and never
re-rendered.

This reduces builds times by upwards of 4 - 5 minutes on larger
projects.
parent 7b48a1e2
import type { Span } from '../trace'
import type { NextConfigComplete } from '../server/config-shared'
import {
TRACE_IGNORES,
type BuildTraceContext,
getFilesMapFromReasons,
} from './webpack/plugins/next-trace-entrypoints-plugin'
import {
TRACE_OUTPUT_VERSION,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
} from '../shared/lib/constants'
import path from 'path'
import fs from 'fs/promises'
import { PageInfo } from './utils'
import { loadBindings } from './swc'
import { nonNullable } from '../lib/non-nullable'
import * as ciEnvironment from '../telemetry/ci-info'
import { isMatch } from 'next/dist/compiled/micromatch'
import { defaultOverrides } from '../server/require-hook'
import { nodeFileTrace } from 'next/dist/compiled/@vercel/nft'
import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
export async function collectBuildTraces({
dir,
config,
distDir,
pageKeys,
pageInfos,
staticPages,
nextBuildSpan,
hasSsrAmpPages,
buildTraceContext,
outputFileTracingRoot,
}: {
dir: string
distDir: string
pageKeys: {
app?: string[]
pages: string[]
}
staticPages: Set<string>
hasSsrAmpPages: boolean
outputFileTracingRoot: string
pageInfos: Map<string, PageInfo>
nextBuildSpan: Span
config: NextConfigComplete
buildTraceContext?: BuildTraceContext
}) {
let turboTasksForTrace: unknown
let bindings = await loadBindings()
const runTurbotrace = async function () {
if (!config.experimental.turbotrace || !buildTraceContext) {
return
}
if (!bindings?.isWasm && typeof bindings.turbo.startTrace === 'function') {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = bindings.turbo.createTurboTasks(
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
1024
)
const { entriesTrace, chunksTrace } = buildTraceContext
if (entriesTrace) {
const {
appDir: buildTraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] = await bindings.turbo.startTrace(
action,
turboTasksForTrace
)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(buildTraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from<[string, string]>(
entryNameMap.entries()
).filter(([k]) => k.startsWith(buildTraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action, outputPath } = chunksTrace
action.input = action.input.filter((f: any) => {
const outputPagesPath = path.join(outputPath, '..', 'pages')
return (
!f.startsWith(outputPagesPath) ||
!staticPages.has(
// strip `outputPagesPath` and file ext from absolute
f.substring(outputPagesPath.length, f.length - 3)
)
)
})
await bindings.turbo.startTrace(action, turboTasksForTrace)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await fs
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await fs.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
const { outputFileTracingIncludes = {}, outputFileTracingExcludes = {} } =
config.experimental
const excludeGlobKeys = Object.keys(outputFileTracingExcludes)
await nextBuildSpan
.traceChild('node-file-trace-build')
.traceAsyncFn(async () => {
const nextServerTraceOutput = path.join(
distDir,
'next-server.js.nft.json'
)
const nextMinimalTraceOutput = path.join(
distDir,
'next-minimal-server.js.nft.json'
)
const root =
config.experimental?.turbotrace?.contextDirectory ??
outputFileTracingRoot
// Under standalone mode, we need to trace the extra IPC server and
// worker files.
const isStandalone = config.output === 'standalone'
const nextServerEntry = require.resolve('next/dist/server/next-server')
const sharedEntriesSet = [
...(config.experimental.turbotrace
? []
: Object.keys(defaultOverrides).map((value) =>
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)),
require.resolve('next/dist/compiled/next-server/app-page.runtime.prod'),
require.resolve(
'next/dist/compiled/next-server/app-route.runtime.prod'
),
require.resolve('next/dist/compiled/next-server/pages.runtime.prod'),
require.resolve(
'next/dist/compiled/next-server/pages-api.runtime.prod'
),
]
const { incrementalCacheHandlerPath } = config.experimental
// ensure we trace any dependencies needed for custom
// incremental cache handler
if (incrementalCacheHandlerPath) {
sharedEntriesSet.push(
require.resolve(
path.isAbsolute(incrementalCacheHandlerPath)
? incrementalCacheHandlerPath
: path.join(dir, incrementalCacheHandlerPath)
)
)
}
const serverEntries = [
...sharedEntriesSet,
...(isStandalone
? [
require.resolve('next/dist/server/lib/start-server'),
require.resolve('next/dist/server/next'),
require.resolve('next/dist/server/require-hook'),
]
: []),
require.resolve('next/dist/server/next-server'),
].filter(Boolean) as string[]
const minimalServerEntries = [
...sharedEntriesSet,
require.resolve('next/dist/compiled/next-server/server.runtime.prod'),
].filter(Boolean)
const additionalIgnores = new Set<string>()
for (const glob of excludeGlobKeys) {
if (isMatch('next-server', glob)) {
outputFileTracingExcludes[glob].forEach((exclude) => {
additionalIgnores.add(exclude)
})
}
}
const ignores = [
'**/*.d.ts',
'**/*.map',
isStandalone ? null : '**/next/dist/compiled/jest-worker/**/*',
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/squoosh/**/*.wasm',
'**/next/dist/server/lib/route-resolver*',
...(ciEnvironment.hasNextSupport
? [
// only ignore image-optimizer code when
// this is being handled outside of next-server
'**/next/dist/server/image-optimizer.js',
'**/node_modules/sharp/**/*',
]
: []),
...(!hasSsrAmpPages
? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
: []),
...additionalIgnores,
...(isStandalone ? [] : TRACE_IGNORES),
...(config.experimental.outputFileTracingIgnores || []),
].filter(nonNullable)
const ignoreFn = (pathname: string) => {
if (path.isAbsolute(pathname) && !pathname.startsWith(root)) {
return true
}
return isMatch(pathname, ignores, {
contains: true,
dot: true,
})
}
const traceContext = path.join(nextServerEntry, '..', '..')
const serverTracedFiles = new Set<string>()
const minimalServerTracedFiles = new Set<string>()
function addToTracedFiles(base: string, file: string, dest: Set<string>) {
dest.add(
path.relative(distDir, path.join(base, file)).replace(/\\/g, '/')
)
}
if (isStandalone) {
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/processChild'),
serverTracedFiles
)
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/threadChild'),
serverTracedFiles
)
}
if (config.experimental.turbotrace) {
await runTurbotrace()
const startTrace = bindings.turbo.startTrace
const makeTrace = async (entries: string[]) =>
startTrace(
{
action: 'print',
input: entries,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace?.logLevel,
processCwd: config.experimental.turbotrace?.processCwd,
logDetail: config.experimental.turbotrace?.logDetail,
showAll: config.experimental.turbotrace?.logAll,
},
turboTasksForTrace
)
// turbotrace does not handle concurrent tracing
const vanillaFiles = await makeTrace(serverEntries)
const minimalFiles = await makeTrace(minimalServerEntries)
for (const [set, files] of [
[serverTracedFiles, vanillaFiles],
[minimalServerTracedFiles, minimalFiles],
] as [Set<string>, string[]][]) {
for (const file of files) {
if (!ignoreFn(path.join(traceContext, file))) {
addToTracedFiles(traceContext, file, set)
}
}
}
} else {
const chunksToTrace: string[] = [
...(buildTraceContext?.chunksTrace?.action.input || []),
...serverEntries,
...minimalServerEntries,
]
const result = await nodeFileTrace(chunksToTrace, {
base: outputFileTracingRoot,
processCwd: dir,
ignore: ignoreFn,
mixedModules: true,
})
const reasons = result.reasons
const fileList = result.fileList
result.esmFileList.forEach((file) => fileList.add(file))
const parentFilesMap = getFilesMapFromReasons(fileList, reasons)
const allEntryFiles = new Set<string>()
// get this from buildTraceContext
const entryFiles = new Set<string>()
entryFiles.forEach((file) => {
parentFilesMap
.get(path.relative(outputFileTracingRoot, file))
?.forEach((child) => {
allEntryFiles.add(path.join(traceContext, child))
})
})
for (const [entries, tracedFiles] of [
[serverEntries, serverTracedFiles],
[minimalServerEntries, minimalServerTracedFiles],
] as Array<[string[], Set<string>]>) {
for (const file of entries) {
const curFiles = parentFilesMap.get(
path.relative(outputFileTracingRoot, file)
)
tracedFiles.add(path.relative(distDir, file))
for (const curFile of curFiles || []) {
const filePath = path.join(outputFileTracingRoot, curFile)
if (
!isMatch(filePath, '**/next/dist/pages/**/*', {
dot: true,
contains: true,
})
) {
tracedFiles.add(path.relative(distDir, filePath))
}
}
}
}
await Promise.all(
[
...(buildTraceContext?.chunksTrace?.entryNameFilesMap.entries() ||
new Map()),
].map(async ([entryName, entryNameFiles]) => {
const isApp = entryName.startsWith('app/')
const isPages = entryName.startsWith('pages/')
let route = entryName
if (isApp) {
route = normalizeAppPath(route.substring('app'.length))
}
if (isPages) {
route = normalizePagePath(route.substring('pages'.length))
}
// we don't need to trace for automatically statically optimized
// pages as they don't have server bundles
if (staticPages.has(route)) {
return
}
const entryOutputPath = path.join(
distDir,
'server',
`${entryName}.js`
)
const traceOutputPath = `${entryOutputPath}.nft.json`
const existingTrace = JSON.parse(
await fs.readFile(traceOutputPath, 'utf8')
)
const traceOutputDir = path.dirname(traceOutputPath)
const curTracedFiles = new Set<string>()
for (const file of [...entryNameFiles, entryOutputPath]) {
const curFiles = parentFilesMap.get(
path.relative(outputFileTracingRoot, file)
)
for (const curFile of curFiles || []) {
curTracedFiles.add(
path.relative(
traceOutputDir,
path.join(outputFileTracingRoot, curFile)
)
)
}
}
for (const file of existingTrace.files || []) {
curTracedFiles.add(file)
}
await fs.writeFile(
traceOutputPath,
JSON.stringify({
...existingTrace,
files: [...curTracedFiles].sort(),
})
)
})
)
}
const moduleTypes = ['app-page', 'pages']
for (const type of moduleTypes) {
const modulePath = require.resolve(
`next/dist/server/future/route-modules/${type}/module.compiled`
)
const relativeModulePath = path.relative(root, modulePath)
const contextDir = path.join(
path.dirname(modulePath),
'vendored',
'contexts'
)
for (const item of await fs.readdir(contextDir)) {
const itemPath = path.relative(root, path.join(contextDir, item))
addToTracedFiles(root, itemPath, serverTracedFiles)
addToTracedFiles(root, itemPath, minimalServerTracedFiles)
}
addToTracedFiles(root, relativeModulePath, serverTracedFiles)
addToTracedFiles(root, relativeModulePath, minimalServerTracedFiles)
}
await Promise.all([
fs.writeFile(
nextServerTraceOutput,
JSON.stringify({
version: 1,
files: Array.from(serverTracedFiles),
} as {
version: number
files: string[]
})
),
fs.writeFile(
nextMinimalTraceOutput,
JSON.stringify({
version: 1,
files: Array.from(minimalServerTracedFiles),
} as {
version: number
files: string[]
})
),
])
})
const includeExcludeSpan = nextBuildSpan.traceChild('apply-include-excludes')
const resolvedTraceIncludes = new Map<string, string[]>()
const includeGlobKeys = Object.keys(outputFileTracingIncludes)
await includeExcludeSpan.traceAsyncFn(async () => {
const globOrig =
require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob')
const glob = (pattern: string): Promise<string[]> => {
return new Promise((resolve, reject) => {
globOrig(
pattern,
{ cwd: dir, nodir: true, dot: true },
(err, files) => {
if (err) {
return reject(err)
}
resolve(files)
}
)
})
}
for (let page of pageKeys.pages) {
// edge routes have no trace files
const pageInfo = pageInfos.get(page)
if (pageInfo?.runtime === 'edge') {
continue
}
const combinedIncludes = new Set<string>()
const combinedExcludes = new Set<string>()
page = normalizePagePath(page)
for (const curGlob of includeGlobKeys) {
if (isMatch(page, [curGlob], { dot: true, contains: true })) {
outputFileTracingIncludes[curGlob].forEach((include) => {
combinedIncludes.add(include)
})
}
}
for (const curGlob of excludeGlobKeys) {
if (isMatch(page, [curGlob], { dot: true, contains: true })) {
outputFileTracingExcludes[curGlob].forEach((exclude) => {
combinedExcludes.add(exclude)
})
}
}
if (!combinedIncludes?.size && !combinedExcludes?.size) {
continue
}
const traceFile = path.join(
distDir,
'server/pages',
`${page}.js.nft.json`
)
const pageDir = path.dirname(traceFile)
const traceContent = JSON.parse(await fs.readFile(traceFile, 'utf8'))
const includes: string[] = []
if (combinedIncludes?.size) {
await Promise.all(
[...combinedIncludes].map(async (includeGlob) => {
const results = await glob(includeGlob)
const resolvedInclude = resolvedTraceIncludes.get(includeGlob) || [
...results.map((file) => {
return path.relative(pageDir, path.join(dir, file))
}),
]
includes.push(...resolvedInclude)
resolvedTraceIncludes.set(includeGlob, resolvedInclude)
})
)
}
const combined = new Set([...traceContent.files, ...includes])
if (combinedExcludes?.size) {
const resolvedGlobs = [...combinedExcludes].map((exclude) =>
path.join(dir, exclude)
)
combined.forEach((file) => {
if (
isMatch(path.join(pageDir, file), resolvedGlobs, {
dot: true,
contains: true,
})
) {
combined.delete(file)
}
})
}
await fs.writeFile(
traceFile,
JSON.stringify({
version: traceContent.version,
files: [...combined],
})
)
}
})
}
import type { NodeFileTraceResult } from '@vercel/nft'
import type { RemotePattern } from '../shared/lib/image-config' import type { RemotePattern } from '../shared/lib/image-config'
import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin' import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin'
import type { PagesManifest } from './webpack/plugins/pages-manifest-plugin' import type { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
...@@ -68,8 +67,6 @@ import { ...@@ -68,8 +67,6 @@ import {
SUBRESOURCE_INTEGRITY_MANIFEST, SUBRESOURCE_INTEGRITY_MANIFEST,
MIDDLEWARE_BUILD_MANIFEST, MIDDLEWARE_BUILD_MANIFEST,
MIDDLEWARE_REACT_LOADABLE_MANIFEST, MIDDLEWARE_REACT_LOADABLE_MANIFEST,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
TRACE_OUTPUT_VERSION,
SERVER_REFERENCE_MANIFEST, SERVER_REFERENCE_MANIFEST,
FUNCTIONS_CONFIG_MANIFEST, FUNCTIONS_CONFIG_MANIFEST,
} from '../shared/lib/constants' } from '../shared/lib/constants'
...@@ -146,9 +143,10 @@ import { startTypeChecking } from './type-check' ...@@ -146,9 +143,10 @@ import { startTypeChecking } from './type-check'
import { generateInterceptionRoutesRewrites } from '../lib/generate-interception-routes-rewrites' import { generateInterceptionRoutesRewrites } from '../lib/generate-interception-routes-rewrites'
import { buildDataRoute } from '../server/lib/router-utils/build-data-route' import { buildDataRoute } from '../server/lib/router-utils/build-data-route'
import { defaultOverrides } from '../server/require-hook'
import { initialize as initializeIncrementalCache } from '../server/lib/incremental-cache-server' import { initialize as initializeIncrementalCache } from '../server/lib/incremental-cache-server'
import { nodeFs } from '../server/lib/node-fs-methods' import { nodeFs } from '../server/lib/node-fs-methods'
import { collectBuildTraces } from './collect-build-traces'
import { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints-plugin'
export type SsgRoute = { export type SsgRoute = {
initialRevalidateSeconds: number | false initialRevalidateSeconds: number | false
...@@ -990,14 +988,13 @@ export default async function build( ...@@ -990,14 +988,13 @@ export default async function build(
ignore: [] as string[], ignore: [] as string[],
})) }))
let binding = await loadBindings()
async function turbopackBuild() { async function turbopackBuild() {
const turboNextBuildStart = process.hrtime() const turboNextBuildStart = process.hrtime()
const turboJson = findUp.sync('turbo.json', { cwd: dir }) const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow // eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir }) const packagePath = findUp.sync('package.json', { cwd: dir })
let binding = await loadBindings()
let root = let root =
turboNextBuildRoot ?? turboNextBuildRoot ??
...@@ -1013,15 +1010,16 @@ export default async function build( ...@@ -1013,15 +1010,16 @@ export default async function build(
}) })
const [duration] = process.hrtime(turboNextBuildStart) const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null } return { duration, buildTraceContext: null }
} }
let buildTraceContext: undefined | BuildTraceContext
let runTurbotrace = async (_staticPages: Set<string>) => {}
let turboTasksForTrace: unknown
if (!isGenerate) { if (!isGenerate) {
const { duration: webpackBuildDuration, turbotraceContext } = const { duration: webpackBuildDuration, ...rest } = turboNextBuild
turboNextBuild ? await turbopackBuild() : await webpackBuild() ? await turbopackBuild()
: await webpackBuild()
buildTraceContext = rest.buildTraceContext
telemetry.record( telemetry.record(
eventBuildCompleted(pagesPaths, { eventBuildCompleted(pagesPaths, {
...@@ -1029,101 +1027,6 @@ export default async function build( ...@@ -1029,101 +1027,6 @@ export default async function build(
totalAppPagesCount, totalAppPagesCount,
}) })
) )
runTurbotrace = async function (staticPages: Set<string>) {
if (!turbotraceContext) {
return
}
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = binding.turbo.createTurboTasks(
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
1024
)
const { entriesTrace, chunksTrace } = turbotraceContext
if (entriesTrace) {
const {
appDir: turbotraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] =
await binding.turbo.startTrace(action, turboTasksForTrace)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(turbotraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from<[string, string]>(
entryNameMap.entries()
).filter(([k]) => k.startsWith(turbotraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action, outputPath } = chunksTrace
action.input = action.input.filter((f: any) => {
const outputPagesPath = path.join(outputPath, '..', 'pages')
return (
!f.startsWith(outputPagesPath) ||
!staticPages.has(
// strip `outputPagesPath` and file ext from absolute
f.substring(outputPagesPath.length, f.length - 3)
)
)
})
await binding.turbo.startTrace(action, turboTasksForTrace)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await fs
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await fs.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
} }
// For app directory, we run type checking after build. // For app directory, we run type checking after build.
...@@ -1799,7 +1702,7 @@ export default async function build( ...@@ -1799,7 +1702,7 @@ export default async function build(
return returnValue return returnValue
}) })
await runTurbotrace(staticPages) if (postCompileSpinner) postCompileSpinner.stopAndPersist()
if (customAppGetInitialProps) { if (customAppGetInitialProps) {
console.warn( console.warn(
...@@ -1844,457 +1747,23 @@ export default async function build( ...@@ -1844,457 +1747,23 @@ export default async function build(
) )
} }
const nextServerTraceOutput = path.join( let buildTracesPromise: Promise<any> | undefined = undefined
distDir,
'next-server.js.nft.json'
)
const nextMinimalTraceOutput = path.join(
distDir,
'next-minimal-server.js.nft.json'
)
if (!isGenerate && config.outputFileTracing) { if (!isGenerate && config.outputFileTracing) {
let nodeFileTrace: any buildTracesPromise = collectBuildTraces({
if (config.experimental.turbotrace) { dir,
if (!binding?.isWasm) { config,
nodeFileTrace = binding.turbo.startTrace
}
}
if (!nodeFileTrace) {
nodeFileTrace =
require('next/dist/compiled/@vercel/nft').nodeFileTrace
}
const includeExcludeSpan = nextBuildSpan.traceChild(
'apply-include-excludes'
)
const resolvedTraceIncludes = new Map<string, string[]>()
const {
outputFileTracingIncludes = {},
outputFileTracingExcludes = {},
} = config.experimental
const includeGlobKeys = Object.keys(outputFileTracingIncludes)
const excludeGlobKeys = Object.keys(outputFileTracingExcludes)
await includeExcludeSpan.traceAsyncFn(async () => {
const globOrig =
require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob')
const glob = (pattern: string): Promise<string[]> => {
return new Promise((resolve, reject) => {
globOrig(
pattern,
{ cwd: dir, nodir: true, dot: true },
(err, files) => {
if (err) {
return reject(err)
}
resolve(files)
}
)
})
}
if (config.outputFileTracing) {
for (let page of pageKeys.pages) {
// edge routes have no trace files
const pageInfo = pageInfos.get(page)
if (pageInfo?.runtime === 'edge') {
continue
}
const combinedIncludes = new Set<string>()
const combinedExcludes = new Set<string>()
page = normalizePagePath(page)
for (const curGlob of includeGlobKeys) {
if (isMatch(page, [curGlob])) {
outputFileTracingIncludes[curGlob].forEach((include) => {
combinedIncludes.add(include)
})
}
}
for (const curGlob of excludeGlobKeys) {
if (isMatch(page, [curGlob])) {
outputFileTracingExcludes[curGlob].forEach((exclude) => {
combinedExcludes.add(exclude)
})
}
}
if (!combinedIncludes?.size && !combinedExcludes?.size) {
continue
}
const traceFile = path.join(
distDir,
'server/pages',
`${page}.js.nft.json`
)
const pageDir = path.dirname(traceFile)
const traceContent = JSON.parse(
await fs.readFile(traceFile, 'utf8')
)
const includes: string[] = []
if (combinedIncludes?.size) {
await Promise.all(
[...combinedIncludes].map(async (includeGlob) => {
const results = await glob(includeGlob)
const resolvedInclude = resolvedTraceIncludes.get(
includeGlob
) || [
...results.map((file) => {
return path.relative(pageDir, path.join(dir, file))
}),
]
includes.push(...resolvedInclude)
resolvedTraceIncludes.set(includeGlob, resolvedInclude)
})
)
}
const combined = new Set([...traceContent.files, ...includes])
if (combinedExcludes?.size) {
const resolvedGlobs = [...combinedExcludes].map((exclude) =>
path.join(dir, exclude)
)
combined.forEach((file) => {
if (isMatch(path.join(pageDir, file), resolvedGlobs)) {
combined.delete(file)
}
})
}
await fs.writeFile(
traceFile,
JSON.stringify({
version: traceContent.version,
files: [...combined],
})
)
}
}
})
await nextBuildSpan
.traceChild('trace-next-server')
.traceAsyncFn(async () => {
let cacheKey: string | undefined
// consider all lockFiles in tree in case user accidentally
// has both package-lock.json and yarn.lock
const lockFiles: string[] = (
await Promise.all(
['package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'].map(
(file) => findUp(file, { cwd: dir })
)
)
).filter(Boolean) as any // TypeScript doesn't like this filter
const cachedTracePath = path.join(
distDir,
'cache/next-server.js.nft.json'
)
const minimalCachedTracePath = path.join(
distDir, distDir,
'cache/next-minimal-server.js.nft.json' pageKeys,
) pageInfos,
staticPages,
if ( nextBuildSpan,
lockFiles.length > 0 && hasSsrAmpPages,
// we can't leverage trace cache if this is configured buildTraceContext,
// currently unless we break this to a separate trace outputFileTracingRoot,
// file }).catch((err) => {
!config.experimental.incrementalCacheHandlerPath console.error(err)
) { process.exit(1)
const cacheHash = (
require('crypto') as typeof import('crypto')
).createHash('sha1')
cacheHash.update(require('next/package').version)
cacheHash.update(hasSsrAmpPages + '')
cacheHash.update(ciEnvironment.hasNextSupport + '')
await Promise.all(
lockFiles.map(async (lockFile) => {
cacheHash.update(await fs.readFile(lockFile))
})
)
cacheKey = cacheHash.digest('hex')
try {
const existingTrace = JSON.parse(
await fs.readFile(cachedTracePath, 'utf8')
)
const existingMinimalTrace = JSON.parse(
await fs.readFile(minimalCachedTracePath, 'utf8')
)
if (
existingTrace.cacheKey === cacheKey &&
existingMinimalTrace.cacheKey === cacheKey
) {
await fs.copyFile(cachedTracePath, nextServerTraceOutput)
await fs.copyFile(
minimalCachedTracePath,
nextMinimalTraceOutput
)
return
}
} catch {}
}
const root =
config.experimental?.turbotrace?.contextDirectory ??
outputFileTracingRoot
// Under standalone mode, we need to trace the extra IPC server and
// worker files.
const isStandalone = config.output === 'standalone'
const nextServerEntry = require.resolve(
'next/dist/server/next-server'
)
const sharedEntriesSet = [
...(config.experimental.turbotrace
? []
: Object.keys(defaultOverrides).map((value) =>
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)),
require.resolve(
'next/dist/compiled/next-server/app-page.runtime.prod'
),
require.resolve(
'next/dist/compiled/next-server/app-route.runtime.prod'
),
require.resolve(
'next/dist/compiled/next-server/pages.runtime.prod'
),
require.resolve(
'next/dist/compiled/next-server/pages-api.runtime.prod'
),
]
// ensure we trace any dependencies needed for custom
// incremental cache handler
if (incrementalCacheHandlerPath) {
sharedEntriesSet.push(
require.resolve(
path.isAbsolute(incrementalCacheHandlerPath)
? incrementalCacheHandlerPath
: path.join(dir, incrementalCacheHandlerPath)
)
)
}
const vanillaServerEntries = [
...sharedEntriesSet,
...(isStandalone
? [
require.resolve('next/dist/server/lib/start-server'),
require.resolve('next/dist/server/next'),
require.resolve('next/dist/server/require-hook'),
]
: []),
require.resolve('next/dist/server/next-server'),
].filter(Boolean) as string[]
const minimalServerEntries = [
...sharedEntriesSet,
require.resolve(
'next/dist/compiled/next-server/server.runtime.prod'
),
].filter(Boolean)
const additionalIgnores = new Set<string>()
for (const glob of excludeGlobKeys) {
if (isMatch('next-server', glob)) {
outputFileTracingExcludes[glob].forEach((exclude) => {
additionalIgnores.add(exclude)
})
}
}
const ignores = [
'**/*.d.ts',
'**/*.map',
'**/next/dist/pages/**/*',
isStandalone ? null : '**/next/dist/compiled/jest-worker/**/*',
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/squoosh/**/*.wasm',
'**/next/dist/server/lib/route-resolver*',
...(ciEnvironment.hasNextSupport
? [
// only ignore image-optimizer code when
// this is being handled outside of next-server
'**/next/dist/server/image-optimizer.js',
'**/node_modules/sharp/**/*',
]
: []),
...(!hasSsrAmpPages
? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
: []),
...additionalIgnores,
].filter(nonNullable)
const ignoreFn = (pathname: string) => {
if (path.isAbsolute(pathname) && !pathname.startsWith(root)) {
return true
}
return isMatch(pathname, ignores, {
contains: true,
dot: true,
})
}
const traceContext = path.join(nextServerEntry, '..', '..')
const tracedFiles = new Set<string>()
const minimalTracedFiles = new Set<string>()
function addToTracedFiles(
base: string,
file: string,
dest: Set<string>
) {
dest.add(
path
.relative(distDir, path.join(base, file))
.replace(/\\/g, '/')
)
}
if (isStandalone) {
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/processChild'),
tracedFiles
)
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/threadChild'),
tracedFiles
)
}
if (config.experimental.turbotrace) {
const makeTrace = async (entries: string[]) =>
nodeFileTrace(
{
action: 'print',
input: entries,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace?.logLevel,
processCwd: config.experimental.turbotrace?.processCwd,
logDetail: config.experimental.turbotrace?.logDetail,
showAll: config.experimental.turbotrace?.logAll,
},
turboTasksForTrace
)
// turbotrace does not handle concurrent tracing
const vanillaFiles = await makeTrace(vanillaServerEntries)
const minimalFiles = await makeTrace(minimalServerEntries)
for (const [set, files] of [
[tracedFiles, vanillaFiles],
[minimalTracedFiles, minimalFiles],
] as [Set<string>, string[]][]) {
for (const file of files) {
if (!ignoreFn(path.join(traceContext, file))) {
addToTracedFiles(traceContext, file, set)
}
}
}
} else {
const makeTrace = async (entries: string[]) =>
nodeFileTrace(entries, {
base: root,
processCwd: dir,
ignore: ignoreFn,
})
const [vanillaFiles, minimalFiles]: NodeFileTraceResult[] =
await Promise.all([
makeTrace(vanillaServerEntries),
makeTrace(minimalServerEntries),
])
for (const [set, traceResult] of [
[tracedFiles, vanillaFiles],
[minimalTracedFiles, minimalFiles],
] as [Set<string>, NodeFileTraceResult][]) {
for (const file of traceResult.fileList) {
addToTracedFiles(root, file, set)
}
}
}
const moduleTypes = ['app-page', 'pages']
for (const type of moduleTypes) {
const modulePath = require.resolve(
`next/dist/server/future/route-modules/${type}/module.compiled`
)
const relativeModulePath = path.relative(root, modulePath)
const contextDir = path.join(
path.dirname(modulePath),
'vendored',
'contexts'
)
for (const item of await fs.readdir(contextDir)) {
const itemPath = path.relative(
root,
path.join(contextDir, item)
)
addToTracedFiles(root, itemPath, tracedFiles)
addToTracedFiles(root, itemPath, minimalTracedFiles)
}
addToTracedFiles(root, relativeModulePath, tracedFiles)
addToTracedFiles(root, relativeModulePath, minimalTracedFiles)
}
await Promise.all([
fs.writeFile(
nextServerTraceOutput,
JSON.stringify({
version: 1,
cacheKey,
files: Array.from(tracedFiles),
} as {
version: number
files: string[]
})
),
fs.writeFile(
nextMinimalTraceOutput,
JSON.stringify({
version: 1,
cacheKey,
files: Array.from(minimalTracedFiles),
} as {
version: number
files: string[]
})
),
])
await fs.unlink(cachedTracePath).catch(() => {})
await fs.unlink(minimalCachedTracePath).catch(() => {})
await fs
.copyFile(nextServerTraceOutput, cachedTracePath)
.catch(() => {})
await fs
.copyFile(nextMinimalTraceOutput, minimalCachedTracePath)
.catch(() => {})
}) })
} }
...@@ -2396,30 +1865,11 @@ export default async function build( ...@@ -2396,30 +1865,11 @@ export default async function build(
) )
) )
if (config.output === 'standalone') {
await nextBuildSpan
.traceChild('copy-traced-files')
.traceAsyncFn(async () => {
await copyTracedFiles(
dir,
distDir,
pageKeys.pages,
denormalizedAppPages,
outputFileTracingRoot,
requiredServerFiles.config,
middlewareManifest,
hasInstrumentationHook
)
})
}
const finalPrerenderRoutes: { [route: string]: SsgRoute } = {} const finalPrerenderRoutes: { [route: string]: SsgRoute } = {}
const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {} const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {}
const tbdPrerenderRoutes: string[] = [] const tbdPrerenderRoutes: string[] = []
let ssgNotFoundPaths: string[] = [] let ssgNotFoundPaths: string[] = []
if (postCompileSpinner) postCompileSpinner.stopAndPersist()
const { i18n } = config const { i18n } = config
const usedStaticStatusPages = STATIC_STATUS_PAGES.filter( const usedStaticStatusPages = STATIC_STATUS_PAGES.filter(
...@@ -3064,6 +2514,8 @@ export default async function build( ...@@ -3064,6 +2514,8 @@ export default async function build(
}) })
} }
let buildTracesSpinner = createSpinner(`Collecting build traces`)
// ensure the worker is not left hanging // ensure the worker is not left hanging
pagesStaticWorkers.close() pagesStaticWorkers.close()
appStaticWorkers?.close() appStaticWorkers?.close()
...@@ -3222,57 +2674,6 @@ export default async function build( ...@@ -3222,57 +2674,6 @@ export default async function build(
return Promise.reject(err) return Promise.reject(err)
}) })
if (config.output === 'standalone') {
for (const file of [
...requiredServerFiles.files,
path.join(config.distDir, SERVER_FILES_MANIFEST),
...loadedEnvFiles.reduce<string[]>((acc, envFile) => {
if (['.env', '.env.production'].includes(envFile.path)) {
acc.push(envFile.path)
}
return acc
}, []),
]) {
const filePath = path.join(dir, file)
const outputPath = path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, filePath)
)
await fs.mkdir(path.dirname(outputPath), {
recursive: true,
})
await fs.copyFile(filePath, outputPath)
}
await recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'
),
{ overwrite: true }
)
if (appDir) {
const originalServerApp = path.join(distDir, SERVER_DIRECTORY, 'app')
if (fsExistsSync(originalServerApp)) {
await recursiveCopy(
originalServerApp,
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'app'
),
{ overwrite: true }
)
}
}
}
await nextBuildSpan.traceChild('print-tree-view').traceAsyncFn(() => await nextBuildSpan.traceChild('print-tree-view').traceAsyncFn(() =>
printTreeView(pageKeys, pageInfos, { printTreeView(pageKeys, pageInfos, {
distPath: distDir, distPath: distDir,
...@@ -3314,6 +2715,11 @@ export default async function build( ...@@ -3314,6 +2715,11 @@ export default async function build(
} }
if (config.output === 'export') { if (config.output === 'export') {
if (buildTracesSpinner) {
buildTracesSpinner?.stop()
buildTracesSpinner = undefined
}
const exportApp: typeof import('../export').default = const exportApp: typeof import('../export').default =
require('../export').default require('../export').default
...@@ -3348,11 +2754,91 @@ export default async function build( ...@@ -3348,11 +2754,91 @@ export default async function build(
pagesWorker.close() pagesWorker.close()
appWorker.close() appWorker.close()
} }
await buildTracesPromise
if (config.output === 'standalone') {
await nextBuildSpan
.traceChild('copy-traced-files')
.traceAsyncFn(async () => {
await copyTracedFiles(
dir,
distDir,
pageKeys.pages,
denormalizedAppPages,
outputFileTracingRoot,
requiredServerFiles.config,
middlewareManifest,
hasInstrumentationHook,
staticPages
)
if (config.output === 'standalone') {
for (const file of [
...requiredServerFiles.files,
path.join(config.distDir, SERVER_FILES_MANIFEST),
...loadedEnvFiles.reduce<string[]>((acc, envFile) => {
if (['.env', '.env.production'].includes(envFile.path)) {
acc.push(envFile.path)
}
return acc
}, []),
]) {
const filePath = path.join(dir, file)
const outputPath = path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, filePath)
)
await fs.mkdir(path.dirname(outputPath), {
recursive: true,
})
await fs.copyFile(filePath, outputPath)
}
await recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'
),
{ overwrite: true }
)
if (appDir) {
const originalServerApp = path.join(
distDir,
SERVER_DIRECTORY,
'app'
)
if (fsExistsSync(originalServerApp)) {
await recursiveCopy(
originalServerApp,
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'app'
),
{ overwrite: true }
)
}
}
}
})
}
if (buildTracesSpinner) {
buildTracesSpinner.stopAndPersist()
buildTracesSpinner = undefined
}
await nextBuildSpan await nextBuildSpan
.traceChild('telemetry-flush') .traceChild('telemetry-flush')
.traceAsyncFn(() => telemetry.flush()) .traceAsyncFn(() => telemetry.flush())
}) })
return buildResult return buildResult
} finally { } finally {
// Ensure we wait for lockfile patching if present // Ensure we wait for lockfile patching if present
......
...@@ -1796,7 +1796,8 @@ export async function copyTracedFiles( ...@@ -1796,7 +1796,8 @@ export async function copyTracedFiles(
tracingRoot: string, tracingRoot: string,
serverConfig: NextConfig, serverConfig: NextConfig,
middlewareManifest: MiddlewareManifest, middlewareManifest: MiddlewareManifest,
hasInstrumentationHook: boolean hasInstrumentationHook: boolean,
staticPages: Set<string>
) { ) {
const outputPath = path.join(distDir, 'standalone') const outputPath = path.join(distDir, 'standalone')
let moduleType = false let moduleType = false
...@@ -1889,6 +1890,12 @@ export async function copyTracedFiles( ...@@ -1889,6 +1890,12 @@ export async function copyTracedFiles(
if (middlewareManifest.functions.hasOwnProperty(page)) { if (middlewareManifest.functions.hasOwnProperty(page)) {
continue continue
} }
const route = normalizePagePath(page)
if (staticPages.has(route)) {
continue
}
const pageFile = path.join( const pageFile = path.join(
distDir, distDir,
'server', 'server',
...@@ -1897,7 +1904,9 @@ export async function copyTracedFiles( ...@@ -1897,7 +1904,9 @@ export async function copyTracedFiles(
) )
const pageTraceFile = `${pageFile}.nft.json` const pageTraceFile = `${pageFile}.nft.json`
await handleTraceFiles(pageTraceFile).catch((err) => { await handleTraceFiles(pageTraceFile).catch((err) => {
if (err.code !== 'ENOENT' || (page !== '/404' && page !== '/500')) {
Log.warn(`Failed to copy traced files for ${pageFile}`, err) Log.warn(`Failed to copy traced files for ${pageFile}`, err)
}
}) })
} }
......
...@@ -24,8 +24,8 @@ import loadConfig from '../../server/config' ...@@ -24,8 +24,8 @@ import loadConfig from '../../server/config'
import { trace } from '../../trace' import { trace } from '../../trace'
import { WEBPACK_LAYERS } from '../../lib/constants' import { WEBPACK_LAYERS } from '../../lib/constants'
import { import {
BuildTraceContext,
TraceEntryPointsPlugin, TraceEntryPointsPlugin,
TurbotraceContext,
} from '../webpack/plugins/next-trace-entrypoints-plugin' } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { UnwrapPromise } from '../../lib/coalesced-function' import { UnwrapPromise } from '../../lib/coalesced-function'
import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin' import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'
...@@ -61,7 +61,7 @@ export async function webpackBuildImpl( ...@@ -61,7 +61,7 @@ export async function webpackBuildImpl(
): Promise<{ ): Promise<{
duration: number duration: number
pluginState: any pluginState: any
turbotraceContext?: TurbotraceContext buildTraceContext?: BuildTraceContext
serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries'] serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries']
}> { }> {
let result: CompilerResult | null = { let result: CompilerResult | null = {
...@@ -183,18 +183,22 @@ export async function webpackBuildImpl( ...@@ -183,18 +183,22 @@ export async function webpackBuildImpl(
let inputFileSystem: any let inputFileSystem: any
if (!compilerName || compilerName === 'server') { if (!compilerName || compilerName === 'server') {
debug('starting server compiler')
const start = Date.now()
;[serverResult, inputFileSystem] = await runCompiler(serverConfig, { ;[serverResult, inputFileSystem] = await runCompiler(serverConfig, {
runWebpackSpan, runWebpackSpan,
inputFileSystem, inputFileSystem,
}) })
debug('server result', serverResult) debug(`server compiler finished ${Date.now() - start}ms`)
} }
if (!compilerName || compilerName === 'edge-server') { if (!compilerName || compilerName === 'edge-server') {
debug('starting edge-server compiler')
const start = Date.now()
;[edgeServerResult, inputFileSystem] = edgeConfig ;[edgeServerResult, inputFileSystem] = edgeConfig
? await runCompiler(edgeConfig, { runWebpackSpan, inputFileSystem }) ? await runCompiler(edgeConfig, { runWebpackSpan, inputFileSystem })
: [null] : [null]
debug('edge server result', edgeServerResult) debug(`edge-server compiler finished ${Date.now() - start}ms`)
} }
// Only continue if there were no errors // Only continue if there were no errors
...@@ -223,11 +227,13 @@ export async function webpackBuildImpl( ...@@ -223,11 +227,13 @@ export async function webpackBuildImpl(
} }
if (!compilerName || compilerName === 'client') { if (!compilerName || compilerName === 'client') {
debug('starting client compiler')
const start = Date.now()
;[clientResult, inputFileSystem] = await runCompiler(clientConfig, { ;[clientResult, inputFileSystem] = await runCompiler(clientConfig, {
runWebpackSpan, runWebpackSpan,
inputFileSystem, inputFileSystem,
}) })
debug('client result', clientResult) debug(`client compiler finished ${Date.now() - start}ms`)
} }
} }
...@@ -319,7 +325,7 @@ export async function webpackBuildImpl( ...@@ -319,7 +325,7 @@ export async function webpackBuildImpl(
return { return {
duration: webpackBuildEnd[0], duration: webpackBuildEnd[0],
turbotraceContext: traceEntryPointsPlugin?.turbotraceContext, buildTraceContext: traceEntryPointsPlugin?.buildTraceContext,
pluginState: getPluginState(), pluginState: getPluginState(),
serializedPagesManifestEntries: { serializedPagesManifestEntries: {
edgeServerPages: pagesPluginModule.edgeServerPages, edgeServerPages: pagesPluginModule.edgeServerPages,
...@@ -360,17 +366,25 @@ export async function workerMain(workerData: { ...@@ -360,17 +366,25 @@ export async function workerMain(workerData: {
NextBuildContext.nextBuildSpan = trace('next-build') NextBuildContext.nextBuildSpan = trace('next-build')
const result = await webpackBuildImpl(workerData.compilerName) const result = await webpackBuildImpl(workerData.compilerName)
const { entriesTrace } = result.turbotraceContext ?? {} const { entriesTrace, chunksTrace } = result.buildTraceContext ?? {}
if (entriesTrace) { if (entriesTrace) {
const { entryNameMap, depModArray } = entriesTrace const { entryNameMap, depModArray } = entriesTrace
if (depModArray) { if (depModArray) {
result.turbotraceContext!.entriesTrace!.depModArray = depModArray result.buildTraceContext!.entriesTrace!.depModArray = depModArray
} }
if (entryNameMap) { if (entryNameMap) {
const entryEntries = Array.from(entryNameMap?.entries() ?? []) const entryEntries = Array.from(entryNameMap?.entries() ?? [])
// @ts-expect-error // @ts-expect-error
result.turbotraceContext.entriesTrace.entryNameMap = entryEntries result.buildTraceContext.entriesTrace.entryNameMap = entryEntries
} }
} }
if (chunksTrace?.entryNameFilesMap) {
const entryNameFilesMap = Array.from(
chunksTrace.entryNameFilesMap.entries() ?? []
)
// @ts-expect-error
result.buildTraceContext!.chunksTrace!.entryNameFilesMap = entryNameFilesMap
}
return result return result
} }
import { COMPILER_INDEXES } from '../../shared/lib/constants' import { COMPILER_INDEXES } from '../../shared/lib/constants'
import * as Log from '../output/log' import * as Log from '../output/log'
import { NextBuildContext } from '../build-context' import { NextBuildContext } from '../build-context'
import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin' import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { Worker } from 'next/dist/compiled/jest-worker' import { Worker } from 'next/dist/compiled/jest-worker'
import origDebug from 'next/dist/compiled/debug' import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process' import { ChildProcess } from 'child_process'
...@@ -50,7 +50,7 @@ async function webpackBuildWithWorker() { ...@@ -50,7 +50,7 @@ async function webpackBuildWithWorker() {
const combinedResult = { const combinedResult = {
duration: 0, duration: 0,
turbotraceContext: {} as TurbotraceContext, buildTraceContext: {} as BuildTraceContext,
} }
// order matters here // order matters here
const ORDERED_COMPILER_NAMES = [ const ORDERED_COMPILER_NAMES = [
...@@ -85,15 +85,28 @@ async function webpackBuildWithWorker() { ...@@ -85,15 +85,28 @@ async function webpackBuildWithWorker() {
combinedResult.duration += curResult.duration combinedResult.duration += curResult.duration
if (curResult.turbotraceContext?.entriesTrace) { if (curResult.buildTraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext const { entryNameMap } = curResult.buildTraceContext.entriesTrace!
const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) { if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map( combinedResult.buildTraceContext.entriesTrace =
curResult.buildTraceContext.entriesTrace
combinedResult.buildTraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap entryNameMap
) )
} }
if (curResult.buildTraceContext?.chunksTrace) {
const { entryNameFilesMap } = curResult.buildTraceContext.chunksTrace!
if (entryNameFilesMap) {
combinedResult.buildTraceContext.chunksTrace =
curResult.buildTraceContext.chunksTrace!
combinedResult.buildTraceContext.chunksTrace!.entryNameFilesMap =
new Map(entryNameFilesMap)
}
}
} }
} }
buildSpinner?.stopAndPersist() buildSpinner?.stopAndPersist()
......
...@@ -23,7 +23,7 @@ import { getModuleBuildInfo } from '../loaders/get-module-build-info' ...@@ -23,7 +23,7 @@ import { getModuleBuildInfo } from '../loaders/get-module-build-info'
import { getPageFilePath } from '../../entries' import { getPageFilePath } from '../../entries'
const PLUGIN_NAME = 'TraceEntryPointsPlugin' const PLUGIN_NAME = 'TraceEntryPointsPlugin'
const TRACE_IGNORES = [ export const TRACE_IGNORES = [
'**/*/next/dist/server/next.js', '**/*/next/dist/server/next.js',
'**/*/next/dist/bin/next', '**/*/next/dist/bin/next',
] ]
...@@ -48,7 +48,7 @@ function getModuleFromDependency( ...@@ -48,7 +48,7 @@ function getModuleFromDependency(
return compilation.moduleGraph.getModule(dep) return compilation.moduleGraph.getModule(dep)
} }
function getFilesMapFromReasons( export function getFilesMapFromReasons(
fileList: Set<string>, fileList: Set<string>,
reasons: NodeFileTraceReasons, reasons: NodeFileTraceReasons,
ignoreFn?: (file: string, parent?: string) => Boolean ignoreFn?: (file: string, parent?: string) => Boolean
...@@ -114,7 +114,7 @@ export interface TurbotraceAction { ...@@ -114,7 +114,7 @@ export interface TurbotraceAction {
memoryLimit?: number memoryLimit?: number
} }
export interface TurbotraceContext { export interface BuildTraceContext {
entriesTrace?: { entriesTrace?: {
action: TurbotraceAction action: TurbotraceAction
appDir: string appDir: string
...@@ -125,11 +125,12 @@ export interface TurbotraceContext { ...@@ -125,11 +125,12 @@ export interface TurbotraceContext {
chunksTrace?: { chunksTrace?: {
action: TurbotraceAction action: TurbotraceAction
outputPath: string outputPath: string
entryNameFilesMap: Map<string, Array<string>>
} }
} }
export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
public turbotraceContext: TurbotraceContext = {} public buildTraceContext: BuildTraceContext = {}
private rootDir: string private rootDir: string
private appDir: string | undefined private appDir: string | undefined
...@@ -140,7 +141,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -140,7 +141,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
private traceIgnores: string[] private traceIgnores: string[]
private esmExternals?: NextConfigComplete['experimental']['esmExternals'] private esmExternals?: NextConfigComplete['experimental']['esmExternals']
private turbotrace?: NextConfigComplete['experimental']['turbotrace'] private turbotrace?: NextConfigComplete['experimental']['turbotrace']
private chunksToTrace: string[] = []
constructor({ constructor({
rootDir, rootDir,
...@@ -174,18 +174,13 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -174,18 +174,13 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
// Here we output all traced assets and webpack chunks to a // Here we output all traced assets and webpack chunks to a
// ${page}.js.nft.json file // ${page}.js.nft.json file
async createTraceAssets( async createTraceAssets(compilation: any, assets: any, span: Span) {
compilation: any,
assets: any,
span: Span,
readlink: any,
stat: any
) {
const outputPath = compilation.outputOptions.path const outputPath = compilation.outputOptions.path
await span.traceChild('create-trace-assets').traceAsyncFn(async () => { await span.traceChild('create-trace-assets').traceAsyncFn(async () => {
const entryFilesMap = new Map<any, Set<string>>() const entryFilesMap = new Map<any, Set<string>>()
const chunksToTrace = new Set<string>() const chunksToTrace = new Set<string>()
const entryNameFilesMap = new Map<string, Array<string>>()
const isTraceable = (file: string) => const isTraceable = (file: string) =>
!NOT_TRACEABLE.some((suffix) => { !NOT_TRACEABLE.some((suffix) => {
...@@ -214,77 +209,30 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -214,77 +209,30 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
} }
} }
entryFilesMap.set(entrypoint, entryFiles) entryFilesMap.set(entrypoint, entryFiles)
entryNameFilesMap.set(entrypoint.name, [...entryFiles])
} }
// startTrace existed and callable // startTrace existed and callable
if (this.turbotrace) { this.buildTraceContext.chunksTrace = {
let binding = await loadBindings() action: {
if ( action: 'annotate',
!binding?.isWasm && input: [...chunksToTrace],
typeof binding.turbo.startTrace === 'function' contextDirectory:
) { this.turbotrace?.contextDirectory ?? this.tracingRoot,
this.chunksToTrace = [...chunksToTrace] processCwd: this.turbotrace?.processCwd ?? this.rootDir,
return showAll: this.turbotrace?.logAll,
} logLevel: this.turbotrace?.logLevel,
}
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
const ignoreFn = (path: string) => {
return isMatch(path, ignores, { contains: true, dot: true })
}
const result = await nodeFileTrace([...chunksToTrace], {
base: this.tracingRoot,
processCwd: this.rootDir,
readFile: async (path) => {
if (chunksToTrace.has(path)) {
const source =
assets[
nodePath.relative(outputPath, path).replace(/\\/g, '/')
]?.source?.()
if (source) return source
}
try {
return await new Promise((resolve, reject) => {
;(
compilation.inputFileSystem
.readFile as typeof import('fs').readFile
)(path, (err, data) => {
if (err) return reject(err)
resolve(data)
})
})
} catch (e) {
if (isError(e) && (e.code === 'ENOENT' || e.code === 'EISDIR')) {
return null
}
throw e
}
}, },
readlink, outputPath,
stat, entryNameFilesMap,
ignore: ignoreFn, }
mixedModules: true,
})
const reasons = result.reasons
const fileList = result.fileList
result.esmFileList.forEach((file) => fileList.add(file))
const parentFilesMap = getFilesMapFromReasons(fileList, reasons)
for (const [entrypoint, entryFiles] of entryFilesMap) { for (const [entrypoint, entryFiles] of entryFilesMap) {
const traceOutputName = `../${entrypoint.name}.js.nft.json` const traceOutputName = `../${entrypoint.name}.js.nft.json`
const traceOutputPath = nodePath.dirname( const traceOutputPath = nodePath.dirname(
nodePath.join(outputPath, traceOutputName) nodePath.join(outputPath, traceOutputName)
) )
const allEntryFiles = new Set<string>()
entryFiles.forEach((file) => {
parentFilesMap
.get(nodePath.relative(this.tracingRoot, file))
?.forEach((child) => {
allEntryFiles.add(nodePath.join(this.tracingRoot, child))
})
})
// don't include the entry itself in the trace // don't include the entry itself in the trace
entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`)) entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`))
...@@ -313,7 +261,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -313,7 +261,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
for (const file of new Set([ for (const file of new Set([
...entryFiles, ...entryFiles,
...allEntryFiles,
...(this.entryTraces.get(entrypoint.name) || []), ...(this.entryTraces.get(entrypoint.name) || []),
])) { ])) {
if (file) { if (file) {
...@@ -473,18 +420,12 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -473,18 +420,12 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
entriesToTrace.push(...curExtraEntries.keys()) entriesToTrace.push(...curExtraEntries.keys())
} }
}) })
// startTrace existed and callable
if (this.turbotrace) {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
const contextDirectory = const contextDirectory =
this.turbotrace?.contextDirectory ?? this.tracingRoot this.turbotrace?.contextDirectory ?? this.tracingRoot
const chunks = [...entriesToTrace] const chunks = [...entriesToTrace]
this.turbotraceContext.entriesTrace = { this.buildTraceContext.entriesTrace = {
action: { action: {
action: 'print', action: 'print',
input: chunks, input: chunks,
...@@ -498,9 +439,20 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -498,9 +439,20 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
entryNameMap, entryNameMap,
outputPath: compilation.outputOptions.path!, outputPath: compilation.outputOptions.path!,
} }
// if we're using turbotrace we can skip tracing
// loader contents as it should be able to capture
// fs usage in final chunks instead
if (this.turbotrace) {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
return return
} }
} }
let fileList: Set<string> let fileList: Set<string>
let reasons: NodeFileTraceReasons let reasons: NodeFileTraceReasons
const ignores = [ const ignores = [
...@@ -513,7 +465,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -513,7 +465,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
} }
await finishModulesSpan await finishModulesSpan
.traceChild('node-file-trace', { .traceChild('node-file-trace-plugin', {
traceEntryCount: entriesToTrace.length + '', traceEntryCount: entriesToTrace.length + '',
}) })
.traceAsyncFn(async () => { .traceAsyncFn(async () => {
...@@ -656,9 +608,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -656,9 +608,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
this.createTraceAssets( this.createTraceAssets(
compilation, compilation,
assets, assets,
traceEntrypointsPluginSpan, traceEntrypointsPluginSpan
readlink,
stat
) )
.then(() => callback()) .then(() => callback())
.catch((err) => callback(err)) .catch((err) => callback(err))
...@@ -822,35 +772,5 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -822,35 +772,5 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
) )
}) })
}) })
if (this.turbotrace) {
compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async () => {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
const ignoreFn = (path: string) => {
return isMatch(path, ignores, { contains: true, dot: true })
}
const chunks = this.chunksToTrace.filter((chunk) => !ignoreFn(chunk))
this.turbotraceContext.chunksTrace = {
action: {
action: 'annotate',
input: chunks,
contextDirectory:
this.turbotrace?.contextDirectory ?? this.tracingRoot,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
},
outputPath: compiler.outputPath,
}
}
})
}
} }
} }
import React from 'react' import React from 'react'
import { World } from 'components/world' import { World } from 'components/world'
// prevent static generation for trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() { export default function HelloPage() {
return ( return (
<div> <div>
......
import React from 'react' import React from 'react'
import { World } from '@c/world' import { World } from '@c/world'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() { export default function HelloPage() {
return ( return (
<div> <div>
......
import React from 'react' import React from 'react'
import api from '@lib/b-only' import api from '@lib/b-only'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() { export default function ResolveOrder() {
return <div>{api()}</div> return <div>{api()}</div>
} }
import React from 'react' import React from 'react'
import api from '@lib/api' import api from '@lib/api'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() { export default function ResolveOrder() {
return <div>{api()}</div> return <div>{api()}</div>
} }
import { Hello } from '@mycomponent' import { Hello } from '@mycomponent'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function SingleAlias() { export default function SingleAlias() {
return ( return (
<div> <div>
......
import myfile from 'mypackage/myfile' import myfile from 'mypackage/myfile'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function WildcardAlias() { export default function WildcardAlias() {
return <div>{myfile.hello}</div> return <div>{myfile.hello}</div>
} }
import Link from 'next/link' import Link from 'next/link'
import Error from 'next/error' import Error from 'next/error'
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page(props) { export default function Page(props) {
return ( return (
<> <>
......
...@@ -8,9 +8,18 @@ if (typeof window === 'undefined') { ...@@ -8,9 +8,18 @@ if (typeof window === 'undefined') {
import('nanoid').then((mod) => console.log(mod.nanoid())) import('nanoid').then((mod) => console.log(mod.nanoid()))
} }
export default () => ( // prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page() {
return (
<div> <div>
<Link href="/about">About Page</Link> <Link href="/about">About Page</Link>
<p className="index-page">Hello World</p> <p className="index-page">Hello World</p>
</div> </div>
) )
}
...@@ -4,9 +4,18 @@ import Link from 'next/link' ...@@ -4,9 +4,18 @@ import Link from 'next/link'
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
import next from 'next' import next from 'next'
export default () => ( // prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page() {
return (
<div> <div>
<Link href="/about">About Page</Link> <Link href="/about">About Page</Link>
<p className="index-page">Hello World</p> <p className="index-page">Hello World</p>
</div> </div>
) )
}
...@@ -106,7 +106,7 @@ createNextDescribe( ...@@ -106,7 +106,7 @@ createNextDescribe(
}) })
it('should respond with 405 for POST to static page', async () => { it('should respond with 405 for POST to static page', async () => {
const res = await fetchViaHTTP(next.appPort, '/', undefined, { const res = await fetchViaHTTP(next.appPort, '/about', undefined, {
method: 'POST', method: 'POST',
}) })
expect(res.status).toBe(405) expect(res.status).toBe(405)
...@@ -114,7 +114,7 @@ createNextDescribe( ...@@ -114,7 +114,7 @@ createNextDescribe(
}) })
it('should contain generated page count in output', async () => { it('should contain generated page count in output', async () => {
const pageCount = 40 const pageCount = 37
expect(next.cliOutput).toContain( expect(next.cliOutput).toContain(
`Generating static pages (0/${pageCount})` `Generating static pages (0/${pageCount})`
) )
...@@ -204,18 +204,6 @@ createNextDescribe( ...@@ -204,18 +204,6 @@ createNextDescribe(
], ],
notTests: [/\0/, /\?/, /!/], notTests: [/\0/, /\?/, /!/],
}, },
{
page: '/dynamic',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{ {
page: '/index', page: '/index',
tests: [ tests: [
...@@ -231,19 +219,6 @@ createNextDescribe( ...@@ -231,19 +219,6 @@ createNextDescribe(
], ],
notTests: [/next\/dist\/pages\/_error\.js/, /\0/, /\?/, /!/], notTests: [/next\/dist\/pages\/_error\.js/, /\0/, /\?/, /!/],
}, },
{
page: '/counter',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/react\/cjs\/react\.development\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{ {
page: '/next-import', page: '/next-import',
tests: [ tests: [
...@@ -298,6 +273,7 @@ createNextDescribe( ...@@ -298,6 +273,7 @@ createNextDescribe(
] ]
for (const check of checks) { for (const check of checks) {
require('console').log('checking', check.page)
const { version, files } = await next.readJSON( const { version, files } = await next.readJSON(
join('.next/server/pages/', check.page + '.js.nft.json') join('.next/server/pages/', check.page + '.js.nft.json')
) )
...@@ -940,7 +916,7 @@ createNextDescribe( ...@@ -940,7 +916,7 @@ createNextDescribe(
}) })
it('should add autoExport for auto pre-rendered pages', async () => { it('should add autoExport for auto pre-rendered pages', async () => {
for (const page of ['/', '/about']) { for (const page of ['/about']) {
const html = await renderViaHTTP(next.appPort, page) const html = await renderViaHTTP(next.appPort, page)
const $ = cheerio.load(html) const $ = cheerio.load(html)
const data = JSON.parse($('#__NEXT_DATA__').html()) const data = JSON.parse($('#__NEXT_DATA__').html())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment