Unverified Commit eba9fca3 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Optimize build trace handling (#56048)

This consolidates our trace handling between turbotrace and the default
`nodeFileTrace` handling to prevent over-tracing. With node-file-trace
it's extremely more efficient to trace in a single `nodeFileTrace` run
so that all resolving/analysis etc can be efficiently cached.

Furthermore, this reduces the amount of chunks we need to trace by
waiting until post build since automatically statically optimized paths
don't need to be traced as they are rendered to purely HTML and never
re-rendered.

This reduces builds times by upwards of 4 - 5 minutes on larger
projects.
parent 7b48a1e2
import type { Span } from '../trace'
import type { NextConfigComplete } from '../server/config-shared'
import {
TRACE_IGNORES,
type BuildTraceContext,
getFilesMapFromReasons,
} from './webpack/plugins/next-trace-entrypoints-plugin'
import {
TRACE_OUTPUT_VERSION,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
} from '../shared/lib/constants'
import path from 'path'
import fs from 'fs/promises'
import { PageInfo } from './utils'
import { loadBindings } from './swc'
import { nonNullable } from '../lib/non-nullable'
import * as ciEnvironment from '../telemetry/ci-info'
import { isMatch } from 'next/dist/compiled/micromatch'
import { defaultOverrides } from '../server/require-hook'
import { nodeFileTrace } from 'next/dist/compiled/@vercel/nft'
import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
export async function collectBuildTraces({
dir,
config,
distDir,
pageKeys,
pageInfos,
staticPages,
nextBuildSpan,
hasSsrAmpPages,
buildTraceContext,
outputFileTracingRoot,
}: {
dir: string
distDir: string
pageKeys: {
app?: string[]
pages: string[]
}
staticPages: Set<string>
hasSsrAmpPages: boolean
outputFileTracingRoot: string
pageInfos: Map<string, PageInfo>
nextBuildSpan: Span
config: NextConfigComplete
buildTraceContext?: BuildTraceContext
}) {
let turboTasksForTrace: unknown
let bindings = await loadBindings()
const runTurbotrace = async function () {
if (!config.experimental.turbotrace || !buildTraceContext) {
return
}
if (!bindings?.isWasm && typeof bindings.turbo.startTrace === 'function') {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = bindings.turbo.createTurboTasks(
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
1024
)
const { entriesTrace, chunksTrace } = buildTraceContext
if (entriesTrace) {
const {
appDir: buildTraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] = await bindings.turbo.startTrace(
action,
turboTasksForTrace
)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(buildTraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from<[string, string]>(
entryNameMap.entries()
).filter(([k]) => k.startsWith(buildTraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action, outputPath } = chunksTrace
action.input = action.input.filter((f: any) => {
const outputPagesPath = path.join(outputPath, '..', 'pages')
return (
!f.startsWith(outputPagesPath) ||
!staticPages.has(
// strip `outputPagesPath` and file ext from absolute
f.substring(outputPagesPath.length, f.length - 3)
)
)
})
await bindings.turbo.startTrace(action, turboTasksForTrace)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await fs
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await fs.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
const { outputFileTracingIncludes = {}, outputFileTracingExcludes = {} } =
config.experimental
const excludeGlobKeys = Object.keys(outputFileTracingExcludes)
await nextBuildSpan
.traceChild('node-file-trace-build')
.traceAsyncFn(async () => {
const nextServerTraceOutput = path.join(
distDir,
'next-server.js.nft.json'
)
const nextMinimalTraceOutput = path.join(
distDir,
'next-minimal-server.js.nft.json'
)
const root =
config.experimental?.turbotrace?.contextDirectory ??
outputFileTracingRoot
// Under standalone mode, we need to trace the extra IPC server and
// worker files.
const isStandalone = config.output === 'standalone'
const nextServerEntry = require.resolve('next/dist/server/next-server')
const sharedEntriesSet = [
...(config.experimental.turbotrace
? []
: Object.keys(defaultOverrides).map((value) =>
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)),
require.resolve('next/dist/compiled/next-server/app-page.runtime.prod'),
require.resolve(
'next/dist/compiled/next-server/app-route.runtime.prod'
),
require.resolve('next/dist/compiled/next-server/pages.runtime.prod'),
require.resolve(
'next/dist/compiled/next-server/pages-api.runtime.prod'
),
]
const { incrementalCacheHandlerPath } = config.experimental
// ensure we trace any dependencies needed for custom
// incremental cache handler
if (incrementalCacheHandlerPath) {
sharedEntriesSet.push(
require.resolve(
path.isAbsolute(incrementalCacheHandlerPath)
? incrementalCacheHandlerPath
: path.join(dir, incrementalCacheHandlerPath)
)
)
}
const serverEntries = [
...sharedEntriesSet,
...(isStandalone
? [
require.resolve('next/dist/server/lib/start-server'),
require.resolve('next/dist/server/next'),
require.resolve('next/dist/server/require-hook'),
]
: []),
require.resolve('next/dist/server/next-server'),
].filter(Boolean) as string[]
const minimalServerEntries = [
...sharedEntriesSet,
require.resolve('next/dist/compiled/next-server/server.runtime.prod'),
].filter(Boolean)
const additionalIgnores = new Set<string>()
for (const glob of excludeGlobKeys) {
if (isMatch('next-server', glob)) {
outputFileTracingExcludes[glob].forEach((exclude) => {
additionalIgnores.add(exclude)
})
}
}
const ignores = [
'**/*.d.ts',
'**/*.map',
isStandalone ? null : '**/next/dist/compiled/jest-worker/**/*',
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/squoosh/**/*.wasm',
'**/next/dist/server/lib/route-resolver*',
...(ciEnvironment.hasNextSupport
? [
// only ignore image-optimizer code when
// this is being handled outside of next-server
'**/next/dist/server/image-optimizer.js',
'**/node_modules/sharp/**/*',
]
: []),
...(!hasSsrAmpPages
? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
: []),
...additionalIgnores,
...(isStandalone ? [] : TRACE_IGNORES),
...(config.experimental.outputFileTracingIgnores || []),
].filter(nonNullable)
const ignoreFn = (pathname: string) => {
if (path.isAbsolute(pathname) && !pathname.startsWith(root)) {
return true
}
return isMatch(pathname, ignores, {
contains: true,
dot: true,
})
}
const traceContext = path.join(nextServerEntry, '..', '..')
const serverTracedFiles = new Set<string>()
const minimalServerTracedFiles = new Set<string>()
function addToTracedFiles(base: string, file: string, dest: Set<string>) {
dest.add(
path.relative(distDir, path.join(base, file)).replace(/\\/g, '/')
)
}
if (isStandalone) {
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/processChild'),
serverTracedFiles
)
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/threadChild'),
serverTracedFiles
)
}
if (config.experimental.turbotrace) {
await runTurbotrace()
const startTrace = bindings.turbo.startTrace
const makeTrace = async (entries: string[]) =>
startTrace(
{
action: 'print',
input: entries,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace?.logLevel,
processCwd: config.experimental.turbotrace?.processCwd,
logDetail: config.experimental.turbotrace?.logDetail,
showAll: config.experimental.turbotrace?.logAll,
},
turboTasksForTrace
)
// turbotrace does not handle concurrent tracing
const vanillaFiles = await makeTrace(serverEntries)
const minimalFiles = await makeTrace(minimalServerEntries)
for (const [set, files] of [
[serverTracedFiles, vanillaFiles],
[minimalServerTracedFiles, minimalFiles],
] as [Set<string>, string[]][]) {
for (const file of files) {
if (!ignoreFn(path.join(traceContext, file))) {
addToTracedFiles(traceContext, file, set)
}
}
}
} else {
const chunksToTrace: string[] = [
...(buildTraceContext?.chunksTrace?.action.input || []),
...serverEntries,
...minimalServerEntries,
]
const result = await nodeFileTrace(chunksToTrace, {
base: outputFileTracingRoot,
processCwd: dir,
ignore: ignoreFn,
mixedModules: true,
})
const reasons = result.reasons
const fileList = result.fileList
result.esmFileList.forEach((file) => fileList.add(file))
const parentFilesMap = getFilesMapFromReasons(fileList, reasons)
const allEntryFiles = new Set<string>()
// get this from buildTraceContext
const entryFiles = new Set<string>()
entryFiles.forEach((file) => {
parentFilesMap
.get(path.relative(outputFileTracingRoot, file))
?.forEach((child) => {
allEntryFiles.add(path.join(traceContext, child))
})
})
for (const [entries, tracedFiles] of [
[serverEntries, serverTracedFiles],
[minimalServerEntries, minimalServerTracedFiles],
] as Array<[string[], Set<string>]>) {
for (const file of entries) {
const curFiles = parentFilesMap.get(
path.relative(outputFileTracingRoot, file)
)
tracedFiles.add(path.relative(distDir, file))
for (const curFile of curFiles || []) {
const filePath = path.join(outputFileTracingRoot, curFile)
if (
!isMatch(filePath, '**/next/dist/pages/**/*', {
dot: true,
contains: true,
})
) {
tracedFiles.add(path.relative(distDir, filePath))
}
}
}
}
await Promise.all(
[
...(buildTraceContext?.chunksTrace?.entryNameFilesMap.entries() ||
new Map()),
].map(async ([entryName, entryNameFiles]) => {
const isApp = entryName.startsWith('app/')
const isPages = entryName.startsWith('pages/')
let route = entryName
if (isApp) {
route = normalizeAppPath(route.substring('app'.length))
}
if (isPages) {
route = normalizePagePath(route.substring('pages'.length))
}
// we don't need to trace for automatically statically optimized
// pages as they don't have server bundles
if (staticPages.has(route)) {
return
}
const entryOutputPath = path.join(
distDir,
'server',
`${entryName}.js`
)
const traceOutputPath = `${entryOutputPath}.nft.json`
const existingTrace = JSON.parse(
await fs.readFile(traceOutputPath, 'utf8')
)
const traceOutputDir = path.dirname(traceOutputPath)
const curTracedFiles = new Set<string>()
for (const file of [...entryNameFiles, entryOutputPath]) {
const curFiles = parentFilesMap.get(
path.relative(outputFileTracingRoot, file)
)
for (const curFile of curFiles || []) {
curTracedFiles.add(
path.relative(
traceOutputDir,
path.join(outputFileTracingRoot, curFile)
)
)
}
}
for (const file of existingTrace.files || []) {
curTracedFiles.add(file)
}
await fs.writeFile(
traceOutputPath,
JSON.stringify({
...existingTrace,
files: [...curTracedFiles].sort(),
})
)
})
)
}
const moduleTypes = ['app-page', 'pages']
for (const type of moduleTypes) {
const modulePath = require.resolve(
`next/dist/server/future/route-modules/${type}/module.compiled`
)
const relativeModulePath = path.relative(root, modulePath)
const contextDir = path.join(
path.dirname(modulePath),
'vendored',
'contexts'
)
for (const item of await fs.readdir(contextDir)) {
const itemPath = path.relative(root, path.join(contextDir, item))
addToTracedFiles(root, itemPath, serverTracedFiles)
addToTracedFiles(root, itemPath, minimalServerTracedFiles)
}
addToTracedFiles(root, relativeModulePath, serverTracedFiles)
addToTracedFiles(root, relativeModulePath, minimalServerTracedFiles)
}
await Promise.all([
fs.writeFile(
nextServerTraceOutput,
JSON.stringify({
version: 1,
files: Array.from(serverTracedFiles),
} as {
version: number
files: string[]
})
),
fs.writeFile(
nextMinimalTraceOutput,
JSON.stringify({
version: 1,
files: Array.from(minimalServerTracedFiles),
} as {
version: number
files: string[]
})
),
])
})
const includeExcludeSpan = nextBuildSpan.traceChild('apply-include-excludes')
const resolvedTraceIncludes = new Map<string, string[]>()
const includeGlobKeys = Object.keys(outputFileTracingIncludes)
await includeExcludeSpan.traceAsyncFn(async () => {
const globOrig =
require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob')
const glob = (pattern: string): Promise<string[]> => {
return new Promise((resolve, reject) => {
globOrig(
pattern,
{ cwd: dir, nodir: true, dot: true },
(err, files) => {
if (err) {
return reject(err)
}
resolve(files)
}
)
})
}
for (let page of pageKeys.pages) {
// edge routes have no trace files
const pageInfo = pageInfos.get(page)
if (pageInfo?.runtime === 'edge') {
continue
}
const combinedIncludes = new Set<string>()
const combinedExcludes = new Set<string>()
page = normalizePagePath(page)
for (const curGlob of includeGlobKeys) {
if (isMatch(page, [curGlob], { dot: true, contains: true })) {
outputFileTracingIncludes[curGlob].forEach((include) => {
combinedIncludes.add(include)
})
}
}
for (const curGlob of excludeGlobKeys) {
if (isMatch(page, [curGlob], { dot: true, contains: true })) {
outputFileTracingExcludes[curGlob].forEach((exclude) => {
combinedExcludes.add(exclude)
})
}
}
if (!combinedIncludes?.size && !combinedExcludes?.size) {
continue
}
const traceFile = path.join(
distDir,
'server/pages',
`${page}.js.nft.json`
)
const pageDir = path.dirname(traceFile)
const traceContent = JSON.parse(await fs.readFile(traceFile, 'utf8'))
const includes: string[] = []
if (combinedIncludes?.size) {
await Promise.all(
[...combinedIncludes].map(async (includeGlob) => {
const results = await glob(includeGlob)
const resolvedInclude = resolvedTraceIncludes.get(includeGlob) || [
...results.map((file) => {
return path.relative(pageDir, path.join(dir, file))
}),
]
includes.push(...resolvedInclude)
resolvedTraceIncludes.set(includeGlob, resolvedInclude)
})
)
}
const combined = new Set([...traceContent.files, ...includes])
if (combinedExcludes?.size) {
const resolvedGlobs = [...combinedExcludes].map((exclude) =>
path.join(dir, exclude)
)
combined.forEach((file) => {
if (
isMatch(path.join(pageDir, file), resolvedGlobs, {
dot: true,
contains: true,
})
) {
combined.delete(file)
}
})
}
await fs.writeFile(
traceFile,
JSON.stringify({
version: traceContent.version,
files: [...combined],
})
)
}
})
}
import type { NodeFileTraceResult } from '@vercel/nft'
import type { RemotePattern } from '../shared/lib/image-config'
import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin'
import type { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
......@@ -68,8 +67,6 @@ import {
SUBRESOURCE_INTEGRITY_MANIFEST,
MIDDLEWARE_BUILD_MANIFEST,
MIDDLEWARE_REACT_LOADABLE_MANIFEST,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
TRACE_OUTPUT_VERSION,
SERVER_REFERENCE_MANIFEST,
FUNCTIONS_CONFIG_MANIFEST,
} from '../shared/lib/constants'
......@@ -146,9 +143,10 @@ import { startTypeChecking } from './type-check'
import { generateInterceptionRoutesRewrites } from '../lib/generate-interception-routes-rewrites'
import { buildDataRoute } from '../server/lib/router-utils/build-data-route'
import { defaultOverrides } from '../server/require-hook'
import { initialize as initializeIncrementalCache } from '../server/lib/incremental-cache-server'
import { nodeFs } from '../server/lib/node-fs-methods'
import { collectBuildTraces } from './collect-build-traces'
import { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints-plugin'
export type SsgRoute = {
initialRevalidateSeconds: number | false
......@@ -990,14 +988,13 @@ export default async function build(
ignore: [] as string[],
}))
let binding = await loadBindings()
async function turbopackBuild() {
const turboNextBuildStart = process.hrtime()
const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir })
let binding = await loadBindings()
let root =
turboNextBuildRoot ??
......@@ -1013,15 +1010,16 @@ export default async function build(
})
const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null }
return { duration, buildTraceContext: null }
}
let runTurbotrace = async (_staticPages: Set<string>) => {}
let turboTasksForTrace: unknown
let buildTraceContext: undefined | BuildTraceContext
if (!isGenerate) {
const { duration: webpackBuildDuration, turbotraceContext } =
turboNextBuild ? await turbopackBuild() : await webpackBuild()
const { duration: webpackBuildDuration, ...rest } = turboNextBuild
? await turbopackBuild()
: await webpackBuild()
buildTraceContext = rest.buildTraceContext
telemetry.record(
eventBuildCompleted(pagesPaths, {
......@@ -1029,101 +1027,6 @@ export default async function build(
totalAppPagesCount,
})
)
runTurbotrace = async function (staticPages: Set<string>) {
if (!turbotraceContext) {
return
}
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = binding.turbo.createTurboTasks(
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
1024
)
const { entriesTrace, chunksTrace } = turbotraceContext
if (entriesTrace) {
const {
appDir: turbotraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] =
await binding.turbo.startTrace(action, turboTasksForTrace)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(turbotraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from<[string, string]>(
entryNameMap.entries()
).filter(([k]) => k.startsWith(turbotraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action, outputPath } = chunksTrace
action.input = action.input.filter((f: any) => {
const outputPagesPath = path.join(outputPath, '..', 'pages')
return (
!f.startsWith(outputPagesPath) ||
!staticPages.has(
// strip `outputPagesPath` and file ext from absolute
f.substring(outputPagesPath.length, f.length - 3)
)
)
})
await binding.turbo.startTrace(action, turboTasksForTrace)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await fs
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await fs.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
}
// For app directory, we run type checking after build.
......@@ -1799,7 +1702,7 @@ export default async function build(
return returnValue
})
await runTurbotrace(staticPages)
if (postCompileSpinner) postCompileSpinner.stopAndPersist()
if (customAppGetInitialProps) {
console.warn(
......@@ -1844,458 +1747,24 @@ export default async function build(
)
}
const nextServerTraceOutput = path.join(
distDir,
'next-server.js.nft.json'
)
const nextMinimalTraceOutput = path.join(
distDir,
'next-minimal-server.js.nft.json'
)
let buildTracesPromise: Promise<any> | undefined = undefined
if (!isGenerate && config.outputFileTracing) {
let nodeFileTrace: any
if (config.experimental.turbotrace) {
if (!binding?.isWasm) {
nodeFileTrace = binding.turbo.startTrace
}
}
if (!nodeFileTrace) {
nodeFileTrace =
require('next/dist/compiled/@vercel/nft').nodeFileTrace
}
const includeExcludeSpan = nextBuildSpan.traceChild(
'apply-include-excludes'
)
const resolvedTraceIncludes = new Map<string, string[]>()
const {
outputFileTracingIncludes = {},
outputFileTracingExcludes = {},
} = config.experimental
const includeGlobKeys = Object.keys(outputFileTracingIncludes)
const excludeGlobKeys = Object.keys(outputFileTracingExcludes)
await includeExcludeSpan.traceAsyncFn(async () => {
const globOrig =
require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob')
const glob = (pattern: string): Promise<string[]> => {
return new Promise((resolve, reject) => {
globOrig(
pattern,
{ cwd: dir, nodir: true, dot: true },
(err, files) => {
if (err) {
return reject(err)
}
resolve(files)
}
)
})
}
if (config.outputFileTracing) {
for (let page of pageKeys.pages) {
// edge routes have no trace files
const pageInfo = pageInfos.get(page)
if (pageInfo?.runtime === 'edge') {
continue
}
const combinedIncludes = new Set<string>()
const combinedExcludes = new Set<string>()
page = normalizePagePath(page)
for (const curGlob of includeGlobKeys) {
if (isMatch(page, [curGlob])) {
outputFileTracingIncludes[curGlob].forEach((include) => {
combinedIncludes.add(include)
})
}
}
for (const curGlob of excludeGlobKeys) {
if (isMatch(page, [curGlob])) {
outputFileTracingExcludes[curGlob].forEach((exclude) => {
combinedExcludes.add(exclude)
})
}
}
if (!combinedIncludes?.size && !combinedExcludes?.size) {
continue
}
const traceFile = path.join(
distDir,
'server/pages',
`${page}.js.nft.json`
)
const pageDir = path.dirname(traceFile)
const traceContent = JSON.parse(
await fs.readFile(traceFile, 'utf8')
)
const includes: string[] = []
if (combinedIncludes?.size) {
await Promise.all(
[...combinedIncludes].map(async (includeGlob) => {
const results = await glob(includeGlob)
const resolvedInclude = resolvedTraceIncludes.get(
includeGlob
) || [
...results.map((file) => {
return path.relative(pageDir, path.join(dir, file))
}),
]
includes.push(...resolvedInclude)
resolvedTraceIncludes.set(includeGlob, resolvedInclude)
})
)
}
const combined = new Set([...traceContent.files, ...includes])
if (combinedExcludes?.size) {
const resolvedGlobs = [...combinedExcludes].map((exclude) =>
path.join(dir, exclude)
)
combined.forEach((file) => {
if (isMatch(path.join(pageDir, file), resolvedGlobs)) {
combined.delete(file)
}
})
}
await fs.writeFile(
traceFile,
JSON.stringify({
version: traceContent.version,
files: [...combined],
})
)
}
}
buildTracesPromise = collectBuildTraces({
dir,
config,
distDir,
pageKeys,
pageInfos,
staticPages,
nextBuildSpan,
hasSsrAmpPages,
buildTraceContext,
outputFileTracingRoot,
}).catch((err) => {
console.error(err)
process.exit(1)
})
await nextBuildSpan
.traceChild('trace-next-server')
.traceAsyncFn(async () => {
let cacheKey: string | undefined
// consider all lockFiles in tree in case user accidentally
// has both package-lock.json and yarn.lock
const lockFiles: string[] = (
await Promise.all(
['package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'].map(
(file) => findUp(file, { cwd: dir })
)
)
).filter(Boolean) as any // TypeScript doesn't like this filter
const cachedTracePath = path.join(
distDir,
'cache/next-server.js.nft.json'
)
const minimalCachedTracePath = path.join(
distDir,
'cache/next-minimal-server.js.nft.json'
)
if (
lockFiles.length > 0 &&
// we can't leverage trace cache if this is configured
// currently unless we break this to a separate trace
// file
!config.experimental.incrementalCacheHandlerPath
) {
const cacheHash = (
require('crypto') as typeof import('crypto')
).createHash('sha1')
cacheHash.update(require('next/package').version)
cacheHash.update(hasSsrAmpPages + '')
cacheHash.update(ciEnvironment.hasNextSupport + '')
await Promise.all(
lockFiles.map(async (lockFile) => {
cacheHash.update(await fs.readFile(lockFile))
})
)
cacheKey = cacheHash.digest('hex')
try {
const existingTrace = JSON.parse(
await fs.readFile(cachedTracePath, 'utf8')
)
const existingMinimalTrace = JSON.parse(
await fs.readFile(minimalCachedTracePath, 'utf8')
)
if (
existingTrace.cacheKey === cacheKey &&
existingMinimalTrace.cacheKey === cacheKey
) {
await fs.copyFile(cachedTracePath, nextServerTraceOutput)
await fs.copyFile(
minimalCachedTracePath,
nextMinimalTraceOutput
)
return
}
} catch {}
}
const root =
config.experimental?.turbotrace?.contextDirectory ??
outputFileTracingRoot
// Under standalone mode, we need to trace the extra IPC server and
// worker files.
const isStandalone = config.output === 'standalone'
const nextServerEntry = require.resolve(
'next/dist/server/next-server'
)
const sharedEntriesSet = [
...(config.experimental.turbotrace
? []
: Object.keys(defaultOverrides).map((value) =>
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)),
require.resolve(
'next/dist/compiled/next-server/app-page.runtime.prod'
),
require.resolve(
'next/dist/compiled/next-server/app-route.runtime.prod'
),
require.resolve(
'next/dist/compiled/next-server/pages.runtime.prod'
),
require.resolve(
'next/dist/compiled/next-server/pages-api.runtime.prod'
),
]
// ensure we trace any dependencies needed for custom
// incremental cache handler
if (incrementalCacheHandlerPath) {
sharedEntriesSet.push(
require.resolve(
path.isAbsolute(incrementalCacheHandlerPath)
? incrementalCacheHandlerPath
: path.join(dir, incrementalCacheHandlerPath)
)
)
}
const vanillaServerEntries = [
...sharedEntriesSet,
...(isStandalone
? [
require.resolve('next/dist/server/lib/start-server'),
require.resolve('next/dist/server/next'),
require.resolve('next/dist/server/require-hook'),
]
: []),
require.resolve('next/dist/server/next-server'),
].filter(Boolean) as string[]
const minimalServerEntries = [
...sharedEntriesSet,
require.resolve(
'next/dist/compiled/next-server/server.runtime.prod'
),
].filter(Boolean)
const additionalIgnores = new Set<string>()
for (const glob of excludeGlobKeys) {
if (isMatch('next-server', glob)) {
outputFileTracingExcludes[glob].forEach((exclude) => {
additionalIgnores.add(exclude)
})
}
}
const ignores = [
'**/*.d.ts',
'**/*.map',
'**/next/dist/pages/**/*',
isStandalone ? null : '**/next/dist/compiled/jest-worker/**/*',
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/squoosh/**/*.wasm',
'**/next/dist/server/lib/route-resolver*',
...(ciEnvironment.hasNextSupport
? [
// only ignore image-optimizer code when
// this is being handled outside of next-server
'**/next/dist/server/image-optimizer.js',
'**/node_modules/sharp/**/*',
]
: []),
...(!hasSsrAmpPages
? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
: []),
...additionalIgnores,
].filter(nonNullable)
const ignoreFn = (pathname: string) => {
if (path.isAbsolute(pathname) && !pathname.startsWith(root)) {
return true
}
return isMatch(pathname, ignores, {
contains: true,
dot: true,
})
}
const traceContext = path.join(nextServerEntry, '..', '..')
const tracedFiles = new Set<string>()
const minimalTracedFiles = new Set<string>()
function addToTracedFiles(
base: string,
file: string,
dest: Set<string>
) {
dest.add(
path
.relative(distDir, path.join(base, file))
.replace(/\\/g, '/')
)
}
if (isStandalone) {
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/processChild'),
tracedFiles
)
addToTracedFiles(
'',
require.resolve('next/dist/compiled/jest-worker/threadChild'),
tracedFiles
)
}
if (config.experimental.turbotrace) {
const makeTrace = async (entries: string[]) =>
nodeFileTrace(
{
action: 'print',
input: entries,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace?.logLevel,
processCwd: config.experimental.turbotrace?.processCwd,
logDetail: config.experimental.turbotrace?.logDetail,
showAll: config.experimental.turbotrace?.logAll,
},
turboTasksForTrace
)
// turbotrace does not handle concurrent tracing
const vanillaFiles = await makeTrace(vanillaServerEntries)
const minimalFiles = await makeTrace(minimalServerEntries)
for (const [set, files] of [
[tracedFiles, vanillaFiles],
[minimalTracedFiles, minimalFiles],
] as [Set<string>, string[]][]) {
for (const file of files) {
if (!ignoreFn(path.join(traceContext, file))) {
addToTracedFiles(traceContext, file, set)
}
}
}
} else {
const makeTrace = async (entries: string[]) =>
nodeFileTrace(entries, {
base: root,
processCwd: dir,
ignore: ignoreFn,
})
const [vanillaFiles, minimalFiles]: NodeFileTraceResult[] =
await Promise.all([
makeTrace(vanillaServerEntries),
makeTrace(minimalServerEntries),
])
for (const [set, traceResult] of [
[tracedFiles, vanillaFiles],
[minimalTracedFiles, minimalFiles],
] as [Set<string>, NodeFileTraceResult][]) {
for (const file of traceResult.fileList) {
addToTracedFiles(root, file, set)
}
}
}
const moduleTypes = ['app-page', 'pages']
for (const type of moduleTypes) {
const modulePath = require.resolve(
`next/dist/server/future/route-modules/${type}/module.compiled`
)
const relativeModulePath = path.relative(root, modulePath)
const contextDir = path.join(
path.dirname(modulePath),
'vendored',
'contexts'
)
for (const item of await fs.readdir(contextDir)) {
const itemPath = path.relative(
root,
path.join(contextDir, item)
)
addToTracedFiles(root, itemPath, tracedFiles)
addToTracedFiles(root, itemPath, minimalTracedFiles)
}
addToTracedFiles(root, relativeModulePath, tracedFiles)
addToTracedFiles(root, relativeModulePath, minimalTracedFiles)
}
await Promise.all([
fs.writeFile(
nextServerTraceOutput,
JSON.stringify({
version: 1,
cacheKey,
files: Array.from(tracedFiles),
} as {
version: number
files: string[]
})
),
fs.writeFile(
nextMinimalTraceOutput,
JSON.stringify({
version: 1,
cacheKey,
files: Array.from(minimalTracedFiles),
} as {
version: number
files: string[]
})
),
])
await fs.unlink(cachedTracePath).catch(() => {})
await fs.unlink(minimalCachedTracePath).catch(() => {})
await fs
.copyFile(nextServerTraceOutput, cachedTracePath)
.catch(() => {})
await fs
.copyFile(nextMinimalTraceOutput, minimalCachedTracePath)
.catch(() => {})
})
}
if (serverPropsPages.size > 0 || ssgPages.size > 0) {
......@@ -2396,30 +1865,11 @@ export default async function build(
)
)
if (config.output === 'standalone') {
await nextBuildSpan
.traceChild('copy-traced-files')
.traceAsyncFn(async () => {
await copyTracedFiles(
dir,
distDir,
pageKeys.pages,
denormalizedAppPages,
outputFileTracingRoot,
requiredServerFiles.config,
middlewareManifest,
hasInstrumentationHook
)
})
}
const finalPrerenderRoutes: { [route: string]: SsgRoute } = {}
const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {}
const tbdPrerenderRoutes: string[] = []
let ssgNotFoundPaths: string[] = []
if (postCompileSpinner) postCompileSpinner.stopAndPersist()
const { i18n } = config
const usedStaticStatusPages = STATIC_STATUS_PAGES.filter(
......@@ -3064,6 +2514,8 @@ export default async function build(
})
}
let buildTracesSpinner = createSpinner(`Collecting build traces`)
// ensure the worker is not left hanging
pagesStaticWorkers.close()
appStaticWorkers?.close()
......@@ -3222,57 +2674,6 @@ export default async function build(
return Promise.reject(err)
})
if (config.output === 'standalone') {
for (const file of [
...requiredServerFiles.files,
path.join(config.distDir, SERVER_FILES_MANIFEST),
...loadedEnvFiles.reduce<string[]>((acc, envFile) => {
if (['.env', '.env.production'].includes(envFile.path)) {
acc.push(envFile.path)
}
return acc
}, []),
]) {
const filePath = path.join(dir, file)
const outputPath = path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, filePath)
)
await fs.mkdir(path.dirname(outputPath), {
recursive: true,
})
await fs.copyFile(filePath, outputPath)
}
await recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'
),
{ overwrite: true }
)
if (appDir) {
const originalServerApp = path.join(distDir, SERVER_DIRECTORY, 'app')
if (fsExistsSync(originalServerApp)) {
await recursiveCopy(
originalServerApp,
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'app'
),
{ overwrite: true }
)
}
}
}
await nextBuildSpan.traceChild('print-tree-view').traceAsyncFn(() =>
printTreeView(pageKeys, pageInfos, {
distPath: distDir,
......@@ -3314,6 +2715,11 @@ export default async function build(
}
if (config.output === 'export') {
if (buildTracesSpinner) {
buildTracesSpinner?.stop()
buildTracesSpinner = undefined
}
const exportApp: typeof import('../export').default =
require('../export').default
......@@ -3348,11 +2754,91 @@ export default async function build(
pagesWorker.close()
appWorker.close()
}
await buildTracesPromise
if (config.output === 'standalone') {
await nextBuildSpan
.traceChild('copy-traced-files')
.traceAsyncFn(async () => {
await copyTracedFiles(
dir,
distDir,
pageKeys.pages,
denormalizedAppPages,
outputFileTracingRoot,
requiredServerFiles.config,
middlewareManifest,
hasInstrumentationHook,
staticPages
)
if (config.output === 'standalone') {
for (const file of [
...requiredServerFiles.files,
path.join(config.distDir, SERVER_FILES_MANIFEST),
...loadedEnvFiles.reduce<string[]>((acc, envFile) => {
if (['.env', '.env.production'].includes(envFile.path)) {
acc.push(envFile.path)
}
return acc
}, []),
]) {
const filePath = path.join(dir, file)
const outputPath = path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, filePath)
)
await fs.mkdir(path.dirname(outputPath), {
recursive: true,
})
await fs.copyFile(filePath, outputPath)
}
await recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'
),
{ overwrite: true }
)
if (appDir) {
const originalServerApp = path.join(
distDir,
SERVER_DIRECTORY,
'app'
)
if (fsExistsSync(originalServerApp)) {
await recursiveCopy(
originalServerApp,
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'app'
),
{ overwrite: true }
)
}
}
}
})
}
if (buildTracesSpinner) {
buildTracesSpinner.stopAndPersist()
buildTracesSpinner = undefined
}
await nextBuildSpan
.traceChild('telemetry-flush')
.traceAsyncFn(() => telemetry.flush())
})
return buildResult
} finally {
// Ensure we wait for lockfile patching if present
......
......@@ -1796,7 +1796,8 @@ export async function copyTracedFiles(
tracingRoot: string,
serverConfig: NextConfig,
middlewareManifest: MiddlewareManifest,
hasInstrumentationHook: boolean
hasInstrumentationHook: boolean,
staticPages: Set<string>
) {
const outputPath = path.join(distDir, 'standalone')
let moduleType = false
......@@ -1889,6 +1890,12 @@ export async function copyTracedFiles(
if (middlewareManifest.functions.hasOwnProperty(page)) {
continue
}
const route = normalizePagePath(page)
if (staticPages.has(route)) {
continue
}
const pageFile = path.join(
distDir,
'server',
......@@ -1897,7 +1904,9 @@ export async function copyTracedFiles(
)
const pageTraceFile = `${pageFile}.nft.json`
await handleTraceFiles(pageTraceFile).catch((err) => {
Log.warn(`Failed to copy traced files for ${pageFile}`, err)
if (err.code !== 'ENOENT' || (page !== '/404' && page !== '/500')) {
Log.warn(`Failed to copy traced files for ${pageFile}`, err)
}
})
}
......
......@@ -24,8 +24,8 @@ import loadConfig from '../../server/config'
import { trace } from '../../trace'
import { WEBPACK_LAYERS } from '../../lib/constants'
import {
BuildTraceContext,
TraceEntryPointsPlugin,
TurbotraceContext,
} from '../webpack/plugins/next-trace-entrypoints-plugin'
import { UnwrapPromise } from '../../lib/coalesced-function'
import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'
......@@ -61,7 +61,7 @@ export async function webpackBuildImpl(
): Promise<{
duration: number
pluginState: any
turbotraceContext?: TurbotraceContext
buildTraceContext?: BuildTraceContext
serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries']
}> {
let result: CompilerResult | null = {
......@@ -183,18 +183,22 @@ export async function webpackBuildImpl(
let inputFileSystem: any
if (!compilerName || compilerName === 'server') {
debug('starting server compiler')
const start = Date.now()
;[serverResult, inputFileSystem] = await runCompiler(serverConfig, {
runWebpackSpan,
inputFileSystem,
})
debug('server result', serverResult)
debug(`server compiler finished ${Date.now() - start}ms`)
}
if (!compilerName || compilerName === 'edge-server') {
debug('starting edge-server compiler')
const start = Date.now()
;[edgeServerResult, inputFileSystem] = edgeConfig
? await runCompiler(edgeConfig, { runWebpackSpan, inputFileSystem })
: [null]
debug('edge server result', edgeServerResult)
debug(`edge-server compiler finished ${Date.now() - start}ms`)
}
// Only continue if there were no errors
......@@ -223,11 +227,13 @@ export async function webpackBuildImpl(
}
if (!compilerName || compilerName === 'client') {
debug('starting client compiler')
const start = Date.now()
;[clientResult, inputFileSystem] = await runCompiler(clientConfig, {
runWebpackSpan,
inputFileSystem,
})
debug('client result', clientResult)
debug(`client compiler finished ${Date.now() - start}ms`)
}
}
......@@ -319,7 +325,7 @@ export async function webpackBuildImpl(
return {
duration: webpackBuildEnd[0],
turbotraceContext: traceEntryPointsPlugin?.turbotraceContext,
buildTraceContext: traceEntryPointsPlugin?.buildTraceContext,
pluginState: getPluginState(),
serializedPagesManifestEntries: {
edgeServerPages: pagesPluginModule.edgeServerPages,
......@@ -360,17 +366,25 @@ export async function workerMain(workerData: {
NextBuildContext.nextBuildSpan = trace('next-build')
const result = await webpackBuildImpl(workerData.compilerName)
const { entriesTrace } = result.turbotraceContext ?? {}
const { entriesTrace, chunksTrace } = result.buildTraceContext ?? {}
if (entriesTrace) {
const { entryNameMap, depModArray } = entriesTrace
if (depModArray) {
result.turbotraceContext!.entriesTrace!.depModArray = depModArray
result.buildTraceContext!.entriesTrace!.depModArray = depModArray
}
if (entryNameMap) {
const entryEntries = Array.from(entryNameMap?.entries() ?? [])
// @ts-expect-error
result.turbotraceContext.entriesTrace.entryNameMap = entryEntries
result.buildTraceContext.entriesTrace.entryNameMap = entryEntries
}
}
if (chunksTrace?.entryNameFilesMap) {
const entryNameFilesMap = Array.from(
chunksTrace.entryNameFilesMap.entries() ?? []
)
// @ts-expect-error
result.buildTraceContext!.chunksTrace!.entryNameFilesMap = entryNameFilesMap
}
return result
}
import { COMPILER_INDEXES } from '../../shared/lib/constants'
import * as Log from '../output/log'
import { NextBuildContext } from '../build-context'
import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { Worker } from 'next/dist/compiled/jest-worker'
import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process'
......@@ -50,7 +50,7 @@ async function webpackBuildWithWorker() {
const combinedResult = {
duration: 0,
turbotraceContext: {} as TurbotraceContext,
buildTraceContext: {} as BuildTraceContext,
}
// order matters here
const ORDERED_COMPILER_NAMES = [
......@@ -85,15 +85,28 @@ async function webpackBuildWithWorker() {
combinedResult.duration += curResult.duration
if (curResult.turbotraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext
if (curResult.buildTraceContext?.entriesTrace) {
const { entryNameMap } = curResult.buildTraceContext.entriesTrace!
const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
combinedResult.buildTraceContext.entriesTrace =
curResult.buildTraceContext.entriesTrace
combinedResult.buildTraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap
)
}
if (curResult.buildTraceContext?.chunksTrace) {
const { entryNameFilesMap } = curResult.buildTraceContext.chunksTrace!
if (entryNameFilesMap) {
combinedResult.buildTraceContext.chunksTrace =
curResult.buildTraceContext.chunksTrace!
combinedResult.buildTraceContext.chunksTrace!.entryNameFilesMap =
new Map(entryNameFilesMap)
}
}
}
}
buildSpinner?.stopAndPersist()
......
......@@ -23,7 +23,7 @@ import { getModuleBuildInfo } from '../loaders/get-module-build-info'
import { getPageFilePath } from '../../entries'
const PLUGIN_NAME = 'TraceEntryPointsPlugin'
const TRACE_IGNORES = [
export const TRACE_IGNORES = [
'**/*/next/dist/server/next.js',
'**/*/next/dist/bin/next',
]
......@@ -48,7 +48,7 @@ function getModuleFromDependency(
return compilation.moduleGraph.getModule(dep)
}
function getFilesMapFromReasons(
export function getFilesMapFromReasons(
fileList: Set<string>,
reasons: NodeFileTraceReasons,
ignoreFn?: (file: string, parent?: string) => Boolean
......@@ -114,7 +114,7 @@ export interface TurbotraceAction {
memoryLimit?: number
}
export interface TurbotraceContext {
export interface BuildTraceContext {
entriesTrace?: {
action: TurbotraceAction
appDir: string
......@@ -125,11 +125,12 @@ export interface TurbotraceContext {
chunksTrace?: {
action: TurbotraceAction
outputPath: string
entryNameFilesMap: Map<string, Array<string>>
}
}
export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
public turbotraceContext: TurbotraceContext = {}
public buildTraceContext: BuildTraceContext = {}
private rootDir: string
private appDir: string | undefined
......@@ -140,7 +141,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
private traceIgnores: string[]
private esmExternals?: NextConfigComplete['experimental']['esmExternals']
private turbotrace?: NextConfigComplete['experimental']['turbotrace']
private chunksToTrace: string[] = []
constructor({
rootDir,
......@@ -174,18 +174,13 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
// Here we output all traced assets and webpack chunks to a
// ${page}.js.nft.json file
async createTraceAssets(
compilation: any,
assets: any,
span: Span,
readlink: any,
stat: any
) {
async createTraceAssets(compilation: any, assets: any, span: Span) {
const outputPath = compilation.outputOptions.path
await span.traceChild('create-trace-assets').traceAsyncFn(async () => {
const entryFilesMap = new Map<any, Set<string>>()
const chunksToTrace = new Set<string>()
const entryNameFilesMap = new Map<string, Array<string>>()
const isTraceable = (file: string) =>
!NOT_TRACEABLE.some((suffix) => {
......@@ -214,77 +209,30 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
}
}
entryFilesMap.set(entrypoint, entryFiles)
entryNameFilesMap.set(entrypoint.name, [...entryFiles])
}
// startTrace existed and callable
if (this.turbotrace) {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
this.chunksToTrace = [...chunksToTrace]
return
}
}
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
const ignoreFn = (path: string) => {
return isMatch(path, ignores, { contains: true, dot: true })
}
const result = await nodeFileTrace([...chunksToTrace], {
base: this.tracingRoot,
processCwd: this.rootDir,
readFile: async (path) => {
if (chunksToTrace.has(path)) {
const source =
assets[
nodePath.relative(outputPath, path).replace(/\\/g, '/')
]?.source?.()
if (source) return source
}
try {
return await new Promise((resolve, reject) => {
;(
compilation.inputFileSystem
.readFile as typeof import('fs').readFile
)(path, (err, data) => {
if (err) return reject(err)
resolve(data)
})
})
} catch (e) {
if (isError(e) && (e.code === 'ENOENT' || e.code === 'EISDIR')) {
return null
}
throw e
}
this.buildTraceContext.chunksTrace = {
action: {
action: 'annotate',
input: [...chunksToTrace],
contextDirectory:
this.turbotrace?.contextDirectory ?? this.tracingRoot,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
},
readlink,
stat,
ignore: ignoreFn,
mixedModules: true,
})
const reasons = result.reasons
const fileList = result.fileList
result.esmFileList.forEach((file) => fileList.add(file))
const parentFilesMap = getFilesMapFromReasons(fileList, reasons)
outputPath,
entryNameFilesMap,
}
for (const [entrypoint, entryFiles] of entryFilesMap) {
const traceOutputName = `../${entrypoint.name}.js.nft.json`
const traceOutputPath = nodePath.dirname(
nodePath.join(outputPath, traceOutputName)
)
const allEntryFiles = new Set<string>()
entryFiles.forEach((file) => {
parentFilesMap
.get(nodePath.relative(this.tracingRoot, file))
?.forEach((child) => {
allEntryFiles.add(nodePath.join(this.tracingRoot, child))
})
})
// don't include the entry itself in the trace
entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`))
......@@ -313,7 +261,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
for (const file of new Set([
...entryFiles,
...allEntryFiles,
...(this.entryTraces.get(entrypoint.name) || []),
])) {
if (file) {
......@@ -473,34 +420,39 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
entriesToTrace.push(...curExtraEntries.keys())
}
})
// startTrace existed and callable
const contextDirectory =
this.turbotrace?.contextDirectory ?? this.tracingRoot
const chunks = [...entriesToTrace]
this.buildTraceContext.entriesTrace = {
action: {
action: 'print',
input: chunks,
contextDirectory,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
logLevel: this.turbotrace?.logLevel,
showAll: this.turbotrace?.logAll,
},
appDir: this.rootDir,
depModArray: Array.from(depModMap.keys()),
entryNameMap,
outputPath: compilation.outputOptions.path!,
}
// if we're using turbotrace we can skip tracing
// loader contents as it should be able to capture
// fs usage in final chunks instead
if (this.turbotrace) {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
const contextDirectory =
this.turbotrace?.contextDirectory ?? this.tracingRoot
const chunks = [...entriesToTrace]
this.turbotraceContext.entriesTrace = {
action: {
action: 'print',
input: chunks,
contextDirectory,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
logLevel: this.turbotrace?.logLevel,
showAll: this.turbotrace?.logAll,
},
appDir: this.rootDir,
depModArray: Array.from(depModMap.keys()),
entryNameMap,
outputPath: compilation.outputOptions.path!,
}
return
}
}
let fileList: Set<string>
let reasons: NodeFileTraceReasons
const ignores = [
......@@ -513,7 +465,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
}
await finishModulesSpan
.traceChild('node-file-trace', {
.traceChild('node-file-trace-plugin', {
traceEntryCount: entriesToTrace.length + '',
})
.traceAsyncFn(async () => {
......@@ -656,9 +608,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
this.createTraceAssets(
compilation,
assets,
traceEntrypointsPluginSpan,
readlink,
stat
traceEntrypointsPluginSpan
)
.then(() => callback())
.catch((err) => callback(err))
......@@ -822,35 +772,5 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
)
})
})
if (this.turbotrace) {
compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async () => {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
const ignoreFn = (path: string) => {
return isMatch(path, ignores, { contains: true, dot: true })
}
const chunks = this.chunksToTrace.filter((chunk) => !ignoreFn(chunk))
this.turbotraceContext.chunksTrace = {
action: {
action: 'annotate',
input: chunks,
contextDirectory:
this.turbotrace?.contextDirectory ?? this.tracingRoot,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
},
outputPath: compiler.outputPath,
}
}
})
}
}
}
import React from 'react'
import { World } from 'components/world'
// prevent static generation for trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() {
return (
<div>
......
import React from 'react'
import { World } from '@c/world'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function HelloPage() {
return (
<div>
......
import React from 'react'
import api from '@lib/b-only'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() {
return <div>{api()}</div>
}
import React from 'react'
import api from '@lib/api'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function ResolveOrder() {
return <div>{api()}</div>
}
import { Hello } from '@mycomponent'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function SingleAlias() {
return (
<div>
......
import myfile from 'mypackage/myfile'
// prevent static generation
export function getServerSideProps() {
return {
props: {},
}
}
export default function WildcardAlias() {
return <div>{myfile.hello}</div>
}
import Link from 'next/link'
import Error from 'next/error'
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page(props) {
return (
<>
......
......@@ -8,9 +8,18 @@ if (typeof window === 'undefined') {
import('nanoid').then((mod) => console.log(mod.nanoid()))
}
export default () => (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page() {
return (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
}
......@@ -4,9 +4,18 @@ import Link from 'next/link'
// eslint-disable-next-line no-unused-vars
import next from 'next'
export default () => (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
// prevent static generation for build trace test
export function getServerSideProps() {
return {
props: {},
}
}
export default function Page() {
return (
<div>
<Link href="/about">About Page</Link>
<p className="index-page">Hello World</p>
</div>
)
}
......@@ -106,7 +106,7 @@ createNextDescribe(
})
it('should respond with 405 for POST to static page', async () => {
const res = await fetchViaHTTP(next.appPort, '/', undefined, {
const res = await fetchViaHTTP(next.appPort, '/about', undefined, {
method: 'POST',
})
expect(res.status).toBe(405)
......@@ -114,7 +114,7 @@ createNextDescribe(
})
it('should contain generated page count in output', async () => {
const pageCount = 40
const pageCount = 37
expect(next.cliOutput).toContain(
`Generating static pages (0/${pageCount})`
)
......@@ -204,18 +204,6 @@ createNextDescribe(
],
notTests: [/\0/, /\?/, /!/],
},
{
page: '/dynamic',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{
page: '/index',
tests: [
......@@ -231,19 +219,6 @@ createNextDescribe(
],
notTests: [/next\/dist\/pages\/_error\.js/, /\0/, /\?/, /!/],
},
{
page: '/counter',
tests: [
/webpack-runtime\.js/,
/chunks\/.*?\.js/,
/node_modules\/react\/index\.js/,
/node_modules\/react\/package\.json/,
/node_modules\/react\/cjs\/react\.production\.min\.js/,
/node_modules\/react\/cjs\/react\.development\.js/,
/node_modules\/next/,
],
notTests: [/\0/, /\?/, /!/],
},
{
page: '/next-import',
tests: [
......@@ -298,6 +273,7 @@ createNextDescribe(
]
for (const check of checks) {
require('console').log('checking', check.page)
const { version, files } = await next.readJSON(
join('.next/server/pages/', check.page + '.js.nft.json')
)
......@@ -940,7 +916,7 @@ createNextDescribe(
})
it('should add autoExport for auto pre-rendered pages', async () => {
for (const page of ['/', '/about']) {
for (const page of ['/about']) {
const html = await renderViaHTTP(next.appPort, page)
const $ = cheerio.load(html)
const data = JSON.parse($('#__NEXT_DATA__').html())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment