Unverified Commit ee60948e authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

remove turbotrace from webpack codepath (#71359)

Closes PACK-3356

### What?

We previously supported a flag `experimental.turbotrace` which would opt in to using turbo-tasks powered machinery for performing the tasks that were previously handled by node-file-trace. There was a bifurcation where both turbopack and webpack would use either node-file-trace or the new turbotrace which lead to untested code paths. This never really gained traction on the webpack side and, with the decision to move turbotrace and nft generation directly into the critical path of a turbo build (as opposed to a post-processing step) is obsolete for turbopack. We rip it out.

### Why?

See above

### How?

Remove the flag, and the code path branches that refer to it. I opted to keep some formatting to preserve the diff but may refactor the now-rustless code later as there is still some control flow that is unnecessary.
parent 482f4ad2
......@@ -815,7 +815,7 @@ impl Project {
emit_event("modularizeImports", config.modularize_imports.is_some());
emit_event("transpilePackages", config.transpile_packages.is_some());
emit_event("turbotrace", config.experimental.turbotrace.is_some());
emit_event("turbotrace", false);
// compiler options
let compiler_options = config.compiler.as_ref();
......
......@@ -501,7 +501,6 @@ pub struct ExperimentalConfig {
pub strict_next_head: Option<bool>,
pub swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
pub turbo: Option<ExperimentalTurboConfig>,
pub turbotrace: Option<serde_json::Value>,
pub external_middleware_rewrites_resolve: Option<bool>,
pub scroll_restoration: Option<bool>,
pub use_deployment_id: Option<bool>,
......
......@@ -8,14 +8,8 @@ import {
getHash,
} from './webpack/plugins/next-trace-entrypoints-plugin'
import {
TRACE_OUTPUT_VERSION,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
} from '../shared/lib/constants'
import path from 'path'
import fs from 'fs/promises'
import { loadBindings } from './swc'
import { nonNullable } from '../lib/non-nullable'
import * as ciEnvironment from '../server/ci-info'
import debugOriginal from 'next/dist/compiled/debug'
......@@ -27,7 +21,6 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
import isError from '../lib/is-error'
import type { NodeFileTraceReasons } from '@vercel/nft'
import type { RoutesUsingEdgeRuntime } from './utils'
import type { ExternalObject, NextTurboTasks } from './swc/generated-native'
const debug = debugOriginal('next:build:build-traces')
......@@ -108,104 +101,6 @@ export async function collectBuildTraces({
}) {
const startTime = Date.now()
debug('starting build traces')
let turboTasksForTrace: ExternalObject<NextTurboTasks>
let bindings = await loadBindings()
const runTurbotrace = async function () {
if (!config.experimental.turbotrace || !buildTraceContext) {
return
}
if (!bindings?.isWasm && typeof bindings.turbo.startTrace === 'function') {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = bindings.turbo.createTurboTasks(
distDir,
false,
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
1024
)
const { entriesTrace, chunksTrace } = buildTraceContext
if (entriesTrace) {
const {
appDir: buildTraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] = await bindings.turbo.startTrace(
action,
turboTasksForTrace
)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(buildTraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from<[string, string]>(
Object.entries(entryNameMap)
).filter(([k]) => k.startsWith(buildTraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action, outputPath } = chunksTrace
action.input = action.input.filter((f: any) => {
const outputPagesPath = path.join(outputPath, '..', 'pages')
return (
!f.startsWith(outputPagesPath) ||
!staticPages.includes(
// strip `outputPagesPath` and file ext from absolute
f.substring(outputPagesPath.length, f.length - 3)
)
)
})
await bindings.turbo.startTrace(action, turboTasksForTrace)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await fs
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await fs.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
const { outputFileTracingIncludes = {}, outputFileTracingExcludes = {} } =
config
......@@ -214,7 +109,7 @@ export async function collectBuildTraces({
await nextBuildSpan
.traceChild('node-file-trace-build', {
isTurbotrace: Boolean(config.experimental.turbotrace) ? 'true' : 'false',
isTurbotrace: 'false', // TODO(arlyon): remove this
})
.traceAsyncFn(async () => {
const nextServerTraceOutput = path.join(
......@@ -225,23 +120,16 @@ export async function collectBuildTraces({
distDir,
'next-minimal-server.js.nft.json'
)
const root =
config.experimental?.turbotrace?.contextDirectory ??
outputFileTracingRoot
const root = outputFileTracingRoot
// Under standalone mode, we need to trace the extra IPC server and
// worker files.
const isStandalone = config.output === 'standalone'
const nextServerEntry = require.resolve('next/dist/server/next-server')
const sharedEntriesSet = [
...(config.experimental.turbotrace
? []
: Object.keys(defaultOverrides).map((value) =>
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)),
]
const sharedEntriesSet = Object.keys(defaultOverrides).map((value) =>
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)
const { cacheHandler } = config
const { cacheHandlers } = config.experimental
......@@ -373,7 +261,6 @@ export async function collectBuildTraces({
const routeIgnoreFn = makeIgnoreFn(routesIgnores)
const traceContext = path.join(nextServerEntry, '..', '..')
const serverTracedFiles = new Set<string>()
const minimalServerTracedFiles = new Set<string>()
......@@ -396,45 +283,7 @@ export async function collectBuildTraces({
)
}
if (config.experimental.turbotrace) {
await runTurbotrace()
const startTrace = bindings.turbo.startTrace
const makeTrace = async (entries: string[]) =>
startTrace(
{
action: 'print',
input: entries,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace?.logLevel,
processCwd: config.experimental.turbotrace?.processCwd,
logDetail: config.experimental.turbotrace?.logDetail,
showAll: config.experimental.turbotrace?.logAll,
},
turboTasksForTrace
)
// turbotrace does not handle concurrent tracing
const vanillaFiles = await makeTrace(serverEntries)
const minimalFiles = await makeTrace(minimalServerEntries)
for (const [set, files] of [
[serverTracedFiles, vanillaFiles],
[minimalServerTracedFiles, minimalFiles],
] as [Set<string>, string[]][]) {
for (const file of files) {
if (
!(
set === minimalServerTracedFiles
? minimalServerIgnoreFn
: serverIgnoreFn
)(path.join(traceContext, file))
) {
addToTracedFiles(traceContext, file, set)
}
}
}
} else {
{
const chunksToTrace: string[] = [
...(buildTraceContext?.chunksTrace?.action.input || []),
...serverEntries,
......
......@@ -1809,7 +1809,6 @@ export default async function getBaseWebpackConfig(
esmExternals: config.experimental.esmExternals,
outputFileTracingRoot: config.outputFileTracingRoot,
appDirEnabled: hasAppDir,
turbotrace: config.experimental.turbotrace,
optOutBundlingPackages,
traceIgnores: [],
flyingShuttle: Boolean(flyingShuttle),
......@@ -1950,7 +1949,6 @@ export default async function getBaseWebpackConfig(
['swcRemoveConsole', !!config.compiler?.removeConsole],
['swcImportSource', !!jsConfig?.compilerOptions?.jsxImportSource],
['swcEmotion', !!config.compiler?.emotion],
['turbotrace', !!config.experimental.turbotrace],
['transpilePackages', !!config.transpilePackages],
[
'skipMiddlewareUrlNormalize',
......
......@@ -17,7 +17,6 @@ import {
NODE_RESOLVE_OPTIONS,
} from '../../webpack-config'
import type { NextConfigComplete } from '../../../server/config-shared'
import { loadBindings } from '../../swc'
import picomatch from 'next/dist/compiled/picomatch'
import { getModuleBuildInfo } from '../loaders/get-module-build-info'
import { getPageFilePath } from '../../entries'
......@@ -110,9 +109,6 @@ export interface TurbotraceAction {
input: string[]
contextDirectory: string
processCwd: string
logLevel?: NonNullable<
NextConfigComplete['experimental']['turbotrace']
>['logLevel']
showAll?: boolean
memoryLimit?: number
}
......@@ -148,7 +144,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
private entryTraces: Map<string, Map<string, { bundled: boolean }>>
private traceIgnores: string[]
private esmExternals?: NextConfigComplete['experimental']['esmExternals']
private turbotrace?: NextConfigComplete['experimental']['turbotrace']
private traceHashes: Map<string, string>
private flyingShuttle?: boolean
private compilerType: CompilerNameValues
......@@ -167,7 +162,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
traceIgnores,
esmExternals,
outputFileTracingRoot,
turbotrace,
flyingShuttle,
swcLoaderConfig,
}: {
......@@ -181,7 +175,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
traceIgnores?: string[]
outputFileTracingRoot?: string
esmExternals?: NextConfigComplete['experimental']['esmExternals']
turbotrace?: NextConfigComplete['experimental']['turbotrace']
swcLoaderConfig: TraceEntryPointsPlugin['swcLoaderConfig']
}) {
this.rootDir = rootDir
......@@ -192,7 +185,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
this.appDirEnabled = appDirEnabled
this.traceIgnores = traceIgnores || []
this.tracingRoot = outputFileTracingRoot || rootDir
this.turbotrace = turbotrace
this.optOutBundlingPackages = optOutBundlingPackages
this.flyingShuttle = flyingShuttle
this.traceHashes = new Map()
......@@ -249,11 +241,8 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
action: {
action: 'annotate',
input: [...chunksToTrace],
contextDirectory:
this.turbotrace?.contextDirectory ?? this.tracingRoot,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
contextDirectory: this.tracingRoot,
processCwd: this.rootDir,
},
outputPath,
entryNameFilesMap: Object.fromEntries(entryNameFilesMap),
......@@ -624,8 +613,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
}
}
const contextDirectory =
this.turbotrace?.contextDirectory ?? this.tracingRoot
const contextDirectory = this.tracingRoot
const chunks = [...entriesToTrace]
this.buildTraceContext.entriesTrace = {
......@@ -633,9 +621,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
action: 'print',
input: chunks,
contextDirectory,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
logLevel: this.turbotrace?.logLevel,
showAll: this.turbotrace?.logAll,
processCwd: this.rootDir,
},
appDir: this.rootDir,
depModArray: Array.from(depModMap.keys()),
......@@ -643,19 +629,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
outputPath: compilation.outputOptions.path!,
}
// if we're using turbotrace we can skip tracing
// loader contents as it should be able to capture
// fs usage in final chunks instead
if (this.turbotrace) {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
return
}
}
let fileList: Set<string>
let reasons: NodeFileTraceReasons
const ignores = [
......
......@@ -415,27 +415,6 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
optimizePackageImports: z.array(z.string()).optional(),
optimizeServerReact: z.boolean().optional(),
clientTraceMetadata: z.array(z.string()).optional(),
turbotrace: z
.object({
logLevel: z
.enum([
'bug',
'fatal',
'error',
'warning',
'hint',
'note',
'suggestions',
'info',
])
.optional(),
logAll: z.boolean().optional(),
logDetail: z.boolean().optional(),
contextDirectory: z.string().optional(),
processCwd: z.string().optional(),
memoryLimit: z.number().int().optional(),
})
.optional(),
serverMinification: z.boolean().optional(),
serverSourceMaps: z.boolean().optional(),
useWasmBinary: z.boolean().optional(),
......
......@@ -355,23 +355,6 @@ export interface ExperimentalConfig {
optimizeServerReact?: boolean
turbo?: ExperimentalTurboOptions
turbotrace?: {
logLevel?:
| 'bug'
| 'fatal'
| 'error'
| 'warning'
| 'hint'
| 'note'
| 'suggestions'
| 'info'
logDetail?: boolean
logAll?: boolean
contextDirectory?: string
processCwd?: string
/** in `MB` */
memoryLimit?: number
}
/**
* For use with `@next/mdx`. Compile MDX files using the new Rust compiler.
......@@ -1138,7 +1121,6 @@ export const defaultConfig: NextConfig = {
amp: undefined,
urlImports: undefined,
turbo: undefined,
turbotrace: undefined,
typedRoutes: false,
typedEnv: false,
clientTraceMetadata: undefined,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment