Unverified Commit ee60948e authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

remove turbotrace from webpack codepath (#71359)

Closes PACK-3356

### What?

We previously supported a flag `experimental.turbotrace` which would opt in to using turbo-tasks powered machinery for performing the tasks that were previously handled by node-file-trace. There was a bifurcation where both turbopack and webpack would use either node-file-trace or the new turbotrace which lead to untested code paths. This never really gained traction on the webpack side and, with the decision to move turbotrace and nft generation directly into the critical path of a turbo build (as opposed to a post-processing step) is obsolete for turbopack. We rip it out.

### Why?

See above

### How?

Remove the flag, and the code path branches that refer to it. I opted to keep some formatting to preserve the diff but may refactor the now-rustless code later as there is still some control flow that is unnecessary.
parent 482f4ad2
...@@ -815,7 +815,7 @@ impl Project { ...@@ -815,7 +815,7 @@ impl Project {
emit_event("modularizeImports", config.modularize_imports.is_some()); emit_event("modularizeImports", config.modularize_imports.is_some());
emit_event("transpilePackages", config.transpile_packages.is_some()); emit_event("transpilePackages", config.transpile_packages.is_some());
emit_event("turbotrace", config.experimental.turbotrace.is_some()); emit_event("turbotrace", false);
// compiler options // compiler options
let compiler_options = config.compiler.as_ref(); let compiler_options = config.compiler.as_ref();
......
...@@ -501,7 +501,6 @@ pub struct ExperimentalConfig { ...@@ -501,7 +501,6 @@ pub struct ExperimentalConfig {
pub strict_next_head: Option<bool>, pub strict_next_head: Option<bool>,
pub swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>, pub swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
pub turbo: Option<ExperimentalTurboConfig>, pub turbo: Option<ExperimentalTurboConfig>,
pub turbotrace: Option<serde_json::Value>,
pub external_middleware_rewrites_resolve: Option<bool>, pub external_middleware_rewrites_resolve: Option<bool>,
pub scroll_restoration: Option<bool>, pub scroll_restoration: Option<bool>,
pub use_deployment_id: Option<bool>, pub use_deployment_id: Option<bool>,
......
...@@ -8,14 +8,8 @@ import { ...@@ -8,14 +8,8 @@ import {
getHash, getHash,
} from './webpack/plugins/next-trace-entrypoints-plugin' } from './webpack/plugins/next-trace-entrypoints-plugin'
import {
TRACE_OUTPUT_VERSION,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
} from '../shared/lib/constants'
import path from 'path' import path from 'path'
import fs from 'fs/promises' import fs from 'fs/promises'
import { loadBindings } from './swc'
import { nonNullable } from '../lib/non-nullable' import { nonNullable } from '../lib/non-nullable'
import * as ciEnvironment from '../server/ci-info' import * as ciEnvironment from '../server/ci-info'
import debugOriginal from 'next/dist/compiled/debug' import debugOriginal from 'next/dist/compiled/debug'
...@@ -27,7 +21,6 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths' ...@@ -27,7 +21,6 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
import isError from '../lib/is-error' import isError from '../lib/is-error'
import type { NodeFileTraceReasons } from '@vercel/nft' import type { NodeFileTraceReasons } from '@vercel/nft'
import type { RoutesUsingEdgeRuntime } from './utils' import type { RoutesUsingEdgeRuntime } from './utils'
import type { ExternalObject, NextTurboTasks } from './swc/generated-native'
const debug = debugOriginal('next:build:build-traces') const debug = debugOriginal('next:build:build-traces')
...@@ -108,104 +101,6 @@ export async function collectBuildTraces({ ...@@ -108,104 +101,6 @@ export async function collectBuildTraces({
}) { }) {
const startTime = Date.now() const startTime = Date.now()
debug('starting build traces') debug('starting build traces')
let turboTasksForTrace: ExternalObject<NextTurboTasks>
let bindings = await loadBindings()
const runTurbotrace = async function () {
if (!config.experimental.turbotrace || !buildTraceContext) {
return
}
if (!bindings?.isWasm && typeof bindings.turbo.startTrace === 'function') {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = bindings.turbo.createTurboTasks(
distDir,
false,
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
1024
)
const { entriesTrace, chunksTrace } = buildTraceContext
if (entriesTrace) {
const {
appDir: buildTraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] = await bindings.turbo.startTrace(
action,
turboTasksForTrace
)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(buildTraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from<[string, string]>(
Object.entries(entryNameMap)
).filter(([k]) => k.startsWith(buildTraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action, outputPath } = chunksTrace
action.input = action.input.filter((f: any) => {
const outputPagesPath = path.join(outputPath, '..', 'pages')
return (
!f.startsWith(outputPagesPath) ||
!staticPages.includes(
// strip `outputPagesPath` and file ext from absolute
f.substring(outputPagesPath.length, f.length - 3)
)
)
})
await bindings.turbo.startTrace(action, turboTasksForTrace)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await fs
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await fs.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
const { outputFileTracingIncludes = {}, outputFileTracingExcludes = {} } = const { outputFileTracingIncludes = {}, outputFileTracingExcludes = {} } =
config config
...@@ -214,7 +109,7 @@ export async function collectBuildTraces({ ...@@ -214,7 +109,7 @@ export async function collectBuildTraces({
await nextBuildSpan await nextBuildSpan
.traceChild('node-file-trace-build', { .traceChild('node-file-trace-build', {
isTurbotrace: Boolean(config.experimental.turbotrace) ? 'true' : 'false', isTurbotrace: 'false', // TODO(arlyon): remove this
}) })
.traceAsyncFn(async () => { .traceAsyncFn(async () => {
const nextServerTraceOutput = path.join( const nextServerTraceOutput = path.join(
...@@ -225,23 +120,16 @@ export async function collectBuildTraces({ ...@@ -225,23 +120,16 @@ export async function collectBuildTraces({
distDir, distDir,
'next-minimal-server.js.nft.json' 'next-minimal-server.js.nft.json'
) )
const root = const root = outputFileTracingRoot
config.experimental?.turbotrace?.contextDirectory ??
outputFileTracingRoot
// Under standalone mode, we need to trace the extra IPC server and // Under standalone mode, we need to trace the extra IPC server and
// worker files. // worker files.
const isStandalone = config.output === 'standalone' const isStandalone = config.output === 'standalone'
const nextServerEntry = require.resolve('next/dist/server/next-server') const sharedEntriesSet = Object.keys(defaultOverrides).map((value) =>
const sharedEntriesSet = [ require.resolve(value, {
...(config.experimental.turbotrace paths: [require.resolve('next/dist/server/require-hook')],
? [] })
: Object.keys(defaultOverrides).map((value) => )
require.resolve(value, {
paths: [require.resolve('next/dist/server/require-hook')],
})
)),
]
const { cacheHandler } = config const { cacheHandler } = config
const { cacheHandlers } = config.experimental const { cacheHandlers } = config.experimental
...@@ -373,7 +261,6 @@ export async function collectBuildTraces({ ...@@ -373,7 +261,6 @@ export async function collectBuildTraces({
const routeIgnoreFn = makeIgnoreFn(routesIgnores) const routeIgnoreFn = makeIgnoreFn(routesIgnores)
const traceContext = path.join(nextServerEntry, '..', '..')
const serverTracedFiles = new Set<string>() const serverTracedFiles = new Set<string>()
const minimalServerTracedFiles = new Set<string>() const minimalServerTracedFiles = new Set<string>()
...@@ -396,45 +283,7 @@ export async function collectBuildTraces({ ...@@ -396,45 +283,7 @@ export async function collectBuildTraces({
) )
} }
if (config.experimental.turbotrace) { {
await runTurbotrace()
const startTrace = bindings.turbo.startTrace
const makeTrace = async (entries: string[]) =>
startTrace(
{
action: 'print',
input: entries,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace?.logLevel,
processCwd: config.experimental.turbotrace?.processCwd,
logDetail: config.experimental.turbotrace?.logDetail,
showAll: config.experimental.turbotrace?.logAll,
},
turboTasksForTrace
)
// turbotrace does not handle concurrent tracing
const vanillaFiles = await makeTrace(serverEntries)
const minimalFiles = await makeTrace(minimalServerEntries)
for (const [set, files] of [
[serverTracedFiles, vanillaFiles],
[minimalServerTracedFiles, minimalFiles],
] as [Set<string>, string[]][]) {
for (const file of files) {
if (
!(
set === minimalServerTracedFiles
? minimalServerIgnoreFn
: serverIgnoreFn
)(path.join(traceContext, file))
) {
addToTracedFiles(traceContext, file, set)
}
}
}
} else {
const chunksToTrace: string[] = [ const chunksToTrace: string[] = [
...(buildTraceContext?.chunksTrace?.action.input || []), ...(buildTraceContext?.chunksTrace?.action.input || []),
...serverEntries, ...serverEntries,
......
...@@ -1809,7 +1809,6 @@ export default async function getBaseWebpackConfig( ...@@ -1809,7 +1809,6 @@ export default async function getBaseWebpackConfig(
esmExternals: config.experimental.esmExternals, esmExternals: config.experimental.esmExternals,
outputFileTracingRoot: config.outputFileTracingRoot, outputFileTracingRoot: config.outputFileTracingRoot,
appDirEnabled: hasAppDir, appDirEnabled: hasAppDir,
turbotrace: config.experimental.turbotrace,
optOutBundlingPackages, optOutBundlingPackages,
traceIgnores: [], traceIgnores: [],
flyingShuttle: Boolean(flyingShuttle), flyingShuttle: Boolean(flyingShuttle),
...@@ -1950,7 +1949,6 @@ export default async function getBaseWebpackConfig( ...@@ -1950,7 +1949,6 @@ export default async function getBaseWebpackConfig(
['swcRemoveConsole', !!config.compiler?.removeConsole], ['swcRemoveConsole', !!config.compiler?.removeConsole],
['swcImportSource', !!jsConfig?.compilerOptions?.jsxImportSource], ['swcImportSource', !!jsConfig?.compilerOptions?.jsxImportSource],
['swcEmotion', !!config.compiler?.emotion], ['swcEmotion', !!config.compiler?.emotion],
['turbotrace', !!config.experimental.turbotrace],
['transpilePackages', !!config.transpilePackages], ['transpilePackages', !!config.transpilePackages],
[ [
'skipMiddlewareUrlNormalize', 'skipMiddlewareUrlNormalize',
......
...@@ -17,7 +17,6 @@ import { ...@@ -17,7 +17,6 @@ import {
NODE_RESOLVE_OPTIONS, NODE_RESOLVE_OPTIONS,
} from '../../webpack-config' } from '../../webpack-config'
import type { NextConfigComplete } from '../../../server/config-shared' import type { NextConfigComplete } from '../../../server/config-shared'
import { loadBindings } from '../../swc'
import picomatch from 'next/dist/compiled/picomatch' import picomatch from 'next/dist/compiled/picomatch'
import { getModuleBuildInfo } from '../loaders/get-module-build-info' import { getModuleBuildInfo } from '../loaders/get-module-build-info'
import { getPageFilePath } from '../../entries' import { getPageFilePath } from '../../entries'
...@@ -110,9 +109,6 @@ export interface TurbotraceAction { ...@@ -110,9 +109,6 @@ export interface TurbotraceAction {
input: string[] input: string[]
contextDirectory: string contextDirectory: string
processCwd: string processCwd: string
logLevel?: NonNullable<
NextConfigComplete['experimental']['turbotrace']
>['logLevel']
showAll?: boolean showAll?: boolean
memoryLimit?: number memoryLimit?: number
} }
...@@ -148,7 +144,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -148,7 +144,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
private entryTraces: Map<string, Map<string, { bundled: boolean }>> private entryTraces: Map<string, Map<string, { bundled: boolean }>>
private traceIgnores: string[] private traceIgnores: string[]
private esmExternals?: NextConfigComplete['experimental']['esmExternals'] private esmExternals?: NextConfigComplete['experimental']['esmExternals']
private turbotrace?: NextConfigComplete['experimental']['turbotrace']
private traceHashes: Map<string, string> private traceHashes: Map<string, string>
private flyingShuttle?: boolean private flyingShuttle?: boolean
private compilerType: CompilerNameValues private compilerType: CompilerNameValues
...@@ -167,7 +162,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -167,7 +162,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
traceIgnores, traceIgnores,
esmExternals, esmExternals,
outputFileTracingRoot, outputFileTracingRoot,
turbotrace,
flyingShuttle, flyingShuttle,
swcLoaderConfig, swcLoaderConfig,
}: { }: {
...@@ -181,7 +175,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -181,7 +175,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
traceIgnores?: string[] traceIgnores?: string[]
outputFileTracingRoot?: string outputFileTracingRoot?: string
esmExternals?: NextConfigComplete['experimental']['esmExternals'] esmExternals?: NextConfigComplete['experimental']['esmExternals']
turbotrace?: NextConfigComplete['experimental']['turbotrace']
swcLoaderConfig: TraceEntryPointsPlugin['swcLoaderConfig'] swcLoaderConfig: TraceEntryPointsPlugin['swcLoaderConfig']
}) { }) {
this.rootDir = rootDir this.rootDir = rootDir
...@@ -192,7 +185,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -192,7 +185,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
this.appDirEnabled = appDirEnabled this.appDirEnabled = appDirEnabled
this.traceIgnores = traceIgnores || [] this.traceIgnores = traceIgnores || []
this.tracingRoot = outputFileTracingRoot || rootDir this.tracingRoot = outputFileTracingRoot || rootDir
this.turbotrace = turbotrace
this.optOutBundlingPackages = optOutBundlingPackages this.optOutBundlingPackages = optOutBundlingPackages
this.flyingShuttle = flyingShuttle this.flyingShuttle = flyingShuttle
this.traceHashes = new Map() this.traceHashes = new Map()
...@@ -249,11 +241,8 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -249,11 +241,8 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
action: { action: {
action: 'annotate', action: 'annotate',
input: [...chunksToTrace], input: [...chunksToTrace],
contextDirectory: contextDirectory: this.tracingRoot,
this.turbotrace?.contextDirectory ?? this.tracingRoot, processCwd: this.rootDir,
processCwd: this.turbotrace?.processCwd ?? this.rootDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
}, },
outputPath, outputPath,
entryNameFilesMap: Object.fromEntries(entryNameFilesMap), entryNameFilesMap: Object.fromEntries(entryNameFilesMap),
...@@ -624,8 +613,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -624,8 +613,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
} }
} }
const contextDirectory = const contextDirectory = this.tracingRoot
this.turbotrace?.contextDirectory ?? this.tracingRoot
const chunks = [...entriesToTrace] const chunks = [...entriesToTrace]
this.buildTraceContext.entriesTrace = { this.buildTraceContext.entriesTrace = {
...@@ -633,9 +621,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -633,9 +621,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
action: 'print', action: 'print',
input: chunks, input: chunks,
contextDirectory, contextDirectory,
processCwd: this.turbotrace?.processCwd ?? this.rootDir, processCwd: this.rootDir,
logLevel: this.turbotrace?.logLevel,
showAll: this.turbotrace?.logAll,
}, },
appDir: this.rootDir, appDir: this.rootDir,
depModArray: Array.from(depModMap.keys()), depModArray: Array.from(depModMap.keys()),
...@@ -643,19 +629,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance { ...@@ -643,19 +629,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
outputPath: compilation.outputOptions.path!, outputPath: compilation.outputOptions.path!,
} }
// if we're using turbotrace we can skip tracing
// loader contents as it should be able to capture
// fs usage in final chunks instead
if (this.turbotrace) {
let binding = await loadBindings()
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
return
}
}
let fileList: Set<string> let fileList: Set<string>
let reasons: NodeFileTraceReasons let reasons: NodeFileTraceReasons
const ignores = [ const ignores = [
......
...@@ -415,27 +415,6 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() => ...@@ -415,27 +415,6 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
optimizePackageImports: z.array(z.string()).optional(), optimizePackageImports: z.array(z.string()).optional(),
optimizeServerReact: z.boolean().optional(), optimizeServerReact: z.boolean().optional(),
clientTraceMetadata: z.array(z.string()).optional(), clientTraceMetadata: z.array(z.string()).optional(),
turbotrace: z
.object({
logLevel: z
.enum([
'bug',
'fatal',
'error',
'warning',
'hint',
'note',
'suggestions',
'info',
])
.optional(),
logAll: z.boolean().optional(),
logDetail: z.boolean().optional(),
contextDirectory: z.string().optional(),
processCwd: z.string().optional(),
memoryLimit: z.number().int().optional(),
})
.optional(),
serverMinification: z.boolean().optional(), serverMinification: z.boolean().optional(),
serverSourceMaps: z.boolean().optional(), serverSourceMaps: z.boolean().optional(),
useWasmBinary: z.boolean().optional(), useWasmBinary: z.boolean().optional(),
......
...@@ -355,23 +355,6 @@ export interface ExperimentalConfig { ...@@ -355,23 +355,6 @@ export interface ExperimentalConfig {
optimizeServerReact?: boolean optimizeServerReact?: boolean
turbo?: ExperimentalTurboOptions turbo?: ExperimentalTurboOptions
turbotrace?: {
logLevel?:
| 'bug'
| 'fatal'
| 'error'
| 'warning'
| 'hint'
| 'note'
| 'suggestions'
| 'info'
logDetail?: boolean
logAll?: boolean
contextDirectory?: string
processCwd?: string
/** in `MB` */
memoryLimit?: number
}
/** /**
* For use with `@next/mdx`. Compile MDX files using the new Rust compiler. * For use with `@next/mdx`. Compile MDX files using the new Rust compiler.
...@@ -1138,7 +1121,6 @@ export const defaultConfig: NextConfig = { ...@@ -1138,7 +1121,6 @@ export const defaultConfig: NextConfig = {
amp: undefined, amp: undefined,
urlImports: undefined, urlImports: undefined,
turbo: undefined, turbo: undefined,
turbotrace: undefined,
typedRoutes: false, typedRoutes: false,
typedEnv: false, typedEnv: false,
clientTraceMetadata: undefined, clientTraceMetadata: undefined,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment