Unverified Commit 89e3493a authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Reduce webpack memory for apps with many routes

Avoid storing per-route copies of bundled trace dependencies and all-route loader options. Fail webpack worker calls when their IPC result cannot be serialized.
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 68406a4c
...@@ -70,6 +70,10 @@ import { getStrictRouteMatchingDefaultWarning } from '../server/lib/router-utils ...@@ -70,6 +70,10 @@ import { getStrictRouteMatchingDefaultWarning } from '../server/lib/router-utils
type ObjectValue<T> = T extends { [key: string]: infer V } ? V : never type ObjectValue<T> = T extends { [key: string]: infer V } ? V : never
import { getStaticInfoIncludingLayouts } from './get-static-info-including-layouts' import { getStaticInfoIncludingLayouts } from './get-static-info-including-layouts'
import { getPageFromPath } from './route-discovery' import { getPageFromPath } from './route-discovery'
import {
createStaticSiblingsIndex,
getStaticSiblingsByParentUrl,
} from './static-siblings-index'
export function getPageFilePath({ export function getPageFilePath({
absolutePagePath, absolutePagePath,
...@@ -534,6 +538,18 @@ export async function createEntrypoints( ...@@ -534,6 +538,18 @@ export async function createEntrypoints(
) )
} }
// Static siblings of dynamic segments are computed across all app routes
// once, and each app entry only receives the parts it can use. Passing all
// routes to every entry would make the loader requests grow quadratically
// with the number of routes.
const staticSiblingsIndex = createStaticSiblingsIndex(
Object.keys(appPathsPerRoute)
)
const getStaticSiblingsLoaderOption = (routeAppPaths: string[] | null) =>
JSON.stringify(
getStaticSiblingsByParentUrl(staticSiblingsIndex, routeAppPaths ?? [])
)
const getEntryHandler = const getEntryHandler =
(mappings: MappedPages, pagesType: PAGE_TYPES): ((page: string) => void) => (mappings: MappedPages, pagesType: PAGE_TYPES): ((page: string) => void) =>
async (page) => { async (page) => {
...@@ -641,7 +657,8 @@ export async function createEntrypoints( ...@@ -641,7 +657,8 @@ export async function createEntrypoints(
pagePath: absolutePagePath, pagePath: absolutePagePath,
appDir, appDir,
appPaths: matchedAppPaths, appPaths: matchedAppPaths,
allNormalizedAppPaths: Object.keys(appPathsPerRoute), staticSiblingsByParentUrl:
getStaticSiblingsLoaderOption(matchedAppPaths),
pageExtensions, pageExtensions,
basePath: config.basePath, basePath: config.basePath,
assetPrefix: config.assetPrefix, assetPrefix: config.assetPrefix,
...@@ -727,7 +744,8 @@ export async function createEntrypoints( ...@@ -727,7 +744,8 @@ export async function createEntrypoints(
pagePath: absolutePagePath, pagePath: absolutePagePath,
appDir: appDir!, appDir: appDir!,
appPaths: matchedAppPaths, appPaths: matchedAppPaths,
allNormalizedAppPaths: Object.keys(appPathsPerRoute), staticSiblingsByParentUrl:
getStaticSiblingsLoaderOption(matchedAppPaths),
pageExtensions, pageExtensions,
basePath: config.basePath, basePath: config.basePath,
assetPrefix: config.assetPrefix, assetPrefix: config.assetPrefix,
......
import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
/**
* Static siblings of dynamic segments, keyed by the URL path of their parent
* (e.g. `/products` -> `['sale', 'new']`). Built once per build from all app
* routes, so that `next-app-loader` doesn't need the full list of routes in
* every entry's loader options (which grows quadratically with the number of
* routes).
*/
export type StaticSiblingsIndex = Map<string, string[]>
/**
* A subset of the {@link StaticSiblingsIndex} with only the parent URL paths
* that a single app entry can query. Passed as a loader option.
*/
export type StaticSiblingsByParentUrl = Record<string, string[]>
export const isDynamicSegment = (segment: string) =>
segment.startsWith('[') && segment.endsWith(']')
function toUrlSegments(appPath: string): string[] {
return normalizeAppPath(appPath).split('/').filter(Boolean)
}
function getParentUrlPath(urlSegments: readonly string[], level: number) {
return level === 0 ? '/' : '/' + urlSegments.slice(0, level).join('/')
}
/**
* Collects, for every parent URL path, the static segment names of its
* children across all app routes (route groups and parallel route slots don't
* contribute to the URL). Segment order follows the order of
* `allNormalizedAppPaths`.
*/
export function createStaticSiblingsIndex(
allNormalizedAppPaths: readonly string[]
): StaticSiblingsIndex {
const index = new Map<string, Set<string>>()
for (const appPath of allNormalizedAppPaths) {
const urlSegments = toUrlSegments(appPath)
for (let level = 0; level < urlSegments.length; level++) {
const segment = urlSegments[level]
if (isDynamicSegment(segment)) {
continue
}
const parentUrlPath = getParentUrlPath(urlSegments, level)
let siblings = index.get(parentUrlPath)
if (!siblings) {
siblings = new Set()
index.set(parentUrlPath, siblings)
}
siblings.add(segment)
}
}
return new Map(
Array.from(index, ([parentUrlPath, siblings]) => [
parentUrlPath,
Array.from(siblings),
])
)
}
/**
* Selects the parts of the index that the loader can query for an entry:
* the parents of every dynamic segment along the entry's own app paths.
*/
export function getStaticSiblingsByParentUrl(
index: StaticSiblingsIndex,
appPaths: readonly string[]
): StaticSiblingsByParentUrl {
const result: StaticSiblingsByParentUrl = {}
for (const appPath of appPaths) {
const urlSegments = toUrlSegments(appPath)
for (let level = 0; level < urlSegments.length; level++) {
if (!isDynamicSegment(urlSegments[level])) {
continue
}
const parentUrlPath = getParentUrlPath(urlSegments, level)
result[parentUrlPath] = index.get(parentUrlPath) ?? []
}
}
return result
}
/**
* For a given segment path (in file system space, e.g.
* `(group)/products/[id]`), returns the static sibling segments at the same
* URL path level. Siblings may live in different parts of the file system
* tree (e.g. other route groups) but at the same URL level:
*
* /app/(marketing)/products/sale/page.tsx -> /products/sale
* /app/(shop)/products/[id]/page.tsx -> /products/[id]
*
* When called with `(shop)/products/[id]`, this returns `['sale']`. Only
* dynamic segments have static siblings; `null` (dev, where not all routes
* are known) means none are known.
*/
export function getStaticSiblingSegments(
staticSiblingsByParentUrl: StaticSiblingsByParentUrl | null | undefined,
segmentPath: string
): string[] {
// Add a trailing /page so normalizeAppPath strips it properly
const currentUrlSegments = toUrlSegments(segmentPath + '/page')
// If the path is empty (root level), there are no siblings
if (currentUrlSegments.length === 0) {
return []
}
const currentSegment = currentUrlSegments[currentUrlSegments.length - 1]
if (!isDynamicSegment(currentSegment) || !staticSiblingsByParentUrl) {
return []
}
const parentUrlPath = getParentUrlPath(
currentUrlSegments,
currentUrlSegments.length - 1
)
// The current segment is dynamic, so it is never one of its static siblings.
return Object.prototype.hasOwnProperty.call(
staticSiblingsByParentUrl,
parentUrlPath
)
? staticSiblingsByParentUrl[parentUrlPath]
: []
}
// jest-worker's child sends results as `[PARENT_MESSAGE_OK, result]`, which
// `process.send` serializes as JSON.
const PARENT_MESSAGE_OK = 0
/**
* The webpack build worker sends its result to the main process as JSON over
* IPC. If the message can't be serialized (e.g. it exceeds the maximum string
* length), the worker's IPC send fails outside of the method call and the
* main process waits for a result forever. Checking up front turns this into
* a rejected worker call, so the build fails with an actionable error.
*/
export function assertWorkerResultSerializable(
result: unknown,
compilerName: string
): void {
try {
JSON.stringify([PARENT_MESSAGE_OK, result])
} catch (err) {
throw new Error(
`The webpack build worker result for the "${compilerName}" compiler could not be sent to the main process: ${
err instanceof Error ? err.message : String(err)
}`,
{ cause: err }
)
}
}
...@@ -45,6 +45,7 @@ import { Telemetry } from '../../telemetry/storage' ...@@ -45,6 +45,7 @@ import { Telemetry } from '../../telemetry/storage'
import { durationToString, hrtimeToSeconds } from '../duration-to-string' import { durationToString, hrtimeToSeconds } from '../duration-to-string'
import { installBindings } from '../swc/install-bindings' import { installBindings } from '../swc/install-bindings'
import { Bundler } from '../../lib/bundler' import { Bundler } from '../../lib/bundler'
import { assertWorkerResultSerializable } from './assert-worker-result-serializable'
const debug = origDebug('next:build:webpack-build') const debug = origDebug('next:build:webpack-build')
...@@ -462,5 +463,7 @@ export async function workerMain(workerData: { ...@@ -462,5 +463,7 @@ export async function workerMain(workerData: {
// Save CPU profile before worker exits // Save CPU profile before worker exits
await saveCpuProfile() await saveCpuProfile()
return { ...result, debugTraceEvents: getTraceEvents() } const workerResult = { ...result, debugTraceEvents: getTraceEvents() }
assertWorkerResultSerializable(workerResult, workerData.compilerName)
return workerResult
} }
...@@ -40,10 +40,13 @@ import type { Compilation } from 'webpack' ...@@ -40,10 +40,13 @@ import type { Compilation } from 'webpack'
import { createAppRouteCode } from './create-app-route-code' import { createAppRouteCode } from './create-app-route-code'
import { MissingDefaultParallelRouteError } from '../../../../shared/lib/errors/missing-default-parallel-route-error' import { MissingDefaultParallelRouteError } from '../../../../shared/lib/errors/missing-default-parallel-route-error'
import { isInterceptionRouteAppPath } from '../../../../shared/lib/router/utils/interception-routes' import { isInterceptionRouteAppPath } from '../../../../shared/lib/router/utils/interception-routes'
import { normalizeAppPath } from '../../../../shared/lib/router/utils/app-paths'
import { normalizePathSep } from '../../../../shared/lib/page-path/normalize-path-sep' import { normalizePathSep } from '../../../../shared/lib/page-path/normalize-path-sep'
import { installBindings } from '../../../swc/install-bindings' import { installBindings } from '../../../swc/install-bindings'
import {
getStaticSiblingSegments as getStaticSiblingSegmentsFromIndex,
type StaticSiblingsByParentUrl,
} from '../../../static-siblings-index'
export type AppLoaderOptions = { export type AppLoaderOptions = {
name: string name: string
...@@ -51,9 +54,10 @@ export type AppLoaderOptions = { ...@@ -51,9 +54,10 @@ export type AppLoaderOptions = {
pagePath: string pagePath: string
appDir: string appDir: string
appPaths: readonly string[] | null appPaths: readonly string[] | null
// All normalized app paths across the entire app, used for computing // JSON-encoded `StaticSiblingsByParentUrl`: the static siblings of the
// static siblings for dynamic segments // dynamic segments of this entry's app paths. Not available in dev, where
allNormalizedAppPaths: readonly string[] | null // routes are compiled on demand.
staticSiblingsByParentUrl: string | undefined
preferredRegion: string | string[] | undefined preferredRegion: string | string[] | undefined
pageExtensions: PageExtensions pageExtensions: PageExtensions
assetPrefix: string assetPrefix: string
...@@ -131,9 +135,6 @@ const normalizeParallelKey = (key: string) => ...@@ -131,9 +135,6 @@ const normalizeParallelKey = (key: string) =>
const isCatchAllSegment = (segment: string) => const isCatchAllSegment = (segment: string) =>
segment.startsWith('[...') || segment.startsWith('[[...') segment.startsWith('[...') || segment.startsWith('[[...')
const isDynamicSegment = (segment: string) =>
segment.startsWith('[') && segment.endsWith(']')
const isDirectory = async (pathname: string) => { const isDirectory = async (pathname: string) => {
try { try {
const stat = await fs.stat(pathname) const stat = await fs.stat(pathname)
...@@ -875,7 +876,7 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -875,7 +876,7 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
name, name,
appDir, appDir,
appPaths, appPaths,
allNormalizedAppPaths: allNormalizedAppPathsOption, staticSiblingsByParentUrl: staticSiblingsByParentUrlOption,
pagePath, pagePath,
pageExtensions, pageExtensions,
rootDir, rootDir,
...@@ -927,8 +928,11 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -927,8 +928,11 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
const normalizedAppPaths = const normalizedAppPaths =
typeof appPaths === 'string' ? [appPaths] : appPaths || [] typeof appPaths === 'string' ? [appPaths] : appPaths || []
// All normalized app paths for computing static siblings across route groups // Static siblings of the dynamic segments of this entry, across route groups
const allNormalizedAppPaths = allNormalizedAppPathsOption ?? [] const staticSiblingsByParentUrl: StaticSiblingsByParentUrl | null =
staticSiblingsByParentUrlOption
? JSON.parse(staticSiblingsByParentUrlOption)
: null
const resolveParallelSegments = ( const resolveParallelSegments = (
pathname: string pathname: string
...@@ -1044,77 +1048,20 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -1044,77 +1048,20 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
* *
* When called with "(shop)/products/[id]", this would return ['sale']. * When called with "(shop)/products/[id]", this would return ['sale'].
* *
* TODO: This function, along with resolveParallelSegments and * The siblings are looked up in an index built once from all app routes by
* hasChildRoutesForSegment, repeatedly scans normalizedAppPaths. A more * `createEntrypoints` (see `static-siblings-index.ts`), which only passes
* optimal approach would build an intermediate tree structure first * the parent URL paths of this entry's dynamic segments.
* (representing the URL namespace with route groups collapsed), then derive *
* all this information in a single pass. The Turbopack implementation * TODO: resolveParallelSegments and hasChildRoutesForSegment repeatedly
* already uses a more tree-oriented approach (DirectoryTree -> * scan normalizedAppPaths. A more optimal approach would build an
* AppPageLoaderTree), so this is less urgent to refactor given Turbopack is * intermediate tree structure first (representing the URL namespace with
* the canonical implementation going forward. * route groups collapsed), then derive all this information in a single
* pass. The Turbopack implementation already uses a more tree-oriented
* approach (DirectoryTree -> AppPageLoaderTree), so this is less urgent to
* refactor given Turbopack is the canonical implementation going forward.
*/ */
const getStaticSiblingSegments = (segmentPath: string): string[] => { const getStaticSiblingSegments = (segmentPath: string): string[] =>
// Normalize the current path to URL space getStaticSiblingSegmentsFromIndex(staticSiblingsByParentUrl, segmentPath)
// Add a trailing /page so normalizeAppPath strips it properly
const currentUrlPath = normalizeAppPath(segmentPath + '/page')
const currentUrlSegments = currentUrlPath.split('/').filter(Boolean)
// If the path is empty (root level), there are no siblings
if (currentUrlSegments.length === 0) {
return []
}
const currentSegment = currentUrlSegments[currentUrlSegments.length - 1]
const parentUrlPath =
currentUrlSegments.length === 1
? '/'
: '/' + currentUrlSegments.slice(0, -1).join('/')
// The URL level at which we're looking for siblings (0-indexed)
const siblingLevel = currentUrlSegments.length - 1
// Only compute siblings for dynamic segments
if (!isDynamicSegment(currentSegment)) {
return []
}
// Use a Set to avoid duplicates (multiple paths may share the same sibling segment)
const siblings = new Set<string>()
for (const appPath of allNormalizedAppPaths) {
// Normalize each path to URL space (strip route groups, parallel routes, and /page suffix)
const urlPath = normalizeAppPath(appPath)
const urlSegments = urlPath.split('/').filter(Boolean)
// Path must have at least enough segments to reach the sibling level
if (urlSegments.length <= siblingLevel) {
continue
}
// Check if the parent path matches (all segments before the sibling level)
const pathParent =
siblingLevel === 0
? '/'
: '/' + urlSegments.slice(0, siblingLevel).join('/')
if (pathParent !== parentUrlPath) {
continue
}
// Get the segment at the same level as the current segment
const segmentAtLevel = urlSegments[siblingLevel]
// Check if this is a sibling: different segment and static
if (
segmentAtLevel !== currentSegment &&
!isDynamicSegment(segmentAtLevel)
) {
siblings.add(segmentAtLevel)
}
}
return Array.from(siblings)
}
const resolveDir: DirResolver = (pathToResolve) => { const resolveDir: DirResolver = (pathToResolve) => {
return createAbsolutePath(appDir, pathToResolve) return createAbsolutePath(appDir, pathToResolve)
......
...@@ -1037,7 +1037,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface { ...@@ -1037,7 +1037,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
name: bundlePath, name: bundlePath,
page, page,
appPaths: entryData.appPaths, appPaths: entryData.appPaths,
allNormalizedAppPaths: null, // Not available in dev mode staticSiblingsByParentUrl: undefined, // Not available in dev mode
pagePath: posix.join( pagePath: posix.join(
APP_DIR_ALIAS, APP_DIR_ALIAS,
relative( relative(
...@@ -1178,7 +1178,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface { ...@@ -1178,7 +1178,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
name: bundlePath, name: bundlePath,
page, page,
appPaths: entryData.appPaths, appPaths: entryData.appPaths,
allNormalizedAppPaths: null, // Not available in dev mode staticSiblingsByParentUrl: undefined, // Not available in dev mode
pagePath, pagePath,
appDir: this.appDir!, appDir: this.appDir!,
pageExtensions: this.config.pageExtensions, pageExtensions: this.config.pageExtensions,
......
/* eslint-env jest */
import {
createStaticSiblingsIndex,
getStaticSiblingsByParentUrl,
getStaticSiblingSegments,
} from 'next/dist/build/static-siblings-index'
import { normalizeAppPath } from 'next/dist/shared/lib/router/utils/app-paths'
const isDynamicSegment = (segment: string) =>
segment.startsWith('[') && segment.endsWith(']')
/**
* Reference implementation: the scan `next-app-loader` used to perform over
* every app route for each dynamic segment.
*/
function referenceStaticSiblings(
allNormalizedAppPaths: readonly string[],
segmentPath: string
): string[] {
const currentUrlPath = normalizeAppPath(segmentPath + '/page')
const currentUrlSegments = currentUrlPath.split('/').filter(Boolean)
if (currentUrlSegments.length === 0) return []
const currentSegment = currentUrlSegments[currentUrlSegments.length - 1]
const parentUrlPath =
currentUrlSegments.length === 1
? '/'
: '/' + currentUrlSegments.slice(0, -1).join('/')
const siblingLevel = currentUrlSegments.length - 1
if (!isDynamicSegment(currentSegment)) return []
const siblings = new Set<string>()
for (const appPath of allNormalizedAppPaths) {
const urlSegments = normalizeAppPath(appPath).split('/').filter(Boolean)
if (urlSegments.length <= siblingLevel) continue
const pathParent =
siblingLevel === 0
? '/'
: '/' + urlSegments.slice(0, siblingLevel).join('/')
if (pathParent !== parentUrlPath) continue
const segmentAtLevel = urlSegments[siblingLevel]
if (
segmentAtLevel !== currentSegment &&
!isDynamicSegment(segmentAtLevel)
) {
siblings.add(segmentAtLevel)
}
}
return Array.from(siblings)
}
/** Every file-system prefix of an app path, as the loader may query it. */
function segmentPathPrefixes(appPath: string): string[] {
const parts = appPath.split('/')
const prefixes: string[] = []
for (let i = 0; i <= parts.length; i++) {
prefixes.push(parts.slice(0, i).join('/'))
}
return prefixes
}
// Shaped like `appPathsPerRoute` in `createEntrypoints`: normalized route ->
// file-system app paths that render it.
const appPathsPerRoute: Record<string, string[]> = {
'/': ['/page'],
'/products/sale': ['/(group1)/products/sale/page'],
'/products/[id]': ['/(shop)/products/[id]/page'],
'/products/new': ['/products/new/page'],
'/items/featured': ['/items/featured/page'],
'/items/[...slug]': ['/items/[...slug]/page'],
'/docs/[[...path]]': ['/docs/[[...path]]/page'],
'/docs/intro': ['/docs/intro/page'],
'/categories/electronics/computers/laptops': [
'/categories/electronics/computers/laptops/page',
],
'/categories/[category]/[sub]/[leaf]': [
'/categories/[category]/[sub]/[leaf]/page',
],
'/categories/[category]': ['/categories/[category]/page'],
'/categories/featured': ['/categories/featured/page'],
'/dashboard': ['/dashboard/@panel/page', '/dashboard/page'],
'/dashboard/settings': [
'/dashboard/@panel/settings/page',
'/dashboard/settings/page',
],
'/dashboard/[tab]': ['/dashboard/@panel/[tab]/page', '/dashboard/[tab]/page'],
'/[lang]': ['/[lang]/page'],
'/[lang]/about': ['/[lang]/about/page'],
'/feed/(.)photo/[id]': ['/feed/(.)photo/[id]/page'],
'/feed/photo/[id]': ['/feed/photo/[id]/page'],
'/feed/photo/latest': ['/feed/photo/latest/page'],
'/api/[id]': ['/api/[id]/route'],
'/api/health': ['/api/health/route'],
}
describe('static siblings index', () => {
const allNormalizedAppPaths = Object.keys(appPathsPerRoute)
const index = createStaticSiblingsIndex(allNormalizedAppPaths)
for (const [route, appPaths] of Object.entries(appPathsPerRoute)) {
it(`matches the reference scan for every segment of ${route}`, () => {
const byParentUrl = getStaticSiblingsByParentUrl(index, appPaths)
for (const appPath of appPaths) {
for (const segmentPath of segmentPathPrefixes(appPath)) {
expect([
segmentPath,
getStaticSiblingSegments(byParentUrl, segmentPath),
]).toEqual([
segmentPath,
referenceStaticSiblings(allNormalizedAppPaths, segmentPath),
])
}
}
})
}
it('finds siblings across route groups', () => {
const byParentUrl = getStaticSiblingsByParentUrl(index, [
'/(shop)/products/[id]/page',
])
expect(
getStaticSiblingSegments(byParentUrl, '/(shop)/products/[id]')
).toEqual(['sale', 'new'])
})
it('only passes parents of dynamic segments of the entry', () => {
expect(
getStaticSiblingsByParentUrl(index, [
'/categories/[category]/[sub]/[leaf]/page',
])
).toEqual({
'/categories': ['electronics', 'featured'],
'/categories/[category]': [],
'/categories/[category]/[sub]': [],
})
})
it('passes nothing for routes without dynamic segments', () => {
expect(
getStaticSiblingsByParentUrl(index, [
'/dashboard/@panel/settings/page',
'/dashboard/settings/page',
])
).toEqual({})
})
it('returns no siblings when the index is not available (dev)', () => {
expect(getStaticSiblingSegments(null, '/(shop)/products/[id]')).toEqual([])
})
it('keeps per-entry data independent of the total number of routes', () => {
const many = Array.from({ length: 2000 }, (_, i) => `/page${i}/static`)
const manyIndex = createStaticSiblingsIndex(many)
expect(
getStaticSiblingsByParentUrl(manyIndex, ['/page1/static/page'])
).toEqual({})
})
})
/* eslint-env jest */
import path from 'path'
import {
createEntryTrace,
getFilesMapFromReasons,
} from 'next/dist/build/webpack/plugins/next-trace-entrypoints-plugin'
type Reasons = Map<
string,
{ type: string[]; ignored: boolean; parents: Set<string> }
>
// Native paths, like the ones the plugin computes with `path.relative`.
const tracingRoot = path.resolve('/project')
const rel = (file: string) => path.join(...file.split('/'))
const abs = (file: string) => path.join(tracingRoot, rel(file))
/**
* Reference implementation: the per-entry trace computation that
* `TraceEntryPointsPlugin` used before it stopped materializing every
* (bundled file x entry) pair. Returns the files that end up in the entry's
* `.nft.json` (before making them relative to the output directory).
*/
function referenceIncludedFiles({
fileList,
reasons,
isIgnored,
entry,
extraEntries,
entryFiles,
}: {
fileList: Set<string>
reasons: Reasons
isIgnored: (file: string) => boolean
entry: string
extraEntries: string[]
entryFiles: string[]
}): string[] {
const parentFilesMap = new Map<string, Map<string, { ignored: boolean }>>()
function propagateToParents(
parents: Set<string>,
file: string,
seen = new Set<string>()
) {
for (const parent of parents || []) {
if (!seen.has(parent)) {
seen.add(parent)
let parentFiles = parentFilesMap.get(parent)
if (!parentFiles) {
parentFiles = new Map()
parentFilesMap.set(parent, parentFiles)
}
parentFiles.set(file, { ignored: Boolean(isIgnored(file)) })
const parentReason = reasons.get(parent)
if (parentReason?.parents) {
propagateToParents(parentReason.parents, file, seen)
}
}
}
}
for (const file of fileList) {
const reason = reasons.get(file)
const isInitial =
reason?.type.length === 1 && reason.type.includes('initial')
if (
!reason ||
!reason.parents ||
(isInitial && reason.parents.size === 0)
) {
continue
}
propagateToParents(reason.parents, file)
}
const finalDeps = new Map<string, { bundled: boolean }>()
finalDeps.set(entry, { bundled: true })
for (const [dep, info] of parentFilesMap
.get(path.relative(tracingRoot, entry))
?.entries() || []) {
finalDeps.set(path.join(tracingRoot, dep), { bundled: info.ignored })
}
for (const extraEntry of extraEntries) {
finalDeps.set(extraEntry, { bundled: false })
for (const [dep, info] of parentFilesMap
.get(path.relative(tracingRoot, extraEntry))
?.entries() || []) {
finalDeps.set(path.join(tracingRoot, dep), { bundled: info.ignored })
}
}
const included: string[] = []
for (const file of new Set([...entryFiles, ...finalDeps.keys()])) {
if (!finalDeps.get(file)?.bundled) included.push(file)
}
return included
}
function newIncludedFiles(opts: Parameters<typeof referenceIncludedFiles>[0]) {
const ignoredFiles = new Set<string>()
const parentFilesMap = getFilesMapFromReasons(
opts.fileList,
opts.reasons as any,
opts.isIgnored,
{ skipIgnored: true, ignoredFiles }
)
const trace = createEntryTrace({
tracingRoot,
entry: opts.entry,
extraEntries: opts.extraEntries,
parentFilesMap,
fileList: opts.fileList,
reasons: opts.reasons as any,
ignoredFiles,
})
const included: string[] = []
for (const file of new Set([...opts.entryFiles, ...trace.files()])) {
if (!trace.isBundled(file)) included.push(file)
}
return included
}
// Small deterministic PRNG so failures are reproducible.
function mulberry32(seed: number) {
return () => {
seed |= 0
seed = (seed + 0x6d2b79f5) | 0
let t = Math.imul(seed ^ (seed >>> 15), 1 | seed)
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
}
function randomCase(seed: number) {
const rand = mulberry32(seed)
const fileCount = 4 + Math.floor(rand() * 20)
const files = Array.from({ length: fileCount }, (_, i) => rel(`src/f${i}.js`))
const reasons: Reasons = new Map()
const rootCount = 1 + Math.floor(rand() * 3)
for (let i = 0; i < fileCount; i++) {
const parents = new Set<string>()
if (i >= rootCount) {
const parentCount = 1 + Math.floor(rand() * 3)
for (let p = 0; p < parentCount; p++) {
// Mostly acyclic, with the occasional back edge.
const j =
rand() < 0.1 ? Math.floor(rand() * fileCount) : Math.floor(rand() * i)
if (j !== i || rand() < 0.5) parents.add(files[j])
}
}
const type =
i < rootCount ? ['initial'] : rand() < 0.2 ? ['asset'] : ['dependency']
// Some files are traced without a reason entry.
if (rand() < 0.05) continue
reasons.set(files[i], { type, ignored: false, parents })
}
const fileList = new Set(files.filter(() => rand() < 0.95))
const ignored = new Set(files.filter(() => rand() < 0.6))
const pick = () => files[Math.floor(rand() * fileCount)]
const entry = abs(files[0])
const extraEntries: string[] = []
const extraCount = Math.floor(rand() * 4)
for (let i = 0; i < extraCount; i++) {
extraEntries.push(abs(rand() < 0.3 ? files[0] : pick()))
}
const entryFiles = [abs('.next/server/chunks/1.js')]
const emittedCount = Math.floor(rand() * 3)
for (let i = 0; i < emittedCount; i++) entryFiles.push(abs(pick()))
return {
fileList,
reasons,
isIgnored: (file: string) => ignored.has(file),
entry,
extraEntries,
entryFiles,
}
}
describe('TraceEntryPointsPlugin entry traces', () => {
it('include the same files as the reference implementation', () => {
for (let seed = 1; seed <= 2000; seed++) {
const testCase = randomCase(seed)
expect([seed, newIncludedFiles(testCase).sort()]).toEqual([
seed,
referenceIncludedFiles(testCase).sort(),
])
}
})
it('handles extra entries that are bundled dependencies of later roots', () => {
const reasons: Reasons = new Map([
[
rel('src/page.js'),
{ type: ['initial'], ignored: false, parents: new Set() },
],
[
rel('src/a.js'),
{ type: ['initial'], ignored: false, parents: new Set() },
],
[
rel('src/b.js'),
{
type: ['dependency'],
ignored: false,
parents: new Set([rel('src/c.js')]),
},
],
[
rel('src/c.js'),
{ type: ['initial'], ignored: false, parents: new Set() },
],
[
rel('node_modules/x/index.js'),
{
type: ['dependency'],
ignored: false,
parents: new Set([rel('src/b.js')]),
},
],
])
const testCase = {
fileList: new Set([...reasons.keys()]),
reasons,
isIgnored: (file: string) => file.startsWith(rel('src') + path.sep),
entry: abs('src/page.js'),
extraEntries: [abs('src/b.js'), abs('src/c.js')],
entryFiles: [abs('.next/server/app/page.js'), abs('src/b.js')],
}
const expected = referenceIncludedFiles(testCase).sort()
expect(expected).toEqual(
[
abs('.next/server/app/page.js'),
abs('node_modules/x/index.js'),
abs('src/c.js'),
].sort()
)
expect(newIncludedFiles(testCase).sort()).toEqual(expected)
})
it('does not record ignored files when skipIgnored is set', () => {
const reasons: Reasons = new Map([
[
rel('src/page.js'),
{ type: ['initial'], ignored: false, parents: new Set() },
],
[
rel('src/comp.js'),
{
type: ['dependency'],
ignored: false,
parents: new Set([rel('src/page.js')]),
},
],
[
rel('node_modules/react/index.js'),
{
type: ['dependency'],
ignored: false,
parents: new Set([rel('src/comp.js')]),
},
],
])
const fileList = new Set([...reasons.keys()])
const isIgnored = jest.fn((file: string) =>
file.startsWith(rel('src') + path.sep)
)
const ignoredFiles = new Set<string>()
const map = getFilesMapFromReasons(fileList, reasons as any, isIgnored, {
skipIgnored: true,
ignoredFiles,
})
expect(
Object.fromEntries([...map].map(([k, v]) => [k, [...v.keys()]]))
).toEqual({
[rel('src/page.js')]: [rel('node_modules/react/index.js')],
[rel('src/comp.js')]: [rel('node_modules/react/index.js')],
})
expect([...ignoredFiles]).toEqual([rel('src/comp.js')])
// Called once per traced file, not once per (file, parent) pair.
expect(isIgnored).toHaveBeenCalledTimes(2)
})
it('keeps the default behavior without skipIgnored', () => {
const reasons: Reasons = new Map([
[
rel('src/page.js'),
{ type: ['initial'], ignored: false, parents: new Set() },
],
[
rel('src/comp.js'),
{
type: ['dependency'],
ignored: false,
parents: new Set([rel('src/page.js')]),
},
],
])
const isIgnored = jest.fn(() => true)
const map = getFilesMapFromReasons(
new Set(reasons.keys()),
reasons as any,
isIgnored
)
expect(map.get(rel('src/page.js'))?.get(rel('src/comp.js'))).toEqual({
ignored: true,
})
expect(isIgnored).toHaveBeenCalledWith(
rel('src/comp.js'),
rel('src/page.js')
)
})
})
/* eslint-env jest */
import path from 'path'
import { Worker } from 'next/dist/lib/worker'
import { assertWorkerResultSerializable } from 'next/dist/build/webpack-build/assert-worker-result-serializable'
describe('assertWorkerResultSerializable', () => {
it('accepts serializable results', () => {
expect(() =>
assertWorkerResultSerializable({ a: [1, 'b', { c: null }] }, 'server')
).not.toThrow()
})
it('throws a descriptive error for a BigInt', () => {
expect(() =>
assertWorkerResultSerializable({ value: BigInt(1) }, 'server')
).toThrow(
/The webpack build worker result for the "server" compiler could not be sent to the main process: .*BigInt/
)
})
it('throws a descriptive error for a cycle', () => {
const cyclic: Record<string, unknown> = {}
cyclic.self = cyclic
expect(() => assertWorkerResultSerializable(cyclic, 'client')).toThrow(
/The webpack build worker result for the "client" compiler could not be sent to the main process: .*circular/i
)
})
it('checks the IPC message and reports string length errors', () => {
const result = { value: 'result' }
const stringify = jest
.spyOn(JSON, 'stringify')
.mockImplementationOnce(() => {
throw new RangeError('Invalid string length')
})
try {
expect(() => assertWorkerResultSerializable(result, 'server')).toThrow(
'The webpack build worker result for the "server" compiler could not be sent to the main process: Invalid string length'
)
expect(stringify).toHaveBeenCalledWith([0, result])
} finally {
stringify.mockRestore()
}
})
})
describe('build worker with an unserializable result', () => {
it('rejects the call instead of hanging', async () => {
const worker = new Worker(
path.join(__dirname, 'webpack-build-worker-result', 'worker.js'),
{
exposedMethods: ['unserializable'],
debuggerPortOffset: -1,
isolatedMemory: true,
numWorkers: 1,
maxRetries: 0,
}
) as Worker & { unserializable: () => Promise<unknown> }
try {
await expect(worker.unserializable()).rejects.toThrow(
/The webpack build worker result for the "server" compiler could not be sent to the main process/
)
} finally {
await worker.end()
}
}, 30_000)
})
const {
assertWorkerResultSerializable,
} = require('next/dist/build/webpack-build/assert-worker-result-serializable')
// Mirrors `workerMain` in `packages/next/src/build/webpack-build/impl.ts`,
// returning a result that jest-worker cannot send over IPC.
exports.unserializable = async function unserializable() {
const result = { value: BigInt(1) }
assertWorkerResultSerializable(result, 'server')
return result
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment