Unverified Commit bdf82000 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add additional handling for experimental tracing (#67785)

This adds additional trace handling under the experimental trace mode.
Outside of the feature flag the bloom filter for app router paths used
in pages to hard navigate was updated to no longer be inlined by default
with webpack to avoid invalidating the main bundle for pages too often.
Instead the bloom filter data is stored in the `_buildManifest` file
which is only used in the pages router and is easier to update as it's
per-build already.

This tracing feature is still very experimental and should not be
leveraged in production or outside of testing.

---------
Co-authored-by: 's avatarZack Tanner <1939140+ztanner@users.noreply.github.com>
parent e48eb5a2
import fs from 'fs'
import path from 'path'
import crypto from 'crypto'
import { getPageFromPath } from '../entries'
import { Sema } from 'next/dist/compiled/async-sema'
export interface DetectedEntriesResult {
app: string[]
pages: string[]
}
let _hasShuttle: undefined | boolean = undefined
export async function hasShuttle(shuttleDir: string) {
if (typeof _hasShuttle === 'boolean') {
return _hasShuttle
}
_hasShuttle = await fs.promises
.access(path.join(shuttleDir, 'server'))
.then(() => true)
.catch(() => false)
return _hasShuttle
}
export async function detectChangedEntries({
appPaths,
pagesPaths,
pageExtensions,
distDir,
shuttleDir,
}: {
appPaths?: string[]
pagesPaths?: string[]
pageExtensions: string[]
distDir: string
shuttleDir: string
}): Promise<{
changed: DetectedEntriesResult
unchanged: DetectedEntriesResult
}> {
const changedEntries: {
app: string[]
pages: string[]
} = {
app: [],
pages: [],
}
const unchangedEntries: typeof changedEntries = {
app: [],
pages: [],
}
if (!(await hasShuttle(shuttleDir))) {
// no shuttle so consider everything changed
console.log(`no shuttle. can't detect changes`)
return {
changed: {
pages: pagesPaths || [],
app: appPaths || [],
},
unchanged: {
pages: [],
app: [],
},
}
}
const hashCache = new Map<string, string>()
async function computeHash(p: string): Promise<string> {
let hash = hashCache.get(p)
if (hash) {
return hash
}
return new Promise((resolve, reject) => {
const hashInst = crypto.createHash('sha1')
const stream = fs.createReadStream(p)
stream.on('error', (err) => reject(err))
stream.on('data', (chunk) => hashInst.update(chunk))
stream.on('end', () => {
const digest = hashInst.digest('hex')
resolve(digest)
hashCache.set(p, digest)
})
})
}
const hashSema = new Sema(16)
let globalEntryChanged = false
async function detectChange({
normalizedEntry,
entry,
type,
}: {
entry: string
normalizedEntry: string
type: keyof typeof changedEntries
}) {
const traceFile = path.join(
shuttleDir,
'server',
type,
`${normalizedEntry}.js.nft.json`
)
let changed = true
// we don't need to check any further entry's dependencies if
// a global entry changed since that invalidates everything
if (!globalEntryChanged) {
try {
const traceData: {
fileHashes: Record<string, string>
} = JSON.parse(await fs.promises.readFile(traceFile, 'utf8'))
if (traceData) {
let changedDependency = false
await Promise.all(
Object.keys(traceData.fileHashes).map(async (file) => {
try {
if (changedDependency) return
await hashSema.acquire()
const relativeTraceFile = path.relative(
path.join(shuttleDir, 'server', type),
traceFile
)
const originalTraceFile = path.join(
distDir,
'server',
type,
relativeTraceFile
)
const absoluteFile = path.join(
path.dirname(originalTraceFile),
file
)
if (absoluteFile.startsWith(distDir)) {
return
}
const prevHash = traceData.fileHashes[file]
const curHash = await computeHash(absoluteFile)
if (prevHash !== curHash) {
console.log('detected change on', {
prevHash,
curHash,
file,
entry: normalizedEntry,
})
changedDependency = true
}
} finally {
hashSema.release()
}
})
)
if (!changedDependency) {
changed = false
}
} else {
console.error('missing trace data', traceFile, normalizedEntry)
}
} catch (err) {
console.error(`Failed to detect change for ${entry}`, err)
}
}
// we always rebuild global entries so we have a version
// that matches the newest build/runtime
const isGlobalEntry = /(_app|_document|_error)/.test(entry)
if (changed || isGlobalEntry) {
// if a global entry changed all entries are changed
if (!globalEntryChanged && isGlobalEntry) {
console.log(`global entry ${entry} changed invalidating all entries`)
globalEntryChanged = true
// move unchanged to changed
changedEntries[type].push(...unchangedEntries[type])
}
changedEntries[type].push(entry)
} else {
unchangedEntries[type].push(entry)
}
}
// loop over entries and their dependency's hashes
// to detect which changed
for (const entry of pagesPaths || []) {
let normalizedEntry = getPageFromPath(entry, pageExtensions)
if (normalizedEntry === '/') {
normalizedEntry = '/index'
}
await detectChange({ entry, normalizedEntry, type: 'pages' })
}
for (const entry of appPaths || []) {
const normalizedEntry = getPageFromPath(entry, pageExtensions)
await detectChange({ entry, normalizedEntry, type: 'app' })
}
console.log(
'changed entries',
JSON.stringify(
{
changedEntries,
unchangedEntries,
},
null,
2
)
)
return {
changed: changedEntries,
unchanged: unchangedEntries,
}
}
This diff is collapsed.
import fs from 'fs'
import path from 'path'
import {
BUILD_MANIFEST,
APP_BUILD_MANIFEST,
REACT_LOADABLE_MANIFEST,
APP_PATH_ROUTES_MANIFEST,
PAGES_MANIFEST,
ROUTES_MANIFEST,
} from '../../shared/lib/constants'
import { recursiveCopy } from '../../lib/recursive-copy'
// we can create a new shuttle with the outputs before env values have
// been inlined, can be done after stitching takes place
export async function storeShuttle({
distDir,
shuttleDir,
}: {
distDir: string
shuttleDir: string
}) {
await fs.promises.rm(shuttleDir, { force: true, recursive: true })
await fs.promises.mkdir(shuttleDir, { recursive: true })
// copy all server entries
await recursiveCopy(
path.join(distDir, 'server'),
path.join(shuttleDir, 'server'),
{
filter(item) {
return !item.match(/\.(rsc|meta|html)$/)
},
}
)
const pagesManifest = JSON.parse(
await fs.promises.readFile(
path.join(shuttleDir, 'server', PAGES_MANIFEST),
'utf8'
)
)
// ensure manifest isn't modified to .html as it's before static gen
for (const key of Object.keys(pagesManifest)) {
pagesManifest[key] = pagesManifest[key].replace(/\.html$/, '.js')
}
await fs.promises.writeFile(
path.join(shuttleDir, 'server', PAGES_MANIFEST),
JSON.stringify(pagesManifest)
)
// copy static assets
await recursiveCopy(
path.join(distDir, 'static'),
path.join(shuttleDir, 'static')
)
// copy manifests not nested in {distDir}/server/
await fs.promises.mkdir(path.join(shuttleDir, 'manifests'), {
recursive: true,
})
for (const item of [
BUILD_MANIFEST,
ROUTES_MANIFEST,
APP_BUILD_MANIFEST,
REACT_LOADABLE_MANIFEST,
APP_PATH_ROUTES_MANIFEST,
]) {
const outputPath = path.join(shuttleDir, 'manifests', item)
await fs.promises.mkdir(path.dirname(outputPath), { recursive: true })
await fs.promises.copyFile(path.join(distDir, item), outputPath)
}
}
...@@ -101,7 +101,7 @@ import { ...@@ -101,7 +101,7 @@ import {
import type { EventBuildFeatureUsage } from '../telemetry/events' import type { EventBuildFeatureUsage } from '../telemetry/events'
import { Telemetry } from '../telemetry/storage' import { Telemetry } from '../telemetry/storage'
import { getPageStaticInfo } from './analysis/get-page-static-info' import { getPageStaticInfo } from './analysis/get-page-static-info'
import { createPagesMapping, sortByPageExts } from './entries' import { createPagesMapping, getPageFromPath, sortByPageExts } from './entries'
import { PAGE_TYPES } from '../lib/page-types' import { PAGE_TYPES } from '../lib/page-types'
import { generateBuildId } from './generate-build-id' import { generateBuildId } from './generate-build-id'
import { isWriteable } from './is-writeable' import { isWriteable } from './is-writeable'
...@@ -187,6 +187,12 @@ import { ...@@ -187,6 +187,12 @@ import {
checkIsAppPPREnabled, checkIsAppPPREnabled,
checkIsRoutePPREnabled, checkIsRoutePPREnabled,
} from '../server/lib/experimental/ppr' } from '../server/lib/experimental/ppr'
import {
detectChangedEntries,
type DetectedEntriesResult,
} from './flying-shuttle/detect-changed-entries'
import { storeShuttle } from './flying-shuttle/store-shuttle'
import { stitchBuilds } from './flying-shuttle/stitch-builds'
interface ExperimentalBypassForInfo { interface ExperimentalBypassForInfo {
experimentalBypassFor?: RouteHas[] experimentalBypassFor?: RouteHas[]
...@@ -717,8 +723,10 @@ export default async function build( ...@@ -717,8 +723,10 @@ export default async function build(
) )
NextBuildContext.buildId = buildId NextBuildContext.buildId = buildId
const shuttleDir = path.join(distDir, 'cache', 'shuttle')
if (config.experimental.flyingShuttle) { if (config.experimental.flyingShuttle) {
await fs.mkdir(path.join(distDir, 'cache', 'shuttle'), { await fs.mkdir(shuttleDir, {
recursive: true, recursive: true,
}) })
} }
...@@ -838,20 +846,32 @@ export default async function build( ...@@ -838,20 +846,32 @@ export default async function build(
appDir appDir
) )
const providedPagePaths: string[] = JSON.parse(
process.env.NEXT_PROVIDED_PAGE_PATHS || '[]'
)
let pagesPaths = let pagesPaths =
providedPagePaths.length > 0 !appDirOnly && pagesDir
? providedPagePaths ? await nextBuildSpan.traceChild('collect-pages').traceAsyncFn(() =>
: !appDirOnly && pagesDir recursiveReadDir(pagesDir, {
? await nextBuildSpan.traceChild('collect-pages').traceAsyncFn(() => pathnameFilter: validFileMatcher.isPageFile,
recursiveReadDir(pagesDir, { })
pathnameFilter: validFileMatcher.isPageFile, )
}) : []
)
: [] let changedPagePathsResult:
| undefined
| {
changed: DetectedEntriesResult
unchanged: DetectedEntriesResult
}
if (pagesPaths && config.experimental.flyingShuttle) {
changedPagePathsResult = await detectChangedEntries({
pagesPaths,
pageExtensions: config.pageExtensions,
distDir,
shuttleDir,
})
console.log({ changedPagePathsResult })
pagesPaths = changedPagePathsResult.changed.pages
}
const middlewareDetectionRegExp = new RegExp( const middlewareDetectionRegExp = new RegExp(
`^${MIDDLEWARE_FILENAME}\\.(?:${config.pageExtensions.join('|')})$` `^${MIDDLEWARE_FILENAME}\\.(?:${config.pageExtensions.join('|')})$`
...@@ -912,27 +932,37 @@ export default async function build( ...@@ -912,27 +932,37 @@ export default async function build(
let mappedAppPages: MappedPages | undefined let mappedAppPages: MappedPages | undefined
let denormalizedAppPages: string[] | undefined let denormalizedAppPages: string[] | undefined
let changedAppPathsResult:
| undefined
| {
changed: DetectedEntriesResult
unchanged: DetectedEntriesResult
}
if (appDir) { if (appDir) {
const providedAppPaths: string[] = JSON.parse( let appPaths = await nextBuildSpan
process.env.NEXT_PROVIDED_APP_PATHS || '[]' .traceChild('collect-app-paths')
) .traceAsyncFn(() =>
recursiveReadDir(appDir, {
pathnameFilter: (absolutePath) =>
validFileMatcher.isAppRouterPage(absolutePath) ||
// For now we only collect the root /not-found page in the app
// directory as the 404 fallback
validFileMatcher.isRootNotFound(absolutePath),
ignorePartFilter: (part) => part.startsWith('_'),
})
)
let appPaths = if (appPaths && config.experimental.flyingShuttle) {
providedAppPaths.length > 0 changedAppPathsResult = await detectChangedEntries({
? providedAppPaths appPaths,
: await nextBuildSpan pageExtensions: config.pageExtensions,
.traceChild('collect-app-paths') distDir,
.traceAsyncFn(() => shuttleDir,
recursiveReadDir(appDir, { })
pathnameFilter: (absolutePath) => console.log({ changedAppPathsResult })
validFileMatcher.isAppRouterPage(absolutePath) || appPaths = changedAppPathsResult.changed.app
// For now we only collect the root /not-found page in the app }
// directory as the 404 fallback
validFileMatcher.isRootNotFound(absolutePath),
ignorePartFilter: (part) => part.startsWith('_'),
})
)
mappedAppPages = await nextBuildSpan mappedAppPages = await nextBuildSpan
.traceChild('create-app-mapping') .traceChild('create-app-mapping')
...@@ -1137,19 +1167,41 @@ export default async function build( ...@@ -1137,19 +1167,41 @@ export default async function build(
), ),
} }
} }
let clientRouterFilters:
| undefined
| ReturnType<typeof createClientRouterFilter>
if (config.experimental.clientRouterFilter) { if (config.experimental.clientRouterFilter) {
const nonInternalRedirects = (config._originalRedirects || []).filter( const nonInternalRedirects = (config._originalRedirects || []).filter(
(r: any) => !r.internal (r: any) => !r.internal
) )
const clientRouterFilters = createClientRouterFilter( const filterPaths: string[] = []
appPaths,
if (config.experimental.flyingShuttle) {
filterPaths.push(
...[
// client filter always has all app paths
...(changedAppPathsResult?.unchanged?.app || []),
...(changedAppPathsResult?.changed?.app || []),
].map((entry) =>
normalizeAppPath(getPageFromPath(entry, config.pageExtensions))
),
...(changedPagePathsResult?.unchanged.pages.length
? changedPagePathsResult.changed?.pages || []
: []
).map((item) => getPageFromPath(item, config.pageExtensions))
)
} else {
filterPaths.push(...appPaths)
}
clientRouterFilters = createClientRouterFilter(
filterPaths,
config.experimental.clientRouterFilterRedirects config.experimental.clientRouterFilterRedirects
? nonInternalRedirects ? nonInternalRedirects
: [], : [],
config.experimental.clientRouterFilterAllowedRate config.experimental.clientRouterFilterAllowedRate
) )
NextBuildContext.clientRouterFilters = clientRouterFilters NextBuildContext.clientRouterFilters = clientRouterFilters
} }
...@@ -1323,7 +1375,7 @@ export default async function build( ...@@ -1323,7 +1375,7 @@ export default async function build(
env: process.env as Record<string, string>, env: process.env as Record<string, string>,
defineEnv: createDefineEnv({ defineEnv: createDefineEnv({
isTurbopack: true, isTurbopack: true,
clientRouterFilters: NextBuildContext.clientRouterFilters, clientRouterFilters,
config, config,
dev, dev,
distDir, distDir,
...@@ -1736,7 +1788,7 @@ export default async function build( ...@@ -1736,7 +1788,7 @@ export default async function build(
const appDynamicParamPaths = new Set<string>() const appDynamicParamPaths = new Set<string>()
const appDefaultConfigs = new Map<string, AppConfig>() const appDefaultConfigs = new Map<string, AppConfig>()
const pageInfos: PageInfos = new Map<string, PageInfo>() const pageInfos: PageInfos = new Map<string, PageInfo>()
const pagesManifest = await readManifest<PagesManifest>(pagesManifestPath) let pagesManifest = await readManifest<PagesManifest>(pagesManifestPath)
const buildManifest = await readManifest<BuildManifest>(buildManifestPath) const buildManifest = await readManifest<BuildManifest>(buildManifestPath)
const appBuildManifest = appDir const appBuildManifest = appDir
? await readManifest<AppBuildManifest>(appBuildManifestPath) ? await readManifest<AppBuildManifest>(appBuildManifestPath)
...@@ -2424,6 +2476,53 @@ export default async function build( ...@@ -2424,6 +2476,53 @@ export default async function build(
path.join(distDir, SERVER_DIRECTORY, MIDDLEWARE_MANIFEST) path.join(distDir, SERVER_DIRECTORY, MIDDLEWARE_MANIFEST)
) )
if (!isGenerateMode) {
if (config.experimental.flyingShuttle) {
console.log('stitching builds...')
const stitchResult = await stitchBuilds(
{
buildId,
distDir,
shuttleDir,
rewrites,
redirects,
edgePreviewProps: {
__NEXT_PREVIEW_MODE_ID:
NextBuildContext.previewProps!.previewModeId,
__NEXT_PREVIEW_MODE_ENCRYPTION_KEY:
NextBuildContext.previewProps!.previewModeEncryptionKey,
__NEXT_PREVIEW_MODE_SIGNING_KEY:
NextBuildContext.previewProps!.previewModeSigningKey,
},
encryptionKey,
allowedErrorRate:
config.experimental.clientRouterFilterAllowedRate,
},
{
changed: {
pages: changedPagePathsResult?.changed.pages || [],
app: changedAppPathsResult?.changed.app || [],
},
unchanged: {
pages: changedPagePathsResult?.unchanged.pages || [],
app: changedAppPathsResult?.unchanged.app || [],
},
pageExtensions: config.pageExtensions,
}
)
// reload pagesManifest since it's been updated on disk
if (stitchResult.pagesManifest) {
pagesManifest = stitchResult.pagesManifest
}
console.log('storing shuttle')
await storeShuttle({
distDir,
shuttleDir,
})
}
}
const finalPrerenderRoutes: { [route: string]: SsgRoute } = {} const finalPrerenderRoutes: { [route: string]: SsgRoute } = {}
const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {} const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {}
const tbdPrerenderRoutes: string[] = [] const tbdPrerenderRoutes: string[] = []
......
...@@ -945,24 +945,11 @@ export default async function getBaseWebpackConfig( ...@@ -945,24 +945,11 @@ export default async function getBaseWebpackConfig(
), ),
], ],
...(config.experimental.flyingShuttle
? {
recordsPath: path.join(distDir, 'cache', 'shuttle', 'records.json'),
}
: {}),
optimization: { optimization: {
emitOnErrors: !dev, emitOnErrors: !dev,
checkWasmTypes: false, checkWasmTypes: false,
nodeEnv: false, nodeEnv: false,
...(config.experimental.flyingShuttle
? {
moduleIds: 'deterministic',
portableRecords: true,
}
: {}),
splitChunks: ((): splitChunks: (():
| Required<webpack.Configuration>['optimization']['splitChunks'] | Required<webpack.Configuration>['optimization']['splitChunks']
| false => { | false => {
...@@ -1019,7 +1006,7 @@ export default async function getBaseWebpackConfig( ...@@ -1019,7 +1006,7 @@ export default async function getBaseWebpackConfig(
if (isNodeServer || isEdgeServer) { if (isNodeServer || isEdgeServer) {
return { return {
filename: `${isEdgeServer ? 'edge-chunks/' : ''}[name].js`, filename: `${isEdgeServer ? `edge-chunks${config.experimental.flyingShuttle ? `-${buildId}` : ''}/` : ''}[name].js`,
chunks: 'all', chunks: 'all',
minChunks: 2, minChunks: 2,
} }
...@@ -1107,6 +1094,7 @@ export default async function getBaseWebpackConfig( ...@@ -1107,6 +1094,7 @@ export default async function getBaseWebpackConfig(
runtimeChunk: isClient runtimeChunk: isClient
? { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } ? { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK }
: undefined, : undefined,
minimize: minimize:
!dev && !dev &&
(isClient || (isClient ||
...@@ -1199,13 +1187,28 @@ export default async function getBaseWebpackConfig( ...@@ -1199,13 +1187,28 @@ export default async function getBaseWebpackConfig(
...(config.experimental.flyingShuttle ...(config.experimental.flyingShuttle
? { ? {
// ensure we only use contenthash as it's more deterministic // ensure we only use contenthash as it's more deterministic
filename: isNodeOrEdgeCompilation filename: (p) => {
? dev || isEdgeServer if (isNodeOrEdgeCompilation) {
? `[name].js` // runtime chunk needs hash so it can be isolated
: `../[name].js` // across builds
: `static/chunks/${isDevFallback ? 'fallback/' : ''}[name]${ const isRuntimeChunk = p.chunk?.name?.match(
dev ? '' : '-[contenthash]' /webpack-(api-runtime|runtime)/
}.js`, )
return `${isEdgeServer ? '' : '../'}[name]${isRuntimeChunk ? `-${buildId}` : ''}.js`
}
// client filename
return `static/chunks/[name]-[contenthash].js`
},
path: isNodeServer
? path.join(outputPath, `chunks-${buildId}`)
: outputPath,
chunkFilename: isNodeOrEdgeCompilation
? `[name].js`
: `static/chunks/[contenthash].js`,
webassemblyModuleFilename: 'static/wasm/[contenthash].wasm',
} }
: {}), : {}),
}, },
...@@ -1796,7 +1799,6 @@ export default async function getBaseWebpackConfig( ...@@ -1796,7 +1799,6 @@ export default async function getBaseWebpackConfig(
}), }),
getDefineEnvPlugin({ getDefineEnvPlugin({
isTurbopack: false, isTurbopack: false,
clientRouterFilters,
config, config,
dev, dev,
distDir, distDir,
...@@ -1894,6 +1896,7 @@ export default async function getBaseWebpackConfig( ...@@ -1894,6 +1896,7 @@ export default async function getBaseWebpackConfig(
rewrites, rewrites,
isDevFallback, isDevFallback,
appDirEnabled: hasAppDir, appDirEnabled: hasAppDir,
clientRouterFilters,
}), }),
new ProfilingPlugin({ runWebpackSpan, rootDir: dir }), new ProfilingPlugin({ runWebpackSpan, rootDir: dir }),
config.optimizeFonts && config.optimizeFonts &&
......
import type { BloomFilter } from '../../../shared/lib/bloom-filter'
import type { Rewrite, CustomRoutes } from '../../../lib/load-custom-routes' import type { Rewrite, CustomRoutes } from '../../../lib/load-custom-routes'
import devalue from 'next/dist/compiled/devalue' import devalue from 'next/dist/compiled/devalue'
import { webpack, sources } from 'next/dist/compiled/webpack/webpack' import { webpack, sources } from 'next/dist/compiled/webpack/webpack'
...@@ -17,6 +18,7 @@ import getRouteFromEntrypoint from '../../../server/get-route-from-entrypoint' ...@@ -17,6 +18,7 @@ import getRouteFromEntrypoint from '../../../server/get-route-from-entrypoint'
import { ampFirstEntryNamesMap } from './next-drop-client-page-plugin' import { ampFirstEntryNamesMap } from './next-drop-client-page-plugin'
import { getSortedRoutes } from '../../../shared/lib/router/utils' import { getSortedRoutes } from '../../../shared/lib/router/utils'
import { spans } from './profiling-plugin' import { spans } from './profiling-plugin'
import { Span } from '../../../trace'
type DeepMutable<T> = { -readonly [P in keyof T]: DeepMutable<T[P]> } type DeepMutable<T> = { -readonly [P in keyof T]: DeepMutable<T[P]> }
...@@ -91,13 +93,22 @@ export function normalizeRewritesForBuildManifest( ...@@ -91,13 +93,22 @@ export function normalizeRewritesForBuildManifest(
// This function takes the asset map generated in BuildManifestPlugin and creates a // This function takes the asset map generated in BuildManifestPlugin and creates a
// reduced version to send to the client. // reduced version to send to the client.
function generateClientManifest( export function generateClientManifest(
compiler: any,
compilation: any,
assetMap: BuildManifest, assetMap: BuildManifest,
rewrites: CustomRoutes['rewrites'] rewrites: CustomRoutes['rewrites'],
clientRouterFilters?: {
staticFilter: ReturnType<BloomFilter['export']>
dynamicFilter: ReturnType<BloomFilter['export']>
},
compiler?: any,
compilation?: any
): string | undefined { ): string | undefined {
const compilationSpan = spans.get(compilation) || spans.get(compiler) const compilationSpan = compilation
? spans.get(compilation)
: compiler
? spans.get(compiler)
: new Span({ name: 'client-manifest' })
const genClientManifestSpan = compilationSpan?.traceChild( const genClientManifestSpan = compilationSpan?.traceChild(
'NextJsBuildManifest-generateClientManifest' 'NextJsBuildManifest-generateClientManifest'
) )
...@@ -105,6 +116,8 @@ function generateClientManifest( ...@@ -105,6 +116,8 @@ function generateClientManifest(
return genClientManifestSpan?.traceFn(() => { return genClientManifestSpan?.traceFn(() => {
const clientManifest: ClientBuildManifest = { const clientManifest: ClientBuildManifest = {
__rewrites: normalizeRewritesForBuildManifest(rewrites) as any, __rewrites: normalizeRewritesForBuildManifest(rewrites) as any,
__routerFilterStatic: clientRouterFilters?.staticFilter as any,
__routerFilterDynamic: clientRouterFilters?.dynamicFilter as any,
} }
const appDependencies = new Set(assetMap.pages['/_app']) const appDependencies = new Set(assetMap.pages['/_app'])
const sortedPageKeys = getSortedRoutes(Object.keys(assetMap.pages)) const sortedPageKeys = getSortedRoutes(Object.keys(assetMap.pages))
...@@ -162,12 +175,14 @@ export default class BuildManifestPlugin { ...@@ -162,12 +175,14 @@ export default class BuildManifestPlugin {
private rewrites: CustomRoutes['rewrites'] private rewrites: CustomRoutes['rewrites']
private isDevFallback: boolean private isDevFallback: boolean
private appDirEnabled: boolean private appDirEnabled: boolean
private clientRouterFilters?: Parameters<typeof generateClientManifest>[2]
constructor(options: { constructor(options: {
buildId: string buildId: string
rewrites: CustomRoutes['rewrites'] rewrites: CustomRoutes['rewrites']
isDevFallback?: boolean isDevFallback?: boolean
appDirEnabled: boolean appDirEnabled: boolean
clientRouterFilters?: Parameters<typeof generateClientManifest>[2]
}) { }) {
this.buildId = options.buildId this.buildId = options.buildId
this.isDevFallback = !!options.isDevFallback this.isDevFallback = !!options.isDevFallback
...@@ -177,6 +192,7 @@ export default class BuildManifestPlugin { ...@@ -177,6 +192,7 @@ export default class BuildManifestPlugin {
fallback: [], fallback: [],
} }
this.appDirEnabled = options.appDirEnabled this.appDirEnabled = options.appDirEnabled
this.clientRouterFilters = options.clientRouterFilters
this.rewrites.beforeFiles = options.rewrites.beforeFiles.map(processRoute) this.rewrites.beforeFiles = options.rewrites.beforeFiles.map(processRoute)
this.rewrites.afterFiles = options.rewrites.afterFiles.map(processRoute) this.rewrites.afterFiles = options.rewrites.afterFiles.map(processRoute)
this.rewrites.fallback = options.rewrites.fallback.map(processRoute) this.rewrites.fallback = options.rewrites.fallback.map(processRoute)
...@@ -195,6 +211,7 @@ export default class BuildManifestPlugin { ...@@ -195,6 +211,7 @@ export default class BuildManifestPlugin {
ampDevFiles: [], ampDevFiles: [],
lowPriorityFiles: [], lowPriorityFiles: [],
rootMainFiles: [], rootMainFiles: [],
rootMainFilesTree: {},
pages: { '/_app': [] }, pages: { '/_app': [] },
ampFirstPages: [], ampFirstPages: [],
} }
...@@ -308,10 +325,11 @@ export default class BuildManifestPlugin { ...@@ -308,10 +325,11 @@ export default class BuildManifestPlugin {
assets[clientManifestPath] = new sources.RawSource( assets[clientManifestPath] = new sources.RawSource(
`self.__BUILD_MANIFEST = ${generateClientManifest( `self.__BUILD_MANIFEST = ${generateClientManifest(
compiler,
compilation,
assetMap, assetMap,
this.rewrites this.rewrites,
this.clientRouterFilters,
compiler,
compilation
)};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()` )};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()`
) )
} }
......
...@@ -197,6 +197,8 @@ export function getDefineEnv({ ...@@ -197,6 +197,8 @@ export function getDefineEnv({
? 5 * 60 // 5 minutes ? 5 * 60 // 5 minutes
: config.experimental.staleTimes?.static : config.experimental.staleTimes?.static
), ),
'process.env.__NEXT_FLYING_SHUTTLE':
config.experimental.flyingShuttle ?? false,
'process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED': 'process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED':
config.experimental.clientRouterFilter ?? true, config.experimental.clientRouterFilter ?? true,
'process.env.__NEXT_CLIENT_ROUTER_S_FILTER': 'process.env.__NEXT_CLIENT_ROUTER_S_FILTER':
......
...@@ -1017,7 +1017,8 @@ async function renderToHTMLOrFlightImpl( ...@@ -1017,7 +1017,8 @@ async function renderToHTMLOrFlightImpl(
renderOpts.crossOrigin, renderOpts.crossOrigin,
subresourceIntegrityManifest, subresourceIntegrityManifest,
getAssetQueryString(ctx, true), getAssetQueryString(ctx, true),
nonce nonce,
renderOpts.page
) )
const rscPayload = await getRSCPayload(tree, ctx, asNotFound) const rscPayload = await getRSCPayload(tree, ctx, asNotFound)
...@@ -1385,7 +1386,8 @@ async function renderToHTMLOrFlightImpl( ...@@ -1385,7 +1386,8 @@ async function renderToHTMLOrFlightImpl(
renderOpts.crossOrigin, renderOpts.crossOrigin,
subresourceIntegrityManifest, subresourceIntegrityManifest,
getAssetQueryString(ctx, false), getAssetQueryString(ctx, false),
nonce nonce,
'/_not-found/page'
) )
const errorRSCPayload = await getErrorRSCPayload(tree, ctx, errorType) const errorRSCPayload = await getErrorRSCPayload(tree, ctx, errorType)
......
...@@ -9,7 +9,8 @@ export function getRequiredScripts( ...@@ -9,7 +9,8 @@ export function getRequiredScripts(
crossOrigin: undefined | '' | 'anonymous' | 'use-credentials', crossOrigin: undefined | '' | 'anonymous' | 'use-credentials',
SRIManifest: undefined | Record<string, string>, SRIManifest: undefined | Record<string, string>,
qs: string, qs: string,
nonce: string | undefined nonce: string | undefined,
pagePath: string
): [ ): [
() => void, () => void,
{ src: string; integrity?: string; crossOrigin?: string | undefined }, { src: string; integrity?: string; crossOrigin?: string | undefined },
...@@ -25,7 +26,9 @@ export function getRequiredScripts( ...@@ -25,7 +26,9 @@ export function getRequiredScripts(
crossOrigin, crossOrigin,
} }
const files = buildManifest.rootMainFiles.map(encodeURIPath) const files = (
buildManifest.rootMainFilesTree?.[pagePath] || buildManifest.rootMainFiles
).map(encodeURIPath)
if (files.length === 0) { if (files.length === 0) {
throw new Error( throw new Error(
'Invariant: missing bootstrap script. This is a bug in Next.js' 'Invariant: missing bootstrap script. This is a bug in Next.js'
......
...@@ -7,6 +7,10 @@ export type BuildManifest = { ...@@ -7,6 +7,10 @@ export type BuildManifest = {
polyfillFiles: readonly string[] polyfillFiles: readonly string[]
lowPriorityFiles: readonly string[] lowPriorityFiles: readonly string[]
rootMainFiles: readonly string[] rootMainFiles: readonly string[]
// this is a separate field for flying shuttle to allow
// different root main files per entries/build (ideally temporary)
// until we can stitch the runtime chunks together safely
rootMainFilesTree: { [appRoute: string]: readonly string[] }
pages: { pages: {
'/_app': readonly string[] '/_app': readonly string[]
[page: string]: readonly string[] [page: string]: readonly string[]
......
...@@ -765,45 +765,6 @@ export default class Router implements BaseRouter { ...@@ -765,45 +765,6 @@ export default class Router implements BaseRouter {
], ],
} }
if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) {
const { BloomFilter } =
require('../../lib/bloom-filter') as typeof import('../../lib/bloom-filter')
type Filter = ReturnType<
import('../../lib/bloom-filter').BloomFilter['export']
>
const routerFilterSValue: Filter | false = process.env
.__NEXT_CLIENT_ROUTER_S_FILTER as any
const staticFilterData: Filter | undefined = routerFilterSValue
? routerFilterSValue
: undefined
const routerFilterDValue: Filter | false = process.env
.__NEXT_CLIENT_ROUTER_D_FILTER as any
const dynamicFilterData: Filter | undefined = routerFilterDValue
? routerFilterDValue
: undefined
if (staticFilterData?.numHashes) {
this._bfl_s = new BloomFilter(
staticFilterData.numItems,
staticFilterData.errorRate
)
this._bfl_s.import(staticFilterData)
}
if (dynamicFilterData?.numHashes) {
this._bfl_d = new BloomFilter(
dynamicFilterData.numItems,
dynamicFilterData.errorRate
)
this._bfl_d.import(dynamicFilterData)
}
}
// Backwards compat for Router.router.events // Backwards compat for Router.router.events
// TODO: Should be remove the following major version as it was never documented // TODO: Should be remove the following major version as it was never documented
this.events = Router.events this.events = Router.events
...@@ -1060,10 +1021,86 @@ export default class Router implements BaseRouter { ...@@ -1060,10 +1021,86 @@ export default class Router implements BaseRouter {
skipNavigate?: boolean skipNavigate?: boolean
) { ) {
if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) { if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) {
if (!this._bfl_s && !this._bfl_d) {
const { BloomFilter } =
require('../../lib/bloom-filter') as typeof import('../../lib/bloom-filter')
type Filter = ReturnType<
import('../../lib/bloom-filter').BloomFilter['export']
>
let staticFilterData: Filter | undefined
let dynamicFilterData: Filter | undefined
try {
;({
__routerFilterStatic: staticFilterData,
__routerFilterDynamic: dynamicFilterData,
} = (await getClientBuildManifest()) as any as {
__routerFilterStatic?: Filter
__routerFilterDynamic?: Filter
})
} catch (err) {
// failed to load build manifest hard navigate
// to be safe
console.error(err)
if (skipNavigate) {
return true
}
handleHardNavigation({
url: addBasePath(
addLocale(as, locale || this.locale, this.defaultLocale)
),
router: this,
})
return new Promise(() => {})
}
const routerFilterSValue: Filter | false = process.env
.__NEXT_CLIENT_ROUTER_S_FILTER as any
if (!staticFilterData && routerFilterSValue) {
staticFilterData = routerFilterSValue ? routerFilterSValue : undefined
}
const routerFilterDValue: Filter | false = process.env
.__NEXT_CLIENT_ROUTER_D_FILTER as any
if (!dynamicFilterData && routerFilterDValue) {
dynamicFilterData = routerFilterDValue
? routerFilterDValue
: undefined
}
if (staticFilterData?.numHashes) {
this._bfl_s = new BloomFilter(
staticFilterData.numItems,
staticFilterData.errorRate
)
this._bfl_s.import(staticFilterData)
}
if (dynamicFilterData?.numHashes) {
this._bfl_d = new BloomFilter(
dynamicFilterData.numItems,
dynamicFilterData.errorRate
)
this._bfl_d.import(dynamicFilterData)
}
}
let matchesBflStatic = false let matchesBflStatic = false
let matchesBflDynamic = false let matchesBflDynamic = false
const pathsToCheck: Array<{ as?: string; allowMatchCurrent?: boolean }> =
[{ as }, { as: resolvedAs }]
if (process.env.__NEXT_FLYING_SHUTTLE) {
// if existing page changed we hard navigate to
// avoid runtime conflict with new page
// TODO: check buildManifest files instead?
pathsToCheck.push({ as: this.asPath, allowMatchCurrent: true })
}
for (const curAs of [as, resolvedAs]) { for (const { as: curAs, allowMatchCurrent } of pathsToCheck) {
if (curAs) { if (curAs) {
const asNoSlash = removeTrailingSlash( const asNoSlash = removeTrailingSlash(
new URL(curAs, 'http://n').pathname new URL(curAs, 'http://n').pathname
...@@ -1073,8 +1110,9 @@ export default class Router implements BaseRouter { ...@@ -1073,8 +1110,9 @@ export default class Router implements BaseRouter {
) )
if ( if (
allowMatchCurrent ||
asNoSlash !== asNoSlash !==
removeTrailingSlash(new URL(this.asPath, 'http://n').pathname) removeTrailingSlash(new URL(this.asPath, 'http://n').pathname)
) { ) {
matchesBflStatic = matchesBflStatic =
matchesBflStatic || matchesBflStatic ||
......
'use client' 'use client'
import { useSearchParams } from 'next/navigation' import { useSearchParams } from 'next/navigation'
import dynamic from 'next/dynamic'
const Button = dynamic(() =>
import('../../../../components/button/button').then((mod) => mod.Button)
)
export default function IdPage({ children, params }) { export default function IdPage({ children, params }) {
return ( return (
...@@ -14,6 +19,7 @@ export default function IdPage({ children, params }) { ...@@ -14,6 +19,7 @@ export default function IdPage({ children, params }) {
<p id="search-params"> <p id="search-params">
{JSON.stringify(Object.fromEntries(useSearchParams()))} {JSON.stringify(Object.fromEntries(useSearchParams()))}
</p> </p>
<Button>Click me!!</Button>
</> </>
) )
} }
'use client'
import * as buttonStyle from './button.module.css'
export function Button({ children }) {
return (
<button
className={buttonStyle.button}
onClick={() => {
console.log('clicked button!')
}}
>
{children}
</button>
)
}
.button {
background: #000;
color: #fff;
}
import fs from 'fs' import fs from 'fs'
import path from 'path' import path from 'path'
import { nextTestSetup, isNextStart } from 'e2e-utils' import { nextTestSetup, isNextStart } from 'e2e-utils'
import { retry } from 'next-test-utils'
// This feature is only relevant to Webpack. // This feature is only relevant to Webpack.
;(process.env.TURBOPACK ? describe.skip : describe)( ;(process.env.TURBOPACK ? describe.skip : describe)(
...@@ -70,5 +71,94 @@ import { nextTestSetup, isNextStart } from 'e2e-utils' ...@@ -70,5 +71,94 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
} }
} }
}) })
async function checkAppPagesNavigation() {
for (const path of [
'/',
'/blog/123',
'/dynamic-client/first/second',
'/dashboard',
'/dashboard/deployments/123',
]) {
require('console').error('checking', path)
const res = await next.fetch(path)
expect(res.status).toBe(200)
const browser = await next.browser(path)
// TODO: check for hydration success properly
await retry(async () => {
expect(await browser.eval('!!window.next.router')).toBe(true)
})
const browserLogs = await browser.log()
expect(
browserLogs.some(({ message }) => message.includes('error'))
).toBeFalse()
}
// TODO: check we hard navigate boundaries properly
}
it('should only rebuild just a changed app route correctly', async () => {
await next.stop()
const dataPath = 'app/dashboard/deployments/[id]/data.json'
const originalContent = await next.readFile(dataPath)
try {
await next.patchFile(dataPath, JSON.stringify({ hello: 'again' }))
await next.start()
await checkAppPagesNavigation()
} finally {
await next.patchFile(dataPath, originalContent)
}
})
it('should only rebuild just a changed pages route correctly', async () => {
await next.stop()
const pagePath = 'pages/index.js'
const originalContent = await next.readFile(pagePath)
try {
await next.patchFile(
pagePath,
originalContent.replace(
'hello from pages/index',
'hello from pages/index!!'
)
)
await next.start()
await checkAppPagesNavigation()
} finally {
await next.patchFile(pagePath, originalContent)
}
})
it('should only rebuild a changed app and pages route correctly', async () => {
await next.stop()
const pagePath = 'pages/index.js'
const originalPageContent = await next.readFile(pagePath)
const dataPath = 'app/dashboard/deployments/[id]/data.json'
const originalDataContent = await next.readFile(dataPath)
try {
await next.patchFile(
pagePath,
originalPageContent.replace(
'hello from pages/index',
'hello from pages/index!!'
)
)
await next.patchFile(dataPath, JSON.stringify({ hello: 'again' }))
await next.start()
await checkAppPagesNavigation()
} finally {
await next.patchFile(pagePath, originalPageContent)
await next.patchFile(dataPath, originalDataContent)
}
})
} }
) )
...@@ -8,6 +8,14 @@ module.exports = { ...@@ -8,6 +8,14 @@ module.exports = {
parallelServerBuildTraces: true, parallelServerBuildTraces: true,
webpackBuildWorker: true, webpackBuildWorker: true,
}, },
webpack(cfg) {
if (process.env.NEXT_PRIVATE_FLYING_SHUTTLE) {
// disable the webpack cache to make sure we're
// deterministic without
cfg.cache = false
}
return cfg
},
// output: 'standalone', // output: 'standalone',
rewrites: async () => { rewrites: async () => {
return { return {
......
import React from 'react' import React from 'react'
import Link from 'next/link' import Link from 'next/link'
import dynamic from 'next/dynamic'
import styles from '../styles/shared.module.css' import styles from '../styles/shared.module.css'
const Button = dynamic(() =>
import('../components/button/button').then((mod) => mod.Button)
)
export default function Page() { export default function Page() {
return ( return (
<> <>
...@@ -10,6 +15,7 @@ export default function Page() { ...@@ -10,6 +15,7 @@ export default function Page() {
</p> </p>
<Link href="/dashboard">Dashboard</Link> <Link href="/dashboard">Dashboard</Link>
<p id="react-version">{React.version}</p> <p id="react-version">{React.version}</p>
<Button>Click me!</Button>
</> </>
) )
} }
import { nextTestSetup } from 'e2e-utils'
import glob from 'glob'
import path from 'path'
describe('Provided page/app paths', () => {
const { next, isNextDev } = nextTestSetup({
files: __dirname,
// Deployments are unable to inspect the `.next` directory.
skipDeployment: true,
dependencies: {
nanoid: '4.0.1',
},
env: {
NEXT_PROVIDED_PAGE_PATHS: JSON.stringify(['/index.js', '/ssg.js']),
NEXT_PROVIDED_APP_PATHS: JSON.stringify([
'/dashboard/page.js',
'/(newroot)/dashboard/another/page.js',
]),
},
})
if (isNextDev) {
it('should skip dev', () => {})
return
}
it('should only build the provided paths', async () => {
const appPaths = await glob.sync('**/*.js', {
cwd: path.join(next.testDir, '.next/server/app'),
})
const pagePaths = await glob.sync('**/*.js', {
cwd: path.join(next.testDir, '.next/server/pages'),
})
expect(appPaths).toEqual([
'_not-found/page_client-reference-manifest.js',
'_not-found/page.js',
'(newroot)/dashboard/another/page_client-reference-manifest.js',
'(newroot)/dashboard/another/page.js',
'dashboard/page_client-reference-manifest.js',
'dashboard/page.js',
])
expect(pagePaths).toEqual([
'_app.js',
'_document.js',
'_error.js',
'ssg.js',
])
for (const pathname of ['/', '/ssg', '/dashboard', '/dashboard/another']) {
const res = await next.fetch(pathname)
expect(res.status).toBe(200)
}
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment