Unverified Commit 953ec519 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Ensure experimental trace invalidates correctly (#68076)

This ensures we invalidate the experimental trace result when Next.js
versions mismatch or our global hash differs as these changes should
re-build everything to be safe we aren't missing an important change.

Closes: NDX-104
parent c21bf9cb
...@@ -3,6 +3,8 @@ import path from 'path' ...@@ -3,6 +3,8 @@ import path from 'path'
import crypto from 'crypto' import crypto from 'crypto'
import { getPageFromPath } from '../entries' import { getPageFromPath } from '../entries'
import { Sema } from 'next/dist/compiled/async-sema' import { Sema } from 'next/dist/compiled/async-sema'
import { generateShuttleManifest } from './store-shuttle'
import type { NextConfigComplete } from '../../server/config-shared'
export interface DetectedEntriesResult { export interface DetectedEntriesResult {
app: string[] app: string[]
...@@ -10,15 +12,47 @@ export interface DetectedEntriesResult { ...@@ -10,15 +12,47 @@ export interface DetectedEntriesResult {
} }
let _hasShuttle: undefined | boolean = undefined let _hasShuttle: undefined | boolean = undefined
export async function hasShuttle(shuttleDir: string) { export async function hasShuttle(
config: NextConfigComplete,
shuttleDir: string
) {
if (typeof _hasShuttle === 'boolean') { if (typeof _hasShuttle === 'boolean') {
return _hasShuttle return _hasShuttle
} }
_hasShuttle = await fs.promises let foundShuttleManifest:
.access(path.join(shuttleDir, 'server')) | ReturnType<typeof generateShuttleManifest>
.then(() => true) | undefined
.catch(() => false)
try {
foundShuttleManifest = JSON.parse(
await fs.promises
.readFile(path.join(shuttleDir, 'shuttle-manifest.json'), 'utf8')
.catch(() => '{}')
)
} catch (err: unknown) {
_hasShuttle = false
return _hasShuttle
}
const currentShuttleManifest = generateShuttleManifest(config)
if (
_hasShuttle !== false &&
currentShuttleManifest.nextVersion !== foundShuttleManifest?.nextVersion
) {
// we don't allow using shuttle from differing Next.js version
// as it could have internal changes
console.log(`shuttle has differing Next.js version, skipping.`)
_hasShuttle = false
}
if (
_hasShuttle !== false &&
currentShuttleManifest.globalHash !== foundShuttleManifest?.globalHash
) {
// if the global hash was invalidated we bypass the cache to be safe
console.log(`shuttle has differing global hash, skipping.`)
_hasShuttle = false
}
return _hasShuttle return _hasShuttle
} }
...@@ -28,12 +62,14 @@ export async function detectChangedEntries({ ...@@ -28,12 +62,14 @@ export async function detectChangedEntries({
pageExtensions, pageExtensions,
distDir, distDir,
shuttleDir, shuttleDir,
config,
}: { }: {
appPaths?: string[] appPaths?: string[]
pagesPaths?: string[] pagesPaths?: string[]
pageExtensions: string[] pageExtensions: string[]
distDir: string distDir: string
shuttleDir: string shuttleDir: string
config: NextConfigComplete
}): Promise<{ }): Promise<{
changed: DetectedEntriesResult changed: DetectedEntriesResult
unchanged: DetectedEntriesResult unchanged: DetectedEntriesResult
...@@ -50,7 +86,7 @@ export async function detectChangedEntries({ ...@@ -50,7 +86,7 @@ export async function detectChangedEntries({
pages: [], pages: [],
} }
if (!(await hasShuttle(shuttleDir))) { if (!(await hasShuttle(config, shuttleDir))) {
// no shuttle so consider everything changed // no shuttle so consider everything changed
console.log(`no shuttle. can't detect changes`) console.log(`no shuttle. can't detect changes`)
return { return {
......
...@@ -31,9 +31,11 @@ import { ...@@ -31,9 +31,11 @@ import {
ROUTES_MANIFEST, ROUTES_MANIFEST,
} from '../../shared/lib/constants' } from '../../shared/lib/constants'
import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths'
import type { NextConfigComplete } from '../../server/config-shared'
export async function stitchBuilds( export async function stitchBuilds(
{ {
config,
distDir, distDir,
shuttleDir, shuttleDir,
buildId, buildId,
...@@ -43,6 +45,7 @@ export async function stitchBuilds( ...@@ -43,6 +45,7 @@ export async function stitchBuilds(
encryptionKey, encryptionKey,
edgePreviewProps, edgePreviewProps,
}: { }: {
config: NextConfigComplete
buildId: string buildId: string
distDir: string distDir: string
shuttleDir: string shuttleDir: string
...@@ -64,7 +67,7 @@ export async function stitchBuilds( ...@@ -64,7 +67,7 @@ export async function stitchBuilds(
): Promise<{ ): Promise<{
pagesManifest?: PagesManifest pagesManifest?: PagesManifest
}> { }> {
if (!(await hasShuttle(shuttleDir))) { if (!(await hasShuttle(config, shuttleDir))) {
// no shuttle directory nothing to stitch // no shuttle directory nothing to stitch
return {} return {}
} }
......
import fs from 'fs' import fs from 'fs'
import path from 'path' import path from 'path'
import crypto from 'crypto'
import { recursiveCopy } from '../../lib/recursive-copy'
import { version as nextVersion } from 'next/package.json'
import type { NextConfigComplete } from '../../server/config-shared'
import { import {
BUILD_MANIFEST, BUILD_MANIFEST,
APP_BUILD_MANIFEST, APP_BUILD_MANIFEST,
...@@ -8,20 +12,57 @@ import { ...@@ -8,20 +12,57 @@ import {
PAGES_MANIFEST, PAGES_MANIFEST,
ROUTES_MANIFEST, ROUTES_MANIFEST,
} from '../../shared/lib/constants' } from '../../shared/lib/constants'
import { recursiveCopy } from '../../lib/recursive-copy' import { getNextPublicEnvironmentVariables } from '../webpack/plugins/define-env-plugin'
export function generateShuttleManifest(config: NextConfigComplete) {
// NEXT_PUBLIC_ changes for now since they are inlined
// and specific next config values that can impact the build
const globalHash = crypto.createHash('sha256')
for (const key in getNextPublicEnvironmentVariables()) {
globalHash.update(`${key}=${process.env[key]}`)
}
const omittedConfigKeys = ['headers', 'rewrites', 'redirects']
for (const key in config) {
if (omittedConfigKeys.includes(key)) {
continue
}
let serializedConfig =
typeof config[key] === 'function'
? config[key].toString()
: JSON.stringify(config[key])
globalHash.update(`${key}=${serializedConfig}`)
}
return {
nextVersion,
globalHash: globalHash.digest('hex'),
}
}
// we can create a new shuttle with the outputs before env values have // we can create a new shuttle with the outputs before env values have
// been inlined, can be done after stitching takes place // been inlined, can be done after stitching takes place
export async function storeShuttle({ export async function storeShuttle({
config,
distDir, distDir,
shuttleDir, shuttleDir,
}: { }: {
distDir: string distDir: string
shuttleDir: string shuttleDir: string
config: NextConfigComplete
}) { }) {
await fs.promises.rm(shuttleDir, { force: true, recursive: true }) await fs.promises.rm(shuttleDir, { force: true, recursive: true })
await fs.promises.mkdir(shuttleDir, { recursive: true }) await fs.promises.mkdir(shuttleDir, { recursive: true })
const shuttleManifest = generateShuttleManifest(config)
await fs.promises.writeFile(
path.join(shuttleDir, 'shuttle-manifest.json'),
JSON.stringify(shuttleManifest)
)
// copy all server entries // copy all server entries
await recursiveCopy( await recursiveCopy(
path.join(distDir, 'server'), path.join(distDir, 'server'),
......
...@@ -869,6 +869,7 @@ export default async function build( ...@@ -869,6 +869,7 @@ export default async function build(
pageExtensions: config.pageExtensions, pageExtensions: config.pageExtensions,
distDir, distDir,
shuttleDir, shuttleDir,
config,
}) })
console.log({ changedPagePathsResult }) console.log({ changedPagePathsResult })
pagesPaths = changedPagePathsResult.changed.pages pagesPaths = changedPagePathsResult.changed.pages
...@@ -960,6 +961,7 @@ export default async function build( ...@@ -960,6 +961,7 @@ export default async function build(
pageExtensions: config.pageExtensions, pageExtensions: config.pageExtensions,
distDir, distDir,
shuttleDir, shuttleDir,
config,
}) })
console.log({ changedAppPathsResult }) console.log({ changedAppPathsResult })
appPaths = changedAppPathsResult.changed.app appPaths = changedAppPathsResult.changed.app
...@@ -2482,6 +2484,7 @@ export default async function build( ...@@ -2482,6 +2484,7 @@ export default async function build(
console.log('stitching builds...') console.log('stitching builds...')
const stitchResult = await stitchBuilds( const stitchResult = await stitchBuilds(
{ {
config,
buildId, buildId,
distDir, distDir,
shuttleDir, shuttleDir,
...@@ -2518,6 +2521,7 @@ export default async function build( ...@@ -2518,6 +2521,7 @@ export default async function build(
console.log('storing shuttle') console.log('storing shuttle')
await storeShuttle({ await storeShuttle({
config,
distDir, distDir,
shuttleDir, shuttleDir,
}) })
......
...@@ -58,7 +58,7 @@ interface SerializedDefineEnv { ...@@ -58,7 +58,7 @@ interface SerializedDefineEnv {
/** /**
* Collects all environment variables that are using the `NEXT_PUBLIC_` prefix. * Collects all environment variables that are using the `NEXT_PUBLIC_` prefix.
*/ */
function getNextPublicEnvironmentVariables(): DefineEnv { export function getNextPublicEnvironmentVariables(): DefineEnv {
const defineEnv: DefineEnv = {} const defineEnv: DefineEnv = {}
for (const key in process.env) { for (const key in process.env) {
if (key.startsWith('NEXT_PUBLIC_')) { if (key.startsWith('NEXT_PUBLIC_')) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment