Unverified Commit 823484a7 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Apply env inlining during generate build mode (#76990)

This brings back the env inlining step separate of compilation used for
the [flying shuttle
experiment](https://github.com/vercel/next.js/pull/73710) so that when
`next build --experimental-build-mode=compile` is used we don't inline
these values so that these artifacts can be cached/reused independent of
the env values. Then `next build --experimental-build-mode=generate` can
be used to inline the values and run prerendering to generate finalized
build.
parent 6a16a33c
......@@ -659,5 +659,7 @@
"658": "Pass `Infinity` instead of `false` if you want to cache on the server forever without checking with the origin.",
"659": "SSG should not return an image cache value",
"660": "Rspack support is only available in Next.js canary.",
"661": "Build failed because of %s errors"
"661": "Build failed because of %s errors",
"662": "Invariant failed to find webpack runtime chunk",
"663": "Invariant: client chunk changed but failed to detect hash %s"
}
......@@ -93,4 +93,5 @@ export const NextBuildContext: Partial<{
previewModeId: string
fetchCacheKeyPrefix?: string
allowedRevalidateHeaderKeys?: string[]
isCompileMode?: boolean
}> = {}
......@@ -206,6 +206,7 @@ import {
} from '../server/lib/router-utils/build-prefetch-segment-data-route'
import { turbopackBuild } from './turbopack-build'
import { inlineStaticEnv, populateStaticEnv } from '../lib/inline-static-env'
type Fallback = null | boolean | string
......@@ -821,6 +822,7 @@ export default async function build(
): Promise<void> {
const isCompileMode = experimentalBuildMode === 'compile'
const isGenerateMode = experimentalBuildMode === 'generate'
NextBuildContext.isCompileMode = isCompileMode
let loadedConfig: NextConfigComplete | undefined
try {
......@@ -880,6 +882,12 @@ export default async function build(
)
NextBuildContext.buildId = buildId
// when using compile mode static env isn't inlined so we
// need to populate in normal runtime env
if (isCompileMode || isGenerateMode) {
populateStaticEnv(config)
}
const customRoutes: CustomRoutes = await nextBuildSpan
.traceChild('load-custom-routes')
.traceAsyncFn(() => loadCustomRoutes(config))
......@@ -2483,6 +2491,20 @@ export default async function build(
requiredServerFilesManifest
)
// we don't need to inline for turbopack build as
// it will handle it's own caching separate of compile
if (isGenerateMode && !turboNextBuild) {
await nextBuildSpan
.traceChild('inline-static-env')
.traceAsyncFn(async () => {
await inlineStaticEnv({
distDir,
config,
buildId,
})
})
}
const middlewareManifest: MiddlewareManifest = await readManifest(
path.join(distDir, SERVER_DIRECTORY, MIDDLEWARE_MANIFEST)
)
......
......@@ -107,6 +107,7 @@ export async function webpackBuildImpl(
const commonWebpackOptions = {
isServer: false,
isCompileMode: NextBuildContext.isCompileMode,
buildId: NextBuildContext.buildId!,
encryptionKey: NextBuildContext.encryptionKey!,
config: config,
......
......@@ -306,7 +306,9 @@ export default async function getBaseWebpackConfig(
clientRouterFilters,
fetchCacheKeyPrefix,
edgePreviewProps,
isCompileMode,
}: {
isCompileMode?: boolean
buildId: string
encryptionKey: string
config: NextConfigComplete
......@@ -1909,6 +1911,7 @@ export default async function getBaseWebpackConfig(
isNodeOrEdgeCompilation,
isNodeServer,
middlewareMatchers,
omitNonDeterministic: isCompileMode,
}),
isClient &&
new ReactLoadablePlugin({
......
......@@ -38,6 +38,7 @@ export interface DefineEnvPluginOptions {
isNodeOrEdgeCompilation: boolean
isNodeServer: boolean
middlewareMatchers: MiddlewareMatcher[] | undefined
omitNonDeterministic?: boolean
}
interface DefineEnv {
......@@ -140,6 +141,7 @@ export function getDefineEnv({
isNodeOrEdgeCompilation,
isNodeServer,
middlewareMatchers,
omitNonDeterministic,
}: DefineEnvPluginOptions): SerializedDefineEnv {
const nextPublicEnv = getNextPublicEnvironmentVariables()
const nextConfigEnv = getNextConfigEnv(config)
......@@ -304,7 +306,30 @@ export function getDefineEnv({
defineEnv[key] = userDefines[key]
}
return serializeDefineEnv(defineEnv)
const serializedDefineEnv = serializeDefineEnv(defineEnv)
// we delay inlining these values until after the build
// with flying shuttle enabled so we can update them
// without invalidating entries
if (!dev && omitNonDeterministic) {
// client uses window. instead of leaving process.env
// in case process isn't polyfilled on client already
// since by this point it won't be added by webpack
const safeKey = (key: string) =>
isClient ? `window.${key.split('.').pop()}` : key
for (const key in nextPublicEnv) {
serializedDefineEnv[key] = safeKey(key)
}
for (const key in nextConfigEnv) {
serializedDefineEnv[key] = safeKey(key)
}
for (const key of ['process.env.NEXT_DEPLOYMENT_ID']) {
serializedDefineEnv[key] = safeKey(key)
}
}
return serializedDefineEnv
}
export function getDefineEnvPlugin(options: DefineEnvPluginOptions) {
......
import fs from 'fs'
import path from 'path'
import crypto from 'crypto'
import { promisify } from 'util'
import globOriginal from 'next/dist/compiled/glob'
import {
getNextConfigEnv,
getNextPublicEnvironmentVariables,
} from '../build/webpack/plugins/define-env-plugin'
import { Sema } from 'next/dist/compiled/async-sema'
import type { NextConfigComplete } from '../server/config-shared'
import { BUILD_MANIFEST } from '../shared/lib/constants'
const glob = promisify(globOriginal)
const getStaticEnv = (config: NextConfigComplete) => {
const staticEnv: Record<string, string | undefined> = {
...getNextPublicEnvironmentVariables(),
...getNextConfigEnv(config),
'process.env.NEXT_DEPLOYMENT_ID': config.deploymentId || '',
}
return staticEnv
}
export function populateStaticEnv(config: NextConfigComplete) {
// since inlining comes after static generation we need
// to ensure this value is assigned to process env so it
// can still be accessed
const staticEnv = getStaticEnv(config)
for (const key in staticEnv) {
const innerKey = key.split('.').pop() || ''
if (!process.env[innerKey]) {
process.env[innerKey] = staticEnv[key] || ''
}
}
}
export async function inlineStaticEnv({
distDir,
config,
buildId,
}: {
distDir: string
buildId: string
config: NextConfigComplete
}) {
const nextConfigEnv = getNextConfigEnv(config)
const staticEnv = getStaticEnv(config)
const serverDir = path.join(distDir, 'server')
const serverChunks = await glob('**/*.js', {
cwd: serverDir,
})
const clientDir = path.join(distDir, 'static')
const clientChunks = await glob('**/*.js', {
cwd: clientDir,
})
const webpackRuntimeFile = clientChunks.find((item) =>
item.match(/webpack-[a-z0-9]{16}/)
)
if (!webpackRuntimeFile) {
throw new Error(`Invariant failed to find webpack runtime chunk`)
}
const inlineSema = new Sema(8)
const nextConfigEnvKeys = Object.keys(nextConfigEnv).map((item) =>
item.split('process.env.').pop()
)
const builtRegEx = new RegExp(
`[\\w]{1,}(\\.env)?\\.(?:NEXT_PUBLIC_[\\w]{1,}${nextConfigEnvKeys.length ? '|' + nextConfigEnvKeys.join('|') : ''})`,
'g'
)
const changedClientFiles: Array<{ file: string; content: string }> = []
for (const [parentDir, files] of [
[serverDir, serverChunks],
[clientDir, clientChunks],
] as const) {
await Promise.all(
files.map(async (file) => {
await inlineSema.acquire()
const filepath = path.join(parentDir, file)
const content = await fs.promises.readFile(filepath, 'utf8')
const newContent = content.replace(builtRegEx, (match) => {
let normalizedMatch = `process.env.${match.split('.').pop()}`
if (staticEnv[normalizedMatch]) {
return JSON.stringify(staticEnv[normalizedMatch])
}
return match
})
await fs.promises.writeFile(filepath, newContent)
if (content !== newContent && parentDir === clientDir) {
changedClientFiles.push({ file, content: newContent })
}
inlineSema.release()
})
)
}
const hashChanges: Array<{
originalHash: string
newHash: string
}> = []
// hashes need updating for any changed client files
for (const { file, content } of changedClientFiles) {
// hash is 16 chars currently for all client chunks
const originalHash = file.match(/([a-z0-9]{16})\./)?.[1] || ''
if (!originalHash) {
throw new Error(
`Invariant: client chunk changed but failed to detect hash ${file}`
)
}
const newHash = crypto
.createHash('sha256')
.update(content)
.digest('hex')
.substring(0, 16)
hashChanges.push({ originalHash, newHash })
const filepath = path.join(clientDir, file)
await fs.promises.rename(filepath, filepath.replace(originalHash, newHash))
}
// update build-manifest and webpack-runtime with new hashes
for (const file of [
path.join(distDir, BUILD_MANIFEST),
path.join(distDir, 'static', webpackRuntimeFile),
path.join(distDir, 'static', buildId, '_buildManifest.js'),
]) {
const content = await fs.promises.readFile(file, 'utf-8')
let newContent = content
for (const { originalHash, newHash } of hashChanges) {
newContent = newContent.replaceAll(originalHash, newHash)
}
if (content !== newContent) {
await fs.promises.writeFile(file, newContent)
}
}
}
......@@ -614,6 +614,17 @@ export default abstract class Server<
reactMaxHeadersLength: this.nextConfig.reactMaxHeadersLength,
}
if (process.env.NEXT_RUNTIME !== 'edge') {
const { populateStaticEnv } =
require('../lib/inline-static-env') as typeof import('../lib/inline-static-env')
// when using compile mode static env isn't inlined so we
// need to populate in normal runtime env
if (this.renderOpts.isExperimentalCompile) {
populateStaticEnv(this.nextConfig)
}
}
// Initialize next/config with the environment configuration
setConfig({
serverRuntimeConfig,
......
......@@ -18,6 +18,9 @@ describe('app dir - basic', () => {
dependencies: {
nanoid: '4.0.1',
},
env: {
NEXT_PUBLIC_TEST_ID: Date.now() + '',
},
})
if (isNextStart) {
......@@ -1788,4 +1791,34 @@ describe('app dir - basic', () => {
})
}
})
// this one comes at the end to not change behavior from above
// assertions with compile mode specifically
// consider breaking out into separate fixture if we expand this any more
if (process.env.NEXT_EXPERIMENTAL_COMPILE) {
it('should run generate command correctly', async () => {
await next.stop()
next.buildCommand = `pnpm next build --experimental-build-mode=generate`
await next.start()
let browser = await next.browser('/')
expect(await browser.elementByCss('#my-env').text()).toBe(
next.env.NEXT_PUBLIC_TEST_ID
)
expect(await browser.elementByCss('#my-other-env').text()).toBe(
`${next.env.NEXT_PUBLIC_TEST_ID}-suffix`
)
browser = await next.browser('/dashboard/deployments/123')
expect(await browser.elementByCss('#my-env').text()).toBe(
next.env.NEXT_PUBLIC_TEST_ID
)
expect(await browser.elementByCss('#my-other-env').text()).toBe(
`${next.env.NEXT_PUBLIC_TEST_ID}-suffix`
)
})
}
})
......@@ -272,6 +272,15 @@ export function nextTestSetup(
const prop = next[property]
return typeof prop === 'function' ? prop.bind(next) : prop
},
set: function (_target, key, value) {
if (!next) {
throw new Error(
'next instance is not initialized yet, make sure you call methods on next instance in test body.'
)
}
next[key] = value
return true
},
})
return {
......
......@@ -65,8 +65,8 @@ export class NextInstance {
protected overrideFiles: ResolvedFileConfig
protected nextConfig?: NextConfig
protected installCommand?: InstallCommand
protected buildCommand?: string
protected buildOptions?: string
public buildCommand?: string
public buildOptions?: string
protected startCommand?: string
protected startOptions?: string[]
protected dependencies?: PackageJson['dependencies'] = {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment