Unverified Commit 4e6d0557 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add hot-reloading for env file changes (#38483)

* Add hot-reloading for env file changes

* update watching

* update test

* update initial env handling

* undo test change

* add comment for entries clearing

* update on-demand-entry-handler restart handling

* lint-fix

* handle bad plugin
parent 970469f3
......@@ -4,12 +4,13 @@ import * as path from 'path'
import * as dotenv from 'dotenv'
import { expand as dotenvExpand } from 'dotenv-expand'
export type Env = { [key: string]: string }
export type Env = { [key: string]: string | undefined }
export type LoadedEnvFiles = Array<{
path: string
contents: string
}>
let initialEnv: Env | undefined = undefined
let combinedEnv: Env | undefined = undefined
let cachedLoadedEnvFiles: LoadedEnvFiles = []
......@@ -21,18 +22,24 @@ type Log = {
export function processEnv(
loadedEnvFiles: LoadedEnvFiles,
dir?: string,
log: Log = console
log: Log = console,
forceReload = false
) {
// don't reload env if we already have since this breaks escaped
// environment values e.g. \$ENV_FILE_KEY
if (process.env.__NEXT_PROCESSED_ENV || loadedEnvFiles.length === 0) {
if (!initialEnv) {
initialEnv = Object.assign({}, process.env)
}
// only reload env when forceReload is specified
if (
!forceReload &&
(process.env.__NEXT_PROCESSED_ENV || loadedEnvFiles.length === 0)
) {
return process.env as Env
}
// flag that we processed the environment values in case a serverless
// function is re-used or we are running in `next start` mode
process.env.__NEXT_PROCESSED_ENV = 'true'
const origEnv = Object.assign({}, process.env)
const origEnv = Object.assign({}, initialEnv)
const parsed: dotenv.DotenvParseOutput = {}
for (const envFile of loadedEnvFiles) {
......@@ -61,21 +68,27 @@ export function processEnv(
)
}
}
return Object.assign(process.env, parsed)
}
export function loadEnvConfig(
dir: string,
dev?: boolean,
log: Log = console
log: Log = console,
forceReload = false
): {
combinedEnv: Env
loadedEnvFiles: LoadedEnvFiles
} {
// don't reload env if we already have since this breaks escaped
// environment values e.g. \$ENV_FILE_KEY
if (combinedEnv) return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
if (!initialEnv) {
initialEnv = Object.assign({}, process.env)
}
// only reload env when forceReload is specified
if (combinedEnv && !forceReload) {
return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
}
process.env = Object.assign({}, initialEnv)
cachedLoadedEnvFiles = []
const isTest = process.env.NODE_ENV === 'test'
const mode = isTest ? 'test' : dev ? 'development' : 'production'
......@@ -112,6 +125,6 @@ export function loadEnvConfig(
}
}
}
combinedEnv = processEnv(cachedLoadedEnvFiles, dir, log)
combinedEnv = processEnv(cachedLoadedEnvFiles, dir, log, forceReload)
return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
}
......@@ -265,7 +265,10 @@ export default abstract class Server<ServerOptions extends Options = Options> {
res: BaseNextResponse
): void
protected abstract loadEnvConfig(params: { dev: boolean }): void
protected abstract loadEnvConfig(params: {
dev: boolean
forceReload?: boolean
}): void
public constructor(options: ServerOptions) {
const {
......
......@@ -31,6 +31,7 @@ import { findPageFile } from '../lib/find-page-file'
import {
BUILDING,
entries,
getInvalidator,
onDemandEntryHandler,
} from './on-demand-entry-handler'
import { denormalizePagePath } from '../../shared/lib/page-path/denormalize-page-path'
......@@ -941,6 +942,10 @@ export default class HotReloader {
edgeServerStats: () => this.edgeServerStats,
}),
]
// trigger invalidation to ensure any previous callbacks
// are handled in the on-demand-entry-handler
getInvalidator()?.invalidate()
}
public async stop(): Promise<void> {
......
......@@ -252,7 +252,7 @@ export default class DevServer extends Server {
)
let resolved = false
return new Promise((resolve, reject) => {
return new Promise(async (resolve, reject) => {
// Watchpack doesn't emit an event for an empty directory
fs.readdir(this.pagesDir, (_, files) => {
if (files?.length) {
......@@ -265,7 +265,9 @@ export default class DevServer extends Server {
}
})
const wp = (this.webpackWatcher = new Watchpack())
const wp = (this.webpackWatcher = new Watchpack({
ignored: /(node_modules|\.next)/,
}))
const pages = [this.pagesDir]
const app = this.appDir ? [this.appDir] : []
const directories = [...pages, ...app]
......@@ -275,7 +277,16 @@ export default class DevServer extends Server {
)
let nestedMiddleware: string[] = []
wp.watch(files, directories, 0)
const envFiles = [
'.env.development.local',
'.env.local',
'.env.development',
'.env',
].map((file) => pathJoin(this.dir, file))
files.push(...envFiles)
wp.watch({ directories: [this.dir], startTime: 0 })
const envFileTimes = new Map()
wp.on('aggregated', async () => {
let middlewareMatcher: RegExp | undefined
......@@ -283,8 +294,27 @@ export default class DevServer extends Server {
const knownFiles = wp.getTimeInfoEntries()
const appPaths: Record<string, string> = {}
const edgeRoutesSet = new Set<string>()
let envChange = false
for (const [fileName, meta] of knownFiles) {
if (
!files.includes(fileName) &&
!directories.some((dir) => fileName.startsWith(dir))
) {
continue
}
if (envFiles.includes(fileName)) {
if (
envFileTimes.get(fileName) &&
envFileTimes.get(fileName) !== meta.timestamp
) {
envChange = true
}
envFileTimes.set(fileName, meta.timestamp)
continue
}
if (
meta?.accuracy === undefined ||
!regexPageExtension.test(fileName)
......@@ -358,6 +388,12 @@ export default class DevServer extends Server {
routedPages.push(pageName)
}
if (envChange) {
this.loadEnvConfig({ dev: true, forceReload: true })
await this.hotReloader?.stop()
await this.hotReloader?.start()
}
if (nestedMiddleware.length > 0) {
Log.error(
new NestedMiddlewareError(nestedMiddleware, this.dir, this.pagesDir)
......
......@@ -127,6 +127,10 @@ export const entries: {
let invalidator: Invalidator
export const getInvalidator = () => invalidator
const doneCallbacks: EventEmitter | null = new EventEmitter()
const lastClientAccessPages = ['']
const lastServerAccessPagesForAppDir = ['']
export function onDemandEntryHandler({
maxInactiveAge,
multiCompiler,
......@@ -145,9 +149,6 @@ export function onDemandEntryHandler({
appDir?: string
}) {
invalidator = new Invalidator(multiCompiler)
const doneCallbacks: EventEmitter | null = new EventEmitter()
const lastClientAccessPages = ['']
const lastServerAccessPagesForAppDir = ['']
const startBuilding = (_compilation: webpack.Compilation) => {
invalidator.startBuilding()
......@@ -223,11 +224,7 @@ export function onDemandEntryHandler({
const pingIntervalTime = Math.max(1000, Math.min(5000, maxInactiveAge))
setInterval(function () {
disposeInactiveEntries(
lastClientAccessPages,
lastServerAccessPagesForAppDir,
maxInactiveAge
)
disposeInactiveEntries(maxInactiveAge)
}, pingIntervalTime + 1000).unref()
function handleAppDirPing(
......@@ -397,11 +394,7 @@ export function onDemandEntryHandler({
}
}
function disposeInactiveEntries(
lastClientAccessPages: string[],
lastServerAccessPagesForAppDir: string[],
maxInactiveAge: number
) {
function disposeInactiveEntries(maxInactiveAge: number) {
Object.keys(entries).forEach((page) => {
const { lastActiveTime, status, dispose, bundlePath } = entries[page]
......@@ -467,19 +460,19 @@ class Invalidator {
this.building = true
if (!keys || keys.length === 0) {
this.multiCompiler.compilers[0].watching.invalidate()
this.multiCompiler.compilers[1].watching.invalidate()
this.multiCompiler.compilers[2].watching.invalidate()
this.multiCompiler.compilers[0].watching?.invalidate()
this.multiCompiler.compilers[1].watching?.invalidate()
this.multiCompiler.compilers[2].watching?.invalidate()
return
}
for (const key of keys) {
if (key === 'client') {
this.multiCompiler.compilers[0].watching.invalidate()
this.multiCompiler.compilers[0].watching?.invalidate()
} else if (key === 'server') {
this.multiCompiler.compilers[1].watching.invalidate()
this.multiCompiler.compilers[1].watching?.invalidate()
} else if (key === 'edgeServer') {
this.multiCompiler.compilers[2].watching.invalidate()
this.multiCompiler.compilers[2].watching?.invalidate()
}
}
}
......
......@@ -160,8 +160,14 @@ export default class NextNodeServer extends BaseServer {
? (compression() as ExpressMiddleware)
: undefined
protected loadEnvConfig({ dev }: { dev: boolean }) {
loadEnvConfig(this.dir, dev, Log)
protected loadEnvConfig({
dev,
forceReload,
}: {
dev: boolean
forceReload?: boolean
}) {
loadEnvConfig(this.dir, dev, Log, forceReload)
}
protected getPublicDir(): string {
......
......@@ -361,6 +361,12 @@ declare module 'next/dist/compiled/watchpack' {
class Watchpack extends EventEmitter {
constructor(options?: any)
watch(params: {
files?: string[]
directories?: string[]
startTime?: number
missing?: string[]
}): void
watch(files: string[], directories: string[], startTime?: number): void
close(): void
......
export default () => <p>{process.env.NEXT_PUBLIC_HELLO_WORLD}</p>
......@@ -21,18 +21,22 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
]
const items = {
nextConfigEnv: process.env.nextConfigEnv,
nextConfigPublicEnv: process.env.nextConfigPublicEnv,
}
export default async (req, res) => {
const items = {
nextConfigEnv: process.env.nextConfigEnv,
nextConfigPublicEnv: process.env.nextConfigPublicEnv,
}
variables.forEach((variable) => {
items[variable] = process.env[variable]
})
variables.forEach((variable) => {
items[variable] = process.env[variable]
})
export default async (req, res) => {
// Only for testing, don't do this...
res.json(items)
}
......@@ -21,6 +21,10 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
]
export async function getStaticProps() {
......
......@@ -21,6 +21,10 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
]
export async function getStaticProps() {
......
......@@ -21,6 +21,10 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
]
export async function getServerSideProps() {
......
......@@ -12,10 +12,12 @@ import {
launchApp,
killApp,
fetchViaHTTP,
check,
} from 'next-test-utils'
let app
let appPort
let output = ''
const appDir = join(__dirname, '../app')
const getEnvFromHtml = async (path) => {
......@@ -27,7 +29,7 @@ const getEnvFromHtml = async (path) => {
return env
}
const runTests = (mode = 'dev') => {
const runTests = (mode = 'dev', didReload = false) => {
const isDevOnly = mode === 'dev'
const isTestEnv = mode === 'test'
const isDev = isDevOnly || isTestEnv
......@@ -56,6 +58,11 @@ const runTests = (mode = 'dev') => {
expect(data.ENV_FILE_EMPTY_FIRST).toBe(isTestEnv ? '' : '$escaped')
expect(data.ENV_FILE_PROCESS_ENV).toBe('env-cli')
if (didReload) {
expect(data.NEW_ENV_KEY).toBe('true')
expect(data.NEW_ENV_LOCAL_KEY).toBe('hello')
expect(data.NEW_ENV_DEV_KEY).toBe('from-dev')
}
expect(data.nextConfigEnv).toBe('hello from next.config.js')
expect(data.nextConfigPublicEnv).toBe('hello again from next.config.js')
}
......@@ -138,17 +145,84 @@ const runTests = (mode = 'dev') => {
describe('Env Config', () => {
describe('dev mode', () => {
beforeAll(async () => {
output = ''
appPort = await findPort()
app = await launchApp(appDir, appPort, {
env: {
PROCESS_ENV_KEY: 'processenvironment',
ENV_FILE_PROCESS_ENV: 'env-cli',
},
onStdout(msg) {
output += msg || ''
},
onStderr(msg) {
output += msg || ''
},
})
await renderViaHTTP(appPort, '/another-global')
})
afterAll(() => killApp(app))
runTests('dev')
describe('with hot reload', () => {
const originalContents = []
beforeAll(async () => {
const outputIndex = output.length
const envToUpdate = [
{
toAdd: 'NEW_ENV_KEY=true',
file: '.env',
},
{
toAdd: 'NEW_ENV_LOCAL_KEY=hello',
file: '.env.local',
},
{
toAdd: 'NEW_ENV_DEV_KEY=from-dev\nNEXT_PUBLIC_HELLO_WORLD=again',
file: '.env.development',
},
]
for (const { file, toAdd } of envToUpdate) {
const content = await fs.readFile(join(appDir, file), 'utf8')
originalContents.push({ file, content })
await fs.writeFile(join(appDir, file), content + '\n' + toAdd)
}
await check(() => {
return output.substring(outputIndex)
}, /Loaded env from/)
})
afterAll(async () => {
for (const { file, content } of originalContents) {
await fs.writeFile(join(appDir, file), content)
}
})
runTests('dev', true)
it('should update inlined values correctly', async () => {
await renderViaHTTP(appPort, '/another-global')
const buildManifest = await fs.readJson(
join(__dirname, '../app/.next/build-manifest.json')
)
const pageFile = buildManifest.pages['/another-global'].find(
(filename) => filename.includes('pages/another-global')
)
// read client bundle contents since a server side render can
// have the value available during render but it not be injected
const bundleContent = await fs.readFile(
join(appDir, '.next', pageFile),
'utf8'
)
expect(bundleContent).toContain('again')
})
})
})
describe('test environment', () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment