Unverified Commit 4e6d0557 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add hot-reloading for env file changes (#38483)

* Add hot-reloading for env file changes

* update watching

* update test

* update initial env handling

* undo test change

* add comment for entries clearing

* update on-demand-entry-handler restart handling

* lint-fix

* handle bad plugin
parent 970469f3
...@@ -4,12 +4,13 @@ import * as path from 'path' ...@@ -4,12 +4,13 @@ import * as path from 'path'
import * as dotenv from 'dotenv' import * as dotenv from 'dotenv'
import { expand as dotenvExpand } from 'dotenv-expand' import { expand as dotenvExpand } from 'dotenv-expand'
export type Env = { [key: string]: string } export type Env = { [key: string]: string | undefined }
export type LoadedEnvFiles = Array<{ export type LoadedEnvFiles = Array<{
path: string path: string
contents: string contents: string
}> }>
let initialEnv: Env | undefined = undefined
let combinedEnv: Env | undefined = undefined let combinedEnv: Env | undefined = undefined
let cachedLoadedEnvFiles: LoadedEnvFiles = [] let cachedLoadedEnvFiles: LoadedEnvFiles = []
...@@ -21,18 +22,24 @@ type Log = { ...@@ -21,18 +22,24 @@ type Log = {
export function processEnv( export function processEnv(
loadedEnvFiles: LoadedEnvFiles, loadedEnvFiles: LoadedEnvFiles,
dir?: string, dir?: string,
log: Log = console log: Log = console,
forceReload = false
) { ) {
// don't reload env if we already have since this breaks escaped if (!initialEnv) {
// environment values e.g. \$ENV_FILE_KEY initialEnv = Object.assign({}, process.env)
if (process.env.__NEXT_PROCESSED_ENV || loadedEnvFiles.length === 0) { }
// only reload env when forceReload is specified
if (
!forceReload &&
(process.env.__NEXT_PROCESSED_ENV || loadedEnvFiles.length === 0)
) {
return process.env as Env return process.env as Env
} }
// flag that we processed the environment values in case a serverless // flag that we processed the environment values in case a serverless
// function is re-used or we are running in `next start` mode // function is re-used or we are running in `next start` mode
process.env.__NEXT_PROCESSED_ENV = 'true' process.env.__NEXT_PROCESSED_ENV = 'true'
const origEnv = Object.assign({}, process.env) const origEnv = Object.assign({}, initialEnv)
const parsed: dotenv.DotenvParseOutput = {} const parsed: dotenv.DotenvParseOutput = {}
for (const envFile of loadedEnvFiles) { for (const envFile of loadedEnvFiles) {
...@@ -61,21 +68,27 @@ export function processEnv( ...@@ -61,21 +68,27 @@ export function processEnv(
) )
} }
} }
return Object.assign(process.env, parsed) return Object.assign(process.env, parsed)
} }
export function loadEnvConfig( export function loadEnvConfig(
dir: string, dir: string,
dev?: boolean, dev?: boolean,
log: Log = console log: Log = console,
forceReload = false
): { ): {
combinedEnv: Env combinedEnv: Env
loadedEnvFiles: LoadedEnvFiles loadedEnvFiles: LoadedEnvFiles
} { } {
// don't reload env if we already have since this breaks escaped if (!initialEnv) {
// environment values e.g. \$ENV_FILE_KEY initialEnv = Object.assign({}, process.env)
if (combinedEnv) return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles } }
// only reload env when forceReload is specified
if (combinedEnv && !forceReload) {
return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
}
process.env = Object.assign({}, initialEnv)
cachedLoadedEnvFiles = []
const isTest = process.env.NODE_ENV === 'test' const isTest = process.env.NODE_ENV === 'test'
const mode = isTest ? 'test' : dev ? 'development' : 'production' const mode = isTest ? 'test' : dev ? 'development' : 'production'
...@@ -112,6 +125,6 @@ export function loadEnvConfig( ...@@ -112,6 +125,6 @@ export function loadEnvConfig(
} }
} }
} }
combinedEnv = processEnv(cachedLoadedEnvFiles, dir, log) combinedEnv = processEnv(cachedLoadedEnvFiles, dir, log, forceReload)
return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles } return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
} }
...@@ -265,7 +265,10 @@ export default abstract class Server<ServerOptions extends Options = Options> { ...@@ -265,7 +265,10 @@ export default abstract class Server<ServerOptions extends Options = Options> {
res: BaseNextResponse res: BaseNextResponse
): void ): void
protected abstract loadEnvConfig(params: { dev: boolean }): void protected abstract loadEnvConfig(params: {
dev: boolean
forceReload?: boolean
}): void
public constructor(options: ServerOptions) { public constructor(options: ServerOptions) {
const { const {
......
...@@ -31,6 +31,7 @@ import { findPageFile } from '../lib/find-page-file' ...@@ -31,6 +31,7 @@ import { findPageFile } from '../lib/find-page-file'
import { import {
BUILDING, BUILDING,
entries, entries,
getInvalidator,
onDemandEntryHandler, onDemandEntryHandler,
} from './on-demand-entry-handler' } from './on-demand-entry-handler'
import { denormalizePagePath } from '../../shared/lib/page-path/denormalize-page-path' import { denormalizePagePath } from '../../shared/lib/page-path/denormalize-page-path'
...@@ -941,6 +942,10 @@ export default class HotReloader { ...@@ -941,6 +942,10 @@ export default class HotReloader {
edgeServerStats: () => this.edgeServerStats, edgeServerStats: () => this.edgeServerStats,
}), }),
] ]
// trigger invalidation to ensure any previous callbacks
// are handled in the on-demand-entry-handler
getInvalidator()?.invalidate()
} }
public async stop(): Promise<void> { public async stop(): Promise<void> {
......
...@@ -252,7 +252,7 @@ export default class DevServer extends Server { ...@@ -252,7 +252,7 @@ export default class DevServer extends Server {
) )
let resolved = false let resolved = false
return new Promise((resolve, reject) => { return new Promise(async (resolve, reject) => {
// Watchpack doesn't emit an event for an empty directory // Watchpack doesn't emit an event for an empty directory
fs.readdir(this.pagesDir, (_, files) => { fs.readdir(this.pagesDir, (_, files) => {
if (files?.length) { if (files?.length) {
...@@ -265,7 +265,9 @@ export default class DevServer extends Server { ...@@ -265,7 +265,9 @@ export default class DevServer extends Server {
} }
}) })
const wp = (this.webpackWatcher = new Watchpack()) const wp = (this.webpackWatcher = new Watchpack({
ignored: /(node_modules|\.next)/,
}))
const pages = [this.pagesDir] const pages = [this.pagesDir]
const app = this.appDir ? [this.appDir] : [] const app = this.appDir ? [this.appDir] : []
const directories = [...pages, ...app] const directories = [...pages, ...app]
...@@ -275,7 +277,16 @@ export default class DevServer extends Server { ...@@ -275,7 +277,16 @@ export default class DevServer extends Server {
) )
let nestedMiddleware: string[] = [] let nestedMiddleware: string[] = []
wp.watch(files, directories, 0) const envFiles = [
'.env.development.local',
'.env.local',
'.env.development',
'.env',
].map((file) => pathJoin(this.dir, file))
files.push(...envFiles)
wp.watch({ directories: [this.dir], startTime: 0 })
const envFileTimes = new Map()
wp.on('aggregated', async () => { wp.on('aggregated', async () => {
let middlewareMatcher: RegExp | undefined let middlewareMatcher: RegExp | undefined
...@@ -283,8 +294,27 @@ export default class DevServer extends Server { ...@@ -283,8 +294,27 @@ export default class DevServer extends Server {
const knownFiles = wp.getTimeInfoEntries() const knownFiles = wp.getTimeInfoEntries()
const appPaths: Record<string, string> = {} const appPaths: Record<string, string> = {}
const edgeRoutesSet = new Set<string>() const edgeRoutesSet = new Set<string>()
let envChange = false
for (const [fileName, meta] of knownFiles) { for (const [fileName, meta] of knownFiles) {
if (
!files.includes(fileName) &&
!directories.some((dir) => fileName.startsWith(dir))
) {
continue
}
if (envFiles.includes(fileName)) {
if (
envFileTimes.get(fileName) &&
envFileTimes.get(fileName) !== meta.timestamp
) {
envChange = true
}
envFileTimes.set(fileName, meta.timestamp)
continue
}
if ( if (
meta?.accuracy === undefined || meta?.accuracy === undefined ||
!regexPageExtension.test(fileName) !regexPageExtension.test(fileName)
...@@ -358,6 +388,12 @@ export default class DevServer extends Server { ...@@ -358,6 +388,12 @@ export default class DevServer extends Server {
routedPages.push(pageName) routedPages.push(pageName)
} }
if (envChange) {
this.loadEnvConfig({ dev: true, forceReload: true })
await this.hotReloader?.stop()
await this.hotReloader?.start()
}
if (nestedMiddleware.length > 0) { if (nestedMiddleware.length > 0) {
Log.error( Log.error(
new NestedMiddlewareError(nestedMiddleware, this.dir, this.pagesDir) new NestedMiddlewareError(nestedMiddleware, this.dir, this.pagesDir)
......
...@@ -127,6 +127,10 @@ export const entries: { ...@@ -127,6 +127,10 @@ export const entries: {
let invalidator: Invalidator let invalidator: Invalidator
export const getInvalidator = () => invalidator export const getInvalidator = () => invalidator
const doneCallbacks: EventEmitter | null = new EventEmitter()
const lastClientAccessPages = ['']
const lastServerAccessPagesForAppDir = ['']
export function onDemandEntryHandler({ export function onDemandEntryHandler({
maxInactiveAge, maxInactiveAge,
multiCompiler, multiCompiler,
...@@ -145,9 +149,6 @@ export function onDemandEntryHandler({ ...@@ -145,9 +149,6 @@ export function onDemandEntryHandler({
appDir?: string appDir?: string
}) { }) {
invalidator = new Invalidator(multiCompiler) invalidator = new Invalidator(multiCompiler)
const doneCallbacks: EventEmitter | null = new EventEmitter()
const lastClientAccessPages = ['']
const lastServerAccessPagesForAppDir = ['']
const startBuilding = (_compilation: webpack.Compilation) => { const startBuilding = (_compilation: webpack.Compilation) => {
invalidator.startBuilding() invalidator.startBuilding()
...@@ -223,11 +224,7 @@ export function onDemandEntryHandler({ ...@@ -223,11 +224,7 @@ export function onDemandEntryHandler({
const pingIntervalTime = Math.max(1000, Math.min(5000, maxInactiveAge)) const pingIntervalTime = Math.max(1000, Math.min(5000, maxInactiveAge))
setInterval(function () { setInterval(function () {
disposeInactiveEntries( disposeInactiveEntries(maxInactiveAge)
lastClientAccessPages,
lastServerAccessPagesForAppDir,
maxInactiveAge
)
}, pingIntervalTime + 1000).unref() }, pingIntervalTime + 1000).unref()
function handleAppDirPing( function handleAppDirPing(
...@@ -397,11 +394,7 @@ export function onDemandEntryHandler({ ...@@ -397,11 +394,7 @@ export function onDemandEntryHandler({
} }
} }
function disposeInactiveEntries( function disposeInactiveEntries(maxInactiveAge: number) {
lastClientAccessPages: string[],
lastServerAccessPagesForAppDir: string[],
maxInactiveAge: number
) {
Object.keys(entries).forEach((page) => { Object.keys(entries).forEach((page) => {
const { lastActiveTime, status, dispose, bundlePath } = entries[page] const { lastActiveTime, status, dispose, bundlePath } = entries[page]
...@@ -467,19 +460,19 @@ class Invalidator { ...@@ -467,19 +460,19 @@ class Invalidator {
this.building = true this.building = true
if (!keys || keys.length === 0) { if (!keys || keys.length === 0) {
this.multiCompiler.compilers[0].watching.invalidate() this.multiCompiler.compilers[0].watching?.invalidate()
this.multiCompiler.compilers[1].watching.invalidate() this.multiCompiler.compilers[1].watching?.invalidate()
this.multiCompiler.compilers[2].watching.invalidate() this.multiCompiler.compilers[2].watching?.invalidate()
return return
} }
for (const key of keys) { for (const key of keys) {
if (key === 'client') { if (key === 'client') {
this.multiCompiler.compilers[0].watching.invalidate() this.multiCompiler.compilers[0].watching?.invalidate()
} else if (key === 'server') { } else if (key === 'server') {
this.multiCompiler.compilers[1].watching.invalidate() this.multiCompiler.compilers[1].watching?.invalidate()
} else if (key === 'edgeServer') { } else if (key === 'edgeServer') {
this.multiCompiler.compilers[2].watching.invalidate() this.multiCompiler.compilers[2].watching?.invalidate()
} }
} }
} }
......
...@@ -160,8 +160,14 @@ export default class NextNodeServer extends BaseServer { ...@@ -160,8 +160,14 @@ export default class NextNodeServer extends BaseServer {
? (compression() as ExpressMiddleware) ? (compression() as ExpressMiddleware)
: undefined : undefined
protected loadEnvConfig({ dev }: { dev: boolean }) { protected loadEnvConfig({
loadEnvConfig(this.dir, dev, Log) dev,
forceReload,
}: {
dev: boolean
forceReload?: boolean
}) {
loadEnvConfig(this.dir, dev, Log, forceReload)
} }
protected getPublicDir(): string { protected getPublicDir(): string {
......
...@@ -361,6 +361,12 @@ declare module 'next/dist/compiled/watchpack' { ...@@ -361,6 +361,12 @@ declare module 'next/dist/compiled/watchpack' {
class Watchpack extends EventEmitter { class Watchpack extends EventEmitter {
constructor(options?: any) constructor(options?: any)
watch(params: {
files?: string[]
directories?: string[]
startTime?: number
missing?: string[]
}): void
watch(files: string[], directories: string[], startTime?: number): void watch(files: string[], directories: string[], startTime?: number): void
close(): void close(): void
......
export default () => <p>{process.env.NEXT_PUBLIC_HELLO_WORLD}</p>
...@@ -21,18 +21,22 @@ const variables = [ ...@@ -21,18 +21,22 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT', 'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED', 'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION', 'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
] ]
const items = { export default async (req, res) => {
nextConfigEnv: process.env.nextConfigEnv, const items = {
nextConfigPublicEnv: process.env.nextConfigPublicEnv, nextConfigEnv: process.env.nextConfigEnv,
} nextConfigPublicEnv: process.env.nextConfigPublicEnv,
}
variables.forEach((variable) => { variables.forEach((variable) => {
items[variable] = process.env[variable] items[variable] = process.env[variable]
}) })
export default async (req, res) => {
// Only for testing, don't do this... // Only for testing, don't do this...
res.json(items) res.json(items)
} }
...@@ -21,6 +21,10 @@ const variables = [ ...@@ -21,6 +21,10 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT', 'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED', 'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION', 'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
] ]
export async function getStaticProps() { export async function getStaticProps() {
......
...@@ -21,6 +21,10 @@ const variables = [ ...@@ -21,6 +21,10 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT', 'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED', 'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION', 'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
] ]
export async function getStaticProps() { export async function getStaticProps() {
......
...@@ -21,6 +21,10 @@ const variables = [ ...@@ -21,6 +21,10 @@ const variables = [
'ENV_FILE_EXPANDED_CONCAT', 'ENV_FILE_EXPANDED_CONCAT',
'ENV_FILE_EXPANDED_ESCAPED', 'ENV_FILE_EXPANDED_ESCAPED',
'ENV_FILE_KEY_EXCLAMATION', 'ENV_FILE_KEY_EXCLAMATION',
'NEW_ENV_KEY',
'NEW_ENV_LOCAL_KEY',
'NEW_ENV_DEV_KEY',
'NEXT_PUBLIC_HELLO_WORLD',
] ]
export async function getServerSideProps() { export async function getServerSideProps() {
......
...@@ -12,10 +12,12 @@ import { ...@@ -12,10 +12,12 @@ import {
launchApp, launchApp,
killApp, killApp,
fetchViaHTTP, fetchViaHTTP,
check,
} from 'next-test-utils' } from 'next-test-utils'
let app let app
let appPort let appPort
let output = ''
const appDir = join(__dirname, '../app') const appDir = join(__dirname, '../app')
const getEnvFromHtml = async (path) => { const getEnvFromHtml = async (path) => {
...@@ -27,7 +29,7 @@ const getEnvFromHtml = async (path) => { ...@@ -27,7 +29,7 @@ const getEnvFromHtml = async (path) => {
return env return env
} }
const runTests = (mode = 'dev') => { const runTests = (mode = 'dev', didReload = false) => {
const isDevOnly = mode === 'dev' const isDevOnly = mode === 'dev'
const isTestEnv = mode === 'test' const isTestEnv = mode === 'test'
const isDev = isDevOnly || isTestEnv const isDev = isDevOnly || isTestEnv
...@@ -56,6 +58,11 @@ const runTests = (mode = 'dev') => { ...@@ -56,6 +58,11 @@ const runTests = (mode = 'dev') => {
expect(data.ENV_FILE_EMPTY_FIRST).toBe(isTestEnv ? '' : '$escaped') expect(data.ENV_FILE_EMPTY_FIRST).toBe(isTestEnv ? '' : '$escaped')
expect(data.ENV_FILE_PROCESS_ENV).toBe('env-cli') expect(data.ENV_FILE_PROCESS_ENV).toBe('env-cli')
if (didReload) {
expect(data.NEW_ENV_KEY).toBe('true')
expect(data.NEW_ENV_LOCAL_KEY).toBe('hello')
expect(data.NEW_ENV_DEV_KEY).toBe('from-dev')
}
expect(data.nextConfigEnv).toBe('hello from next.config.js') expect(data.nextConfigEnv).toBe('hello from next.config.js')
expect(data.nextConfigPublicEnv).toBe('hello again from next.config.js') expect(data.nextConfigPublicEnv).toBe('hello again from next.config.js')
} }
...@@ -138,17 +145,84 @@ const runTests = (mode = 'dev') => { ...@@ -138,17 +145,84 @@ const runTests = (mode = 'dev') => {
describe('Env Config', () => { describe('Env Config', () => {
describe('dev mode', () => { describe('dev mode', () => {
beforeAll(async () => { beforeAll(async () => {
output = ''
appPort = await findPort() appPort = await findPort()
app = await launchApp(appDir, appPort, { app = await launchApp(appDir, appPort, {
env: { env: {
PROCESS_ENV_KEY: 'processenvironment', PROCESS_ENV_KEY: 'processenvironment',
ENV_FILE_PROCESS_ENV: 'env-cli', ENV_FILE_PROCESS_ENV: 'env-cli',
}, },
onStdout(msg) {
output += msg || ''
},
onStderr(msg) {
output += msg || ''
},
}) })
await renderViaHTTP(appPort, '/another-global')
}) })
afterAll(() => killApp(app)) afterAll(() => killApp(app))
runTests('dev') runTests('dev')
describe('with hot reload', () => {
const originalContents = []
beforeAll(async () => {
const outputIndex = output.length
const envToUpdate = [
{
toAdd: 'NEW_ENV_KEY=true',
file: '.env',
},
{
toAdd: 'NEW_ENV_LOCAL_KEY=hello',
file: '.env.local',
},
{
toAdd: 'NEW_ENV_DEV_KEY=from-dev\nNEXT_PUBLIC_HELLO_WORLD=again',
file: '.env.development',
},
]
for (const { file, toAdd } of envToUpdate) {
const content = await fs.readFile(join(appDir, file), 'utf8')
originalContents.push({ file, content })
await fs.writeFile(join(appDir, file), content + '\n' + toAdd)
}
await check(() => {
return output.substring(outputIndex)
}, /Loaded env from/)
})
afterAll(async () => {
for (const { file, content } of originalContents) {
await fs.writeFile(join(appDir, file), content)
}
})
runTests('dev', true)
it('should update inlined values correctly', async () => {
await renderViaHTTP(appPort, '/another-global')
const buildManifest = await fs.readJson(
join(__dirname, '../app/.next/build-manifest.json')
)
const pageFile = buildManifest.pages['/another-global'].find(
(filename) => filename.includes('pages/another-global')
)
// read client bundle contents since a server side render can
// have the value available during render but it not be injected
const bundleContent = await fs.readFile(
join(appDir, '.next', pageFile),
'utf8'
)
expect(bundleContent).toContain('again')
})
})
}) })
describe('test environment', () => { describe('test environment', () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment