Commit adb87bfa authored by Will Binns-Smith's avatar Will Binns-Smith

wip

parent d49212c8
......@@ -37,7 +37,7 @@ import {
} from './middleware-turbopack'
import { PageNotFoundError } from '../../shared/lib/utils'
import { debounce } from '../utils'
import { deleteCache } from './require-cache'
import { deleteCache, ExternalWatcher } from './require-cache'
import {
clearAllModuleContexts,
clearModuleContext,
......@@ -358,6 +358,19 @@ export async function createHotReloaderTurbopack(
const fileLogger = getFileLogger()
fileLogger.initialize(distDir, mcpServerEnabled)
// Watch external (non-distDir) modules loaded by Turbopack chunks so that
// changes to serverExternalPackages modules are reflected on the next request
// without needing a recompile.
const externalWatcher = new ExternalWatcher(distDir)
externalWatcher.onExternalChanged = () => {
if (process.env.NEXT_HMR_EXTERNAL_DEBUG) {
console.log('[ExternalWatcher] external changed — clearing chunk cache')
}
if (typeof __next__clear_chunk_cache__ === 'function') {
__next__clear_chunk_cache__()
}
}
const encryptionKey = await generateEncryptionKeyBase64({
isBuild: false,
distDir,
......@@ -629,6 +642,11 @@ export async function createHotReloaderTurbopack(
// Reset the fetch patch so patchFetch() can re-wrap on the next request.
if (serverPaths.length > 0) {
resetFetch()
// Discover any external (non-distDir) modules that Turbopack chunks have
// loaded via require() and start watching them for changes. When a watched
// file changes on disk it is immediately evicted from require.cache so the
// next request picks up the fresh version — without needing a recompile.
externalWatcher.scan()
}
// Clear Turbopack's chunk-loading cache so chunks are re-required from disk on
......@@ -1545,6 +1563,12 @@ export async function createHotReloaderTurbopack(
if (inputPage === '/src/instrumentation') return
}
// Scan for any external modules that were loaded by Turbopack chunks
// during previous requests and start watching them. This is cheap (just
// iterating require.cache) and ensures we catch externals that were first
// loaded after the last chunk write.
externalWatcher.scan()
return hotReloaderSpan
.traceChild('ensure-page', {
inputPage,
......@@ -1705,6 +1729,8 @@ export async function createHotReloaderTurbopack(
})
},
close() {
externalWatcher.dispose()
// Report MCP telemetry if MCP server is enabled
recordMcpTelemetry(opts.telemetry)
......
import { statSync } from 'fs'
import isError from '../../lib/is-error'
import { realpathSync } from '../../lib/realpath'
import { clearManifestCache } from '../load-manifest.external'
import Watchpack from 'next/dist/compiled/watchpack'
function deleteFromRequireCache(filePath: string) {
try {
......@@ -33,3 +35,142 @@ export function deleteCache(filePath: string) {
deleteFromRequireCache(filePath)
}
/**
* Watches external (non-distDir) modules that are loaded as children of
* Turbopack output chunks, evicting them from require.cache when they change.
*
* Turbopack doesn't watch node_modules, so changes to serverExternalPackages
* modules never trigger a recompile. This watcher fills that gap: after each
* chunk write we scan distDir chunks for non-distDir children (externals) and
* start watching any we haven't seen before. When a watched file changes on
* disk we immediately evict it from require.cache so the next request picks up
* the fresh version.
*
* Evicting the external alone is not enough: Turbopack's devModuleCache (a
* closure variable in [turbopack]_runtime.js) caches instantiated modules and
* is not cleared by clearChunkCache(). To force re-instantiation we also evict
* the runtime chunk and all page chunks that reference it. This causes the
* runtime to be re-required on the next request with a fresh devModuleCache,
* so page modules are re-instantiated and pick up the new external.
*
* We evict ONLY the runtime chunk itself (not its children) to avoid evicting
* async storage externals (work-async-storage.external.js etc.) which are
* children of the runtime's SSR chunks and must remain in require.cache.
*/
export class ExternalWatcher {
// Watchpack handles fs.watch lifecycle, error recovery, and platform quirks.
private wp = new Watchpack({ aggregateTimeout: 200 })
private watched = new Set<string>()
/**
* Called whenever an external module is evicted from require.cache due to a
* detected change. The hot reloader uses this to also clear Turbopack's chunk
* cache so the runtime chunk re-executes and picks up the fresh external.
*/
onExternalChanged: (() => void) | undefined
constructor(private distDir: string) {
this.wp.on('aggregated', (changes: Set<string>) => {
for (const filePath of changes) {
if (this.watched.has(filePath)) {
this.evictExternal(filePath)
}
}
})
}
/**
* Scan all currently-loaded distDir chunks for non-distDir children.
* Newly-discovered externals are added to the Watchpack watch list.
* Already-watched externals are checked for mtime changes that occurred
* before the watcher fired (stale-at-discovery race).
*/
scan() {
const newFiles: string[] = []
for (const [filePath, mod] of Object.entries(require.cache)) {
if (!filePath.startsWith(this.distDir) || !mod) continue
for (const child of mod.children) {
if (child.filename.startsWith(this.distDir)) continue
if (this.watched.has(child.filename)) continue
this.watched.add(child.filename)
newFiles.push(child.filename)
}
}
if (newFiles.length === 0) return
// Re-watch with the updated file list. Watchpack diffs internally so
// existing watchers are preserved; only new files get new watchers.
// startTime: Date.now() means only future changes fire the aggregated event.
this.wp.watch({ files: [...this.watched], startTime: Date.now() })
// Stale-at-discovery check: if a file was modified after the distDir chunk
// that loaded it was written (i.e. between first render and this scan),
// Watchpack won't fire for it. Evict it immediately.
for (const filePath of newFiles) {
const mod = require.cache[filePath]
if (!mod) continue
const parentMod = Object.values(require.cache).find(
(m) => m?.filename.startsWith(this.distDir) && m.children.includes(mod)
)
if (!parentMod) continue
try {
const extStat = statSync(filePath, { throwIfNoEntry: false })
const parentStat = statSync(parentMod.filename, {
throwIfNoEntry: false,
})
if (extStat && parentStat && extStat.mtimeMs > parentStat.mtimeMs) {
this.evictExternal(filePath)
}
} catch {
// Ignore stat errors.
}
}
}
private evictExternal(filePath: string) {
// Evict the external itself.
deleteFromRequireCache(filePath)
// Find the Turbopack runtime chunk directly by filename pattern — it's the
// single distDir chunk whose name contains "[turbopack]_runtime". Evicting
// it (but NOT its children) forces a fresh devModuleCache on the next
// request. The runtime's children (SSR chunks + async storage externals)
// remain in require.cache so in-flight requests keep their storage context.
// Pass 1: find the Turbopack runtime chunk by filename pattern.
let runtimeChunk: NodeJS.Module | undefined
for (const [path, mod] of Object.entries(require.cache)) {
if (
mod &&
path.startsWith(this.distDir) &&
path.includes('[turbopack]_runtime')
) {
runtimeChunk = mod
break
}
}
// Pass 2: find all page chunks that have the runtime as a direct child.
const pageChunks: NodeJS.Module[] = []
if (runtimeChunk) {
for (const [path, mod] of Object.entries(require.cache)) {
if (
mod &&
path.startsWith(this.distDir) &&
mod.children.includes(runtimeChunk)
) {
pageChunks.push(mod)
}
}
}
if (runtimeChunk) deleteFromRequireCache(runtimeChunk.filename)
for (const pageChunk of pageChunks) {
deleteFromRequireCache(pageChunk.filename)
}
this.onExternalChanged?.()
}
dispose() {
this.wp.close()
this.watched.clear()
}
}
import { value } from 'hmr-external'
import { fileValue } from '../../lib/hmr-file-external'
export default function ExternalPage() {
return (
<>
<p id="value">{value}</p>
<p id="file-value">{fileValue}</p>
</>
)
}
module.exports = { fileValue: 'file-external-value: 0' }
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {}
const nextConfig = {
serverExternalPackages: ['hmr-external', './lib/hmr-file-external'],
}
module.exports = nextConfig
{
"name": "hmr-external",
"version": "0.0.1",
"main": "index.js"
}
......@@ -156,6 +156,48 @@ describe('server-hmr', () => {
)
})
describe('external package hmr', () => {
itTurbopackDev(
'reflects changes to a node_modules serverExternalPackages module on the next request',
async () => {
const initial = await next.render$('/external')
expect(initial('#value').text()).toBe('external-value: 0')
// Patch the external package directly — it lives outside Turbopack's
// module graph so server HMR cannot patch it in-place. The runtime must
// fall back to clearing require.cache so the updated file is re-required
// on the next request.
await next.patchFile('node_modules/hmr-external/index.js', (content) =>
content.replace('external-value: 0', 'external-value: 1')
)
await retry(async () => {
const updated = await next.render$('/external')
expect(updated('#value').text()).toBe('external-value: 1')
})
}
)
itTurbopackDev(
'reflects changes to a non-node_modules serverExternalPackages file on the next request',
async () => {
const initial = await next.render$('/external')
expect(initial('#file-value').text()).toBe('file-external-value: 0')
// Patch the external file directly — it lives outside Turbopack's
// module graph so server HMR cannot patch it in-place.
await next.patchFile('lib/hmr-file-external.js', (content) =>
content.replace('file-external-value: 0', 'file-external-value: 1')
)
await retry(async () => {
const updated = await next.render$('/external')
expect(updated('#file-value').text()).toBe('file-external-value: 1')
})
}
)
})
describe('route handler hmr', () => {
function getText(res: Response) {
return res.ok
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment