Unverified Commit 6204bf03 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Turbopack: preserve chunk load promise identity (#100421)

## Summary

#98836 added an on-demand `__turbopack_ensure_chunk__` hook to the
Turbopack Node dev runtime by overriding `__turbopack_context__.l`. The
override returned `ensured.then(() => loadChunkAsync(...))`, so every
call that hit a not-yet-cached chunk produced a **new promise**, and the
first caller's promise never matched the one later callers got from
`chunkCache`. That broke referential identity for `__next_chunk_load__`
(which wraps `.l`), which React relies on. It also created extra
promises for React's dev async tracking to process.

This change makes `loadChunkAsync` itself aware of
`__turbopack_ensure_chunk__`:

- The hook is called synchronously on a cache miss. If it returns
`undefined` (nothing to compile), the chunk is required synchronously
and the shared pre-resolved promise is cached, exactly as without the
hook.
- If it returns a promise (the chunk is being compiled on demand), a
single `ensure → require/install` promise is cached in `chunkCache`
immediately. Concurrent and later callers all get the same promise until
`clearChunkCache()`.
- If ensuring the chunk fails (a synchronous throw or an async
rejection), the failure is not cached, so the next load retries.
`require` failures stay cached as `ChunkLoadError`, as before.
- The separate `.l` override and `chunksBeingEnsured` map in
`dev-base.ts` are removed. The Node runtime snapshots are regenerated.

On the Next.js side, the dev hook is now a plain function. It returns
synchronously for chunks that aren't lazy-compilation chunks, or that
have no matching entries, and returns a promise only when it actually
activates a lazy chunk.

## Verification

- `pnpm test-dev-turbo
test/development/app-dir/lazy-dynamic-imports/lazy-dynamic-imports.test.ts`
(12/12 pass)
- A scratch Node script against the generated dev and build runtimes,
with a mocked ensure hook. It checks:
- async ensure: same promise for concurrent and post-settle calls,
ensure called once, modules installed only after ensure, and eviction
plus retry after an ensure failure
- sync ensure: modules installed synchronously with the shared resolved
promise, and no caching after a synchronous throw
  - cached `ChunkLoadError`, non-JS chunks, and `clearChunkCache` reset

The identity and sync-path checks fail on the previous runtime and pass
with this change.

<!-- NEXT_JS_LLM -->


<!-- fleet a53f5f3f-06ac-4812-a068-facaf3b95fe5 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent a0e452b9
......@@ -2394,11 +2394,18 @@ export async function createHotReloaderTurbopack(
if (lazyDynamicImportsSSR) {
const previousEnsureChunk = globalThis.__turbopack_ensure_chunk__
const ensureChunk = async (chunkPath: string) => {
// Synchronous when there is nothing to compile, so the runtime can load
// the chunk without creating a new promise for it.
const ensureChunk = (chunkPath: string): void | Promise<void> => {
if (!chunkPath.includes('lazy-compilation-')) return
const keys = serverAssetMapper.getKeysByAsset(chunkPath)
if (keys.length === 0 || !(await project.activateLazyChunk(chunkPath))) {
if (keys.length === 0) return
return activateLazyChunk(chunkPath, keys)
}
const activateLazyChunk = async (chunkPath: string, keys: EntryKey[]) => {
if (!(await project.activateLazyChunk(chunkPath))) {
return
}
......
......@@ -32,31 +32,6 @@ nodeDevContextPrototype.c = moduleCache
nodeDevContextPrototype.R = resolvePathFromModule
nodeDevContextPrototype.C = clearChunkCache
if (globalThis.__turbopack_ensure_chunk__ !== undefined) {
const chunksBeingEnsured = new Map<ChunkPath, Promise<void>>()
function loadChunkAsyncOnDemand<TModule extends Module>(
this: TurbopackBaseContext<TModule>,
chunkData: ChunkData
): Promise<void> {
const chunkPath = typeof chunkData === 'string' ? chunkData : chunkData.path
const ensureChunk = globalThis.__turbopack_ensure_chunk__
if (ensureChunk === undefined || chunkCache.has(chunkPath)) {
return loadChunkAsync.call(this, chunkData)
}
const ensured =
chunksBeingEnsured.get(chunkPath) ??
Promise.resolve()
.then(() => ensureChunk(chunkPath))
.finally(() => chunksBeingEnsured.delete(chunkPath))
chunksBeingEnsured.set(chunkPath, ensured)
return ensured.then(() => loadChunkAsync.call(this, chunkData))
}
nodeDevContextPrototype.l = loadChunkAsyncOnDemand
}
markDevModulesLoaded = true
function interceptDevModuleExecution(
......
......@@ -108,27 +108,67 @@ function loadChunkAsync<TModule extends Module>(
let entry = chunkCache.get(chunkPath)
if (entry === undefined) {
try {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules: CompressedModuleFactories = require(resolved)
installCompressedModuleFactories(chunkModules, 0)
entry = loadedChunk
} catch (cause) {
const errorMessage = `Failed to load chunk ${chunkPath} from module ${this.m.id}`
const error = new Error(errorMessage, { cause })
error.name = 'ChunkLoadError'
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(error)
const sourceModuleId = this.m.id
// `ensureChunk` is conditionally installed to support lazy dynamic imports during SSR
// It can be used to call back into turbopack to compile the chunk we are about to reference.
const ensureChunk = globalThis.__turbopack_ensure_chunk__
let ensuring: void | Promise<void> = undefined
if (ensureChunk !== undefined) {
try {
// Returns `undefined` when the chunk can be required right away.
ensuring = ensureChunk(chunkPath)
} catch (error) {
// Don't cache a failure to compile the chunk, so that the next load
// retries it.
return Promise.reject(error)
}
}
if (ensuring === undefined) {
entry = requireChunk(chunkPath, sourceModuleId)
} else {
// The chunk is being compiled on demand before it can be required.
// Cache the combined promise right away so that every caller (including
// concurrent ones) gets the same promise for this chunk.
const ensured: Promise<void> = Promise.resolve(ensuring).then(
() => requireChunk(chunkPath, sourceModuleId),
(error) => {
// Don't cache a failure to compile the chunk, so that the next load
// retries it.
if (chunkCache.get(chunkPath) === ensured) {
chunkCache.delete(chunkPath)
}
throw error
}
)
entry = ensured
}
chunkCache.set(chunkPath, entry)
}
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
return entry
}
function requireChunk(
chunkPath: ChunkPath,
sourceModuleId: ModuleId
): Promise<void> {
try {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules: CompressedModuleFactories = require(resolved)
installCompressedModuleFactories(chunkModules, 0)
return loadedChunk
} catch (cause) {
const errorMessage = `Failed to load chunk ${chunkPath} from module ${sourceModuleId}`
const error = new Error(errorMessage, { cause })
error.name = 'ChunkLoadError'
// Cache the failure promise, future requests will also get this same rejection
return Promise.reject(error)
}
}
contextPrototype.l = loadChunkAsync
function loadChunkAsyncByUrl<TModule extends Module>(
......
......@@ -818,28 +818,61 @@ function loadChunkAsync(chunkData) {
}
let entry = chunkCache.get(chunkPath);
if (entry === undefined) {
try {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules = require(resolved);
installCompressedModuleFactories(chunkModules, 0);
entry = loadedChunk;
} catch (cause) {
const errorMessage = `Failed to load chunk ${chunkPath} from module ${this.m.id}`;
const error = new Error(errorMessage, {
cause
const sourceModuleId = this.m.id;
// `ensureChunk` is conditionally installed to support lazy dynamic imports during SSR
// It can be used to call back into turbopack to compile the chunk we are about to reference.
const ensureChunk = globalThis.__turbopack_ensure_chunk__;
let ensuring = undefined;
if (ensureChunk !== undefined) {
try {
// Returns `undefined` when the chunk can be required right away.
ensuring = ensureChunk(chunkPath);
} catch (error) {
// Don't cache a failure to compile the chunk, so that the next load
// retries it.
return Promise.reject(error);
}
}
if (ensuring === undefined) {
entry = requireChunk(chunkPath, sourceModuleId);
} else {
// The chunk is being compiled on demand before it can be required.
// Cache the combined promise right away so that every caller (including
// concurrent ones) gets the same promise for this chunk.
const ensured = Promise.resolve(ensuring).then(()=>requireChunk(chunkPath, sourceModuleId), (error)=>{
// Don't cache a failure to compile the chunk, so that the next load
// retries it.
if (chunkCache.get(chunkPath) === ensured) {
chunkCache.delete(chunkPath);
}
throw error;
});
error.name = 'ChunkLoadError';
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(error);
entry = ensured;
}
chunkCache.set(chunkPath, entry);
}
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
return entry;
}
function requireChunk(chunkPath, sourceModuleId) {
try {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules = require(resolved);
installCompressedModuleFactories(chunkModules, 0);
return loadedChunk;
} catch (cause) {
const errorMessage = `Failed to load chunk ${chunkPath} from module ${sourceModuleId}`;
const error = new Error(errorMessage, {
cause
});
error.name = 'ChunkLoadError';
// Cache the failure promise, future requests will also get this same rejection
return Promise.reject(error);
}
}
contextPrototype.l = loadChunkAsync;
function loadChunkAsyncByUrl(chunkUrl) {
const path1 = url.fileURLToPath(new URL(chunkUrl, RUNTIME_ROOT));
......@@ -1580,20 +1613,6 @@ nodeDevContextPrototype.q = exportUrl;
nodeDevContextPrototype.c = moduleCache;
nodeDevContextPrototype.R = resolvePathFromModule;
nodeDevContextPrototype.C = clearChunkCache;
if (globalThis.__turbopack_ensure_chunk__ !== undefined) {
const chunksBeingEnsured = new Map();
function loadChunkAsyncOnDemand(chunkData) {
const chunkPath = typeof chunkData === 'string' ? chunkData : chunkData.path;
const ensureChunk = globalThis.__turbopack_ensure_chunk__;
if (ensureChunk === undefined || chunkCache.has(chunkPath)) {
return loadChunkAsync.call(this, chunkData);
}
const ensured = chunksBeingEnsured.get(chunkPath) ?? Promise.resolve().then(()=>ensureChunk(chunkPath)).finally(()=>chunksBeingEnsured.delete(chunkPath));
chunksBeingEnsured.set(chunkPath, ensured);
return ensured.then(()=>loadChunkAsync.call(this, chunkData));
}
nodeDevContextPrototype.l = loadChunkAsyncOnDemand;
}
markDevModulesLoaded = true;
function interceptDevModuleExecution(_module) {
// There are no react refresh hooks server side so this is a no-op
......
......@@ -818,28 +818,61 @@ function loadChunkAsync(chunkData) {
}
let entry = chunkCache.get(chunkPath);
if (entry === undefined) {
try {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules = require(resolved);
installCompressedModuleFactories(chunkModules, 0);
entry = loadedChunk;
} catch (cause) {
const errorMessage = `Failed to load chunk ${chunkPath} from module ${this.m.id}`;
const error = new Error(errorMessage, {
cause
const sourceModuleId = this.m.id;
// `ensureChunk` is conditionally installed to support lazy dynamic imports during SSR
// It can be used to call back into turbopack to compile the chunk we are about to reference.
const ensureChunk = globalThis.__turbopack_ensure_chunk__;
let ensuring = undefined;
if (ensureChunk !== undefined) {
try {
// Returns `undefined` when the chunk can be required right away.
ensuring = ensureChunk(chunkPath);
} catch (error) {
// Don't cache a failure to compile the chunk, so that the next load
// retries it.
return Promise.reject(error);
}
}
if (ensuring === undefined) {
entry = requireChunk(chunkPath, sourceModuleId);
} else {
// The chunk is being compiled on demand before it can be required.
// Cache the combined promise right away so that every caller (including
// concurrent ones) gets the same promise for this chunk.
const ensured = Promise.resolve(ensuring).then(()=>requireChunk(chunkPath, sourceModuleId), (error)=>{
// Don't cache a failure to compile the chunk, so that the next load
// retries it.
if (chunkCache.get(chunkPath) === ensured) {
chunkCache.delete(chunkPath);
}
throw error;
});
error.name = 'ChunkLoadError';
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(error);
entry = ensured;
}
chunkCache.set(chunkPath, entry);
}
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
return entry;
}
function requireChunk(chunkPath, sourceModuleId) {
try {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules = require(resolved);
installCompressedModuleFactories(chunkModules, 0);
return loadedChunk;
} catch (cause) {
const errorMessage = `Failed to load chunk ${chunkPath} from module ${sourceModuleId}`;
const error = new Error(errorMessage, {
cause
});
error.name = 'ChunkLoadError';
// Cache the failure promise, future requests will also get this same rejection
return Promise.reject(error);
}
}
contextPrototype.l = loadChunkAsync;
function loadChunkAsyncByUrl(chunkUrl) {
const path1 = url.fileURLToPath(new URL(chunkUrl, RUNTIME_ROOT));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment