Unverified Commit 8a8888ca authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] use `require` to load chunks in our node runtime (#81738)

## Use `require` to load chunks in node instead of readFile+eval

The benefit of this is that we can slightly simplify chunk loading and make it easier for hosting services to cache bytecode.

Much like [browsers](https://v8.dev/blog/code-caching-for-devs), server environments often leverage VM features to cache parsed javascript as bytecode (or maybe binary ast?) to accelerate cold starts ([example](https://vercel.com/blog/introducing-bytecode-caching-for-vercel-functions)), however, these systems generally avoid caching highly dynamic content like inline scripts and `eval` parameters.  

So just use `require` which is what platforms expect!  This should help platforms achieve code caching, is generally simpler, and by not adding a prefix we fix some subtle off-by-one issue with source maps. 

A slightly better solution imho would be `import()` since it would enable node to do some file reading parsing async with the main thread which could be important if there are multiple `preloadModules`, however this breaks dev mode hot reloading.

Fixes #PACK-5062
parent ae2c750d
...@@ -657,14 +657,16 @@ describe('Cache Components Errors', () => { ...@@ -657,14 +657,16 @@ describe('Cache Components Errors', () => {
} else { } else {
expect(output).toMatchInlineSnapshot(` expect(output).toMatchInlineSnapshot(`
"Error: Route "/dynamic-root": A component accessed data, headers, params, searchParams, or a short-lived cache without a Suspense boundary nor a "use cache" above it. See more info: https://nextjs.org/docs/messages/next-prerender-missing-suspense "Error: Route "/dynamic-root": A component accessed data, headers, params, searchParams, or a short-lived cache without a Suspense boundary nor a "use cache" above it. See more info: https://nextjs.org/docs/messages/next-prerender-missing-suspense
at c (turbopack:///[project]/app/dynamic-root/indirection.tsx:9:1) at c (turbopack:///[project]/app/dynamic-root/indirection.tsx:7:34)
at main (<anonymous>) at main (<anonymous>)
at body (<anonymous>) at body (<anonymous>)
at html (<anonymous>) at html (<anonymous>)
7 | export function IndirectionTwo({ children }) { 5 | }
6 |
> 7 | export function IndirectionTwo({ children }) {
| ^
8 | return children 8 | return children
> 9 | } 9 | }
| ^
10 | 10 |
To get a more detailed stack trace and pinpoint the issue, try one of the following: To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/dynamic-root" in your browser to investigate the error. - Start the app in development mode by running \`next dev\`, then open "/dynamic-root" in your browser to investigate the error.
...@@ -1680,14 +1682,14 @@ describe('Cache Components Errors', () => { ...@@ -1680,14 +1682,14 @@ describe('Cache Components Errors', () => {
} else { } else {
expect(output).toMatchInlineSnapshot(` expect(output).toMatchInlineSnapshot(`
"Error: Route "/sync-attribution/guarded-async-unguarded-clientsync" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client "Error: Route "/sync-attribution/guarded-async-unguarded-clientsync" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client
at c (turbopack:///[project]/app/sync-attribution/guarded-async-unguarded-clientsync/client.tsx:9:7) at c (turbopack:///[project]/app/sync-attribution/guarded-async-unguarded-clientsync/client.tsx:5:16)
7 | return ( 3 | export function SyncIO() {
8 | <main> 4 | // This is a sync IO access that should not cause an error
> 9 | <h1>Sync IO Access</h1> > 5 | const data = new Date().toISOString()
| ^ | ^
10 | <p suppressHydrationWarning>Current date and time: {data}</p> 6 |
11 | </main> 7 | return (
12 | ) 8 | <main>
To get a more detailed stack trace and pinpoint the issue, try one of the following: To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/sync-attribution/guarded-async-unguarded-clientsync" in your browser to investigate the error. - Start the app in development mode by running \`next dev\`, then open "/sync-attribution/guarded-async-unguarded-clientsync" in your browser to investigate the error.
- Rerun the production build with \`next build --debug-prerender\` to generate better stack traces. - Rerun the production build with \`next build --debug-prerender\` to generate better stack traces.
...@@ -1989,14 +1991,14 @@ describe('Cache Components Errors', () => { ...@@ -1989,14 +1991,14 @@ describe('Cache Components Errors', () => {
} else { } else {
expect(output).toMatchInlineSnapshot(` expect(output).toMatchInlineSnapshot(`
"Error: Route "/sync-attribution/unguarded-async-unguarded-clientsync" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client "Error: Route "/sync-attribution/unguarded-async-unguarded-clientsync" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client
at c (turbopack:///[project]/app/sync-attribution/unguarded-async-unguarded-clientsync/client.tsx:9:7) at c (turbopack:///[project]/app/sync-attribution/unguarded-async-unguarded-clientsync/client.tsx:5:16)
7 | return ( 3 | export function SyncIO() {
8 | <main> 4 | // This is a sync IO access that should not cause an error
> 9 | <h1>Sync IO Access</h1> > 5 | const data = new Date().toISOString()
| ^ | ^
10 | <p suppressHydrationWarning>Current date and time: {data}</p> 6 |
11 | </main> 7 | return (
12 | ) 8 | <main>
To get a more detailed stack trace and pinpoint the issue, try one of the following: To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/sync-attribution/unguarded-async-unguarded-clientsync" in your browser to investigate the error. - Start the app in development mode by running \`next dev\`, then open "/sync-attribution/unguarded-async-unguarded-clientsync" in your browser to investigate the error.
- Rerun the production build with \`next build --debug-prerender\` to generate better stack traces. - Rerun the production build with \`next build --debug-prerender\` to generate better stack traces.
......
...@@ -230,14 +230,15 @@ describe('app-dir - server source maps', () => { ...@@ -230,14 +230,15 @@ describe('app-dir - server source maps', () => {
} }
} else { } else {
if (isTurbopack) { if (isTurbopack) {
// TODO(veil): Sourcemapping line off
// TODO(veil): Sourcemap names // TODO(veil): Sourcemap names
// TODO(veil): relative paths // TODO(veil): relative paths
expect(normalizeCliOutput(next.cliOutput)).toContain( expect(normalizeCliOutput(next.cliOutput)).toContain(
'(bundler:///app/ssr-error-log-ignore-listed/page.js:23:5)' '(bundler:///app/ssr-error-log-ignore-listed/page.js:9:17)'
) )
expect(normalizeCliOutput(next.cliOutput)).toContain( expect(normalizeCliOutput(next.cliOutput)).toContain(
'\n> 23 | })\n | ^' '\n' +
"> 9 | const error = new Error('ssr-error-log-ignore-listed')\n" +
' | ^\n'
) )
} else { } else {
// TODO(veil): line/column numbers are flaky in Webpack // TODO(veil): line/column numbers are flaky in Webpack
......
...@@ -82,11 +82,12 @@ describe('build-output-prerender', () => { ...@@ -82,11 +82,12 @@ describe('build-output-prerender', () => {
// TODO(veil): Why is the location incomplete unless we enable --no-mangling? // TODO(veil): Why is the location incomplete unless we enable --no-mangling?
expect(getPrerenderOutput(next.cliOutput)).toMatchInlineSnapshot(` expect(getPrerenderOutput(next.cliOutput)).toMatchInlineSnapshot(`
"Error: Route "/client" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client "Error: Route "/client" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client
at c (bundler:///app/client/page.tsx:5:1) at c (bundler:///app/client/page.tsx:4:28)
2 |
3 | export default function Page() { 3 | export default function Page() {
4 | return <p>Current time: {new Date().toISOString()}</p> > 4 | return <p>Current time: {new Date().toISOString()}</p>
> 5 | } | ^
| ^ 5 | }
6 | 6 |
To get a more detailed stack trace and pinpoint the issue, try one of the following: To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/client" in your browser to investigate the error. - Start the app in development mode by running \`next dev\`, then open "/client" in your browser to investigate the error.
......
...@@ -86,16 +86,16 @@ function createResolvePathFromModule( ...@@ -86,16 +86,16 @@ function createResolvePathFromModule(
function loadChunk(chunkData: ChunkData, source?: SourceInfo): void { function loadChunk(chunkData: ChunkData, source?: SourceInfo): void {
if (typeof chunkData === 'string') { if (typeof chunkData === 'string') {
return loadChunkPath(chunkData, source) loadChunkPath(chunkData, source)
} else { } else {
return loadChunkPath(chunkData.path, source) loadChunkPath(chunkData.path, source)
} }
} }
const loadedChunks = new Set<ChunkPath>() const loadedChunks = new Set<ChunkPath>()
const unsupportedLoadChunk = Promise.resolve(undefined) const unsupportedLoadChunk = Promise.resolve(undefined)
const loadedChunk = Promise.resolve(undefined) const loadedChunk: Promise<void> = Promise.resolve(undefined)
const chunkCache = new Map<ChunkPath, Promise<any> | typeof loadedChunk>() const chunkCache = new Map<ChunkPath, Promise<void>>()
function clearChunkCache() { function clearChunkCache() {
chunkCache.clear() chunkCache.clear()
...@@ -119,7 +119,7 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void { ...@@ -119,7 +119,7 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void {
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) { for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) { if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) { if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory const [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) { for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn moduleFactories[otherModuleId] = moduleFactoryFn
...@@ -143,63 +143,32 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void { ...@@ -143,63 +143,32 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void {
} }
} }
async function loadChunkAsyncUncached( function loadChunkUncached(chunkPath: ChunkPath) {
source: SourceInfo, // resolve to an absolute path to simplify `require` handling
chunkPath: ChunkPath
): Promise<void> {
const resolved = path.resolve(RUNTIME_ROOT, chunkPath) const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
try { // TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
const contents = await fs.readFile(resolved, 'utf-8') // However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules: CompressedModuleFactories = require(resolved)
const localRequire = (id: string) => { for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
let resolvedId = require.resolve(id, { paths: [path.dirname(resolved)] }) if (!moduleFactories[moduleId]) {
return require(resolvedId) if (Array.isArray(moduleFactory)) {
} const [moduleFactoryFn, otherIds] = moduleFactory
const module = { moduleFactories[moduleId] = moduleFactoryFn
exports: {}, for (const otherModuleId of otherIds) {
} moduleFactories[otherModuleId] = moduleFactoryFn
// TODO: Use vm.runInThisContext once our minimal supported Node.js version includes https://github.com/nodejs/node/pull/52153
// eslint-disable-next-line no-eval -- Can't use vm.runInThisContext due to https://github.com/nodejs/node/issues/52102
;(0, eval)(
'(function(module, exports, require, __dirname, __filename) {' +
contents +
'\n})' +
'\n//# sourceURL=' +
url.pathToFileURL(resolved)
)(module, module.exports, localRequire, path.dirname(resolved), resolved)
const chunkModules: CompressedModuleFactories = module.exports
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
}
} else {
moduleFactories[moduleId] = moduleFactory
} }
} else {
moduleFactories[moduleId] = moduleFactory
} }
} }
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`
}
throw new Error(errorMessage, {
cause: e,
})
} }
} }
function loadChunkAsync( function loadChunkAsync(
source: SourceInfo, source: SourceInfo,
chunkData: ChunkData chunkData: ChunkData
): Promise<any> { ): Promise<void> {
const chunkPath = typeof chunkData === 'string' ? chunkData : chunkData.path const chunkPath = typeof chunkData === 'string' ? chunkData : chunkData.path
if (!isJs(chunkPath)) { if (!isJs(chunkPath)) {
// We only support loading JS chunks in Node.js. // We only support loading JS chunks in Node.js.
...@@ -209,10 +178,23 @@ function loadChunkAsync( ...@@ -209,10 +178,23 @@ function loadChunkAsync(
let entry = chunkCache.get(chunkPath) let entry = chunkCache.get(chunkPath)
if (entry === undefined) { if (entry === undefined) {
const resolve = chunkCache.set.bind(chunkCache, chunkPath, loadedChunk) try {
// A new Promise ensures callers that don't handle rejection will still trigger one unhandled rejection. // Load the chunk synchronously
// Handling the rejection will not trigger unhandled rejections. loadChunkUncached(chunkPath)
entry = loadChunkAsyncUncached(source, chunkPath).then(resolve) entry = loadedChunk
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`
}
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(
new Error(errorMessage, {
cause: e,
})
)
}
chunkCache.set(chunkPath, entry) chunkCache.set(chunkPath, entry)
} }
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality. // TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
......
...@@ -490,9 +490,9 @@ const moduleCache = Object.create(null); ...@@ -490,9 +490,9 @@ const moduleCache = Object.create(null);
} }
function loadChunk(chunkData, source) { function loadChunk(chunkData, source) {
if (typeof chunkData === 'string') { if (typeof chunkData === 'string') {
return loadChunkPath(chunkData, source); loadChunkPath(chunkData, source);
} else { } else {
return loadChunkPath(chunkData.path, source); loadChunkPath(chunkData.path, source);
} }
} }
const loadedChunks = new Set(); const loadedChunks = new Set();
...@@ -517,7 +517,7 @@ function loadChunkPath(chunkPath, source) { ...@@ -517,7 +517,7 @@ function loadChunkPath(chunkPath, source) {
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){ for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
if (!moduleFactories[moduleId]) { if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) { if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory; const [moduleFactoryFn, otherIds] = moduleFactory;
moduleFactories[moduleId] = moduleFactoryFn; moduleFactories[moduleId] = moduleFactoryFn;
for (const otherModuleId of otherIds){ for (const otherModuleId of otherIds){
moduleFactories[otherModuleId] = moduleFactoryFn; moduleFactories[otherModuleId] = moduleFactoryFn;
...@@ -538,44 +538,24 @@ function loadChunkPath(chunkPath, source) { ...@@ -538,44 +538,24 @@ function loadChunkPath(chunkPath, source) {
}); });
} }
} }
async function loadChunkAsyncUncached(source, chunkPath) { function loadChunkUncached(chunkPath) {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath); const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
try { // TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
const contents = await fs.readFile(resolved, 'utf-8'); // However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const localRequire = (id)=>{ const chunkModules = require(resolved);
let resolvedId = require.resolve(id, { for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
paths: [ if (!moduleFactories[moduleId]) {
path.dirname(resolved) if (Array.isArray(moduleFactory)) {
] const [moduleFactoryFn, otherIds] = moduleFactory;
}); moduleFactories[moduleId] = moduleFactoryFn;
return require(resolvedId); for (const otherModuleId of otherIds){
}; moduleFactories[otherModuleId] = moduleFactoryFn;
const module1 = {
exports: {}
};
(0, eval)('(function(module, exports, require, __dirname, __filename) {' + contents + '\n})' + '\n//# sourceURL=' + url.pathToFileURL(resolved))(module1, module1.exports, localRequire, path.dirname(resolved), resolved);
const chunkModules = module1.exports;
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory;
moduleFactories[moduleId] = moduleFactoryFn;
for (const otherModuleId of otherIds){
moduleFactories[otherModuleId] = moduleFactoryFn;
}
} else {
moduleFactories[moduleId] = moduleFactory;
} }
} else {
moduleFactories[moduleId] = moduleFactory;
} }
} }
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`;
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`;
}
throw new Error(errorMessage, {
cause: e
});
} }
} }
function loadChunkAsync(source, chunkData) { function loadChunkAsync(source, chunkData) {
...@@ -587,10 +567,20 @@ function loadChunkAsync(source, chunkData) { ...@@ -587,10 +567,20 @@ function loadChunkAsync(source, chunkData) {
} }
let entry = chunkCache.get(chunkPath); let entry = chunkCache.get(chunkPath);
if (entry === undefined) { if (entry === undefined) {
const resolve = chunkCache.set.bind(chunkCache, chunkPath, loadedChunk); try {
// A new Promise ensures callers that don't handle rejection will still trigger one unhandled rejection. // Load the chunk synchronously
// Handling the rejection will not trigger unhandled rejections. loadChunkUncached(chunkPath);
entry = loadChunkAsyncUncached(source, chunkPath).then(resolve); entry = loadedChunk;
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`;
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`;
}
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(new Error(errorMessage, {
cause: e
}));
}
chunkCache.set(chunkPath, entry); chunkCache.set(chunkPath, entry);
} }
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality. // TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment