Unverified Commit 8a8888ca authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] use `require` to load chunks in our node runtime (#81738)

## Use `require` to load chunks in node instead of readFile+eval

The benefit of this is that we can slightly simplify chunk loading and make it easier for hosting services to cache bytecode.

Much like [browsers](https://v8.dev/blog/code-caching-for-devs), server environments often leverage VM features to cache parsed javascript as bytecode (or maybe binary ast?) to accelerate cold starts ([example](https://vercel.com/blog/introducing-bytecode-caching-for-vercel-functions)), however, these systems generally avoid caching highly dynamic content like inline scripts and `eval` parameters.  

So just use `require` which is what platforms expect!  This should help platforms achieve code caching, is generally simpler, and by not adding a prefix we fix some subtle off-by-one issue with source maps. 

A slightly better solution imho would be `import()` since it would enable node to do some file reading parsing async with the main thread which could be important if there are multiple `preloadModules`, however this breaks dev mode hot reloading.

Fixes #PACK-5062
parent ae2c750d
......@@ -657,14 +657,16 @@ describe('Cache Components Errors', () => {
} else {
expect(output).toMatchInlineSnapshot(`
"Error: Route "/dynamic-root": A component accessed data, headers, params, searchParams, or a short-lived cache without a Suspense boundary nor a "use cache" above it. See more info: https://nextjs.org/docs/messages/next-prerender-missing-suspense
at c (turbopack:///[project]/app/dynamic-root/indirection.tsx:9:1)
at c (turbopack:///[project]/app/dynamic-root/indirection.tsx:7:34)
at main (<anonymous>)
at body (<anonymous>)
at html (<anonymous>)
7 | export function IndirectionTwo({ children }) {
5 | }
6 |
> 7 | export function IndirectionTwo({ children }) {
| ^
8 | return children
> 9 | }
| ^
9 | }
10 |
To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/dynamic-root" in your browser to investigate the error.
......@@ -1680,14 +1682,14 @@ describe('Cache Components Errors', () => {
} else {
expect(output).toMatchInlineSnapshot(`
"Error: Route "/sync-attribution/guarded-async-unguarded-clientsync" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client
at c (turbopack:///[project]/app/sync-attribution/guarded-async-unguarded-clientsync/client.tsx:9:7)
7 | return (
8 | <main>
> 9 | <h1>Sync IO Access</h1>
| ^
10 | <p suppressHydrationWarning>Current date and time: {data}</p>
11 | </main>
12 | )
at c (turbopack:///[project]/app/sync-attribution/guarded-async-unguarded-clientsync/client.tsx:5:16)
3 | export function SyncIO() {
4 | // This is a sync IO access that should not cause an error
> 5 | const data = new Date().toISOString()
| ^
6 |
7 | return (
8 | <main>
To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/sync-attribution/guarded-async-unguarded-clientsync" in your browser to investigate the error.
- Rerun the production build with \`next build --debug-prerender\` to generate better stack traces.
......@@ -1989,14 +1991,14 @@ describe('Cache Components Errors', () => {
} else {
expect(output).toMatchInlineSnapshot(`
"Error: Route "/sync-attribution/unguarded-async-unguarded-clientsync" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client
at c (turbopack:///[project]/app/sync-attribution/unguarded-async-unguarded-clientsync/client.tsx:9:7)
7 | return (
8 | <main>
> 9 | <h1>Sync IO Access</h1>
| ^
10 | <p suppressHydrationWarning>Current date and time: {data}</p>
11 | </main>
12 | )
at c (turbopack:///[project]/app/sync-attribution/unguarded-async-unguarded-clientsync/client.tsx:5:16)
3 | export function SyncIO() {
4 | // This is a sync IO access that should not cause an error
> 5 | const data = new Date().toISOString()
| ^
6 |
7 | return (
8 | <main>
To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/sync-attribution/unguarded-async-unguarded-clientsync" in your browser to investigate the error.
- Rerun the production build with \`next build --debug-prerender\` to generate better stack traces.
......
......@@ -230,14 +230,15 @@ describe('app-dir - server source maps', () => {
}
} else {
if (isTurbopack) {
// TODO(veil): Sourcemapping line off
// TODO(veil): Sourcemap names
// TODO(veil): relative paths
expect(normalizeCliOutput(next.cliOutput)).toContain(
'(bundler:///app/ssr-error-log-ignore-listed/page.js:23:5)'
'(bundler:///app/ssr-error-log-ignore-listed/page.js:9:17)'
)
expect(normalizeCliOutput(next.cliOutput)).toContain(
'\n> 23 | })\n | ^'
'\n' +
"> 9 | const error = new Error('ssr-error-log-ignore-listed')\n" +
' | ^\n'
)
} else {
// TODO(veil): line/column numbers are flaky in Webpack
......
......@@ -82,11 +82,12 @@ describe('build-output-prerender', () => {
// TODO(veil): Why is the location incomplete unless we enable --no-mangling?
expect(getPrerenderOutput(next.cliOutput)).toMatchInlineSnapshot(`
"Error: Route "/client" used \`new Date()\` inside a Client Component without a Suspense boundary above it. See more info here: https://nextjs.org/docs/messages/next-prerender-current-time-client
at c (bundler:///app/client/page.tsx:5:1)
at c (bundler:///app/client/page.tsx:4:28)
2 |
3 | export default function Page() {
4 | return <p>Current time: {new Date().toISOString()}</p>
> 5 | }
| ^
> 4 | return <p>Current time: {new Date().toISOString()}</p>
| ^
5 | }
6 |
To get a more detailed stack trace and pinpoint the issue, try one of the following:
- Start the app in development mode by running \`next dev\`, then open "/client" in your browser to investigate the error.
......
......@@ -86,16 +86,16 @@ function createResolvePathFromModule(
function loadChunk(chunkData: ChunkData, source?: SourceInfo): void {
if (typeof chunkData === 'string') {
return loadChunkPath(chunkData, source)
loadChunkPath(chunkData, source)
} else {
return loadChunkPath(chunkData.path, source)
loadChunkPath(chunkData.path, source)
}
}
const loadedChunks = new Set<ChunkPath>()
const unsupportedLoadChunk = Promise.resolve(undefined)
const loadedChunk = Promise.resolve(undefined)
const chunkCache = new Map<ChunkPath, Promise<any> | typeof loadedChunk>()
const loadedChunk: Promise<void> = Promise.resolve(undefined)
const chunkCache = new Map<ChunkPath, Promise<void>>()
function clearChunkCache() {
chunkCache.clear()
......@@ -119,7 +119,7 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void {
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory
const [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
......@@ -143,63 +143,32 @@ function loadChunkPath(chunkPath: ChunkPath, source?: SourceInfo): void {
}
}
async function loadChunkAsyncUncached(
source: SourceInfo,
chunkPath: ChunkPath
): Promise<void> {
function loadChunkUncached(chunkPath: ChunkPath) {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
try {
const contents = await fs.readFile(resolved, 'utf-8')
const localRequire = (id: string) => {
let resolvedId = require.resolve(id, { paths: [path.dirname(resolved)] })
return require(resolvedId)
}
const module = {
exports: {},
}
// TODO: Use vm.runInThisContext once our minimal supported Node.js version includes https://github.com/nodejs/node/pull/52153
// eslint-disable-next-line no-eval -- Can't use vm.runInThisContext due to https://github.com/nodejs/node/issues/52102
;(0, eval)(
'(function(module, exports, require, __dirname, __filename) {' +
contents +
'\n})' +
'\n//# sourceURL=' +
url.pathToFileURL(resolved)
)(module, module.exports, localRequire, path.dirname(resolved), resolved)
const chunkModules: CompressedModuleFactories = module.exports
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
}
} else {
moduleFactories[moduleId] = moduleFactory
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules: CompressedModuleFactories = require(resolved)
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
const [moduleFactoryFn, otherIds] = moduleFactory
moduleFactories[moduleId] = moduleFactoryFn
for (const otherModuleId of otherIds) {
moduleFactories[otherModuleId] = moduleFactoryFn
}
} else {
moduleFactories[moduleId] = moduleFactory
}
}
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`
}
throw new Error(errorMessage, {
cause: e,
})
}
}
function loadChunkAsync(
source: SourceInfo,
chunkData: ChunkData
): Promise<any> {
): Promise<void> {
const chunkPath = typeof chunkData === 'string' ? chunkData : chunkData.path
if (!isJs(chunkPath)) {
// We only support loading JS chunks in Node.js.
......@@ -209,10 +178,23 @@ function loadChunkAsync(
let entry = chunkCache.get(chunkPath)
if (entry === undefined) {
const resolve = chunkCache.set.bind(chunkCache, chunkPath, loadedChunk)
// A new Promise ensures callers that don't handle rejection will still trigger one unhandled rejection.
// Handling the rejection will not trigger unhandled rejections.
entry = loadChunkAsyncUncached(source, chunkPath).then(resolve)
try {
// Load the chunk synchronously
loadChunkUncached(chunkPath)
entry = loadedChunk
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`
}
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(
new Error(errorMessage, {
cause: e,
})
)
}
chunkCache.set(chunkPath, entry)
}
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
......
......@@ -490,9 +490,9 @@ const moduleCache = Object.create(null);
}
function loadChunk(chunkData, source) {
if (typeof chunkData === 'string') {
return loadChunkPath(chunkData, source);
loadChunkPath(chunkData, source);
} else {
return loadChunkPath(chunkData.path, source);
loadChunkPath(chunkData.path, source);
}
}
const loadedChunks = new Set();
......@@ -517,7 +517,7 @@ function loadChunkPath(chunkPath, source) {
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory;
const [moduleFactoryFn, otherIds] = moduleFactory;
moduleFactories[moduleId] = moduleFactoryFn;
for (const otherModuleId of otherIds){
moduleFactories[otherModuleId] = moduleFactoryFn;
......@@ -538,44 +538,24 @@ function loadChunkPath(chunkPath, source) {
});
}
}
async function loadChunkAsyncUncached(source, chunkPath) {
function loadChunkUncached(chunkPath) {
// resolve to an absolute path to simplify `require` handling
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
try {
const contents = await fs.readFile(resolved, 'utf-8');
const localRequire = (id)=>{
let resolvedId = require.resolve(id, {
paths: [
path.dirname(resolved)
]
});
return require(resolvedId);
};
const module1 = {
exports: {}
};
(0, eval)('(function(module, exports, require, __dirname, __filename) {' + contents + '\n})' + '\n//# sourceURL=' + url.pathToFileURL(resolved))(module1, module1.exports, localRequire, path.dirname(resolved), resolved);
const chunkModules = module1.exports;
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
let [moduleFactoryFn, otherIds] = moduleFactory;
moduleFactories[moduleId] = moduleFactoryFn;
for (const otherModuleId of otherIds){
moduleFactories[otherModuleId] = moduleFactoryFn;
}
} else {
moduleFactories[moduleId] = moduleFactory;
// TODO: consider switching to `import()` to enable concurrent chunk loading and async file io
// However this is incompatible with hot reloading (since `import` doesn't use the require cache)
const chunkModules = require(resolved);
for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
if (!moduleFactories[moduleId]) {
if (Array.isArray(moduleFactory)) {
const [moduleFactoryFn, otherIds] = moduleFactory;
moduleFactories[moduleId] = moduleFactoryFn;
for (const otherModuleId of otherIds){
moduleFactories[otherModuleId] = moduleFactoryFn;
}
} else {
moduleFactories[moduleId] = moduleFactory;
}
}
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`;
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`;
}
throw new Error(errorMessage, {
cause: e
});
}
}
function loadChunkAsync(source, chunkData) {
......@@ -587,10 +567,20 @@ function loadChunkAsync(source, chunkData) {
}
let entry = chunkCache.get(chunkPath);
if (entry === undefined) {
const resolve = chunkCache.set.bind(chunkCache, chunkPath, loadedChunk);
// A new Promise ensures callers that don't handle rejection will still trigger one unhandled rejection.
// Handling the rejection will not trigger unhandled rejections.
entry = loadChunkAsyncUncached(source, chunkPath).then(resolve);
try {
// Load the chunk synchronously
loadChunkUncached(chunkPath);
entry = loadedChunk;
} catch (e) {
let errorMessage = `Failed to load chunk ${chunkPath}`;
if (source) {
errorMessage += ` from ${stringifySourceInfo(source)}`;
}
// Cache the failure promise, future requests will also get this same rejection
entry = Promise.reject(new Error(errorMessage, {
cause: e
}));
}
chunkCache.set(chunkPath, entry);
}
// TODO: Return an instrumented Promise that React can use instead of relying on referential equality.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment