Unverified Commit eb14325e authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: remove deployment id suffix from client reference manifest chunks (#88741)

Closes PACK-6539

When `experimental.runtimeServerDeploymentId` is enabled, read the `process.env.NEXT_DEPLOYMENT_ID` and append to the chunks when evaluating the `client-reference-manifest.js`

Originally, I tried to do it after the manifest is loaded (in `loadComponents` or in the `RouteModule.prepare`) to "hydrate" the manifest, but that is too late because the manifest needs to be initialized immediately to make module-evaluation-time server actions work (which need the manifest immediately)
parent 689f6fe3
...@@ -968,6 +968,8 @@ pub struct ExperimentalConfig { ...@@ -968,6 +968,8 @@ pub struct ExperimentalConfig {
cache_components: Option<bool>, cache_components: Option<bool>,
use_cache: Option<bool>, use_cache: Option<bool>,
root_params: Option<bool>, root_params: Option<bool>,
runtime_server_deployment_id: Option<bool>,
// --- // ---
// UNSUPPORTED // UNSUPPORTED
// --- // ---
...@@ -1815,6 +1817,15 @@ impl NextConfig { ...@@ -1815,6 +1817,15 @@ impl NextConfig {
) )
} }
#[turbo_tasks::function]
pub fn runtime_server_deployment_id_available(&self) -> Vc<bool> {
Vc::cell(
self.experimental
.runtime_server_deployment_id
.unwrap_or(false),
)
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn cache_kinds(&self) -> Vc<CacheKinds> { pub fn cache_kinds(&self) -> Vc<CacheKinds> {
let mut cache_kinds = CacheKinds::default(); let mut cache_kinds = CacheKinds::default();
......
...@@ -167,12 +167,15 @@ async fn build_manifest( ...@@ -167,12 +167,15 @@ async fn build_manifest(
async move { async move {
let mut entry_manifest: SerializedClientReferenceManifest = Default::default(); let mut entry_manifest: SerializedClientReferenceManifest = Default::default();
let mut references = FxIndexSet::default(); let mut references = FxIndexSet::default();
let chunk_suffix_path = next_config.chunk_suffix_path().owned().await?;
let prefix_path = next_config.computed_asset_prefix().owned().await?; let prefix_path = next_config.computed_asset_prefix().owned().await?;
let suffix_path = chunk_suffix_path.unwrap_or_default(); let runtime_server_deployment_id_available =
*next_config.runtime_server_deployment_id_available().await?;
// TODO: Add `suffix` to the manifest for React to use. let suffix_path = if !runtime_server_deployment_id_available {
// entry_manifest.module_loading.prefix = prefix_path; let chunk_suffix_path = next_config.chunk_suffix_path().owned().await?;
chunk_suffix_path.unwrap_or_default()
} else {
rcstr!("")
};
entry_manifest.module_loading.cross_origin = next_config.cross_origin().owned().await?; entry_manifest.module_loading.cross_origin = next_config.cross_origin().owned().await?;
let ClientReferencesChunks { let ClientReferencesChunks {
...@@ -469,15 +472,28 @@ async fn build_manifest( ...@@ -469,15 +472,28 @@ async fn build_manifest(
FileContent::Content(File::from(formatdoc! { FileContent::Content(File::from(formatdoc! {
r#" r#"
globalThis.__RSC_MANIFEST = globalThis.__RSC_MANIFEST || {{}}; globalThis.__RSC_MANIFEST = globalThis.__RSC_MANIFEST || {{}};
globalThis.__RSC_MANIFEST[{entry_name}] = {manifest} globalThis.__RSC_MANIFEST[{entry_name}] = {manifest};
{suffix}
"#,
entry_name = StringifyJs(&normalized_manifest_entry),
manifest = &client_reference_manifest_json,
suffix = if runtime_server_deployment_id_available {
formatdoc!{
r#"
for (const key in globalThis.__RSC_MANIFEST[{entry_name}].clientModules) {{
const val = {{ ...globalThis.__RSC_MANIFEST[{entry_name}].clientModules[key] }}
globalThis.__RSC_MANIFEST[{entry_name}].clientModules[key] = val
val.chunks = val.chunks.map((c) => `${{c}}?dpl=${{process.env.NEXT_DEPLOYMENT_ID}}`)
}}
"#, "#,
entry_name = StringifyJs(&normalized_manifest_entry), entry_name = StringifyJs(&normalized_manifest_entry),
manifest = &client_reference_manifest_json }
} else {
"".to_string()
}
})) }))
.cell(), .cell(),
) ).to_resolved().await?,
.to_resolved()
.await?,
references: ResolvedVc::cell(references.into_iter().collect()), references: ResolvedVc::cell(references.into_iter().collect()),
} }
.cell()) .cell())
......
...@@ -114,7 +114,6 @@ async function requestHandler( ...@@ -114,7 +114,6 @@ async function requestHandler(
buildManifest, buildManifest,
prerenderManifest, prerenderManifest,
reactLoadableManifest, reactLoadableManifest,
clientReferenceManifest,
subresourceIntegrityManifest, subresourceIntegrityManifest,
dynamicCssManifest, dynamicCssManifest,
} = prepareResult } = prepareResult
...@@ -168,7 +167,6 @@ async function requestHandler( ...@@ -168,7 +167,6 @@ async function requestHandler(
buildManifest, buildManifest,
subresourceIntegrityManifest, subresourceIntegrityManifest,
reactLoadableManifest, reactLoadableManifest,
clientReferenceManifest,
dynamicCssManifest, dynamicCssManifest,
}, },
} }
......
...@@ -610,7 +610,7 @@ export class ClientReferenceManifestPlugin { ...@@ -610,7 +610,7 @@ export class ClientReferenceManifestPlugin {
new sources.RawSource( new sources.RawSource(
`globalThis.__RSC_MANIFEST=(globalThis.__RSC_MANIFEST||{});globalThis.__RSC_MANIFEST[${JSON.stringify( `globalThis.__RSC_MANIFEST=(globalThis.__RSC_MANIFEST||{});globalThis.__RSC_MANIFEST[${JSON.stringify(
pagePath.slice('app'.length) pagePath.slice('app'.length)
)}]=${json}` )}]=${json};`
) as unknown as webpack.sources.RawSource ) as unknown as webpack.sources.RawSource
) )
} }
......
...@@ -146,7 +146,7 @@ async function tryLoadClientReferenceManifest( ...@@ -146,7 +146,7 @@ async function tryLoadClientReferenceManifest(
manifestPath: string, manifestPath: string,
entryName: string, entryName: string,
attempts?: number attempts?: number
) { ): Promise<DeepReadonly<ClientReferenceManifest> | undefined> {
try { try {
const context = await evalManifestWithRetries<{ const context = await evalManifestWithRetries<{
__RSC_MANIFEST: { [key: string]: ClientReferenceManifest } __RSC_MANIFEST: { [key: string]: ClientReferenceManifest }
......
...@@ -92,7 +92,9 @@ export function evalManifest<T extends object>( ...@@ -92,7 +92,9 @@ export function evalManifest<T extends object>(
throw new Error('Manifest file is empty') throw new Error('Manifest file is empty')
} }
let contextObject = {} let contextObject = {
process: { env: { NEXT_DEPLOYMENT_ID: process.env.NEXT_DEPLOYMENT_ID } },
}
runInNewContext(content, contextObject) runInNewContext(content, contextObject)
// Freeze the context object so it cannot be modified if we're caching it. // Freeze the context object so it cannot be modified if we're caching it.
......
import { type NextInstance, nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { type ClientReferenceManifest } from 'next/dist/build/webpack/plugins/flight-manifest-plugin' import { getClientReferenceManifest } from 'next-test-utils'
async function loadClientReferenceManifest(
next: NextInstance,
page: string
): Promise<ClientReferenceManifest> {
return JSON.parse(
(
await next.readFile(
`${next.distDir}/server/app${page}page_client-reference-manifest.js`
)
).match(/]\s*=\s*([\S\s]+)$/)[1]
)
}
describe('client-reference-chunking', () => { describe('client-reference-chunking', () => {
const { next } = nextTestSetup({ const { next } = nextTestSetup({
...@@ -28,8 +15,8 @@ describe('client-reference-chunking', () => { ...@@ -28,8 +15,8 @@ describe('client-reference-chunking', () => {
'Welcome to the Issue Page' 'Welcome to the Issue Page'
) )
let rootManifest = await loadClientReferenceManifest(next, '/') let rootManifest = getClientReferenceManifest(next, '/page')
let issueManifest = await loadClientReferenceManifest(next, '/issue/') let issueManifest = getClientReferenceManifest(next, '/issue/page')
// These two routes have the same client component references, so these should be exactly the // These two routes have the same client component references, so these should be exactly the
// same (especially the `chunks` field) // same (especially the `chunks` field)
......
import path from 'path' import path from 'path'
import { check, getDistDir } from 'next-test-utils' import {
check,
getClientReferenceManifest,
getDistDir,
retry,
} from 'next-test-utils'
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import cheerio from 'cheerio' import cheerio from 'cheerio'
import { import {
...@@ -37,25 +42,20 @@ describe('app dir - rsc basics', () => { ...@@ -37,25 +42,20 @@ describe('app dir - rsc basics', () => {
if (isNextDev && !isTurbopack) { if (isNextDev && !isTurbopack) {
it('should have correct client references keys in manifest', async () => { it('should have correct client references keys in manifest', async () => {
await next.render('/') await next.render('/')
await check(async () => { await retry(() => {
// Check that the client-side manifest is correct before any requests // Check that the client-side manifest is correct before any requests
const clientReferenceManifest = JSON.parse( const clientReferenceManifest = getClientReferenceManifest(
( next,
await next.readFile( '/page'
`${getDistDir()}/server/app/page_client-reference-manifest.js`
)
).match(/]=(.+)$/)[1]
) )
const clientModulesNames = Object.keys( const clientModulesNames = Object.keys(
clientReferenceManifest.clientModules clientReferenceManifest.clientModules
) )
clientModulesNames.every((name) => { expect(clientModulesNames).toSatisfyAll((name) => {
const [, key] = name.split('#', 2) const [, key] = name.split('#', 2)
return key === undefined || key === '' || key === 'default' return key === undefined || key === '' || key === 'default'
}) })
})
return 'success'
}, 'success')
}) })
} }
......
...@@ -35,6 +35,8 @@ import escapeRegex from 'escape-string-regexp' ...@@ -35,6 +35,8 @@ import escapeRegex from 'escape-string-regexp'
// TODO: Create dedicated Jest environment that sets up these matchers // TODO: Create dedicated Jest environment that sets up these matchers
// Edge Runtime unit tests fail with "EvalError: Code generation from strings disallowed for this context" if these matchers are imported in those tests. // Edge Runtime unit tests fail with "EvalError: Code generation from strings disallowed for this context" if these matchers are imported in those tests.
import './add-redbox-matchers' import './add-redbox-matchers'
import { NextInstance } from 'e2e-utils'
import { ClientReferenceManifest } from 'next/dist/build/webpack/plugins/flight-manifest-plugin'
export { shouldUseTurbopack } export { shouldUseTurbopack }
...@@ -1921,3 +1923,41 @@ export function getDistDir(): '.next' | '.next/dev' { ...@@ -1921,3 +1923,41 @@ export function getDistDir(): '.next' | '.next/dev' {
? '.next/dev' ? '.next/dev'
: '.next' : '.next'
} }
/**
* Loads and returns the client reference manifest for a given route
*/
export function getClientReferenceManifest(
next: NextInstance,
route: string
): ClientReferenceManifest {
const manifestPath = path.join(
next.testDir,
next.distDir,
`server/app${route}_client-reference-manifest.js`
)
const modulePath = require.resolve(manifestPath)
// Clear global
delete (globalThis as any).__RSC_MANIFEST
// Need to use jest.isolateModules because Jest messes with require.cache and `delete
// require.cache[modulePath]` doesn't actually work anymore
jest.isolateModules(() => {
// Load the manifest (it sets globalThis.__RSC_MANIFEST)
require(modulePath)
})
const manifest = (globalThis as any).__RSC_MANIFEST[
route
] as ClientReferenceManifest
// Sanity check
expect(
manifest.clientModules ||
manifest.ssrModuleMapping ||
manifest.rscModuleMapping
).toBeDefined()
return manifest
}
import { promises as fs } from 'fs'
import { join } from 'path' import { join } from 'path'
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { retry } from 'next-test-utils' import { getClientReferenceManifest, retry } from 'next-test-utils'
import { ClientReferenceManifest } from 'next/dist/build/webpack/plugins/flight-manifest-plugin'
function getServerReferenceIdsFromBundle(source: string): string[] { function getServerReferenceIdsFromBundle(source: string): string[] {
// Reference IDs are strings with [0-9a-f] that are at least 32 characters long. // Reference IDs are strings with [0-9a-f] that are at least 32 characters long.
...@@ -33,29 +31,20 @@ describe('app-dir - client-actions-tree-shaking', () => { ...@@ -33,29 +31,20 @@ describe('app-dir - client-actions-tree-shaking', () => {
/** /**
* Parses the client reference manifest for a given route and returns the client chunks * Parses the client reference manifest for a given route and returns the client chunks
*/ */
function getClientChunks(appDir: string, route: string): Array<string> { function getClientChunks(route: string): Array<string> {
const modulePath = require.resolve( const clientManifest = getClientReferenceManifest(next, route)
join(appDir, `.next/server/app/${route}_client-reference-manifest.js`)
)
require(modulePath)
const clientManfiest = globalThis.__RSC_MANIFEST[
route
] as ClientReferenceManifest
delete globalThis.__RSC_MANIFEST[route]
delete require.cache[modulePath]
const chunks = new Set<string>() const chunks = new Set<string>()
if (process.env.IS_TURBOPACK_TEST) { if (process.env.IS_TURBOPACK_TEST) {
// These only exist for turbopack and are encoded as files // These only exist for turbopack and are encoded as files
// entryJSFiles is a map of moduel name to a set of chunks relative to `.next` // entryJSFiles is a map of module name to a set of chunks relative to `.next`
for (const entries of Object.values(clientManfiest.entryJSFiles)) { for (const entries of Object.values(clientManifest.entryJSFiles)) {
for (const chunk of entries) { for (const chunk of entries) {
chunks.add(chunk) chunks.add(chunk)
} }
} }
// client mmodules is a mapping from module name to a set of chunks releative to `/_next/` // client mmodules is a mapping from module name to a set of chunks releative to `/_next/`
// So strip that prefix and add it to the chunks // So strip that prefix and add it to the chunks
for (const clientModule of Object.values(clientManfiest.clientModules)) { for (const clientModule of Object.values(clientManifest.clientModules)) {
for (const chunk of clientModule.chunks) { for (const chunk of clientModule.chunks) {
chunks.add(chunk.replace('/_next/', '')) chunks.add(chunk.replace('/_next/', ''))
} }
...@@ -63,7 +52,7 @@ describe('app-dir - client-actions-tree-shaking', () => { ...@@ -63,7 +52,7 @@ describe('app-dir - client-actions-tree-shaking', () => {
} else { } else {
// webpack doens't use entryJSFiles, so we need to use clientModules but the format is different. // webpack doens't use entryJSFiles, so we need to use clientModules but the format is different.
// chunks is a sequence of 'chunk-id', chunk-path pairs, so we need to skip the chunk-id // chunks is a sequence of 'chunk-id', chunk-path pairs, so we need to skip the chunk-id
for (const clientModule of Object.values(clientManfiest.clientModules)) { for (const clientModule of Object.values(clientManifest.clientModules)) {
for (let i = 1; i < clientModule.chunks.length; i += 2) { for (let i = 1; i < clientModule.chunks.length; i += 2) {
chunks.add(clientModule.chunks[i]) chunks.add(clientModule.chunks[i])
} }
...@@ -73,25 +62,23 @@ describe('app-dir - client-actions-tree-shaking', () => { ...@@ -73,25 +62,23 @@ describe('app-dir - client-actions-tree-shaking', () => {
} }
it('should not bundle unused server reference id in client bundles', async () => { it('should not bundle unused server reference id in client bundles', async () => {
const appDir = next.testDir const bundle1Files = getClientChunks('/route-1/page')
const bundle2Files = getClientChunks('/route-2/page')
const bundle1Files = getClientChunks(appDir, '/route-1/page') const bundle3Files = getClientChunks('/route-3/page')
const bundle2Files = getClientChunks(appDir, '/route-2/page')
const bundle3Files = getClientChunks(appDir, '/route-3/page')
const bundle1Contents = await Promise.all( const bundle1Contents = await Promise.all(
bundle1Files.map((file: string) => bundle1Files.map((file: string) =>
fs.readFile(join(appDir, '.next', file), 'utf8') next.readFile(join(next.distDir, file))
) )
) )
const bundle2Contents = await Promise.all( const bundle2Contents = await Promise.all(
bundle2Files.map((file: string) => bundle2Files.map((file: string) =>
fs.readFile(join(appDir, '.next', file), 'utf8') next.readFile(join(next.distDir, file))
) )
) )
const bundle3Contents = await Promise.all( const bundle3Contents = await Promise.all(
bundle3Files.map((file: string) => bundle3Files.map((file: string) =>
fs.readFile(join(appDir, '.next', file), 'utf8') next.readFile(join(next.distDir, file))
) )
) )
......
import { join } from 'path'
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { getClientReferenceManifest } from 'next-test-utils'
import type { ClientReferenceManifest } from 'next/dist/build/webpack/plugins/flight-manifest-plugin' import type { ClientReferenceManifest } from 'next/dist/build/webpack/plugins/flight-manifest-plugin'
describe('route-handler-manifest-size', () => { describe('route-handler-manifest-size', () => {
...@@ -11,38 +11,6 @@ describe('route-handler-manifest-size', () => { ...@@ -11,38 +11,6 @@ describe('route-handler-manifest-size', () => {
if (skipped) return if (skipped) return
/**
* Loads and returns the client reference manifest for a given route
*/
function getClientReferenceManifest(route: string): ClientReferenceManifest {
const appDir = next.testDir
const manifestPath = join(
appDir,
`.next/server/app${route}_client-reference-manifest.js`
)
const modulePath = require.resolve(manifestPath)
// Clear any cached version
delete require.cache[modulePath]
// Initialize global if needed
if (!(globalThis as any).__RSC_MANIFEST) {
;(globalThis as any).__RSC_MANIFEST = {}
}
// Load the manifest (it sets globalThis.__RSC_MANIFEST)
require(modulePath)
const manifest = (globalThis as any).__RSC_MANIFEST[
route
] as ClientReferenceManifest
// Clean up require cache but keep manifest for potential re-reads
delete require.cache[modulePath]
return manifest
}
/** /**
* Gets the module paths from clientModules * Gets the module paths from clientModules
*/ */
...@@ -52,7 +20,7 @@ describe('route-handler-manifest-size', () => { ...@@ -52,7 +20,7 @@ describe('route-handler-manifest-size', () => {
if (isNextStart) { if (isNextStart) {
it('should not include page client components in pure route handler manifest', () => { it('should not include page client components in pure route handler manifest', () => {
const manifest = getClientReferenceManifest('/api/hello/route') const manifest = getClientReferenceManifest(next, '/api/hello/route')
const modulePaths = getClientModulePaths(manifest) const modulePaths = getClientModulePaths(manifest)
// The pure route handler should NOT contain client components from the page // The pure route handler should NOT contain client components from the page
...@@ -66,7 +34,7 @@ describe('route-handler-manifest-size', () => { ...@@ -66,7 +34,7 @@ describe('route-handler-manifest-size', () => {
}) })
it('should include page client components in page manifest', () => { it('should include page client components in page manifest', () => {
const manifest = getClientReferenceManifest('/page') const manifest = getClientReferenceManifest(next, '/page')
const modulePaths = getClientModulePaths(manifest) const modulePaths = getClientModulePaths(manifest)
// The page should contain its client components // The page should contain its client components
...@@ -80,8 +48,8 @@ describe('route-handler-manifest-size', () => { ...@@ -80,8 +48,8 @@ describe('route-handler-manifest-size', () => {
}) })
it('should have significantly smaller manifest for pure route handler compared to page', () => { it('should have significantly smaller manifest for pure route handler compared to page', () => {
const routeManifest = getClientReferenceManifest('/api/hello/route') const routeManifest = getClientReferenceManifest(next, '/api/hello/route')
const pageManifest = getClientReferenceManifest('/page') const pageManifest = getClientReferenceManifest(next, '/page')
const routeModuleCount = Object.keys(routeManifest.clientModules).length const routeModuleCount = Object.keys(routeManifest.clientModules).length
const pageModuleCount = Object.keys(pageManifest.clientModules).length const pageModuleCount = Object.keys(pageManifest.clientModules).length
...@@ -92,7 +60,10 @@ describe('route-handler-manifest-size', () => { ...@@ -92,7 +60,10 @@ describe('route-handler-manifest-size', () => {
}) })
it('should not include page components in route handler that imports client module', () => { it('should not include page components in route handler that imports client module', () => {
const manifest = getClientReferenceManifest('/api-with-client/route') const manifest = getClientReferenceManifest(
next,
'/api-with-client/route'
)
const modulePaths = getClientModulePaths(manifest) const modulePaths = getClientModulePaths(manifest)
// Route handler should NOT have unrelated client components from the page // Route handler should NOT have unrelated client components from the page
......
...@@ -3,5 +3,6 @@ module.exports = { ...@@ -3,5 +3,6 @@ module.exports = {
deploymentId: process.env.CUSTOM_DEPLOYMENT_ID, deploymentId: process.env.CUSTOM_DEPLOYMENT_ID,
experimental: { experimental: {
useSkewCookie: Boolean(process.env.COOKIE_SKEW), useSkewCookie: Boolean(process.env.COOKIE_SKEW),
runtimeServerDeploymentId: !!process.env.RUNTIME_SERVER_DEPLOYMENT_ID,
}, },
} }
...@@ -2,14 +2,21 @@ import { nextTestSetup } from 'e2e-utils' ...@@ -2,14 +2,21 @@ import { nextTestSetup } from 'e2e-utils'
import { retry } from 'next-test-utils' import { retry } from 'next-test-utils'
import { join } from 'node:path' import { join } from 'node:path'
describe.each(['NEXT_DEPLOYMENT_ID', 'CUSTOM_DEPLOYMENT_ID'])( describe.each([
'deployment-id-handling enabled with %s', ['NEXT_DEPLOYMENT_ID', ''],
(envKey) => { ['CUSTOM_DEPLOYMENT_ID', ''],
['NEXT_DEPLOYMENT_ID', ' and runtimeServerDeploymentId'],
])(
'deployment-id-handling enabled with %s%s',
(envKey, runtimeServerDeploymentId) => {
const deploymentId = Date.now() + '' const deploymentId = Date.now() + ''
const { next } = nextTestSetup({ const { next } = nextTestSetup({
files: join(__dirname, 'app'), files: join(__dirname, 'app'),
env: { env: {
[envKey]: deploymentId, [envKey]: deploymentId,
RUNTIME_SERVER_DEPLOYMENT_ID: runtimeServerDeploymentId
? '1'
: undefined,
}, },
}) })
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment