Unverified Commit fef4c28b authored by Josh Story's avatar Josh Story Committed by GitHub

Add RDC compression rollout controls (#97247)

## Summary

Warn during prerendering when the exact UTF-8 size of the uncompressed
postponed-state body exceeds `experimental.maxPostponedStateSize` while
Resume Data Cache compression is enabled.

Add `experimental.disableResumeDataCacheCompression` as an opt-in
rollout flag. It defaults to `false`, preserving the existing compressed
representation. When enabled, both persistence and parsing use raw JSON,
allowing a controlled minimal-mode rollout without format negotiation.

RDC serialization now happens in explicit steps: stringify, check the
raw body size, then conditionally deflate. This avoids compression-ratio
estimates and duplicate serialization.

This is the lower PR in a two-PR stack. It can land first so the raw
representation can be enabled selectively for minimal-mode deployments
and observed on canary before compression is removed.

## Verification

- `pnpm exec jest
packages/next/src/server/resume-data-cache/resume-data-cache.test.ts
packages/next/src/server/app-render/postponed-state.test.ts --runInBand`
- `pnpm --filter=next types`
- `pnpm --filter=next build`

<!-- NEXT_JS_LLM -->
parent 3782922b
......@@ -1467,5 +1467,6 @@
"1466": "TypeScript %s does not provide the compiler API required by Next.js. Set %s to true in your Next.js config to use the TypeScript CLI, or install TypeScript 6 instead.",
"1467": "TypeScript %s does not provide the compiler API required by Next.js. Set %s back to true in your Next.js config to use the TypeScript CLI, or install TypeScript 6 instead.",
"1468": "Missing workUnitStore in createComponentTree",
"1469": "The payload's transport tree is missing a slot that exists in the loader tree: %s"
"1469": "The payload's transport tree is missing a slot that exists in the loader tree: %s",
"1470": "`deflateResumeDataCache` should not be called in edge runtime."
}
......@@ -953,6 +953,9 @@ export function createAppPageEntrypoint({
maxPostponedStateSizeBytes: parseMaxPostponedStateSize(
nextConfig.experimental.maxPostponedStateSize
),
disableResumeDataCacheCompression:
nextConfig.experimental.disableResumeDataCacheCompression ??
false,
exposeTestingApi,
},
......
......@@ -184,6 +184,8 @@ async function requestHandler(
maxPostponedStateSizeBytes: parseMaxPostponedStateSize(
nextConfig.experimental.maxPostponedStateSize
),
disableResumeDataCacheCompression:
nextConfig.experimental.disableResumeDataCacheCompression ?? false,
exposeTestingApi:
nextConfig.cacheComponents === true &&
(pageRouteModule.isDev === true ||
......
......@@ -521,6 +521,8 @@ async function exportAppImpl(
maxPostponedStateSizeBytes: parseMaxPostponedStateSize(
nextConfig.experimental.maxPostponedStateSize
),
disableResumeDataCacheCompression:
nextConfig.experimental.disableResumeDataCacheCompression ?? false,
exposeTestingApi:
nextConfig.experimental.exposeTestingApiInProductionBuild === true,
},
......
......@@ -28,7 +28,10 @@ import { AfterRunner } from '../../server/after/run-with-after'
import type { RequestLifecycleOpts } from '../../server/base-server'
import type { AppSharedContext } from '../../server/app-render/app-render'
import type { MultiFileWriter } from '../../lib/multi-file-writer'
import { stringifyResumeDataCache } from '../../server/resume-data-cache/resume-data-cache'
import {
deflateResumeDataCache,
stringifyResumeDataCache,
} from '../../server/resume-data-cache/resume-data-cache'
import {
UNDERSCORE_GLOBAL_ERROR_ROUTE_ENTRY,
UNDERSCORE_NOT_FOUND_ROUTE_ENTRY,
......@@ -228,6 +231,19 @@ export async function exportAppPage(
JSON.stringify(meta, null, 2)
)
let serializedRenderResumeDataCache: string | undefined
if (renderResumeDataCache) {
serializedRenderResumeDataCache = await stringifyResumeDataCache(
renderResumeDataCache,
renderOpts.cacheComponents
)
if (!renderOpts.experimental.disableResumeDataCacheCompression) {
serializedRenderResumeDataCache = deflateResumeDataCache(
serializedRenderResumeDataCache
)
}
}
return {
// Filter the metadata if the environment does not have next support.
metadata: hasNextSupport
......@@ -243,12 +259,7 @@ export async function exportAppPage(
hasStaticRsc,
cacheControl,
fetchMetrics,
renderResumeDataCache: renderResumeDataCache
? await stringifyResumeDataCache(
renderResumeDataCache,
renderOpts.cacheComponents
)
: undefined,
renderResumeDataCache: serializedRenderResumeDataCache,
}
} catch (err) {
if (!isDynamicUsageError(err)) {
......
......@@ -415,7 +415,8 @@ export async function exportPages(
const renderResumeDataCache = renderResumeDataCachesByPage[pageKey]
? createRenderResumeDataCache(
renderResumeDataCachesByPage[pageKey],
renderOpts.experimental.maxPostponedStateSizeBytes
renderOpts.experimental.maxPostponedStateSizeBytes,
renderOpts.experimental.disableResumeDataCacheCompression
)
: undefined
......
......@@ -3311,14 +3311,16 @@ function prepareAppPage(
type: DynamicState.DATA,
renderResumeDataCache: parseResumeDataCacheFromPostponedState(
renderOpts.postponed,
renderOpts.experimental.maxPostponedStateSizeBytes
renderOpts.experimental.maxPostponedStateSizeBytes,
renderOpts.experimental.disableResumeDataCacheCompression
),
}
} else {
postponedState = parsePostponedState(
renderOpts.postponed,
interpolatedParams,
renderOpts.experimental.maxPostponedStateSizeBytes
renderOpts.experimental.maxPostponedStateSizeBytes,
renderOpts.experimental.disableResumeDataCacheCompression
)
}
}
......@@ -9423,12 +9425,16 @@ async function prerenderToStream(
: DynamicHTMLPreludeState.Full,
fallbackRouteParams,
resumeDataCache,
cacheComponents
cacheComponents,
renderOpts.experimental.maxPostponedStateSizeBytes,
renderOpts.experimental.disableResumeDataCacheCompression
)
} else {
metadata.postponed = await getDynamicDataPostponedState(
resumeDataCache,
cacheComponents
cacheComponents,
renderOpts.experimental.maxPostponedStateSizeBytes,
renderOpts.experimental.disableResumeDataCacheCompression
)
}
reactServerResult.consume()
......@@ -9952,7 +9958,9 @@ async function prerenderToStream(
if (originalFlightPrerenderResultIsDynamic) {
metadata.postponed = await getDynamicDataPostponedState(
originalResumeDataCache,
cacheComponents
cacheComponents,
renderOpts.experimental.maxPostponedStateSizeBytes,
renderOpts.experimental.disableResumeDataCacheCompression
)
originalFlightPrerenderResult.consume()
errorServerResult.consume()
......
import { createPrerenderResumeDataCache } from '../resume-data-cache/resume-data-cache'
import {
createPrerenderResumeDataCache,
stringifyResumeDataCache,
} from '../resume-data-cache/resume-data-cache'
import {
streamFromString,
streamToString,
......@@ -15,6 +18,7 @@ import type {
OpaqueFallbackRouteParams,
OpaqueFallbackRouteParamValue,
} from '../request/fallback-params'
import { CachedRouteKind } from '../response-cache/types'
export function createMockOpaqueFallbackRouteParams(
params: Record<string, OpaqueFallbackRouteParamValue>
......@@ -145,6 +149,79 @@ describe('getDynamicDataPostponedState', () => {
)
expect(state).toMatchInlineSnapshot(`"4:nullnull"`)
})
it('serializes and parses an uncompressed cache when compression is disabled', async () => {
const resumeDataCache = createPrerenderResumeDataCache()
resumeDataCache.fetch.set('cache-key', {
kind: CachedRouteKind.FETCH,
data: {
headers: {},
body: 'cached body',
url: 'https://example.com',
},
revalidate: 60,
})
const serializedResumeDataCache = await stringifyResumeDataCache(
resumeDataCache,
isCacheComponentsEnabled
)
const state = await getDynamicDataPostponedState(
resumeDataCache,
isCacheComponentsEnabled,
undefined,
true
)
expect(state).toBe(`4:null${serializedResumeDataCache}`)
const parsed = parsePostponedState(state, {}, undefined, true)
expect(parsed.renderResumeDataCache.fetch.get('cache-key')).toEqual(
resumeDataCache.fetch.get('cache-key')
)
})
it('warns when the uncompressed state would exceed the size limit', async () => {
const resumeDataCache = createPrerenderResumeDataCache()
resumeDataCache.fetch.set('cache-key', {
kind: CachedRouteKind.FETCH,
data: {
headers: {},
body: '💥'.repeat(2048),
url: 'https://example.com',
},
revalidate: 60,
})
const serializedResumeDataCache = await stringifyResumeDataCache(
resumeDataCache,
isCacheComponentsEnabled
)
const uncompressedStateByteLength = Buffer.byteLength(
`4:null${serializedResumeDataCache}`
)
const warn = jest.spyOn(console, 'warn').mockImplementation(() => {})
await getDynamicDataPostponedState(
resumeDataCache,
isCacheComponentsEnabled,
uncompressedStateByteLength
)
expect(warn).not.toHaveBeenCalled()
await getDynamicDataPostponedState(
resumeDataCache,
isCacheComponentsEnabled,
uncompressedStateByteLength - 1
)
expect(warn).toHaveBeenCalledWith(
expect.stringContaining(
`The uncompressed postponed state is ${uncompressedStateByteLength} bytes`
)
)
warn.mockRestore()
})
})
describe('parseResumeDataCacheFromPostponedState', () => {
......
......@@ -7,10 +7,11 @@ import type { Params } from '../request/params'
import {
createPrerenderResumeDataCache,
createRenderResumeDataCache,
deflateResumeDataCache,
stringifyResumeDataCache,
type PrerenderResumeDataCache,
type RenderResumeDataCache,
} from '../resume-data-cache/resume-data-cache'
import { stringifyResumeDataCache } from '../resume-data-cache/resume-data-cache'
export enum DynamicState {
/**
......@@ -75,12 +76,47 @@ export type PostponedState =
| DynamicDataPostponedState
| DynamicHTMLPostponedState
async function serializePostponedState(
postponedString: string,
resumeDataCache: PrerenderResumeDataCache | RenderResumeDataCache,
isCacheComponentsEnabled: boolean,
maxPostponedStateSizeBytes: number | undefined,
disableResumeDataCacheCompression: boolean
): Promise<string> {
const prefix = `${postponedString.length}:${postponedString}`
let serializedResumeDataCache = await stringifyResumeDataCache(
resumeDataCache,
isCacheComponentsEnabled
)
if (!disableResumeDataCacheCompression) {
if (maxPostponedStateSizeBytes !== undefined) {
const uncompressedPostponedStateByteLength =
Buffer.byteLength(prefix) + Buffer.byteLength(serializedResumeDataCache)
if (uncompressedPostponedStateByteLength > maxPostponedStateSizeBytes) {
console.warn(
`The uncompressed postponed state is ${uncompressedPostponedStateByteLength} bytes, which exceeds the configured experimental.maxPostponedStateSize limit of ${maxPostponedStateSizeBytes} bytes. Next.js currently compresses the Resume Data Cache before persisting the postponed state, but this compression will be removed in a future release. Increase experimental.maxPostponedStateSize to ensure this route can still be resumed after that change.`
)
}
}
serializedResumeDataCache = deflateResumeDataCache(
serializedResumeDataCache
)
}
return prefix + serializedResumeDataCache
}
export async function getDynamicHTMLPostponedState(
postponed: ReactPostponed,
preludeState: DynamicHTMLPreludeState,
fallbackRouteParams: OpaqueFallbackRouteParams | null,
resumeDataCache: PrerenderResumeDataCache | RenderResumeDataCache,
isCacheComponentsEnabled: boolean
isCacheComponentsEnabled: boolean,
maxPostponedStateSizeBytes?: number,
disableResumeDataCacheCompression = false
): Promise<string> {
const data: DynamicHTMLPostponedState['data'] = [preludeState, postponed]
const dataString = JSON.stringify(data)
......@@ -89,10 +125,13 @@ export async function getDynamicHTMLPostponedState(
// state as is.
if (!fallbackRouteParams || fallbackRouteParams.size === 0) {
// Serialized as `<postponedString.length>:<postponedString><renderResumeDataCache>`
return `${dataString.length}:${dataString}${await stringifyResumeDataCache(
createRenderResumeDataCache(resumeDataCache),
isCacheComponentsEnabled
)}`
return serializePostponedState(
dataString,
resumeDataCache,
isCacheComponentsEnabled,
maxPostponedStateSizeBytes,
disableResumeDataCacheCompression
)
}
const replacements: OpaqueFallbackRouteParamEntries = Array.from(
......@@ -104,19 +143,34 @@ export async function getDynamicHTMLPostponedState(
const postponedString = `${replacementsString.length}${replacementsString}${dataString}`
// Serialized as `<postponedString.length>:<postponedString><renderResumeDataCache>`
return `${postponedString.length}:${postponedString}${await stringifyResumeDataCache(resumeDataCache, isCacheComponentsEnabled)}`
return serializePostponedState(
postponedString,
resumeDataCache,
isCacheComponentsEnabled,
maxPostponedStateSizeBytes,
disableResumeDataCacheCompression
)
}
export async function getDynamicDataPostponedState(
resumeDataCache: PrerenderResumeDataCache | RenderResumeDataCache,
isCacheComponentsEnabled: boolean
isCacheComponentsEnabled: boolean,
maxPostponedStateSizeBytes?: number,
disableResumeDataCacheCompression = false
): Promise<string> {
return `4:null${await stringifyResumeDataCache(createRenderResumeDataCache(resumeDataCache), isCacheComponentsEnabled)}`
return serializePostponedState(
'null',
resumeDataCache,
isCacheComponentsEnabled,
maxPostponedStateSizeBytes,
disableResumeDataCacheCompression
)
}
function parsePostponedStateParts(
state: string,
maxPostponedStateSizeBytes: number | undefined
maxPostponedStateSizeBytes: number | undefined,
disableResumeDataCacheCompression: boolean
): {
postponedString: string
renderResumeDataCache: RenderResumeDataCache
......@@ -139,18 +193,23 @@ function parsePostponedStateParts(
),
renderResumeDataCache: createRenderResumeDataCache(
state.slice(tailStart),
maxPostponedStateSizeBytes
maxPostponedStateSizeBytes,
disableResumeDataCacheCompression
),
}
}
export function parseResumeDataCacheFromPostponedState(
state: string,
maxPostponedStateSizeBytes: number | undefined
maxPostponedStateSizeBytes: number | undefined,
disableResumeDataCacheCompression = false
): RenderResumeDataCache {
try {
return parsePostponedStateParts(state, maxPostponedStateSizeBytes)
.renderResumeDataCache
return parsePostponedStateParts(
state,
maxPostponedStateSizeBytes,
disableResumeDataCacheCompression
).renderResumeDataCache
} catch (err) {
console.error(
'Failed to parse postponed state',
......@@ -163,12 +222,14 @@ export function parseResumeDataCacheFromPostponedState(
export function parsePostponedState(
state: string,
interpolatedParams: Params,
maxPostponedStateSizeBytes: number | undefined
maxPostponedStateSizeBytes: number | undefined,
disableResumeDataCacheCompression = false
): PostponedState {
try {
const { postponedString, renderResumeDataCache } = parsePostponedStateParts(
state,
maxPostponedStateSizeBytes
maxPostponedStateSizeBytes,
disableResumeDataCacheCompression
)
try {
......@@ -257,8 +318,9 @@ export function parsePostponedState(
* sensitive) serialized contents. Every field is a size, a structural flag, or
* an error code, never the state bytes themselves.
*
* The serialized layout is `<N>:<postponedString><base64-deflate cache>`, so
* these fields distinguish the failure shapes:
* The serialized layout is `<N>:<postponedString><cache>`. The cache is a
* base64-deflate string by default and raw JSON when RDC compression is
* disabled, so these fields distinguish the failure shapes:
* - `postponedStringComplete: false`: the declared length `N` exceeds what
* actually arrived, i.e. the postponed string itself was truncated.
* - `errorCode: 'Z_BUF_ERROR'` with an empty or short tail: the
......
......@@ -176,6 +176,11 @@ export interface RenderOptsPartial {
*/
maxPostponedStateSizeBytes: number | undefined
/**
* Whether the Resume Data Cache should be persisted without compression.
*/
disableResumeDataCacheCompression: boolean
/**
* Whether the Instant Navigation Testing API is exposed (dev mode or the
* `exposeTestingApiInProductionBuild` flag). When true, the prerendered
......
......@@ -611,6 +611,9 @@ export default abstract class Server<
maxPostponedStateSizeBytes: parseMaxPostponedStateSize(
this.nextConfig.experimental.maxPostponedStateSize
),
disableResumeDataCacheCompression:
this.nextConfig.experimental.disableResumeDataCacheCompression ??
false,
exposeTestingApi:
this.nextConfig.cacheComponents === true &&
(this.dev === true ||
......
......@@ -251,6 +251,7 @@ export const experimentalSchema = {
})
.optional(),
maxPostponedStateSize: zSizeLimit.optional(),
disableResumeDataCacheCompression: z.boolean().optional(),
// The original type was Record<string, any>
extensionAlias: z.record(z.string(), z.any()).optional(),
externalDir: z.boolean().optional(),
......
......@@ -1182,6 +1182,13 @@ export interface ExperimentalConfig {
*/
maxPostponedStateSize?: SizeLimit
/**
* Disables compression of the Resume Data Cache (RDC) when persisting
* postponed state.
* @default false
*/
disableResumeDataCacheCompression?: boolean
/**
* enables the minification of server code.
*
......@@ -2299,6 +2306,7 @@ export const defaultConfig = Object.freeze({
globalNotFound: false,
browserDebugInfoInTerminal: 'warn',
lockDistDir: true,
disableResumeDataCacheCompression: false,
proxyClientMaxBodySize: 10_485_760, // 10MB
hideLogsAfterAbort: false,
mcpServer: true,
......@@ -2410,6 +2418,7 @@ export interface NextConfigRuntime {
| 'testProxy'
| 'runtimeServerDeploymentId'
| 'maxPostponedStateSize'
| 'disableResumeDataCacheCompression'
| 'cachedNavigations'
| 'exposeTestingApiInProductionBuild'
| 'instantInsights'
......@@ -2477,6 +2486,7 @@ export function getNextConfigRuntime(
testProxy: ex.testProxy,
runtimeServerDeploymentId: ex.runtimeServerDeploymentId,
maxPostponedStateSize: ex.maxPostponedStateSize,
disableResumeDataCacheCompression: ex.disableResumeDataCacheCompression,
cachedNavigations: ex.cachedNavigations,
exposeTestingApiInProductionBuild: ex.exposeTestingApiInProductionBuild,
instantInsights: ex.instantInsights,
......
import {
deflateResumeDataCache,
stringifyResumeDataCache,
createRenderResumeDataCache,
} from './resume-data-cache'
import { createPrerenderResumeDataCache } from './resume-data-cache'
import { streamFromString } from '../stream-utils/node-web-streams-helper'
import { inflateSync } from 'node:zlib'
const isCacheComponentsEnabled = process.env.__NEXT_CACHE_COMPONENTS === 'true'
......@@ -79,6 +79,28 @@ function createMockedCacheWithEntryThatFails() {
}
describe('stringifyResumeDataCache', () => {
it('throws in the edge runtime before serializing an empty cache', async () => {
const nextRuntime = process.env.NEXT_RUNTIME
process.env.NEXT_RUNTIME = 'edge'
try {
await expect(
stringifyResumeDataCache(
createPrerenderResumeDataCache(),
isCacheComponentsEnabled
)
).rejects.toThrow(
'`stringifyResumeDataCache` should not be called in edge runtime.'
)
} finally {
if (nextRuntime === undefined) {
delete process.env.NEXT_RUNTIME
} else {
process.env.NEXT_RUNTIME = nextRuntime
}
}
})
it('serializes an empty cache', async () => {
const cache = createPrerenderResumeDataCache()
expect(
......@@ -89,24 +111,17 @@ describe('stringifyResumeDataCache', () => {
it('only serializes cache entries that were not excluded from the prerender result', async () => {
const cache = createMockedCache()
const compressed = await stringifyResumeDataCache(
const serialized = await stringifyResumeDataCache(
cache,
isCacheComponentsEnabled
)
// We have to decompress the output because the compressed string is not
// deterministic. If it fails here it's because the compressed string is
// different.
const decompressed = inflateSync(
Buffer.from(compressed, 'base64')
).toString('utf-8')
if (isCacheComponentsEnabled) {
expect(decompressed).toMatchInlineSnapshot(
expect(serialized).toMatchInlineSnapshot(
`"{"store":{"fetch":{},"cache":{"success":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":300,"revalidate":1}},"encryptedBoundArgs":{}}}"`
)
} else {
expect(decompressed).toMatchInlineSnapshot(
expect(serialized).toMatchInlineSnapshot(
`"{"store":{"fetch":{},"cache":{"success":{"entry":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":300,"revalidate":1},"hasExplicitRevalidate":true,"hasExplicitExpire":true},"dynamic-expire":{"entry":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":299,"revalidate":1},"hasExplicitRevalidate":true,"hasExplicitExpire":true},"zero-revalidate":{"entry":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":300,"revalidate":0},"hasExplicitRevalidate":true,"hasExplicitExpire":true}},"encryptedBoundArgs":{}}}"`
)
}
......@@ -115,26 +130,19 @@ describe('stringifyResumeDataCache', () => {
it('serializes a cache with an entry that fails', async () => {
const cache = createMockedCacheWithEntryThatFails()
const compressed = await stringifyResumeDataCache(
const serialized = await stringifyResumeDataCache(
cache,
isCacheComponentsEnabled
)
// We have to decompress the output because the compressed string is not
// deterministic. If it fails here it's because the compressed string is
// different.
const decompressed = inflateSync(
Buffer.from(compressed, 'base64')
).toString('utf-8')
// We expect that the cache will still contain the successful entries
// but the failed entry will be ignored and omitted from the output.
if (isCacheComponentsEnabled) {
expect(decompressed).toMatchInlineSnapshot(
expect(serialized).toMatchInlineSnapshot(
`"{"store":{"fetch":{},"cache":{"success":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":300,"revalidate":1}},"encryptedBoundArgs":{}}}"`
)
} else {
expect(decompressed).toMatchInlineSnapshot(
expect(serialized).toMatchInlineSnapshot(
`"{"store":{"fetch":{},"cache":{"success":{"entry":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":300,"revalidate":1},"hasExplicitRevalidate":true,"hasExplicitExpire":true},"dynamic-expire":{"entry":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":299,"revalidate":1},"hasExplicitRevalidate":true,"hasExplicitExpire":true},"zero-revalidate":{"entry":{"value":"dmFsdWU=","tags":[],"stale":0,"timestamp":0,"expire":300,"revalidate":0},"hasExplicitRevalidate":true,"hasExplicitExpire":true}},"encryptedBoundArgs":{}}}"`
)
}
......@@ -142,6 +150,25 @@ describe('stringifyResumeDataCache', () => {
})
describe('parseResumeDataCache', () => {
it('throws in the edge runtime before handling an uncompressed cache', () => {
const nextRuntime = process.env.NEXT_RUNTIME
process.env.NEXT_RUNTIME = 'edge'
try {
expect(() =>
createRenderResumeDataCache('null', undefined, true)
).toThrow(
'`createRenderResumeDataCache` should not be called in edge runtime.'
)
} finally {
if (nextRuntime === undefined) {
delete process.env.NEXT_RUNTIME
} else {
process.env.NEXT_RUNTIME = nextRuntime
}
}
})
it('parses an empty cache', () => {
const parsed = createRenderResumeDataCache('null', undefined)
expect(parsed.cache).toEqual(new Map())
......@@ -151,16 +178,26 @@ describe('parseResumeDataCache', () => {
expect(parsed.dynamicCacheKeys).toBeUndefined()
})
it('parses a filled cache', async () => {
const cache = createMockedCache()
const serialized = await stringifyResumeDataCache(
cache,
isCacheComponentsEnabled
)
it.each([false, true])(
'parses a filled cache with compression disabled: %s',
async (disableResumeDataCacheCompression) => {
const cache = createMockedCache()
const serialized = await stringifyResumeDataCache(
cache,
isCacheComponentsEnabled
)
const parsed = createRenderResumeDataCache(serialized, undefined)
const persisted = disableResumeDataCacheCompression
? serialized
: deflateResumeDataCache(serialized)
const parsed = createRenderResumeDataCache(
persisted,
undefined,
disableResumeDataCacheCompression
)
expect(parsed.cache.size).toBe(isCacheComponentsEnabled ? 1 : 3)
expect(parsed.fetch.size).toBe(0)
})
expect(parsed.cache.size).toBe(isCacheComponentsEnabled ? 1 : 3)
expect(parsed.fetch.size).toBe(0)
}
)
})
......@@ -170,36 +170,51 @@ export async function stringifyResumeDataCache(
throw new InvariantError(
'`stringifyResumeDataCache` should not be called in edge runtime.'
)
} else {
if (resumeDataCache.fetch.size === 0 && resumeDataCache.cache.size === 0) {
return 'null'
}
}
if (resumeDataCache.fetch.size === 0 && resumeDataCache.cache.size === 0) {
return 'null'
}
const json: ResumeStoreSerialized = {
store: {
fetch: Object.fromEntries(Array.from(resumeDataCache.fetch.entries())),
cache: Object.fromEntries(
(
await serializeUseCacheCacheStore(
resumeDataCache.cache.entries(),
isCacheComponentsEnabled
)
).filter(
(entry): entry is [string, UseCacheCacheStoreSerialized] =>
entry !== null
const json: ResumeStoreSerialized = {
store: {
fetch: Object.fromEntries(resumeDataCache.fetch),
cache: Object.fromEntries(
(
await serializeUseCacheCacheStore(
resumeDataCache.cache.entries(),
isCacheComponentsEnabled
)
),
encryptedBoundArgs: Object.fromEntries(
Array.from(resumeDataCache.encryptedBoundArgs.entries())
),
},
).filter(
(entry): entry is [string, UseCacheCacheStoreSerialized] =>
entry !== null
)
),
encryptedBoundArgs: Object.fromEntries(
resumeDataCache.encryptedBoundArgs
),
},
}
return JSON.stringify(json)
}
export function deflateResumeDataCache(
serializedResumeDataCache: string
): string {
if (process.env.NEXT_RUNTIME === 'edge') {
throw new InvariantError(
'`deflateResumeDataCache` should not be called in edge runtime.'
)
} else {
if (serializedResumeDataCache === 'null') {
return serializedResumeDataCache
}
// Compress the JSON string using zlib. As the data we already want to
// decompress is in memory, we use the synchronous deflateSync function.
// Compress the JSON string using zlib. As the data is already in memory,
// we use the synchronous deflateSync function.
const { deflateSync } = require('node:zlib') as typeof import('node:zlib')
return deflateSync(JSON.stringify(json)).toString('base64')
return deflateSync(serializedResumeDataCache).toString('base64')
}
}
......@@ -247,6 +262,7 @@ export function createPrerenderResumeDataCache(
* @param prerenderResumeDataCache - A PrerenderResumeDataCache instance to convert to immutable
* @param persistedCache - A serialized cache string to parse
* @param maxPostponedStateSizeBytes - The max compressed size limit in bytes (used to calculate 5x decompression limit)
* @param disableResumeDataCacheCompression - Whether the persisted cache is raw JSON instead of compressed
* @returns An immutable RenderResumeDataCache instance
*/
export function createRenderResumeDataCache(
......@@ -254,11 +270,13 @@ export function createRenderResumeDataCache(
): RenderResumeDataCache
export function createRenderResumeDataCache(
persistedCache: string,
maxPostponedStateSizeBytes: number | undefined
maxPostponedStateSizeBytes: number | undefined,
disableResumeDataCacheCompression?: boolean
): RenderResumeDataCache
export function createRenderResumeDataCache(
resumeDataCacheOrPersistedCache: ResumeDataCache | string,
maxPostponedStateSizeBytes?: number | undefined
maxPostponedStateSizeBytes?: number | undefined,
disableResumeDataCacheCompression = false
): RenderResumeDataCache {
if (process.env.NEXT_RUNTIME === 'edge') {
throw new InvariantError(
......@@ -287,38 +305,43 @@ export function createRenderResumeDataCache(
}
}
// This should be a compressed string. Let's decompress it using zlib.
// As the data we already want to decompress is in memory, we use the
// synchronous inflateSync function.
const { inflateSync } = require('node:zlib') as typeof import('node:zlib')
let serializedResumeDataCache = resumeDataCacheOrPersistedCache
if (!disableResumeDataCacheCompression) {
// This should be a compressed string. Let's decompress it using zlib.
// As the data we already want to decompress is in memory, we use the
// synchronous inflateSync function.
const { inflateSync } = require('node:zlib') as typeof import('node:zlib')
// Limit decompressed size to prevent zipbomb attacks. This is 5x the
// configured maxPostponedStateSize, allowing reasonable compression
// ratios while preventing extreme decompression bombs.
// Default is 500MB (5x the default 100MB compressed limit).
const maxDecompressedSize = maxPostponedStateSizeBytes
? maxPostponedStateSizeBytes * 5
: 500 * 1024 * 1024
// Limit decompressed size to prevent zipbomb attacks. This is 5x the
// configured maxPostponedStateSize, allowing reasonable compression
// ratios while preventing extreme decompression bombs.
// Default is 500MB (5x the default 100MB compressed limit).
const maxDecompressedSize = maxPostponedStateSizeBytes
? maxPostponedStateSizeBytes * 5
: 500 * 1024 * 1024
let json: ResumeStoreSerialized
try {
json = JSON.parse(
inflateSync(Buffer.from(resumeDataCacheOrPersistedCache, 'base64'), {
maxOutputLength: maxDecompressedSize,
}).toString('utf-8')
)
} catch (err: unknown) {
if (
err instanceof RangeError &&
(err as NodeJS.ErrnoException).code === 'ERR_BUFFER_TOO_LARGE'
) {
throw new Error(
`Decompressed resume data cache exceeded ${maxDecompressedSize} byte limit`
)
try {
serializedResumeDataCache = inflateSync(
Buffer.from(serializedResumeDataCache, 'base64'),
{
maxOutputLength: maxDecompressedSize,
}
).toString('utf-8')
} catch (err: unknown) {
if (
err instanceof RangeError &&
(err as NodeJS.ErrnoException).code === 'ERR_BUFFER_TOO_LARGE'
) {
throw new Error(
`Decompressed resume data cache exceeded ${maxDecompressedSize} byte limit`
)
}
throw err
}
throw err
}
const json = JSON.parse(serializedResumeDataCache) as ResumeStoreSerialized
return {
mutable: false,
cache: parseUseCacheCacheStore(Object.entries(json.store.cache)),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment