Commit 5a224361 authored by Janka Uryga's avatar Janka Uryga

use length-encoding util for postponed state

parent ff399348
import { lengthEncodeTuple, lengthDecodeTuple } from './length-encoding'
const cases: { description: string; input: string[] }[] = [
[],
// empty strings
[''],
['', ''],
['foo', ''],
// parts that look like last-element delimiters
[':'],
['', ':'],
[':', ':'],
['foo', ':'],
// mostly benign inputs
['foo'],
['foo', 'bar'],
['foo', 'baar', 'zaaaap'],
['a', 'bb', 'ccc', 'dddd', 'eeeee', 'ffffff'],
// parts that look length-prefixed
['0:foo', '1:baar', '3:zaaaap'],
[lengthEncodeTuple(['a', 'bb', 'ccc', 'dddd', 'eeeee', 'ffffff'])],
['0:', '1:', '22:', '333:', '4444:', '55555:'],
['0:', '1:', ':22', ':333', '4444:', ':55555:'],
['0', '1', '22', '333', '4444', ':55555'],
].map((input) => ({ description: JSON.stringify(input), input }))
it.each(cases)(
'can encode and decode tuples without changes - $description',
({ input }) => {
const encoded: string = lengthEncodeTuple(input)
const decoded: string[] = lengthDecodeTuple(encoded)
expect(decoded).toEqual(input)
}
)
import { InvariantError } from '../../shared/lib/invariant-error'
/**
* Encodes a tuple of strings into a single string,
* where each part except the last is prefixed with its length.
* */
export function lengthEncodeTuple(parts: string[]) {
if (parts.length === 0) {
return ''
}
// Fast path for single-element tuples
if (parts.length === 1) {
return ':' + parts[0]
}
let encoded = ''
const lastIx = parts.length - 1
for (let i = 0; i < parts.length; i++) {
const part = parts[i]
if (i < lastIx) {
// All parts (other than the last one) are prefixed with their length --
// we have to know where one part ends and the next part begins.
encoded += `${part.length}:${part}`
} else {
// The last part does not need a length, because there are no more parts after it,
// and we can just take the rest of the string.
// But it needs a delimiter so that we don't accidentally interpret contents like "3:"
// as another length-prefixed part.
//
// This could be optimized to only add an escape character if
// - it's empty (would be skipped when deserializing)
// - it starts with a ':' (would be mistaken for an escape)
// - it starts with a number (could be mistaken for a length prefix)
// but skipping one `.slice()` call when decoding doesn't feel
// like it's worth the added complexity.
encoded += `:${part}`
}
}
return encoded
}
export function lengthDecodeTuple(encoded: string): string[] {
if (!encoded) {
return []
}
let remaining = encoded
const parts: string[] = []
while (true) {
// Check for a length header. If we don't find one, we'll check for a last-part marker.
// If we don't find that either, the input is malformed.
const lengthHeaderMatch = remaining.match(/^([0-9]+):/)
if (lengthHeaderMatch) {
// This is a length-prefixed part. It can't be the last part.
// '<length>:<part><moreParts>'
// Parse the length header
const partLength = Number.parseInt(lengthHeaderMatch[1])
if (Number.isNaN(partLength)) {
throw new InvariantError(
`Part length cannot be parsed as a number: '${remaining}'`
)
}
// Extract the part, i.e. the next `partLength` characters following the header
const partContentsStartIx = lengthHeaderMatch[0].length
const nextPartStartIx = partContentsStartIx + partLength
const partContents = remaining.slice(partContentsStartIx, nextPartStartIx)
if (partContents.length !== partLength) {
throw new InvariantError(
`Actual part length ${partContents.length} does not match length ${partLength} from header: '${remaining}'`
)
}
parts.push(partContents)
// Advance to the next part
remaining = remaining.slice(nextPartStartIx)
continue
} else if (remaining && remaining[0] === ':') {
// The final part is prefixed with a ':', without a length
// ':<lastPart>'
parts.push(remaining.slice(1))
// This was the final part, so we're done parsing
break
} else {
throw new InvariantError(
`Unexpected part without length header or last-part indicator: '${remaining}'`
)
}
}
return parts
}
...@@ -9,12 +9,12 @@ import { ...@@ -9,12 +9,12 @@ import {
getDynamicHTMLPostponedState, getDynamicHTMLPostponedState,
parsePostponedState, parsePostponedState,
DynamicHTMLPreludeState, DynamicHTMLPreludeState,
type PostponedState,
} from './postponed-state' } from './postponed-state'
describe('getDynamicHTMLPostponedState', () => { it('can serialize and deserialize a HTML postponed state with fallback params and a nonempty resume data cache', async () => {
it('serializes a HTML postponed state with fallback params', async () => { const fallbackParam = '%%drp:slug:e9615126684e5%%'
const key = '%%drp:slug:e9615126684e5%%' const fallbackRouteParams = new Map([['slug', fallbackParam]])
const fallbackRouteParams = new Map([['slug', key]])
const prerenderResumeDataCache = createPrerenderResumeDataCache() const prerenderResumeDataCache = createPrerenderResumeDataCache()
prerenderResumeDataCache.cache.set( prerenderResumeDataCache.cache.set(
...@@ -30,126 +30,112 @@ describe('getDynamicHTMLPostponedState', () => { ...@@ -30,126 +30,112 @@ describe('getDynamicHTMLPostponedState', () => {
) )
const state = await getDynamicHTMLPostponedState( const state = await getDynamicHTMLPostponedState(
{ [key]: key, nested: { [key]: key } }, {
[fallbackParam]: fallbackParam,
nested: { [fallbackParam]: fallbackParam },
},
DynamicHTMLPreludeState.Full, DynamicHTMLPreludeState.Full,
fallbackRouteParams, fallbackRouteParams,
prerenderResumeDataCache prerenderResumeDataCache
) )
const parsed = parsePostponedState(state, { slug: '123' }) const paramValue = '123'
expect(parsed).toMatchInlineSnapshot(` const parsed = parsePostponedState(state, { slug: paramValue })
{ expect(parsed).toEqual({
"data": [ type: DynamicState.HTML,
1, data: [
DynamicHTMLPreludeState.Full,
{ {
"123": "123", [paramValue]: paramValue,
"nested": { nested: {
"123": "123", [paramValue]: paramValue,
}, },
}, },
], ],
"renderResumeDataCache": { renderResumeDataCache: expect.any(Object),
"cache": Map { } satisfies PostponedState)
"1" => Promise {},
},
"decryptedBoundArgs": Map {},
"encryptedBoundArgs": Map {},
"fetch": Map {},
},
"type": 2,
}
`)
const value = await parsed.renderResumeDataCache.cache.get('1')
expect(value).toBeDefined() // Ensure that the replacement worked and removed all the placeholders.
expect(JSON.stringify(parsed)).not.toContain(fallbackParam)
await expect(streamToString(value!.value)).resolves.toEqual('hello')
})
it('serializes a HTML postponed state without fallback params', async () => { const cacheValue = await parsed.renderResumeDataCache.cache.get('1')
const state = await getDynamicHTMLPostponedState( expect(cacheValue).toBeDefined()
{ key: 'value' }, await expect(streamToString(cacheValue!.value)).resolves.toEqual('hello')
DynamicHTMLPreludeState.Full, })
null,
createPrerenderResumeDataCache()
)
expect(state).toMatchInlineSnapshot(`"19:[1,{"key":"value"}]null"`)
})
it('can serialize and deserialize a HTML postponed state with fallback params', async () => { it('can serialize and deserialize a HTML postponed state with fallback params and an empty resume data cache', async () => {
const key = '%%drp:slug:e9615126684e5%%' const fallbackParam = '%%drp:slug:e9615126684e5%%'
const fallbackRouteParams = new Map([['slug', key]]) const fallbackRouteParams = new Map([['slug', fallbackParam]])
const state = await getDynamicHTMLPostponedState( const state = await getDynamicHTMLPostponedState(
{ [key]: key }, { [fallbackParam]: fallbackParam },
DynamicHTMLPreludeState.Full, DynamicHTMLPreludeState.Full,
fallbackRouteParams, fallbackRouteParams,
createPrerenderResumeDataCache() createPrerenderResumeDataCache()
) )
const value = 'hello' const paramValue = 'hello'
const params = { slug: value } const params = { slug: paramValue }
const parsed = parsePostponedState(state, params) const parsed = parsePostponedState(state, params)
expect(parsed).toEqual({ expect(parsed).toEqual({
type: DynamicState.HTML, type: DynamicState.HTML,
data: [1, { [value]: value }], data: [DynamicHTMLPreludeState.Full, { [paramValue]: paramValue }],
renderResumeDataCache: createPrerenderResumeDataCache(), renderResumeDataCache: createPrerenderResumeDataCache(),
}) } satisfies PostponedState)
// The replacements have been replaced. // The replacements have been replaced.
expect(JSON.stringify(parsed)).not.toMatch(key) expect(JSON.stringify(parsed)).not.toContain(fallbackParam)
})
}) })
describe('getDynamicDataPostponedState', () => { it.each([
it('serializes a data postponed state with fallback params', async () => { {
const state = await getDynamicDataPostponedState( description: 'empty prelude',
preludeState: DynamicHTMLPreludeState.Empty,
},
{
description: 'non-empty prelude',
preludeState: DynamicHTMLPreludeState.Full,
},
])(
'can serialize and deserialize a HTML postponed state with no fallback params - $description',
async ({ preludeState }) => {
const fallbackRouteParams = new Map()
const postponed = { fake: 'state' }
const state = await getDynamicHTMLPostponedState(
postponed,
preludeState,
fallbackRouteParams,
createPrerenderResumeDataCache() createPrerenderResumeDataCache()
) )
expect(state).toMatchInlineSnapshot(`"4:nullnull"`)
})
})
describe('parsePostponedState', () => { const parsed = parsePostponedState(state, {})
it('parses a HTML postponed state with fallback params', () => {
const state = `2593:39[["slug","%%drp:slug:e9615126684e5%%"]][1,{"t":2,"d":{"nextSegmentId":2,"rootFormatContext":{"insertionMode":0,"selectedValue":null,"tagScope":0},"progressiveChunkSize":12800,"resumableState":{"idPrefix":"","nextFormID":0,"streamingFormat":0,"instructions":0,"hasBody":true,"hasHtml":true,"unknownResources":{},"dnsResources":{},"connectResources":{"default":{},"anonymous":{},"credentials":{}},"imageResources":{},"styleResources":{},"scriptResources":{"/_next/static/chunks/webpack-6b2534a6458c6fe5.js":null,"/_next/static/chunks/f5e865f6-5e04edf75402c5e9.js":null,"/_next/static/chunks/9440-26a4cfbb73347735.js":null,"/_next/static/chunks/main-app-315ef55d588dbeeb.js":null,"/_next/static/chunks/8630-8e01a4bea783c651.js":null,"/_next/static/chunks/app/layout-1b900e1a3caf3737.js":null},"moduleUnknownResources":{},"moduleScriptResources":{"/_next/static/chunks/webpack-6b2534a6458c6fe5.js":null}},"replayNodes":[["oR",0,[["Context.Provider",0,[["ServerInsertedHTMLProvider",0,[["Context.Provider",0,[["n7",0,[["nU",0,[["nF",0,[["n9",0,[["Fragment",0,[["Context.Provider",2,[["Context.Provider",0,[["Context.Provider",0,[["Context.Provider",0,[["Context.Provider",0,[["Context.Provider",0,[["nY",0,[["nX",0,[["Fragment","c",[["Fragment",0,[["html",1,[["body",0,[["main",3,[["j",0,[["Fragment",0,[["Context.Provider","validation",[["i",2,[["Fragment",0,[["E",0,[["R",0,[["h",0,[["Fragment",0,[["O",0,[["Fragment",0,[["s",0,[["c",0,[["s",0,[["c",0,[["v",0,[["Context.Provider",0,[["Fragment","c",[["j",1,[["Fragment",0,[["Context.Provider","slug|%%drp:slug:e9615126684e5%%|d",[["i",2,[["Fragment",0,[["E",0,[["R",0,[["h",0,[["Fragment",0,[["O",0,[["Fragment",0,[["s",0,[["Fragment",0,[["s",0,[["c",0,[["v",0,[["Context.Provider",0,[["Fragment","c",[["j",1,[["Fragment",0,[["Context.Provider","__PAGE__",[["i",2,[["Fragment",0,[["E",0,[["R",0,[["h",0,[["Fragment",0,[["O",0,[["Suspense",0,[["s",0,[["Fragment",0,[["s",0,[["c",0,[["v",0,[["Context.Provider",0,[["Fragment","c",[["Fragment",0,[],{"1":1}]],null]],null]],null]],null]],null]],null]],null]],null,["Suspense Fallback",0,[],null],0]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],null]],"replaySlots":null}}]null`
const params = {
slug: Math.random().toString(16).slice(3),
}
const parsed = parsePostponedState(state, params)
// Ensure that it parsed it correctly.
expect(parsed).toEqual({ expect(parsed).toEqual({
type: DynamicState.HTML, type: DynamicState.HTML,
data: expect.any(Object), data: [preludeState, postponed],
renderResumeDataCache: createPrerenderResumeDataCache(), renderResumeDataCache: createPrerenderResumeDataCache(),
}) } satisfies PostponedState)
}
)
// Ensure that the replacement worked and removed all the placeholders. it('can serialize and deserialize a data postponed state', async () => {
expect(JSON.stringify(parsed)).not.toMatch(/%%drp:slug:e9615126684e5%%/) const prerenderResumeDataCache = createPrerenderResumeDataCache()
prerenderResumeDataCache.cache.set(
'1',
Promise.resolve({
value: streamFromString('hello'),
tags: [],
stale: 0,
timestamp: 0,
expire: 0,
revalidate: 0,
}) })
)
it('parses a HTML postponed state without fallback params', () => { const state = await getDynamicDataPostponedState(prerenderResumeDataCache)
const state = `2:{}null` const parsed = parsePostponedState(state, {})
const params = {}
const parsed = parsePostponedState(state, params)
// Ensure that it parsed it correctly.
expect(parsed).toEqual({
type: DynamicState.HTML,
data: expect.any(Object),
renderResumeDataCache: createPrerenderResumeDataCache(),
})
})
it('parses a data postponed state', () => { expect(parsed.type).toBe(DynamicState.DATA)
const state = '4:nullnull'
const parsed = parsePostponedState(state, undefined)
// Ensure that it parsed it correctly. const cacheValue = await parsed.renderResumeDataCache.cache.get('1')
expect(parsed).toEqual({ expect(cacheValue).toBeDefined()
type: DynamicState.DATA, await expect(streamToString(cacheValue!.value)).resolves.toEqual('hello')
renderResumeDataCache: createPrerenderResumeDataCache(),
})
})
}) })
import type { FallbackRouteParams } from '../../server/request/fallback-params' import type { FallbackRouteParams } from '../../server/request/fallback-params'
import { InvariantError } from '../../shared/lib/invariant-error'
import type { Params } from '../request/params' import type { Params } from '../request/params'
import { import {
createPrerenderResumeDataCache, createPrerenderResumeDataCache,
...@@ -7,6 +8,7 @@ import { ...@@ -7,6 +8,7 @@ import {
type RenderResumeDataCache, type RenderResumeDataCache,
} from '../resume-data-cache/resume-data-cache' } from '../resume-data-cache/resume-data-cache'
import { stringifyResumeDataCache } from '../resume-data-cache/resume-data-cache' import { stringifyResumeDataCache } from '../resume-data-cache/resume-data-cache'
import { lengthDecodeTuple, lengthEncodeTuple } from './length-encoding'
export enum DynamicState { export enum DynamicState {
/** /**
...@@ -71,6 +73,19 @@ export type PostponedState = ...@@ -71,6 +73,19 @@ export type PostponedState =
| DynamicDataPostponedState | DynamicDataPostponedState
| DynamicHTMLPostponedState | DynamicHTMLPostponedState
type SerializedStateParts =
| SerializedDynamicData
| SerializedDynamicHTML
| SerializedDynamicHTMLWithReplacements
type SerializedDynamicData = [resumeDataCache: string]
type SerializedDynamicHTML = [postponed: string, resumeDataCache: string]
type SerializedDynamicHTMLWithReplacements = [
replacements: string,
postponed: string,
resumeDataCache: string,
]
export async function getDynamicHTMLPostponedState( export async function getDynamicHTMLPostponedState(
postponed: ReactPostponed, postponed: ReactPostponed,
preludeState: DynamicHTMLPreludeState, preludeState: DynamicHTMLPreludeState,
...@@ -81,26 +96,32 @@ export async function getDynamicHTMLPostponedState( ...@@ -81,26 +96,32 @@ export async function getDynamicHTMLPostponedState(
const dataString = JSON.stringify(data) const dataString = JSON.stringify(data)
if (!fallbackRouteParams || fallbackRouteParams.size === 0) { if (!fallbackRouteParams || fallbackRouteParams.size === 0) {
// Serialized as `<postponedString.length>:<postponedString><renderResumeDataCache>` return lengthEncodeTuple([
return `${dataString.length}:${dataString}${await stringifyResumeDataCache( dataString,
await stringifyResumeDataCache(
createRenderResumeDataCache(resumeDataCache) createRenderResumeDataCache(resumeDataCache)
)}` ),
] satisfies SerializedDynamicHTML)
} }
const replacements: Array<[string, string]> = Array.from(fallbackRouteParams) const replacements: Array<[string, string]> = Array.from(fallbackRouteParams)
const replacementsString = JSON.stringify(replacements) const replacementsString = JSON.stringify(replacements)
// Serialized as `<replacements.length><replacements><data>` return lengthEncodeTuple([
const postponedString = `${replacementsString.length}${replacementsString}${dataString}` replacementsString,
dataString,
// Serialized as `<postponedString.length>:<postponedString><renderResumeDataCache>` await stringifyResumeDataCache(resumeDataCache),
return `${postponedString.length}:${postponedString}${await stringifyResumeDataCache(resumeDataCache)}` ] satisfies SerializedDynamicHTMLWithReplacements)
} }
export async function getDynamicDataPostponedState( export async function getDynamicDataPostponedState(
resumeDataCache: PrerenderResumeDataCache | RenderResumeDataCache resumeDataCache: PrerenderResumeDataCache | RenderResumeDataCache
): Promise<string> { ): Promise<string> {
return `4:null${await stringifyResumeDataCache(createRenderResumeDataCache(resumeDataCache))}` return lengthEncodeTuple([
await stringifyResumeDataCache(
createRenderResumeDataCache(resumeDataCache)
),
] satisfies SerializedDynamicData)
} }
export function parsePostponedState( export function parsePostponedState(
...@@ -108,58 +129,46 @@ export function parsePostponedState( ...@@ -108,58 +129,46 @@ export function parsePostponedState(
params: Params | undefined params: Params | undefined
): PostponedState { ): PostponedState {
try { try {
const postponedStringLengthMatch = state.match(/^([0-9]*):/)?.[1] const parts = lengthDecodeTuple(state) as SerializedStateParts
if (!postponedStringLengthMatch) { if (parts.length === 1) {
throw new Error(`Invariant: invalid postponed state ${state}`) parts satisfies SerializedDynamicData
} const [resumeDataCacheString] = parts
const postponedStringLength = parseInt(postponedStringLengthMatch)
// We add a `:` to the end of the length as the first character of the
// postponed string is the length of the replacement entries.
const postponedString = state.slice(
postponedStringLengthMatch.length + 1,
postponedStringLengthMatch.length + postponedStringLength + 1
)
const renderResumeDataCache = createRenderResumeDataCache( const renderResumeDataCache = createRenderResumeDataCache(
state.slice(postponedStringLengthMatch.length + postponedStringLength + 1) resumeDataCacheString
) )
return {
try { type: DynamicState.DATA,
if (postponedString === 'null') { renderResumeDataCache,
return { type: DynamicState.DATA, renderResumeDataCache }
} }
} else if (parts.length === 2 || parts.length === 3) {
if (/^[0-9]/.test(postponedString)) { let replacementsString: string | null = null,
const match = postponedString.match(/^([0-9]*)/)?.[1] postponedString: string,
if (!match) { resumeDataCacheString: string
throw new Error(
`Invariant: invalid postponed state ${JSON.stringify(postponedString)}` if (parts.length === 2) {
) parts satisfies SerializedDynamicHTML
;[postponedString, resumeDataCacheString] = parts
} else {
parts satisfies SerializedDynamicHTMLWithReplacements
;[replacementsString, postponedString, resumeDataCacheString] = parts
} }
// This is the length of the replacements entries. const renderResumeDataCache = createRenderResumeDataCache(
const length = parseInt(match) resumeDataCacheString
const replacements = JSON.parse(
postponedString.slice(
match.length,
// We then go to the end of the string.
match.length + length
) )
) as ReadonlyArray<[string, string]>
let postponed = postponedString.slice(match.length + length) try {
if (replacementsString !== null) {
const replacements = JSON.parse(replacementsString) as ReadonlyArray<
[string, string]
>
for (const [key, searchValue] of replacements) { for (const [key, searchValue] of replacements) {
const value = params?.[key] ?? '' const value = params?.[key] ?? ''
const replaceValue = Array.isArray(value) ? value.join('/') : value const replaceValue = Array.isArray(value) ? value.join('/') : value
postponed = postponed.replaceAll(searchValue, replaceValue) postponedString = postponedString.replaceAll(
} searchValue,
replaceValue
return { )
type: DynamicState.HTML,
data: JSON.parse(postponed),
renderResumeDataCache,
} }
} }
...@@ -172,6 +181,10 @@ export function parsePostponedState( ...@@ -172,6 +181,10 @@ export function parsePostponedState(
console.error('Failed to parse postponed state', err) console.error('Failed to parse postponed state', err)
return { type: DynamicState.DATA, renderResumeDataCache } return { type: DynamicState.DATA, renderResumeDataCache }
} }
} else {
parts satisfies never
throw new InvariantError('Postponed state tuple has invalid length')
}
} catch (err) { } catch (err) {
console.error('Failed to parse postponed state', err) console.error('Failed to parse postponed state', err)
return { return {
......
/// <reference path="../../../../packages/next/types/react-dom.d.ts" />
// in order to use `parsePostponedState` here, we need to bring the extended type definition
// for `import('react-dom/static').PrerenderResult` into the compilation, because that module relies on it
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { retry } from 'next-test-utils' import { retry } from 'next-test-utils'
import stripAnsi from 'strip-ansi' import stripAnsi from 'strip-ansi'
import { format } from 'util' import { format } from 'util'
import { Playwright } from 'next-webdriver' import { Playwright } from 'next-webdriver'
import { import type { RenderResumeDataCache } from 'next/dist/server/resume-data-cache/resume-data-cache'
createRenderResumeDataCache,
RenderResumeDataCache,
} from 'next/dist/server/resume-data-cache/resume-data-cache'
import { PrerenderManifest } from 'next/dist/build' import { PrerenderManifest } from 'next/dist/build'
import { parsePostponedState } from 'next/dist/server/app-render/postponed-state'
const GENERIC_RSC_ERROR = const GENERIC_RSC_ERROR =
'An error occurred in the Server Components render. The specific message is omitted in production builds to avoid leaking sensitive details. A digest property is included on this error instance which may provide additional details about the nature of the error.' 'An error occurred in the Server Components render. The specific message is omitted in production builds to avoid leaking sensitive details. A digest property is included on this error instance which may provide additional details about the nature of the error.'
...@@ -1042,12 +1044,8 @@ async function getSanitizedLogs(browser: Playwright): Promise<string[]> { ...@@ -1042,12 +1044,8 @@ async function getSanitizedLogs(browser: Playwright): Promise<string[]> {
} }
function extractResumeDataCacheFromPostponedState( function extractResumeDataCacheFromPostponedState(
state: string serializedState: string
): RenderResumeDataCache { ): RenderResumeDataCache {
const postponedStringLengthMatch = state.match(/^([0-9]*):/)![1] const state = parsePostponedState(serializedState, {})
const postponedStringLength = parseInt(postponedStringLengthMatch) return state.renderResumeDataCache
return createRenderResumeDataCache(
state.slice(postponedStringLengthMatch.length + postponedStringLength + 1)
)
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment