Unverified Commit 025bf4a5 authored by Sebastian "Sebbie" Silbermann's avatar Sebastian "Sebbie" Silbermann Committed by GitHub

fix(incremental-cache): byte-exact fetch cache key for binary bodies (#96008)

https://github.com/vercel/next.js/security/advisories/GHSA-4633-3j49-mh5qCo-authored-by: 's avatarKarim Rahal <karim@karimrahal.com>
parent 025e8535
...@@ -1440,5 +1440,6 @@ ...@@ -1440,5 +1440,6 @@
"1439": "Route \"%s\": Next.js encountered URL data during prerendering or a navigation.\\n\\n\\`params\\` or \\`searchParams\\` accessed outside of \\`<Suspense>\\` may prevent the navigation from being instant, leading to a slower user experience.\\n\\nWays to fix this:\\n - [stream] Provide a placeholder with \\`<Suspense fallback={...}>\\` around the data access\\n - [block] Set \\`export const instant = false\\` to allow a blocking route\\n\\nLearn more: https://nextjs.org/docs/messages/instant-shell-url-data", "1439": "Route \"%s\": Next.js encountered URL data during prerendering or a navigation.\\n\\n\\`params\\` or \\`searchParams\\` accessed outside of \\`<Suspense>\\` may prevent the navigation from being instant, leading to a slower user experience.\\n\\nWays to fix this:\\n - [stream] Provide a placeholder with \\`<Suspense fallback={...}>\\` around the data access\\n - [block] Set \\`export const instant = false\\` to allow a blocking route\\n\\nLearn more: https://nextjs.org/docs/messages/instant-shell-url-data",
"1440": "Route \"%s\": Next.js encountered uncached data during prerendering.\\n\\n\\`fetch(...)\\` or \\`connection()\\` accessed outside of \\`<Suspense>\\` prevents the route from being prerendered, blocking the page load and leading to a slower user experience.\\n\\nWays to fix this:\\n - [stream] Provide a placeholder with \\`<Suspense fallback={...}>\\` around the data access\\n - [cache] Cache the data access with \\`\"use cache\"\\` (does not apply to \\`connection()\\`)\\n - [block] Set \\`export const instant = false\\` to allow a blocking route\\n\\nLearn more: https://nextjs.org/docs/messages/blocking-prerender-dynamic", "1440": "Route \"%s\": Next.js encountered uncached data during prerendering.\\n\\n\\`fetch(...)\\` or \\`connection()\\` accessed outside of \\`<Suspense>\\` prevents the route from being prerendered, blocking the page load and leading to a slower user experience.\\n\\nWays to fix this:\\n - [stream] Provide a placeholder with \\`<Suspense fallback={...}>\\` around the data access\\n - [cache] Cache the data access with \\`\"use cache\"\\` (does not apply to \\`connection()\\`)\\n - [block] Set \\`export const instant = false\\` to allow a blocking route\\n\\nLearn more: https://nextjs.org/docs/messages/blocking-prerender-dynamic",
"1441": "DevValidationScheduler requires at least one active validation", "1441": "DevValidationScheduler requires at least one active validation",
"1442": "The Server Reference ID did not match the expected format. Received %s.\\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action" "1442": "The Server Reference ID did not match the expected format. Received %s.\\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action",
"1443": "Unsupported body type: %s"
} }
...@@ -55,6 +55,50 @@ export interface CacheHandlerValue { ...@@ -55,6 +55,50 @@ export interface CacheHandlerValue {
value: IncrementalCacheValue | null value: IncrementalCacheValue | null
} }
function toHex(buffer: ArrayBufferView | ArrayBuffer): string {
// Hex-encode body bytes losslessly: decoding as UTF-8 would collapse
// distinct bytes (0xff/0xfe to U+FFFD) and collide; Buffer isn't on edge.
const bytes = isArrayBuffer(buffer)
? new Uint8Array(buffer)
: new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
let hex = ''
for (const byte of bytes) {
hex += byte.toString(16).padStart(2, '0')
}
return hex
}
type Body = NonNullable<RequestInit['body'] | Request['body']>
// Duck typing to support Edge runtime
// TODO: Switch to instanceof checks once Edge runtime is removed.
function isArrayBuffer(
buffer: ArrayBuffer | ArrayBufferView
): buffer is ArrayBuffer {
return !('buffer' in buffer)
}
function isBodyByteSequence(
body: Body
): body is ArrayBufferView<ArrayBuffer> | ArrayBuffer {
return typeof body === 'object' && 'byteLength' in body
}
function isBodyReadableStream(body: Body): body is ReadableStream {
return typeof (body as any).getReader === 'function'
}
function isBodyFormDataOrURLSearchParams(
body: Body
): body is FormData | URLSearchParams {
return typeof (body as any).keys === 'function'
}
function isBodyBlob(body: Body): body is Blob {
return typeof (body as any).arrayBuffer === 'function'
}
export class CacheHandler { export class CacheHandler {
// eslint-disable-next-line // eslint-disable-next-line
constructor(_ctx: CacheHandlerContext) {} constructor(_ctx: CacheHandlerContext) {}
...@@ -80,6 +124,21 @@ export class CacheHandler { ...@@ -80,6 +124,21 @@ export class CacheHandler {
public resetRequestCache(): void {} public resetRequestCache(): void {}
} }
async function hashString(cacheString: string): Promise<string> {
if (process.env.NEXT_RUNTIME === 'edge') {
const encoder = new TextEncoder()
const buffer = encoder.encode(cacheString)
return toHex(await crypto.subtle.digest('SHA-256', buffer))
} else {
const crypto = require('crypto') as typeof import('crypto')
return crypto.createHash('sha256').update(cacheString).digest('hex')
}
}
// this should be bumped anytime a fix is made to cache entries
// that should bust the cache
const MAIN_KEY_PREFIX = 'v4'
export class IncrementalCache implements IncrementalCacheType { export class IncrementalCache implements IncrementalCacheType {
readonly dev?: boolean readonly dev?: boolean
readonly disableForTestmode?: boolean readonly disableForTestmode?: boolean
...@@ -288,28 +347,34 @@ export class IncrementalCache implements IncrementalCacheType { ...@@ -288,28 +347,34 @@ export class IncrementalCache implements IncrementalCacheType {
return this.cacheHandler?.revalidateTag(tags, durations) return this.cacheHandler?.revalidateTag(tags, durations)
} }
async generateSimpleCacheKey(input: string): Promise<string> {
const cacheString = JSON.stringify([
MAIN_KEY_PREFIX,
this.fetchCacheKeyPrefix || '',
input,
])
return hashString(cacheString)
}
// x-ref: https://github.com/facebook/react/blob/2655c9354d8e1c54ba888444220f63e836925caa/packages/react/src/ReactFetch.js#L23 // x-ref: https://github.com/facebook/react/blob/2655c9354d8e1c54ba888444220f63e836925caa/packages/react/src/ReactFetch.js#L23
async generateCacheKey( async generateCacheKey(
url: string, url: string,
init: RequestInit | Request = {} init: RequestInit | Request = {}
): Promise<string> { ): Promise<string> {
// this should be bumped anytime a fix is made to cache entries
// that should bust the cache
const MAIN_KEY_PREFIX = 'v3'
const bodyChunks: string[] = [] const bodyChunks: string[] = []
const encoder = new TextEncoder() const encoder = new TextEncoder()
const decoder = new TextDecoder()
if (init.body) { // Will be set implementing https://fetch.spec.whatwg.org/#concept-bodyinit-extract
// handle Uint8Array body let bodyType: string | null = null
if (init.body instanceof Uint8Array) { const body = init.body
bodyChunks.push(decoder.decode(init.body)) if (body) {
;(init as any)._ogBody = init.body if (isBodyByteSequence(body)) {
} // handle ReadableStream body bodyChunks.push(`bytes:${toHex(body)}`)
else if (typeof (init.body as any).getReader === 'function') { ;(init as any)._ogBody = body
const readableBody = init.body as ReadableStream<Uint8Array | string> } else if (isBodyReadableStream(body)) {
const readableBody = body
const chunks: Uint8Array[] = [] const chunks: Uint8Array[] = []
...@@ -317,20 +382,13 @@ export class IncrementalCache implements IncrementalCacheType { ...@@ -317,20 +382,13 @@ export class IncrementalCache implements IncrementalCacheType {
await readableBody.pipeTo( await readableBody.pipeTo(
new WritableStream({ new WritableStream({
write(chunk) { write(chunk) {
if (typeof chunk === 'string') { chunks.push(
chunks.push(encoder.encode(chunk)) typeof chunk === 'string' ? encoder.encode(chunk) : chunk
bodyChunks.push(chunk) )
} else {
chunks.push(chunk)
bodyChunks.push(decoder.decode(chunk, { stream: true }))
}
}, },
}) })
) )
// Flush the decoder.
bodyChunks.push(decoder.decode())
// Create a new buffer with all the chunks. // Create a new buffer with all the chunks.
const length = chunks.reduce((total, arr) => total + arr.length, 0) const length = chunks.reduce((total, arr) => total + arr.length, 0)
const arrayBuffer = new Uint8Array(length) const arrayBuffer = new Uint8Array(length)
...@@ -342,46 +400,54 @@ export class IncrementalCache implements IncrementalCacheType { ...@@ -342,46 +400,54 @@ export class IncrementalCache implements IncrementalCacheType {
offset += chunk.length offset += chunk.length
} }
bodyChunks.push(`bytes:${toHex(arrayBuffer)}`)
;(init as any)._ogBody = arrayBuffer ;(init as any)._ogBody = arrayBuffer
} catch (err) { } catch (err) {
console.error('Problem reading body', err) console.error('Problem reading body', err)
} }
} // handle FormData or URLSearchParams bodies } else if (isBodyFormDataOrURLSearchParams(body)) {
else if (typeof (init.body as any).keys === 'function') { bodyType =
const formData = init.body as FormData String(body) === '[object FormData]'
;(init as any)._ogBody = init.body ? // We don't need a boundary because we're not actually using this for a Content-Type header
for (const key of new Set([...formData.keys()])) { 'multipart/form-data; boundary='
const values = formData.getAll(key) : 'application/x-www-form-urlencoded;charset=UTF-8'
bodyChunks.push( const iterable = body
`${key}=${( ;(init as any)._ogBody = body
await Promise.all( // Separate, tagged chunks so `["a","b"]` can't collide with `["a,b"]`.
values.map(async (val) => { for (const [key, val] of iterable.entries()) {
if (typeof val === 'string') { bodyChunks.push(`key:${key}`)
return val if (typeof val === 'string') {
} else { bodyChunks.push(`str:${val}`)
return await val.text() } else {
} bodyChunks.push(
}) 'file',
) val.name,
).join(',')}` val.type,
) `bytes:${toHex(await val.arrayBuffer())}`
)
}
} }
// handle blob body // handle blob body
} else if (typeof (init.body as any).arrayBuffer === 'function') { } else if (isBodyBlob(body)) {
const blob = init.body as Blob const blob = body
const arrayBuffer = await blob.arrayBuffer() const arrayBuffer = await blob.arrayBuffer()
bodyChunks.push(await blob.text()) bodyChunks.push('blob', blob.type, `bytes:${toHex(arrayBuffer)}`)
;(init as any)._ogBody = new Blob([arrayBuffer], { type: blob.type }) ;(init as any)._ogBody = new Blob([arrayBuffer], { type: blob.type })
} else if (typeof init.body === 'string') { bodyType = blob.type
bodyChunks.push(init.body) } else if (typeof body === 'string') {
;(init as any)._ogBody = init.body bodyChunks.push(`str:${body}`)
;(init as any)._ogBody = body
bodyType = 'text/plain;charset=UTF-8'
} else {
body satisfies never
throw new Error(`Unsupported body type: ${typeof body}`)
} }
} }
const headers = const headers =
typeof (init.headers || {}).keys === 'function' typeof (init.headers || {}).keys === 'function'
? Object.fromEntries(init.headers as Headers) ? Object.fromEntries(init.headers as Headers)
: Object.assign({}, init.headers) : Object.assign({} as Record<string, string>, init.headers)
// w3c trace context headers can break request caching and deduplication // w3c trace context headers can break request caching and deduplication
// so we remove them from the cache key // so we remove them from the cache key
...@@ -393,6 +459,9 @@ export class IncrementalCache implements IncrementalCacheType { ...@@ -393,6 +459,9 @@ export class IncrementalCache implements IncrementalCacheType {
this.fetchCacheKeyPrefix || '', this.fetchCacheKeyPrefix || '',
url, url,
init.method, init.method,
// Ensures default Content-Type is part of the cache key
// TODO: Only necessary when headers are not used from the Request instance
bodyType,
headers, headers,
init.mode, init.mode,
init.redirect, init.redirect,
...@@ -404,18 +473,7 @@ export class IncrementalCache implements IncrementalCacheType { ...@@ -404,18 +473,7 @@ export class IncrementalCache implements IncrementalCacheType {
bodyChunks, bodyChunks,
]) ])
if (process.env.NEXT_RUNTIME === 'edge') { return hashString(cacheString)
function bufferToHex(buffer: ArrayBuffer): string {
return Array.prototype.map
.call(new Uint8Array(buffer), (b) => b.toString(16).padStart(2, '0'))
.join('')
}
const buffer = encoder.encode(cacheString)
return bufferToHex(await crypto.subtle.digest('SHA-256', buffer))
} else {
const crypto = require('crypto') as typeof import('crypto')
return crypto.createHash('sha256').update(cacheString).digest('hex')
}
} }
async get( async get(
......
...@@ -820,8 +820,8 @@ export function createPatchedFetcher( ...@@ -820,8 +820,8 @@ export function createPatchedFetcher(
fetchUrl, fetchUrl,
isRequestInput ? (input as RequestInit) : init isRequestInput ? (input as RequestInit) : init
) )
} catch (err) { } catch (cause) {
console.error(`Failed to generate cache key for`, input) console.error(`Failed to generate cache key for`, input, cause)
} }
} }
......
...@@ -134,7 +134,8 @@ export function unstable_cache<T extends Callback>( ...@@ -134,7 +134,8 @@ export function unstable_cache<T extends Callback>(
// @TODO stringify is likely not safe here. We will coerce undefined to null which will make // @TODO stringify is likely not safe here. We will coerce undefined to null which will make
// the keyspace smaller than the execution space // the keyspace smaller than the execution space
const invocationKey = `${fixedKey}-${JSON.stringify(args)}` const invocationKey = `${fixedKey}-${JSON.stringify(args)}`
const cacheKey = await incrementalCache.generateCacheKey(invocationKey) const cacheKey =
await incrementalCache.generateSimpleCacheKey(invocationKey)
// $urlWithPath,$sortedQueryStringKeys,$hashOfEveryThingElse // $urlWithPath,$sortedQueryStringKeys,$hashOfEveryThingElse
const fetchUrl = `unstable_cache ${fetchUrlPrefix} ${cb.name ? ` ${cb.name}` : cacheKey}` const fetchUrl = `unstable_cache ${fetchUrlPrefix} ${cb.name ? ` ${cb.name}` : cacheKey}`
const fetchIdx = const fetchIdx =
......
...@@ -4064,6 +4064,53 @@ describe('app-dir static/dynamic handling', () => { ...@@ -4064,6 +4064,53 @@ describe('app-dir static/dynamic handling', () => {
}, 'success') }, 'success')
}) })
it('should not cache similar iterable request bodies', async () => {
const clearCacheResponse = await next.fetch(
'/variable-revalidate/post-method-iterable/revalidate',
{ method: 'POST' }
)
expect(clearCacheResponse.status).toBe(200)
await retry(
async () => {
const res = await next.fetch(
'/variable-revalidate/post-method-iterable'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const dataEmptyBody1 = $('#data-empty-body1').text()
const dataEmptyBody2 = $('#data-empty-body2').text()
const dataEmptyBody3 = $('#data-empty-body3').text()
const dataIterableBody1 = $('#data-iterable-body1').text()
const dataIterableBody2 = $('#data-iterable-body2').text()
expect(dataEmptyBody1).not.toBe(dataEmptyBody2)
expect(dataEmptyBody1).not.toBe(dataEmptyBody3)
expect(dataEmptyBody2).not.toBe(dataEmptyBody3)
expect(dataIterableBody1).not.toBe(dataIterableBody2)
const res2 = await next.fetch(
'/variable-revalidate/post-method-iterable'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#data-empty-body1').text()).toBe(dataEmptyBody1)
expect($2('#data-empty-body2').text()).toBe(dataEmptyBody2)
expect($2('#data-empty-body3').text()).toBe(dataEmptyBody3)
expect($2('#data-iterable-body1').text()).toBe(dataIterableBody1)
expect($2('#data-iterable-body2').text()).toBe(dataIterableBody2)
},
// TODO: Unreliable on Vercel
isNextDeploy ? 3000 : 500,
500
)
})
it('should cache correctly with post method and revalidate edge', async () => { it('should cache correctly with post method and revalidate edge', async () => {
await check(async () => { await check(async () => {
const res = await fetchViaHTTP( const res = await fetchViaHTTP(
......
...@@ -2,7 +2,117 @@ import { fetchRetry } from '../../../lib/fetch-retry' ...@@ -2,7 +2,117 @@ import { fetchRetry } from '../../../lib/fetch-retry'
export const runtime = 'edge' export const runtime = 'edge'
export default async function Page() { async function EmptyBodiesOne() {
// We expect the same cache key so we're doing these in pairs
// 3 or more requests will start to get a miss due to too low revalidation time
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: '',
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: new FormData(),
}
).then((res) => res.text())
return (
<>
<p id="data-empty-body1">{dataWithBody1}</p>
<p id="data-empty-body2">{dataWithBody2}</p>
</>
)
}
async function EmptyBodiesTwo() {
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: '',
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: new URLSearchParams(),
}
).then((res) => res.text())
return (
<>
<p id="data-empty-body3">{dataWithBody1}</p>
<p id="data-empty-body4">{dataWithBody2}</p>
</>
)
}
function EmptyBodies() {
return (
<>
<EmptyBodiesOne />
<EmptyBodiesTwo />
</>
)
}
async function IterableBodies() {
const entries = [
['myParam', 'myValue'],
['myParam', 'anotherValue'],
]
const urlSearchParams = new URLSearchParams(entries)
const formData = new FormData()
entries.forEach(([key, value]) => formData.append(key, value))
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: urlSearchParams,
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: formData,
}
).then((res) => res.text())
return (
<>
<p id="data-iterable-body1">{dataWithBody1}</p>
<p id="data-iterable-body2">{dataWithBody2}</p>
</>
)
}
async function DataBodies() {
const data = await fetchRetry( const data = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random', 'https://next-data-api-endpoint.vercel.app/api/random',
{ {
...@@ -41,18 +151,13 @@ export default async function Page() { ...@@ -41,18 +151,13 @@ export default async function Page() {
} }
).then((res) => res.text()) ).then((res) => res.text())
const formData = new FormData()
formData.append('hello', 'value')
formData.append('another', new Blob(['some text'], { type: 'text/plain' }))
formData.append('another', 'text')
const dataWithBody3 = await fetchRetry( const dataWithBody3 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random', 'https://next-data-api-endpoint.vercel.app/api/random',
{ {
method: 'POST', method: 'POST',
body: formData, body: new URLSearchParams('myParam=myValue&myParam=anotherValue'),
next: { next: {
revalidate: 10, revalidate: 30,
}, },
} }
).then((res) => res.text()) ).then((res) => res.text())
...@@ -61,7 +166,18 @@ export default async function Page() { ...@@ -61,7 +166,18 @@ export default async function Page() {
'https://next-data-api-endpoint.vercel.app/api/random', 'https://next-data-api-endpoint.vercel.app/api/random',
{ {
method: 'POST', method: 'POST',
body: new URLSearchParams('myParam=myValue&myParam=anotherValue'), body: new URLSearchParams('myParam=myValue&myParam=diffValue'),
next: {
revalidate: 30,
},
}
).then((res) => res.text())
const dataWithBody5 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new TextEncoder().encode(JSON.stringify({ hi: 'there' })),
next: { next: {
revalidate: 30, revalidate: 30,
}, },
...@@ -70,12 +186,23 @@ export default async function Page() { ...@@ -70,12 +186,23 @@ export default async function Page() {
return ( return (
<> <>
<p id="page">/variable-revalidate/post-method-cached</p>
<p id="page-data">{data}</p> <p id="page-data">{data}</p>
<p id="data-body1">{dataWithBody1}</p> <p id="data-body1">{dataWithBody1}</p>
<p id="data-body2">{dataWithBody2}</p> <p id="data-body2">{dataWithBody2}</p>
<p id="data-body3">{dataWithBody3}</p> <p id="data-body3">{dataWithBody3}</p>
<p id="data-body4">{dataWithBody4}</p> <p id="data-body4">{dataWithBody4}</p>
<p id="data-body5">{dataWithBody5}</p>
</>
)
}
export default function Page() {
return (
<>
<p id="page">/variable-revalidate/post-method-cached</p>
<DataBodies />
<EmptyBodies />
<IterableBodies />
</> </>
) )
} }
import { Suspense } from 'react'
import { fetchRetry } from '../../../lib/fetch-retry'
import { connection } from 'next/server'
async function EmptyBodies() {
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 3600,
tags: ['variable-revalidate-edge-post-method-iterable'],
},
body: '',
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 3600,
tags: ['variable-revalidate-edge-post-method-iterable'],
},
body: new URLSearchParams(),
}
).then((res) => res.text())
const dataWithBody3 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 3600,
tags: ['variable-revalidate-edge-post-method-iterable'],
},
body: new FormData(),
}
).then((res) => res.text())
return (
<>
<p id="data-empty-body1">{dataWithBody1}</p>
<p id="data-empty-body2">{dataWithBody2}</p>
<p id="data-empty-body3">{dataWithBody3}</p>
</>
)
}
async function IterableBodies() {
const entries = [
['myParam', 'myValue'],
['myParam', 'anotherValue'],
]
const urlSearchParams = new URLSearchParams(entries)
const formData = new FormData()
entries.forEach(([key, value]) => formData.append(key, value))
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 3600,
tags: ['variable-revalidate-edge-post-method-iterable'],
},
body: urlSearchParams,
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 3600,
tags: ['variable-revalidate-edge-post-method-iterable'],
},
body: formData,
}
).then((res) => res.text())
return (
<>
<p id="data-iterable-body1">{dataWithBody1}</p>
<p id="data-iterable-body2">{dataWithBody2}</p>
</>
)
}
async function Dynamic({ children }) {
await connection()
return children
}
export default function Page() {
return (
<>
<p id="page">/variable-revalidate/post-method-iterable</p>
<Suspense fallback={<p>Getting fresh data</p>}>
<Dynamic>
<EmptyBodies />
<IterableBodies />
</Dynamic>
</Suspense>
</>
)
}
import { revalidateTag } from 'next/cache'
export async function POST() {
await revalidateTag('variable-revalidate-edge-post-method-iterable', {
expire: 0,
})
return new Response('ok')
}
...@@ -2,7 +2,117 @@ import { fetchRetry } from '../../../lib/fetch-retry' ...@@ -2,7 +2,117 @@ import { fetchRetry } from '../../../lib/fetch-retry'
export const dynamic = 'force-dynamic' export const dynamic = 'force-dynamic'
export default async function Page() { async function EmptyBodiesOne() {
// We expect the same cache key so we're doing these in pairs
// 3 or more requests will start to get a miss due to too low revalidation time
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: '',
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: new FormData(),
}
).then((res) => res.text())
return (
<>
<p id="data-empty-body1">{dataWithBody1}</p>
<p id="data-empty-body2">{dataWithBody2}</p>
</>
)
}
async function EmptyBodiesTwo() {
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: '',
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: new URLSearchParams(),
}
).then((res) => res.text())
return (
<>
<p id="data-empty-body3">{dataWithBody1}</p>
<p id="data-empty-body4">{dataWithBody2}</p>
</>
)
}
function EmptyBodies() {
return (
<>
<EmptyBodiesOne />
<EmptyBodiesTwo />
</>
)
}
async function IterableBodies() {
const entries = [
['myParam', 'myValue'],
['myParam', 'anotherValue'],
]
const urlSearchParams = new URLSearchParams(entries)
const formData = new FormData()
entries.forEach(([key, value]) => formData.append(key, value))
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: urlSearchParams,
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
body: formData,
}
).then((res) => res.text())
return (
<>
<p id="data-iterable-body1">{dataWithBody1}</p>
<p id="data-iterable-body2">{dataWithBody2}</p>
</>
)
}
async function DataBodies() {
const data = await fetchRetry( const data = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random', 'https://next-data-api-endpoint.vercel.app/api/random',
{ {
...@@ -76,7 +186,6 @@ export default async function Page() { ...@@ -76,7 +186,6 @@ export default async function Page() {
return ( return (
<> <>
<p id="page">/variable-revalidate/post-method-cached</p>
<p id="page-data">{data}</p> <p id="page-data">{data}</p>
<p id="data-body1">{dataWithBody1}</p> <p id="data-body1">{dataWithBody1}</p>
<p id="data-body2">{dataWithBody2}</p> <p id="data-body2">{dataWithBody2}</p>
...@@ -86,3 +195,14 @@ export default async function Page() { ...@@ -86,3 +195,14 @@ export default async function Page() {
</> </>
) )
} }
export default function Page() {
return (
<>
<p id="page">/variable-revalidate/post-method-cached</p>
<DataBodies />
<EmptyBodies />
<IterableBodies />
</>
)
}
import { IncrementalCache } from 'next/dist/server/lib/incremental-cache'
function createCache() {
return new IncrementalCache({
dev: false,
requestHeaders: {},
getPrerenderManifest: () =>
({
version: 4,
routes: {},
dynamicRoutes: {},
notFoundRoutes: [],
preview: {
previewModeId: 'id',
previewModeSigningKey: 'key',
previewModeEncryptionKey: 'key',
},
}) as any,
})
}
describe('IncrementalCache.generateCacheKey', () => {
const cache = createCache()
const url = 'https://example.com/api'
it('distinguishes binary bodies that UTF-8 decoding would collapse', async () => {
// 0xff and 0xfe both decode to U+FFFD as UTF-8; the key must tell them
// apart by their raw bytes.
const a = await cache.generateCacheKey(url, {
body: new Uint8Array([0xff]),
})
const b = await cache.generateCacheKey(url, {
body: new Uint8Array([0xfe]),
})
expect(a).not.toBe(b)
})
it('distinguishes a string body from its UTF-8 encoded bytes', async () => {
// A string and the bytes it encodes to are different request shapes.
const a = await cache.generateCacheKey(url, { body: 'hello' })
const b = await cache.generateCacheKey(url, {
body: new TextEncoder().encode('hello'),
})
expect(a).not.toBe(b)
})
it('is deterministic for identical bodies', async () => {
const a = await cache.generateCacheKey(url, {
body: new Uint8Array([1, 2, 3]),
})
const b = await cache.generateCacheKey(url, {
body: new Uint8Array([1, 2, 3]),
})
expect(a).toBe(b)
})
it('does not collide FormData multi-values with a comma-joined string', async () => {
// The two values ['a', 'b'] must not hash the same as the single 'a,b'.
const multi = new FormData()
multi.append('x', 'a')
multi.append('x', 'b')
const joined = new FormData()
joined.append('x', 'a,b')
const a = await cache.generateCacheKey(url, { body: multi })
const b = await cache.generateCacheKey(url, { body: joined })
expect(a).not.toBe(b)
})
it('distinguishes blobs that differ only by content type', async () => {
const bytes = new Uint8Array([1, 2, 3])
const a = await cache.generateCacheKey(url, {
body: new Blob([bytes], { type: 'text/plain' }),
})
const b = await cache.generateCacheKey(url, {
body: new Blob([bytes], { type: 'application/json' }),
})
expect(a).not.toBe(b)
})
it('distinguishes ArrayBuffers', async () => {
const a = await cache.generateCacheKey(url, {
body: new Uint8Array([1, 2, 3, 4]).buffer,
})
const b = await cache.generateCacheKey(url, {
body: new Uint8Array([1, 2, 3]).buffer,
})
expect(a).not.toBe(b)
})
it('collides identical byte sequences in different typed arrays', async () => {
const uint8View = new Uint8Array([1, 2, 3, 4])
const uint16View = new Uint16Array(
uint8View.buffer,
uint8View.byteOffset,
uint8View.byteLength / 2
)
const a = await cache.generateCacheKey(url, {
body: uint8View,
})
const b = await cache.generateCacheKey(url, {
body: uint16View,
})
expect(a).toBe(b)
})
it('does not collide FormData with differently interleaved values', async () => {
const a = new FormData()
a.append('x', 'a')
a.append('x', 'b')
a.append('y', 'a')
const b = new FormData()
b.append('x', 'a')
b.append('y', 'a')
b.append('x', 'b')
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide FormData whose value forges quoted entry delimiters', async () => {
// Regression guard: this passes today and must keep passing. The `x` value
// below contains `"` bytes chosen so that a quote-wrapped encoding
// (`key:"${key}"` / `str:"${val}"`) would hash both bodies to
// `key:"x"str:"a"key:"y"str:"b"` and collide them. Distinct bodies must get
// distinct keys, so the encoding has to escape `"`/`\` or length-prefix
// each segment rather than rely on unescaped quotes.
const a = new FormData()
a.append('x', 'a"key:"y"str:"b')
const b = new FormData()
b.append('x', 'a')
b.append('y', 'b')
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide FormData whose value spans an entry boundary', async () => {
// Current behavior (a bug): body segments are concatenated into the hash
// without delimiters, so a single field whose value reproduces the tag
// bytes of a following entry hashes identically to the two-field body it
// imitates. Both produce `key:xstr:akey:ystr:b`. Flip this to not.toBe once
// the encoding is made unambiguous.
const a = new FormData()
a.append('x', 'akey:ystr:b')
const b = new FormData()
b.append('x', 'a')
b.append('y', 'b')
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide FormData files that differ only by name/type split', async () => {
// The file branch hashes `file:` + name + type + `bytes:` + content with no
// length prefixes, so name="ab"/type="c" and name="a"/type="bc" both hash
// `file:abc` + `bytes:` + content. Fails until the file name/type/content
// segments are length-prefixed like the string entries now are.
const bytes = new Uint8Array([1, 2, 3])
const a = new FormData()
a.append('f', new Blob([bytes], { type: 'c' }), 'ab')
const b = new FormData()
b.append('f', new Blob([bytes], { type: 'bc' }), 'a')
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide a file whose content forges a following entry', async () => {
// File content is not length-prefixed, so a file body ending in the exact
// bytes a later string entry emits (`key:1kstr:1v`) hashes the same as that
// file plus the real entry. Fails until file content is length-prefixed.
const enc = new TextEncoder()
const a = new FormData()
a.append('f', new Blob([enc.encode('AAAkey:1kstr:1v')], { type: '' }), 'n')
const b = new FormData()
b.append('f', new Blob([enc.encode('AAA')], { type: '' }), 'n')
b.append('k', 'v')
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide blobs that differ only by type/content split', async () => {
// The blob branch hashes `blob:` + type + `bytes:` + content with no length
// prefixes, so type="bytes:x"/content="y" and type=""/content="xbytes:y"
// both hash `blob:bytes:xbytes:y`. Fails until blob type/content are
// length-prefixed.
const enc = new TextEncoder()
const a = new Blob([enc.encode('y')], { type: 'bytes:x' })
const b = new Blob([enc.encode('xbytes:y')], { type: '' })
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide a FormData value whose length digits absorb a forged entry', async () => {
// The previous encoding wrote length prefixes as `byteLength.toString()`
// with no separator before the payload, so a 2-digit length was
// indistinguishable from a 1-digit length followed by a payload digit. Under
// it, both bodies below hashed the identical stream `key:1xstr:11key:1astr:0`:
// A: one entry x="key:1astr:0" -> str: + "11"(len) + "key:1astr:0"
// B: entries x="1", a="" -> str: + "1"(len) + "1"(data) + key:1a...
// B's value "1" emits `str:11`, which A reads as a length-11 value that then
// spells out B's second entry verbatim.
const a = new FormData()
a.append('x', 'key:1astr:0')
const b = new FormData()
b.append('x', '1')
b.append('a', '')
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
it('does not collide blobs whose type-length digits absorb the content', async () => {
// The same separator-less length-prefix ambiguity on the blob branch. Under
// the previous encoding both bodies hashed the identical stream
// `blob:11bytes:aaaaabytes:`:
// A: type "bytes:aaaaa" (len 11), empty content
// -> blob: + "11"(len) + "bytes:aaaaa" + bytes:
// B: type "1" (len 1), content "aaaaabytes:"
// -> blob: + "1"(len) + "1"(type) + bytes: + "aaaaabytes:"
// Lowercase is required because the Blob constructor normalizes `type`.
const enc = new TextEncoder()
const a = new Blob([new Uint8Array(0)], { type: 'bytes:aaaaa' })
const b = new Blob([enc.encode('aaaaabytes:')], { type: '1' })
const keyA = await cache.generateCacheKey(url, { body: a })
const keyB = await cache.generateCacheKey(url, { body: b })
expect(keyA).not.toBe(keyB)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment