Unverified Commit ebed7396 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Ensure fetch fields are detected with RequestInit (#46398)

This ensures we properly detect fetch fields when a `new Request()` is passed to fetch instead of a separate `init` param. 

Fixes: [slack thread](https://vercel.slack.com/archives/C035J346QQL/p1677264109548949)
Closes: https://github.com/vercel/next.js/issues/46349

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)
parent 821c251a
......@@ -8,6 +8,8 @@ import {
IncrementalCacheValue,
IncrementalCacheEntry,
} from '../../response-cache'
import { encode } from '../../../shared/lib/bloom-filter/base64-arraybuffer'
import { encodeText } from '../../node-web-streams-helper'
function toRoute(pathname: string): string {
return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/'
......@@ -146,6 +148,73 @@ export class IncrementalCache {
// x-ref: https://github.com/facebook/react/blob/2655c9354d8e1c54ba888444220f63e836925caa/packages/react/src/ReactFetch.js#L23
async fetchCacheKey(url: string, init: RequestInit = {}): Promise<string> {
let cacheKey: string
const bodyChunks: string[] = []
try {
if (init.body) {
// handle ReadableStream body
if (typeof (init.body as any).getReader === 'function') {
const readableBody = init.body as ReadableStream
const [origBody, newBody] = readableBody.tee()
init.body = origBody
const reader = newBody.getReader()
function processValue({
done,
value,
}: {
done?: boolean
value?: ArrayBuffer | string
}): any {
if (done) {
return
}
if (value) {
try {
bodyChunks.push(
typeof value === 'string' ? value : encode(value)
)
} catch (err) {
console.error(err)
}
}
return reader.read().then(processValue)
}
await reader.read().then(processValue)
} // handle FormData or URLSearchParams bodies
else if (typeof (init.body as any).keys === 'function') {
const formData = init.body as FormData
for (const key of new Set([...formData.keys()])) {
const values = formData.getAll(key)
bodyChunks.push(
`${key}=${(
await Promise.all(
values.map(async (val) => {
if (typeof val === 'string') {
return val
} else {
return await val.text()
}
})
)
).join(',')}`
)
}
// handle blob body
} else if (typeof (init.body as any).arrayBuffer === 'function') {
const blob = init.body as Blob
const arrayBuffer = await blob.arrayBuffer()
bodyChunks.push(encode(await (init.body as Blob).arrayBuffer()))
init.body = new Blob([arrayBuffer], { type: blob.type })
} else if (typeof init.body === 'string') {
bodyChunks.push(init.body)
}
// TODO: handle ReadableStream body?
}
} catch (err) {
console.error(err)
}
const cacheString = JSON.stringify([
this.fetchCacheKey || '',
url,
......@@ -159,8 +228,8 @@ export class IncrementalCache {
init.integrity,
init.next,
init.cache,
bodyChunks,
])
let cacheKey: string
if (process.env.NEXT_RUNTIME === 'edge') {
function bufferToHex(buffer: ArrayBuffer): string {
......@@ -168,7 +237,7 @@ export class IncrementalCache {
.call(new Uint8Array(buffer), (b) => b.toString(16).padStart(2, '0'))
.join('')
}
const buffer = new TextEncoder().encode(cacheString)
const buffer = encodeText(cacheString)
cacheKey = bufferToHex(await crypto.subtle.digest('SHA-256', buffer))
} else {
const crypto = require('crypto') as typeof import('crypto')
......
......@@ -29,6 +29,11 @@ export function patchFetch({
},
async (input: RequestInfo | URL, init: RequestInit | undefined) => {
const staticGenerationStore = staticGenerationAsyncStorage.getStore()
const isRequestInput = input && typeof input === 'object'
const getRequestMeta = (field: string) => {
let value = isRequestInput ? (input as any)[field] : null
return value || (init as any)?.[field]
}
// If the staticGenerationStore is not available, we can't do any
// special treatment of fetch, therefore fallback to the original
......@@ -38,12 +43,15 @@ export function patchFetch({
}
let revalidate: number | undefined | false = undefined
// RequestInit doesn't keep extra fields e.g. next so it's
// only available if init is used separate
let curRevalidate = init?.next?.revalidate
const _cache = getRequestMeta('cache')
if (init?.cache === 'force-cache') {
if (_cache === 'force-cache') {
curRevalidate = false
}
if (['no-cache', 'no-store'].includes(init?.cache || '')) {
if (['no-cache', 'no-store'].includes(_cache || '')) {
curRevalidate = 0
}
if (typeof curRevalidate === 'number') {
......@@ -53,16 +61,18 @@ export function patchFetch({
if (curRevalidate === false) {
revalidate = CACHE_ONE_YEAR
}
const _headers = getRequestMeta('headers')
const initHeaders: Headers =
typeof (init?.headers as Headers)?.get === 'function'
? (init?.headers as Headers)
: new Headers(init?.headers || {})
typeof _headers?.get === 'function'
? _headers
: new Headers(_headers || {})
const hasUnCacheableHeader =
initHeaders.get('authorization') || initHeaders.get('cookie')
const isUnCacheableMethod = !['get', 'head'].includes(
init?.method?.toLowerCase() || 'get'
getRequestMeta('method')?.toLowerCase() || 'get'
)
if (typeof revalidate === 'undefined') {
......@@ -88,6 +98,16 @@ export function patchFetch({
}
let cacheKey: string | undefined
if (
staticGenerationStore.incrementalCache &&
typeof revalidate === 'number' &&
revalidate > 0
) {
cacheKey = await staticGenerationStore.incrementalCache.fetchCacheKey(
isRequestInput ? (input as Request).url : input.toString(),
isRequestInput ? (input as RequestInit) : init
)
}
const doOriginalFetch = async () => {
return originFetch(input, init).then(async (res) => {
......@@ -97,18 +117,15 @@ export function patchFetch({
typeof revalidate === 'number' &&
revalidate > 0
) {
const clonedRes = res.clone()
let base64Body = ''
const arrayBuffer = await res.arrayBuffer()
if (process.env.NEXT_RUNTIME === 'edge') {
const { encode } =
require('../../shared/lib/bloom-filter/base64-arraybuffer') as typeof import('../../shared/lib/bloom-filter/base64-arraybuffer')
base64Body = encode(await clonedRes.arrayBuffer())
base64Body = encode(arrayBuffer)
} else {
base64Body = Buffer.from(await clonedRes.arrayBuffer()).toString(
'base64'
)
base64Body = Buffer.from(arrayBuffer).toString('base64')
}
try {
......@@ -117,7 +134,7 @@ export function patchFetch({
{
kind: 'FETCH',
data: {
headers: Object.fromEntries(clonedRes.headers.entries()),
headers: Object.fromEntries(res.headers.entries()),
body: base64Body,
},
revalidate,
......@@ -128,22 +145,16 @@ export function patchFetch({
} catch (err) {
console.warn(`Failed to set fetch cache`, input, err)
}
return new Response(arrayBuffer, {
headers: res.headers,
status: res.status,
})
}
return res
})
}
if (
staticGenerationStore.incrementalCache &&
typeof revalidate === 'number' &&
revalidate > 0
) {
cacheKey = await staticGenerationStore.incrementalCache.fetchCacheKey(
input && typeof input === 'object'
? (input as Request).url
: input.toString(),
init
)
if (cacheKey && staticGenerationStore?.incrementalCache) {
const entry = await staticGenerationStore.incrementalCache.get(
cacheKey,
true
......
......@@ -84,6 +84,8 @@ createNextDescribe(
'static-to-dynamic-error/[id]/page.js',
'variable-revalidate-edge/encoding/page.js',
'variable-revalidate-edge/no-store/page.js',
'variable-revalidate-edge/post-method-cached/page.js',
'variable-revalidate-edge/post-method-request/page.js',
'variable-revalidate-edge/revalidate-3/page.js',
'variable-revalidate/authorization-cached.html',
'variable-revalidate/authorization-cached.rsc',
......@@ -222,7 +224,7 @@ createNextDescribe(
},
'/variable-revalidate/post-method-cached': {
dataRoute: '/variable-revalidate/post-method-cached.rsc',
initialRevalidateSeconds: 3,
initialRevalidateSeconds: 10,
srcRoute: '/variable-revalidate/post-method-cached',
},
'/variable-revalidate/revalidate-3': {
......@@ -502,6 +504,104 @@ createNextDescribe(
}
})
it('should not cache correctly with POST method request init', async () => {
const res = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-request'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const pageData = $('#page-data').text()
for (let i = 0; i < 3; i++) {
const res2 = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-request'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#page-data').text()).not.toBe(pageData)
}
})
it('should cache correctly with post method and revalidate', async () => {
await check(async () => {
const res = await fetchViaHTTP(
next.url,
'/variable-revalidate/post-method-cached'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const layoutData = $('#layout-data').text()
const pageData = $('#page-data').text()
const dataBody1 = $('#data-body1').text()
const dataBody2 = $('#data-body2').text()
const dataBody3 = $('#data-body3').text()
const dataBody4 = $('#data-body4').text()
expect(dataBody1).not.toBe(dataBody2)
expect(dataBody2).not.toBe(dataBody3)
expect(dataBody3).not.toBe(dataBody4)
const res2 = await fetchViaHTTP(
next.url,
'/variable-revalidate/post-method-cached'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#layout-data').text()).toBe(layoutData)
expect($2('#page-data').text()).toBe(pageData)
expect($2('#data-body1').text()).toBe(dataBody1)
expect($2('#data-body2').text()).toBe(dataBody2)
expect($2('#data-body3').text()).toBe(dataBody3)
expect($2('#data-body4').text()).toBe(dataBody4)
return 'success'
}, 'success')
})
it('should cache correctly with post method and revalidate edge', async () => {
await check(async () => {
const res = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-cached'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const layoutData = $('#layout-data').text()
const pageData = $('#page-data').text()
const dataBody1 = $('#data-body1').text()
const dataBody2 = $('#data-body2').text()
const dataBody3 = $('#data-body3').text()
const dataBody4 = $('#data-body4').text()
const res2 = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-cached'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#layout-data').text()).toBe(layoutData)
expect($2('#page-data').text()).toBe(pageData)
expect($2('#data-body1').text()).toBe(dataBody1)
expect($2('#data-body2').text()).toBe(dataBody2)
expect($2('#data-body3').text()).toBe(dataBody3)
expect($2('#data-body4').text()).toBe(dataBody4)
return 'success'
}, 'success')
})
it('should cache correctly with POST method and revalidate', async () => {
await check(async () => {
const res = await fetchViaHTTP(
......
export const runtime = 'experimental-edge'
const fetchRetry = async (url, init) => {
for (let i = 0; i < 5; i++) {
try {
return await fetch(url, init)
} catch (err) {
if (i === 4) {
throw err
}
console.log(`Failed to fetch`, err, `retrying...`)
}
}
}
export default async function Page() {
const data = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: JSON.stringify({ hello: 'world' }),
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new ReadableStream({
start(controller) {
controller.enqueue(JSON.stringify({ another: 'one' }))
controller.close()
},
}),
duplex: 'half',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const formData = new FormData()
formData.append('hello', 'value')
formData.append('another', new Blob(['some text'], { type: 'text/plain' }))
formData.append('another', 'text')
const dataWithBody3 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: formData,
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody4 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new URLSearchParams('myParam=myValue&myParam=anotherValue'),
next: {
revalidate: 30,
},
}
).then((res) => res.text())
return (
<>
<p id="page">/variable-revalidate/post-method-cached</p>
<p id="page-data">{data}</p>
<p id="data-body1">{dataWithBody1}</p>
<p id="data-body2">{dataWithBody2}</p>
<p id="data-body3">{dataWithBody3}</p>
<p id="data-body4">{dataWithBody4}</p>
</>
)
}
export const runtime = 'experimental-edge'
export default async function Page() {
const data = await fetch(
new Request('https://next-data-api-endpoint.vercel.app/api/random', {
method: 'POST',
})
).then((res) => res.text())
return (
<>
<p id="page">/variable-revalidate/post-method</p>
<p id="page-data">{data}</p>
</>
)
}
import { FormData, Blob } from 'next/dist/compiled/@edge-runtime/primitives'
const fetchRetry = async (url, init) => {
for (let i = 0; i < 5; i++) {
try {
return await fetch(url, init)
} catch (err) {
if (i === 4) {
throw err
}
console.log(`Failed to fetch`, err, `retrying...`)
}
}
}
export default async function Page() {
const data = await fetch(
const data = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: JSON.stringify({ hello: 'world' }),
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new ReadableStream({
start(controller) {
controller.enqueue(JSON.stringify({ another: 'one' }))
controller.close()
},
}),
duplex: 'half',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const formData = new FormData()
formData.append('hello', 'value')
formData.append('another', new Blob(['some text'], { type: 'text/plain' }))
formData.append('another', 'text')
const dataWithBody3 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: formData,
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody4 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new URLSearchParams('myParam=myValue&myParam=anotherValue'),
next: {
revalidate: 3,
revalidate: 30,
},
}
).then((res) => res.text())
......@@ -13,6 +83,10 @@ export default async function Page() {
<>
<p id="page">/variable-revalidate/post-method-cached</p>
<p id="page-data">{data}</p>
<p id="data-body1">{dataWithBody1}</p>
<p id="data-body2">{dataWithBody2}</p>
<p id="data-body3">{dataWithBody3}</p>
<p id="data-body4">{dataWithBody4}</p>
</>
)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment