Unverified Commit ebed7396 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Ensure fetch fields are detected with RequestInit (#46398)

This ensures we properly detect fetch fields when a `new Request()` is passed to fetch instead of a separate `init` param. 

Fixes: [slack thread](https://vercel.slack.com/archives/C035J346QQL/p1677264109548949)
Closes: https://github.com/vercel/next.js/issues/46349

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)
parent 821c251a
...@@ -8,6 +8,8 @@ import { ...@@ -8,6 +8,8 @@ import {
IncrementalCacheValue, IncrementalCacheValue,
IncrementalCacheEntry, IncrementalCacheEntry,
} from '../../response-cache' } from '../../response-cache'
import { encode } from '../../../shared/lib/bloom-filter/base64-arraybuffer'
import { encodeText } from '../../node-web-streams-helper'
function toRoute(pathname: string): string { function toRoute(pathname: string): string {
return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/' return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/'
...@@ -146,6 +148,73 @@ export class IncrementalCache { ...@@ -146,6 +148,73 @@ export class IncrementalCache {
// x-ref: https://github.com/facebook/react/blob/2655c9354d8e1c54ba888444220f63e836925caa/packages/react/src/ReactFetch.js#L23 // x-ref: https://github.com/facebook/react/blob/2655c9354d8e1c54ba888444220f63e836925caa/packages/react/src/ReactFetch.js#L23
async fetchCacheKey(url: string, init: RequestInit = {}): Promise<string> { async fetchCacheKey(url: string, init: RequestInit = {}): Promise<string> {
let cacheKey: string
const bodyChunks: string[] = []
try {
if (init.body) {
// handle ReadableStream body
if (typeof (init.body as any).getReader === 'function') {
const readableBody = init.body as ReadableStream
const [origBody, newBody] = readableBody.tee()
init.body = origBody
const reader = newBody.getReader()
function processValue({
done,
value,
}: {
done?: boolean
value?: ArrayBuffer | string
}): any {
if (done) {
return
}
if (value) {
try {
bodyChunks.push(
typeof value === 'string' ? value : encode(value)
)
} catch (err) {
console.error(err)
}
}
return reader.read().then(processValue)
}
await reader.read().then(processValue)
} // handle FormData or URLSearchParams bodies
else if (typeof (init.body as any).keys === 'function') {
const formData = init.body as FormData
for (const key of new Set([...formData.keys()])) {
const values = formData.getAll(key)
bodyChunks.push(
`${key}=${(
await Promise.all(
values.map(async (val) => {
if (typeof val === 'string') {
return val
} else {
return await val.text()
}
})
)
).join(',')}`
)
}
// handle blob body
} else if (typeof (init.body as any).arrayBuffer === 'function') {
const blob = init.body as Blob
const arrayBuffer = await blob.arrayBuffer()
bodyChunks.push(encode(await (init.body as Blob).arrayBuffer()))
init.body = new Blob([arrayBuffer], { type: blob.type })
} else if (typeof init.body === 'string') {
bodyChunks.push(init.body)
}
// TODO: handle ReadableStream body?
}
} catch (err) {
console.error(err)
}
const cacheString = JSON.stringify([ const cacheString = JSON.stringify([
this.fetchCacheKey || '', this.fetchCacheKey || '',
url, url,
...@@ -159,8 +228,8 @@ export class IncrementalCache { ...@@ -159,8 +228,8 @@ export class IncrementalCache {
init.integrity, init.integrity,
init.next, init.next,
init.cache, init.cache,
bodyChunks,
]) ])
let cacheKey: string
if (process.env.NEXT_RUNTIME === 'edge') { if (process.env.NEXT_RUNTIME === 'edge') {
function bufferToHex(buffer: ArrayBuffer): string { function bufferToHex(buffer: ArrayBuffer): string {
...@@ -168,7 +237,7 @@ export class IncrementalCache { ...@@ -168,7 +237,7 @@ export class IncrementalCache {
.call(new Uint8Array(buffer), (b) => b.toString(16).padStart(2, '0')) .call(new Uint8Array(buffer), (b) => b.toString(16).padStart(2, '0'))
.join('') .join('')
} }
const buffer = new TextEncoder().encode(cacheString) const buffer = encodeText(cacheString)
cacheKey = bufferToHex(await crypto.subtle.digest('SHA-256', buffer)) cacheKey = bufferToHex(await crypto.subtle.digest('SHA-256', buffer))
} else { } else {
const crypto = require('crypto') as typeof import('crypto') const crypto = require('crypto') as typeof import('crypto')
......
...@@ -29,6 +29,11 @@ export function patchFetch({ ...@@ -29,6 +29,11 @@ export function patchFetch({
}, },
async (input: RequestInfo | URL, init: RequestInit | undefined) => { async (input: RequestInfo | URL, init: RequestInit | undefined) => {
const staticGenerationStore = staticGenerationAsyncStorage.getStore() const staticGenerationStore = staticGenerationAsyncStorage.getStore()
const isRequestInput = input && typeof input === 'object'
const getRequestMeta = (field: string) => {
let value = isRequestInput ? (input as any)[field] : null
return value || (init as any)?.[field]
}
// If the staticGenerationStore is not available, we can't do any // If the staticGenerationStore is not available, we can't do any
// special treatment of fetch, therefore fallback to the original // special treatment of fetch, therefore fallback to the original
...@@ -38,12 +43,15 @@ export function patchFetch({ ...@@ -38,12 +43,15 @@ export function patchFetch({
} }
let revalidate: number | undefined | false = undefined let revalidate: number | undefined | false = undefined
// RequestInit doesn't keep extra fields e.g. next so it's
// only available if init is used separate
let curRevalidate = init?.next?.revalidate let curRevalidate = init?.next?.revalidate
const _cache = getRequestMeta('cache')
if (init?.cache === 'force-cache') { if (_cache === 'force-cache') {
curRevalidate = false curRevalidate = false
} }
if (['no-cache', 'no-store'].includes(init?.cache || '')) { if (['no-cache', 'no-store'].includes(_cache || '')) {
curRevalidate = 0 curRevalidate = 0
} }
if (typeof curRevalidate === 'number') { if (typeof curRevalidate === 'number') {
...@@ -53,16 +61,18 @@ export function patchFetch({ ...@@ -53,16 +61,18 @@ export function patchFetch({
if (curRevalidate === false) { if (curRevalidate === false) {
revalidate = CACHE_ONE_YEAR revalidate = CACHE_ONE_YEAR
} }
const _headers = getRequestMeta('headers')
const initHeaders: Headers = const initHeaders: Headers =
typeof (init?.headers as Headers)?.get === 'function' typeof _headers?.get === 'function'
? (init?.headers as Headers) ? _headers
: new Headers(init?.headers || {}) : new Headers(_headers || {})
const hasUnCacheableHeader = const hasUnCacheableHeader =
initHeaders.get('authorization') || initHeaders.get('cookie') initHeaders.get('authorization') || initHeaders.get('cookie')
const isUnCacheableMethod = !['get', 'head'].includes( const isUnCacheableMethod = !['get', 'head'].includes(
init?.method?.toLowerCase() || 'get' getRequestMeta('method')?.toLowerCase() || 'get'
) )
if (typeof revalidate === 'undefined') { if (typeof revalidate === 'undefined') {
...@@ -88,6 +98,16 @@ export function patchFetch({ ...@@ -88,6 +98,16 @@ export function patchFetch({
} }
let cacheKey: string | undefined let cacheKey: string | undefined
if (
staticGenerationStore.incrementalCache &&
typeof revalidate === 'number' &&
revalidate > 0
) {
cacheKey = await staticGenerationStore.incrementalCache.fetchCacheKey(
isRequestInput ? (input as Request).url : input.toString(),
isRequestInput ? (input as RequestInit) : init
)
}
const doOriginalFetch = async () => { const doOriginalFetch = async () => {
return originFetch(input, init).then(async (res) => { return originFetch(input, init).then(async (res) => {
...@@ -97,18 +117,15 @@ export function patchFetch({ ...@@ -97,18 +117,15 @@ export function patchFetch({
typeof revalidate === 'number' && typeof revalidate === 'number' &&
revalidate > 0 revalidate > 0
) { ) {
const clonedRes = res.clone()
let base64Body = '' let base64Body = ''
const arrayBuffer = await res.arrayBuffer()
if (process.env.NEXT_RUNTIME === 'edge') { if (process.env.NEXT_RUNTIME === 'edge') {
const { encode } = const { encode } =
require('../../shared/lib/bloom-filter/base64-arraybuffer') as typeof import('../../shared/lib/bloom-filter/base64-arraybuffer') require('../../shared/lib/bloom-filter/base64-arraybuffer') as typeof import('../../shared/lib/bloom-filter/base64-arraybuffer')
base64Body = encode(await clonedRes.arrayBuffer()) base64Body = encode(arrayBuffer)
} else { } else {
base64Body = Buffer.from(await clonedRes.arrayBuffer()).toString( base64Body = Buffer.from(arrayBuffer).toString('base64')
'base64'
)
} }
try { try {
...@@ -117,7 +134,7 @@ export function patchFetch({ ...@@ -117,7 +134,7 @@ export function patchFetch({
{ {
kind: 'FETCH', kind: 'FETCH',
data: { data: {
headers: Object.fromEntries(clonedRes.headers.entries()), headers: Object.fromEntries(res.headers.entries()),
body: base64Body, body: base64Body,
}, },
revalidate, revalidate,
...@@ -128,22 +145,16 @@ export function patchFetch({ ...@@ -128,22 +145,16 @@ export function patchFetch({
} catch (err) { } catch (err) {
console.warn(`Failed to set fetch cache`, input, err) console.warn(`Failed to set fetch cache`, input, err)
} }
return new Response(arrayBuffer, {
headers: res.headers,
status: res.status,
})
} }
return res return res
}) })
} }
if ( if (cacheKey && staticGenerationStore?.incrementalCache) {
staticGenerationStore.incrementalCache &&
typeof revalidate === 'number' &&
revalidate > 0
) {
cacheKey = await staticGenerationStore.incrementalCache.fetchCacheKey(
input && typeof input === 'object'
? (input as Request).url
: input.toString(),
init
)
const entry = await staticGenerationStore.incrementalCache.get( const entry = await staticGenerationStore.incrementalCache.get(
cacheKey, cacheKey,
true true
......
...@@ -84,6 +84,8 @@ createNextDescribe( ...@@ -84,6 +84,8 @@ createNextDescribe(
'static-to-dynamic-error/[id]/page.js', 'static-to-dynamic-error/[id]/page.js',
'variable-revalidate-edge/encoding/page.js', 'variable-revalidate-edge/encoding/page.js',
'variable-revalidate-edge/no-store/page.js', 'variable-revalidate-edge/no-store/page.js',
'variable-revalidate-edge/post-method-cached/page.js',
'variable-revalidate-edge/post-method-request/page.js',
'variable-revalidate-edge/revalidate-3/page.js', 'variable-revalidate-edge/revalidate-3/page.js',
'variable-revalidate/authorization-cached.html', 'variable-revalidate/authorization-cached.html',
'variable-revalidate/authorization-cached.rsc', 'variable-revalidate/authorization-cached.rsc',
...@@ -222,7 +224,7 @@ createNextDescribe( ...@@ -222,7 +224,7 @@ createNextDescribe(
}, },
'/variable-revalidate/post-method-cached': { '/variable-revalidate/post-method-cached': {
dataRoute: '/variable-revalidate/post-method-cached.rsc', dataRoute: '/variable-revalidate/post-method-cached.rsc',
initialRevalidateSeconds: 3, initialRevalidateSeconds: 10,
srcRoute: '/variable-revalidate/post-method-cached', srcRoute: '/variable-revalidate/post-method-cached',
}, },
'/variable-revalidate/revalidate-3': { '/variable-revalidate/revalidate-3': {
...@@ -502,6 +504,104 @@ createNextDescribe( ...@@ -502,6 +504,104 @@ createNextDescribe(
} }
}) })
it('should not cache correctly with POST method request init', async () => {
const res = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-request'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const pageData = $('#page-data').text()
for (let i = 0; i < 3; i++) {
const res2 = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-request'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#page-data').text()).not.toBe(pageData)
}
})
it('should cache correctly with post method and revalidate', async () => {
await check(async () => {
const res = await fetchViaHTTP(
next.url,
'/variable-revalidate/post-method-cached'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const layoutData = $('#layout-data').text()
const pageData = $('#page-data').text()
const dataBody1 = $('#data-body1').text()
const dataBody2 = $('#data-body2').text()
const dataBody3 = $('#data-body3').text()
const dataBody4 = $('#data-body4').text()
expect(dataBody1).not.toBe(dataBody2)
expect(dataBody2).not.toBe(dataBody3)
expect(dataBody3).not.toBe(dataBody4)
const res2 = await fetchViaHTTP(
next.url,
'/variable-revalidate/post-method-cached'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#layout-data').text()).toBe(layoutData)
expect($2('#page-data').text()).toBe(pageData)
expect($2('#data-body1').text()).toBe(dataBody1)
expect($2('#data-body2').text()).toBe(dataBody2)
expect($2('#data-body3').text()).toBe(dataBody3)
expect($2('#data-body4').text()).toBe(dataBody4)
return 'success'
}, 'success')
})
it('should cache correctly with post method and revalidate edge', async () => {
await check(async () => {
const res = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-cached'
)
expect(res.status).toBe(200)
const html = await res.text()
const $ = cheerio.load(html)
const layoutData = $('#layout-data').text()
const pageData = $('#page-data').text()
const dataBody1 = $('#data-body1').text()
const dataBody2 = $('#data-body2').text()
const dataBody3 = $('#data-body3').text()
const dataBody4 = $('#data-body4').text()
const res2 = await fetchViaHTTP(
next.url,
'/variable-revalidate-edge/post-method-cached'
)
expect(res2.status).toBe(200)
const html2 = await res2.text()
const $2 = cheerio.load(html2)
expect($2('#layout-data').text()).toBe(layoutData)
expect($2('#page-data').text()).toBe(pageData)
expect($2('#data-body1').text()).toBe(dataBody1)
expect($2('#data-body2').text()).toBe(dataBody2)
expect($2('#data-body3').text()).toBe(dataBody3)
expect($2('#data-body4').text()).toBe(dataBody4)
return 'success'
}, 'success')
})
it('should cache correctly with POST method and revalidate', async () => { it('should cache correctly with POST method and revalidate', async () => {
await check(async () => { await check(async () => {
const res = await fetchViaHTTP( const res = await fetchViaHTTP(
......
export const runtime = 'experimental-edge'
const fetchRetry = async (url, init) => {
for (let i = 0; i < 5; i++) {
try {
return await fetch(url, init)
} catch (err) {
if (i === 4) {
throw err
}
console.log(`Failed to fetch`, err, `retrying...`)
}
}
}
export default async function Page() {
const data = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: JSON.stringify({ hello: 'world' }),
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new ReadableStream({
start(controller) {
controller.enqueue(JSON.stringify({ another: 'one' }))
controller.close()
},
}),
duplex: 'half',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const formData = new FormData()
formData.append('hello', 'value')
formData.append('another', new Blob(['some text'], { type: 'text/plain' }))
formData.append('another', 'text')
const dataWithBody3 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: formData,
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody4 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new URLSearchParams('myParam=myValue&myParam=anotherValue'),
next: {
revalidate: 30,
},
}
).then((res) => res.text())
return (
<>
<p id="page">/variable-revalidate/post-method-cached</p>
<p id="page-data">{data}</p>
<p id="data-body1">{dataWithBody1}</p>
<p id="data-body2">{dataWithBody2}</p>
<p id="data-body3">{dataWithBody3}</p>
<p id="data-body4">{dataWithBody4}</p>
</>
)
}
export const runtime = 'experimental-edge'
export default async function Page() {
const data = await fetch(
new Request('https://next-data-api-endpoint.vercel.app/api/random', {
method: 'POST',
})
).then((res) => res.text())
return (
<>
<p id="page">/variable-revalidate/post-method</p>
<p id="page-data">{data}</p>
</>
)
}
import { FormData, Blob } from 'next/dist/compiled/@edge-runtime/primitives'
const fetchRetry = async (url, init) => {
for (let i = 0; i < 5; i++) {
try {
return await fetch(url, init)
} catch (err) {
if (i === 4) {
throw err
}
console.log(`Failed to fetch`, err, `retrying...`)
}
}
}
export default async function Page() { export default async function Page() {
const data = await fetch( const data = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody1 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: JSON.stringify({ hello: 'world' }),
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody2 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: new ReadableStream({
start(controller) {
controller.enqueue(JSON.stringify({ another: 'one' }))
controller.close()
},
}),
duplex: 'half',
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const formData = new FormData()
formData.append('hello', 'value')
formData.append('another', new Blob(['some text'], { type: 'text/plain' }))
formData.append('another', 'text')
const dataWithBody3 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random',
{
method: 'POST',
body: formData,
next: {
revalidate: 10,
},
}
).then((res) => res.text())
const dataWithBody4 = await fetchRetry(
'https://next-data-api-endpoint.vercel.app/api/random', 'https://next-data-api-endpoint.vercel.app/api/random',
{ {
method: 'POST', method: 'POST',
body: new URLSearchParams('myParam=myValue&myParam=anotherValue'),
next: { next: {
revalidate: 3, revalidate: 30,
}, },
} }
).then((res) => res.text()) ).then((res) => res.text())
...@@ -13,6 +83,10 @@ export default async function Page() { ...@@ -13,6 +83,10 @@ export default async function Page() {
<> <>
<p id="page">/variable-revalidate/post-method-cached</p> <p id="page">/variable-revalidate/post-method-cached</p>
<p id="page-data">{data}</p> <p id="page-data">{data}</p>
<p id="data-body1">{dataWithBody1}</p>
<p id="data-body2">{dataWithBody2}</p>
<p id="data-body3">{dataWithBody3}</p>
<p id="data-body4">{dataWithBody4}</p>
</> </>
) )
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment