Unverified Commit fc05711d authored by Andrew Clark's avatar Andrew Clark Committed by GitHub

Verify cache-busting param during segment prefetch (#79563)

Not all CDNs respect the Vary header when caching. We must assume that
only the URL is used to vary the responses. The Next client computes a
hash of the header values and sends it as a search param.

Before responding to a request, we must verify that the hash matches the
expected value. Neglecting to do this properly can lead to cache
poisoning attacks on certain CDNs.

In this initial PR, the verification I've added only runs during per-
segment prefetch requests, since those are the only ones that both vary
on a custom header and are also cacheable. But for safety, we should run
this verification for all requests, once we confirm the behavior is
correct. Will need to update our test suite, since there are a handful
of unit tests that send fetch requests with custom headers but without a
corresponding cache-busting search param.

It would be even better if we stopped using custom request headers, and
instead fully encoded everything into the search param. At least for
cacheable requests.
parent 7187dc24
'use client'
import { hexHash } from '../../../shared/lib/hash'
import { computeCacheBustingSearchParam } from '../../../shared/lib/router/utils/cache-busting-search-param'
import {
NEXT_ROUTER_PREFETCH_HEADER,
NEXT_ROUTER_SEGMENT_PREFETCH_HEADER,
......@@ -29,14 +30,17 @@ export const setCacheBustingSearchParam = (
url: URL,
headers: RequestHeaders
): void => {
const uniqueCacheKey = hexHash(
[
headers[NEXT_ROUTER_PREFETCH_HEADER] || '0',
headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER] || '0',
headers[NEXT_ROUTER_STATE_TREE_HEADER],
headers[NEXT_URL],
].join(',')
const uniqueCacheKey = computeCacheBustingSearchParam(
headers[NEXT_ROUTER_PREFETCH_HEADER],
headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER],
headers[NEXT_ROUTER_STATE_TREE_HEADER],
headers[NEXT_URL]
)
if (uniqueCacheKey === null) {
// None of our custom request headers are present. We don't need to set a
// cache-busting search param.
return
}
/**
* Note that we intentionally do not use `url.searchParams.set` here:
......
......@@ -178,6 +178,7 @@ import { InvariantError } from '../shared/lib/invariant-error'
import { decodeQueryPathParameter } from './lib/decode-query-path-parameter'
import { getCacheHandlers } from './use-cache/handlers'
import { fixMojibake } from './lib/fix-mojibake'
import { computeCacheBustingSearchParam } from '../shared/lib/router/utils/cache-busting-search-param'
export type FindComponentsResult = {
components: LoadComponentsReturnType
......@@ -2074,6 +2075,42 @@ export default abstract class Server<
const hasGetInitialProps = !!components.Component?.getInitialProps
let isSSG = !!components.getStaticProps
// Not all CDNs respect the Vary header when caching. We must assume that
// only the URL is used to vary the responses. The Next client computes a
// hash of the header values and sends it as a search param. Before
// responding to a request, we must verify that the hash matches the
// expected value. Neglecting to do this properly can lead to cache
// poisoning attacks on certain CDNs.
// TODO: This is verification only runs during per-segment prefetch
// requests, since those are the only ones that both vary on a custom
// header and are cacheable. But for safety, we should run this
// verification for all requests, once we confirm the behavior is correct.
// Will need to update our test suite, since there are a handlful of unit
// tests that send fetch requests with custom headers but without a
// corresponding cache-busting search param.
// TODO: Consider not using custom request headers at all, and instead fully
// encode everything into the search param.
if (
this.isAppSegmentPrefetchEnabled &&
getRequestMeta(req, 'segmentPrefetchRSCRequest')
) {
const headers = req.headers
const expectedHash = computeCacheBustingSearchParam(
headers[NEXT_ROUTER_PREFETCH_HEADER.toLowerCase()],
headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER.toLowerCase()],
headers[NEXT_ROUTER_STATE_TREE_HEADER.toLowerCase()],
headers[NEXT_URL.toLowerCase()]
)
const actualHash = getRequestMeta(req, 'cacheBustingSearchParam') ?? null
if (expectedHash !== actualHash) {
// The hash sent by the client does not match the expected value.
// Respond with an error.
res.statusCode = 400
res.body('Bad Request').send()
return null
}
}
// Compute the iSSG cache key. We use the rewroteUrl since
// pages with fallback: false are allowed to be rewritten to
// and we need to look up the path by the rewritten path
......@@ -3830,6 +3867,11 @@ export default abstract class Server<
let page = pathname
const bubbleNoFallback =
getRequestMeta(ctx.req, 'bubbleNoFallback') ?? false
addRequestMeta(
ctx.req,
'cacheBustingSearchParam',
query[NEXT_RSC_UNION_QUERY]
)
delete query[NEXT_RSC_UNION_QUERY]
const options: MatchOptions = {
......
......@@ -90,6 +90,17 @@ export interface RequestMeta {
*/
isRSCRequest?: true
/**
* A search param set by the Next.js client when performing RSC requests.
* Because some CDNs do not vary their cache entries on our custom headers,
* this search param represents a hash of the header values. For any cached
* RSC request, we should verify that the hash matches before responding.
* Otherwise this can lead to cache poisoning.
* TODO: Consider not using custom request headers at all, and instead encode
* everything into the search param.
*/
cacheBustingSearchParam?: string
/**
* True when the request is for the `/_next/data` route using the pages
* router.
......
import { hexHash } from '../../hash'
export function computeCacheBustingSearchParam(
prefetchHeader: string | string[] | undefined,
segmentPrefetchHeader: string | string[] | undefined,
stateTreeHeader: string | string[] | undefined,
nextUrlHeader: string | string[] | undefined
): string | null {
if (
prefetchHeader === undefined &&
segmentPrefetchHeader === undefined &&
stateTreeHeader === undefined &&
nextUrlHeader === undefined
) {
return null
}
return hexHash(
[
prefetchHeader || '0',
segmentPrefetchHeader || '0',
stateTreeHeader || '0',
nextUrlHeader || '0',
].join(',')
)
}
import { TestPage } from '../../../components/page'
export default async function Page(props) {
const params = await props.params
const { locale } = params
return <TestPage pathname={`/${locale}/about`} />
}
import { locales } from '../../components/page'
export async function generateStaticParams() {
return locales.map((locale) => ({ locale }))
}
export default async function Layout(props) {
const params = await props.params
const { locale } = params
const { children } = props
return (
<html lang={locale}>
<body>{children}</body>
</html>
)
}
import { TestPage } from '../../components/page'
export default async function Page(props) {
const params = await props.params
const { locale } = params
return <TestPage pathname={`/${locale}`} />
}
export default ({ children }) => {
return children
}
import { redirect } from 'next/navigation'
import { locales } from '../components/page'
export default () => {
// Redirect to the default locale
return redirect(`/${locales[0]}`)
}
import { unstable_noStore } from 'next/cache'
import Link from 'next/link'
import { Suspense } from 'react'
export const locales = ['en', 'fr']
export const links = [
{ href: '/', text: 'Home' },
...locales
.map((locale) => {
return [
{ href: `/${locale}`, text: locale },
{ href: `/${locale}/about`, text: `${locale} - About` },
]
})
.flat(),
]
function Dynamic() {
unstable_noStore()
return <div id="dynamic">Dynamic</div>
}
export function TestPage({ pathname }) {
return (
<div>
<ul>
{links.map(({ href, text }) => (
<li key={href}>
<Link href={href}>{text}</Link>
</li>
))}
</ul>
<code data-value={pathname}>{pathname}</code>
<Suspense fallback={<div>Loading...</div>}>
<Dynamic />
</Suspense>
</div>
)
}
import { nextTestSetup } from 'e2e-utils'
describe('per segment prefetching', () => {
const { next, isNextDev, isNextDeploy } = nextTestSetup({
files: __dirname,
})
if (isNextDev || isNextDeploy) {
test('ppr is disabled', () => {})
return
}
// This feature is only partially implemented; the client does not yet issue
// these types of requests. This tests that the server responds correctly.
// TODO: Replace with e2e tests once more is implemented.
function prefetch(pageUrl, segmentPath) {
return next.fetch(pageUrl, {
headers: {
RSC: '1',
'Next-Router-Prefetch': '1',
'Next-Router-Segment-Prefetch': segmentPath,
},
})
}
function extractPseudoJSONFromFlightResponse(flightText: string) {
// This is a cheat that takes advantage of the fact that the roots of the
// Flight responses in this test are JSON. Again, this is just a temporary
// smoke test until the client part is implemented; we shouldn't rely on
// this as a general testing strategy.
const match = flightText.match(/^0:(.*)$/m)
if (match) {
return JSON.parse(match[1])
}
return null
}
it('basic route tree prefetch', async () => {
// To perform a prefetch a page, the client first fetches the route tree.
// The response is used to construct prefetches of individual segments.
const routeTreeResponse = await prefetch('/en', '/_tree')
const routeTreeResponseText = await routeTreeResponse.text()
const routeTree = extractPseudoJSONFromFlightResponse(routeTreeResponseText)
// Confirm that the prefetch was successful. This is a basic check to ensure
// that the name of an expected field is present in the response.
expect(typeof routeTree.staleTime).toBe('number')
// The root segment is a shared segment. Demonstrate that fetching the root
// segment for two different pages results in the same response.
const enResponse = await prefetch('/en', '/')
const enResponseText = await enResponse.text()
const frResponse = await prefetch('/fr', '/')
const frResponseText = await frResponse.text()
expect(enResponseText).toEqual(frResponseText)
})
it('respond with 204 if the segment does not have prefetch data', async () => {
const response = await prefetch('/en', '/does-not-exist')
expect(response.status).toBe(204)
const responseText = await response.text()
expect(responseText.trim()).toBe('')
})
})
import { nextTestSetup } from 'e2e-utils'
import { locales } from './components/page'
describe('ppr-navigations simple', () => {
const { next } = nextTestSetup({
files: __dirname,
})
it('can navigate between all the links and back', async () => {
const browser = await next.browser('/')
try {
for (const href of ['/fr/about', '/fr', '/en/about', '/en', '/']) {
// Find the link element for the href and click it.
await browser.elementByCss(`a[href="${href}"]`).click()
// Wait for that page to load.
if (href === '/') {
// The root page redirects to the first locale.
await browser.waitForElementByCss(`[data-value="/${locales[0]}"]`)
} else {
await browser.waitForElementByCss(`[data-value="${href}"]`)
}
await browser.elementByCss('#dynamic')
}
} finally {
await browser.close()
}
})
})
export default function RootLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html lang="en">
<body>{children}</body>
</html>
)
}
import { LinkAccordion } from '../components/link-accordion'
export default function Page() {
return (
<ul>
<li>
<LinkAccordion href="/target-page">Target page</LinkAccordion>
</li>
</ul>
)
}
export default function TargetPage() {
return <div id="target-page">Target page</div>
}
import type * as Playwright from 'playwright'
import webdriver from 'next-webdriver'
import { createRouterAct } from '../router-act'
import { findPort, nextBuild } from 'next-test-utils'
import { isNextDeploy, isNextDev } from 'e2e-utils'
import { start } from './server.mjs'
describe('segment cache (CDN cache busting)', () => {
if (isNextDev || isNextDeploy) {
test('should not run during dev or deploy test runs', () => {})
return
}
// To debug these tests locally, run:
// node start.mjs
//
// This will start the Next app and also a proxy server that simulates a CDN.
// Like certain real-world CDNs, our fake CDN doesn't respect the Vary header.
// It only uses the URL.
let cleanup: () => Promise<void>
let port: number
beforeAll(async () => {
const appDir = __dirname
await nextBuild(appDir, undefined, { cwd: appDir })
const proxyPort = (port = await findPort())
const nextPort = await findPort()
cleanup = await start(proxyPort, nextPort)
})
afterAll(async () => {
await cleanup()
})
it(
"perform fully prefetched navigation with a CDN that doesn't respect " +
'the Vary header',
async () => {
let act
const browser = await webdriver(port, '/', {
beforePageLoad(p: Playwright.Page) {
act = createRouterAct(p)
},
})
// Initiate a prefetch. Each segment will be prefetched individually,
// using the pathname of the target page and a custom header specifying
// the segment. If we didn't also set a cache-busting search param, then
// the fake CDN used by this test suite would incorrectly use the same
// entry for every segment, poisoning the cache.
await act(
async () => {
const linkToggle = await browser.elementByCss(
'[data-link-accordion="/target-page"]'
)
await linkToggle.click()
},
{
includes: 'Target page',
}
)
// Navigate to the prefetched target page.
await act(async () => {
const link = await browser.elementByCss('a[href="/target-page"]')
await link.click()
// The page was prefetched, so we're able to render the target
// page immediately.
const div = await browser.elementById('target-page')
expect(await div.text()).toBe('Target page')
}, 'no-requests')
}
)
it(
'prevent cache poisoning attacks by responding with an error if a custom ' +
'header is sent during a prefetch without a corresponding cache-busting ' +
'search param',
async () => {
const browser = await webdriver(port, '/')
const { status, text } = await browser.eval(async () => {
const res = await fetch('/target-page', {
headers: {
RSC: '1',
'Next-Router-Prefetch': '1',
'Next-Router-Segment-Prefetch': '/_tree',
},
})
return { status: res.status, text: await res.text() }
})
expect(status).toBe(400)
expect(text).toContain('Bad Request')
}
)
})
'use client'
import Link from 'next/link'
import { useState } from 'react'
export function LinkAccordion({
href,
children,
prefetch,
}: {
href: string
children: React.ReactNode
prefetch?: boolean
}) {
const [isVisible, setIsVisible] = useState(false)
return (
<>
<input
type="checkbox"
checked={isVisible}
onChange={() => setIsVisible(!isVisible)}
data-link-accordion={href}
/>
{isVisible ? (
<Link href={href} prefetch={prefetch}>
{children}
</Link>
) : (
<>{children} (link is hidden)</>
)}
</>
)
}
module.exports = {
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {
experimental: {
ppr: true,
clientSegmentCache: true,
},
}
module.exports = nextConfig
import { fileURLToPath } from 'url'
import { dirname } from 'path'
import { spawn } from 'child_process'
import { createServer } from 'node:http'
import httpProxy from 'http-proxy'
import process from 'node:process'
import { createGunzip } from 'zlib'
const dir = dirname(fileURLToPath(import.meta.url))
async function spawnNext(port) {
const child = spawn('pnpm', ['next', 'start', '-p', port, dir], {
env: process.env,
stdio: ['inherit', 'pipe', 'inherit'],
})
child.stdout.pipe(process.stdout)
// Wait until the server is listening.
return new Promise((resolve, reject) => {
child.stdout.on('data', (data) => {
if (data.toString().includes('Ready')) {
resolve(child)
}
})
child.on('exit', (code) => {
if (code === 0) {
resolve(child)
} else {
reject(new Error(`Next.js server exited with code ${code}`))
}
})
})
}
function isCacheableRequest(req) {
return (
req.method === 'GET' && !req.headers['cache-control']?.includes('no-store')
)
}
function isCacheableResponse(res) {
return !res.headers['cache-control']?.includes('no-store')
}
async function createFakeCDN(destPort) {
const fakeCDNCache = new Map()
const proxy = httpProxy.createProxyServer()
const cdnServer = createServer(async (req, res) => {
if (isCacheableRequest(req)) {
// Serve from our fake CDN if there's a matching entry.
const entry = await fakeCDNCache.get(req.url)
if (entry) {
console.log('Serving from fake CDN:', req.url)
res.writeHead(entry.statusCode, entry.statusMessage, entry.headers)
res.end(entry.data)
return
} else {
// No existing entry. Proxy to the Next.js server and then store
// the response in the cache.
proxy.web(req, res, {
target: `http://localhost:${destPort}`,
selfHandleResponse: true,
})
}
} else {
// This request isn't cacheable.
proxy.web(req, res, { target: `http://localhost:${destPort}` })
}
})
proxy.on('proxyRes', function (proxyRes, req, res) {
// If the response is cacheable, store it in a map to simulate a CDN.
//
// Note that we only key the entry on the URL, not any of the headers. i.e.
// we don't respect the Vary header. This is true of certain real CDNs, so
// Next.js must not rely on Vary.
//
// For the purposes of this test we don't respect max-age et al. Every
// entry is cached indefinitely.
if (isCacheableRequest(req) && isCacheableResponse(proxyRes)) {
let resolveCDNEntry
fakeCDNCache.set(req.url, new Promise((res) => (resolveCDNEntry = res)))
// Decompress the original response stream, if needed
let source
if (proxyRes.headers['content-encoding'] === 'gzip') {
source = proxyRes.pipe(createGunzip())
// Just store the uncompressed body and serve that from cache.
// Good enough for the purposes of this test app.
delete proxyRes.headers['content-encoding']
} else {
source = proxyRes
}
const chunks = []
source.on('data', (chunk) => {
chunks.push(chunk)
})
source.on('end', () => {
const data = Buffer.concat(chunks)
// Send response after we've collected all chunks
res.writeHead(
proxyRes.statusCode || 200,
proxyRes.statusMessage,
proxyRes.headers
)
res.end(data)
// Store the raw data for later use
const entry = {
data,
statusCode: proxyRes.statusCode,
statusMessage: proxyRes.statusMessage,
headers: proxyRes.headers,
}
resolveCDNEntry(entry)
})
return
}
// If the response isn't cacheable, pipe it through to the client.
proxyRes.pipe(res)
return
})
return cdnServer
}
export async function start(cdnPort = 3000, nextPort = cdnPort + 1) {
const next = await spawnNext(nextPort)
const cdnServer = await createFakeCDN(nextPort)
const onTerminate = () => {
cdnServer.close()
next.kill()
process.exit(0)
}
process.on('SIGINT', onTerminate)
process.on('SIGTERM', onTerminate)
const cleanup = async () => {
next.kill()
await new Promise((resolve) => cdnServer.close(resolve))
}
return new Promise((resolve, reject) => {
cdnServer.on('error', reject)
cdnServer.listen(cdnPort, () => {
console.log('Server is listening', cdnPort)
resolve(cleanup)
})
})
}
import { start } from './server.mjs'
await start()
import { nextTestSetup } from 'e2e-utils'
import { computeCacheBustingSearchParam } from '../../../../../packages/next/src/shared/lib/router/utils/cache-busting-search-param'
describe('conflicting routes', () => {
const { next, isNextDev, isNextDeploy } = nextTestSetup({
files: __dirname,
......@@ -9,19 +11,34 @@ describe('conflicting routes', () => {
return
}
it.each([
'/en/vercel/~/monitoring',
'/fr/vercel/~/monitoring',
'/es/vercel/~/monitoring',
])('%s matches the right route', async (path) => {
const res = await next.fetch(path, {
// The server will reject prefetch requests that don't set a cache-busting
// search param.
// TODO: We should try to avoid unit tests that simulate internal client
// router requests. There are only a handful of cases that do this currently.
// In the meantime, consider moving this into a shared testing utility.
async function segmentPrefetch(url: string, segmentPath: string) {
const fetchUrl = new URL(url, 'http://localhost')
const searchParams = new URLSearchParams(fetchUrl.search)
searchParams.set(
'_rsc',
computeCacheBustingSearchParam('1', segmentPath, undefined, undefined)
)
fetchUrl.search = searchParams.toString()
return await next.fetch(fetchUrl.pathname + fetchUrl.search, {
headers: {
RSC: '1',
'Next-Router-Prefetch': '1',
'Next-Router-Segment-Prefetch': '/_tree',
'Next-Router-Segment-Prefetch': segmentPath,
},
})
}
it.each([
'/en/vercel/~/monitoring',
'/fr/vercel/~/monitoring',
'/es/vercel/~/monitoring',
])('%s matches the right route', async (path) => {
const res = await segmentPrefetch(path, '/_tree')
expect(res.status).toBe(200)
if (isNextDeploy) {
......@@ -32,14 +49,7 @@ describe('conflicting routes', () => {
})
it('matches the right route when the original route has no dynamic params, is dynamic, and PPR is disabled', async () => {
const res = await next.fetch('/prefetch-tests/dynamic', {
headers: {
RSC: '1',
'Next-Router-Prefetch': '1',
'Next-Router-Segment-Prefetch': '/_tree',
},
})
const res = await segmentPrefetch('/prefetch-tests/dynamic', '/_tree')
expect(res.status).toBe(200)
if (isNextDeploy) {
......@@ -50,13 +60,7 @@ describe('conflicting routes', () => {
})
it('handles conflict between App Router and Pages Router routes', async () => {
const res = await next.fetch('/new/templates', {
headers: {
RSC: '1',
'Next-Router-Prefetch': '1',
'Next-Router-Segment-Prefetch': '/_tree',
},
})
const res = await segmentPrefetch('/new/templates', '/_tree')
// Should match the route defined at pages/new/templates/[[...slug]].js,
// not the one at app/new/[teamSlug]/page.tsx
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment