Unverified Commit 3eb1d035 authored by Andrew Clark's avatar Andrew Clark Committed by GitHub

Fix: Prefetching lazily generated param (#73715)

Fixes an oversight in #72168 where the segment data was not correctly
transferred from the render result to the cache entry in the case where
a prerender is lazily generated after build.

I also took the opportunity to remove one of the intermediate types used
by the various layers that the segment data passes through. For some
reason, in the original PR I made the type of `segmentData` on
CachedAppPageValue an object while the corresponding type on
AppPageRenderResultMetadata was a map. (I didn't really notice before
because in the case where the entry is generated at build time, it gets
written to disk then read back out, so there's some data conversion
happening anyway.)
parent ed7fba76
...@@ -146,7 +146,7 @@ export async function exportAppPage( ...@@ -146,7 +146,7 @@ export async function exportAppPage(
postponed, postponed,
fetchTags, fetchTags,
fetchMetrics, fetchMetrics,
segmentFlightData, segmentData,
} = metadata } = metadata
// Ensure we don't postpone without having PPR enabled. // Ensure we don't postpone without having PPR enabled.
...@@ -200,7 +200,7 @@ export async function exportAppPage( ...@@ -200,7 +200,7 @@ export async function exportAppPage(
flightData flightData
) )
if (segmentFlightData) { if (segmentData) {
// Emit the per-segment prefetch data. We emit them as separate files // Emit the per-segment prefetch data. We emit them as separate files
// so that the cache handler has the option to treat each as a // so that the cache handler has the option to treat each as a
// separate entry. // separate entry.
...@@ -210,7 +210,7 @@ export async function exportAppPage( ...@@ -210,7 +210,7 @@ export async function exportAppPage(
RSC_SEGMENTS_DIR_SUFFIX RSC_SEGMENTS_DIR_SUFFIX
) )
const tasks = [] const tasks = []
for (const [segmentPath, buffer] of segmentFlightData.entries()) { for (const [segmentPath, buffer] of segmentData) {
segmentPaths.push(segmentPath) segmentPaths.push(segmentPath)
const segmentDataFilePath = const segmentDataFilePath =
segmentPath === '/' segmentPath === '/'
......
...@@ -2844,7 +2844,7 @@ async function prerenderToStream( ...@@ -2844,7 +2844,7 @@ async function prerenderToStream(
const flightData = await streamToBuffer(reactServerResult.asStream()) const flightData = await streamToBuffer(reactServerResult.asStream())
metadata.flightData = flightData metadata.flightData = flightData
metadata.segmentFlightData = await collectSegmentData( metadata.segmentData = await collectSegmentData(
flightData, flightData,
finalRenderPrerenderStore, finalRenderPrerenderStore,
ComponentMod, ComponentMod,
...@@ -3323,7 +3323,7 @@ async function prerenderToStream( ...@@ -3323,7 +3323,7 @@ async function prerenderToStream(
serverPrerenderStreamResult.asStream() serverPrerenderStreamResult.asStream()
) )
metadata.flightData = flightData metadata.flightData = flightData
metadata.segmentFlightData = await collectSegmentData( metadata.segmentData = await collectSegmentData(
flightData, flightData,
finalClientPrerenderStore, finalClientPrerenderStore,
ComponentMod, ComponentMod,
...@@ -3455,7 +3455,7 @@ async function prerenderToStream( ...@@ -3455,7 +3455,7 @@ async function prerenderToStream(
if (shouldGenerateStaticFlightData(workStore)) { if (shouldGenerateStaticFlightData(workStore)) {
metadata.flightData = flightData metadata.flightData = flightData
metadata.segmentFlightData = await collectSegmentData( metadata.segmentData = await collectSegmentData(
flightData, flightData,
ssrPrerenderStore, ssrPrerenderStore,
ComponentMod, ComponentMod,
...@@ -3647,7 +3647,7 @@ async function prerenderToStream( ...@@ -3647,7 +3647,7 @@ async function prerenderToStream(
if (shouldGenerateStaticFlightData(workStore)) { if (shouldGenerateStaticFlightData(workStore)) {
const flightData = await streamToBuffer(reactServerResult.asStream()) const flightData = await streamToBuffer(reactServerResult.asStream())
metadata.flightData = flightData metadata.flightData = flightData
metadata.segmentFlightData = await collectSegmentData( metadata.segmentData = await collectSegmentData(
flightData, flightData,
prerenderLegacyStore, prerenderLegacyStore,
ComponentMod, ComponentMod,
...@@ -3801,7 +3801,7 @@ async function prerenderToStream( ...@@ -3801,7 +3801,7 @@ async function prerenderToStream(
reactServerPrerenderResult.asStream() reactServerPrerenderResult.asStream()
) )
metadata.flightData = flightData metadata.flightData = flightData
metadata.segmentFlightData = await collectSegmentData( metadata.segmentData = await collectSegmentData(
flightData, flightData,
prerenderLegacyStore, prerenderLegacyStore,
ComponentMod, ComponentMod,
......
...@@ -2749,7 +2749,7 @@ export default abstract class Server< ...@@ -2749,7 +2749,7 @@ export default abstract class Server<
rscData: metadata.flightData, rscData: metadata.flightData,
postponed: metadata.postponed, postponed: metadata.postponed,
status: res.statusCode, status: res.statusCode,
segmentData: undefined, segmentData: metadata.segmentData,
} satisfies CachedAppPageValue, } satisfies CachedAppPageValue,
revalidate: metadata.revalidate, revalidate: metadata.revalidate,
isFallback: !!fallbackRouteParams, isFallback: !!fallbackRouteParams,
...@@ -3057,8 +3057,9 @@ export default abstract class Server< ...@@ -3057,8 +3057,9 @@ export default abstract class Server<
// it's a 404 — either the segment is fully dynamic, or an invalid segment // it's a 404 — either the segment is fully dynamic, or an invalid segment
// path was requested. // path was requested.
if (cacheEntry.value.segmentData) { if (cacheEntry.value.segmentData) {
const matchedSegment = const matchedSegment = cacheEntry.value.segmentData.get(
cacheEntry.value.segmentData[segmentPrefetchHeader] segmentPrefetchHeader
)
if (matchedSegment !== undefined) { if (matchedSegment !== undefined) {
return { return {
type: 'rsc', type: 'rsc',
......
...@@ -194,14 +194,14 @@ export default class FileSystemCache implements CacheHandler { ...@@ -194,14 +194,14 @@ export default class FileSystemCache implements CacheHandler {
) )
} catch {} } catch {}
let maybeSegmentData: { [segmentPath: string]: string } | undefined let maybeSegmentData: Map<string, Buffer> | undefined
if (meta?.segmentPaths) { if (meta?.segmentPaths) {
// Collect all the segment data for this page. // Collect all the segment data for this page.
// TODO: To optimize file system reads, we should consider creating // TODO: To optimize file system reads, we should consider creating
// separate cache entries for each segment, rather than storing them // separate cache entries for each segment, rather than storing them
// all on the page's entry. Though the behavior is // all on the page's entry. Though the behavior is
// identical regardless. // identical regardless.
const segmentData: { [segmentPath: string]: string } = {} const segmentData: Map<string, Buffer> = new Map()
maybeSegmentData = segmentData maybeSegmentData = segmentData
const segmentsDir = key + RSC_SEGMENTS_DIR_SUFFIX const segmentsDir = key + RSC_SEGMENTS_DIR_SUFFIX
await Promise.all( await Promise.all(
...@@ -213,9 +213,9 @@ export default class FileSystemCache implements CacheHandler { ...@@ -213,9 +213,9 @@ export default class FileSystemCache implements CacheHandler {
IncrementalCacheKind.APP_PAGE IncrementalCacheKind.APP_PAGE
) )
try { try {
segmentData[segmentPath] = await this.fs.readFile( segmentData.set(
segmentDataFilePath, segmentPath,
'utf8' await this.fs.readFile(segmentDataFilePath)
) )
} catch { } catch {
// This shouldn't happen, but if for some reason we fail to // This shouldn't happen, but if for some reason we fail to
......
...@@ -34,7 +34,7 @@ export type AppPageRenderResultMetadata = { ...@@ -34,7 +34,7 @@ export type AppPageRenderResultMetadata = {
fetchTags?: string fetchTags?: string
fetchMetrics?: FetchMetrics fetchMetrics?: FetchMetrics
segmentFlightData?: Map<string, Buffer> segmentData?: Map<string, Buffer>
/** /**
* In development, the cache is warmed up before the render. This is attached * In development, the cache is warmed up before the render. This is attached
......
...@@ -77,7 +77,7 @@ export interface CachedAppPageValue { ...@@ -77,7 +77,7 @@ export interface CachedAppPageValue {
status: number | undefined status: number | undefined
postponed: string | undefined postponed: string | undefined
headers: OutgoingHttpHeaders | undefined headers: OutgoingHttpHeaders | undefined
segmentData: { [segmentPath: string]: string } | undefined segmentData: Map<string, Buffer> | undefined
} }
export interface CachedPageValue { export interface CachedPageValue {
...@@ -118,7 +118,7 @@ export interface IncrementalCachedAppPageValue { ...@@ -118,7 +118,7 @@ export interface IncrementalCachedAppPageValue {
headers: OutgoingHttpHeaders | undefined headers: OutgoingHttpHeaders | undefined
postponed: string | undefined postponed: string | undefined
status: number | undefined status: number | undefined
segmentData: { [segmentPath: string]: string } | undefined segmentData: Map<string, Buffer> | undefined
} }
export interface IncrementalCachedPageValue { export interface IncrementalCachedPageValue {
......
import { Suspense } from 'react'
async function Content({ params }) {
const { param } = await params
return <div id="target-page-with-lazily-generated-param">Param: {param}</div>
}
export default async function Target({ params }) {
return (
<Suspense fallback="Loading...">
<Content params={params} />
</Suspense>
)
}
import Link from 'next/link'
// TODO: Once the appropriate API exists/is implemented, configure the param to
// be statically generated on demand but not at build time (`dynamicParams =
// true` isn't supported when `dynamicIO` is enabled.) For now this test case
// seems to work without extra configuration but it might not in the future.
export default function LazilyGeneratedParamsStartPage() {
return (
<>
<p>
Demonstrates that we can prefetch param that is not generated at build
time but is lazily generated on demand
</p>
<ul>
<li>
<Link href="/lazily-generated-params/some-param-value">Target</Link>
</li>
</ul>
</>
)
}
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import type { Page, Route } from 'playwright' import type * as Playwright from 'playwright'
describe('segment cache (basic tests)', () => { describe('segment cache (basic tests)', () => {
const { next, isNextDev, skipped } = nextTestSetup({ const { next, isNextDev, skipped } = nextTestSetup({
...@@ -19,9 +19,9 @@ describe('segment cache (basic tests)', () => { ...@@ -19,9 +19,9 @@ describe('segment cache (basic tests)', () => {
const prefetchLock = interceptor.lockPrefetches() const prefetchLock = interceptor.lockPrefetches()
const browser = await next.browser('/', { const browser = await next.browser('/', {
beforePageLoad(page: Page) { beforePageLoad(page: Playwright.Page) {
page.route('**/*', async (route: Route) => { page.route('**/*', async (route: Playwright.Route) => {
await interceptor.interceptRoute(route) await interceptor.interceptRoute(page, route)
}) })
}, },
}) })
...@@ -43,9 +43,9 @@ describe('segment cache (basic tests)', () => { ...@@ -43,9 +43,9 @@ describe('segment cache (basic tests)', () => {
it('navigate with prefetched data', async () => { it('navigate with prefetched data', async () => {
const interceptor = createRequestInterceptor() const interceptor = createRequestInterceptor()
const browser = await next.browser('/', { const browser = await next.browser('/', {
beforePageLoad(page: Page) { beforePageLoad(page: Playwright.Page) {
page.route('**/*', async (route: Route) => { page.route('**/*', async (route: Playwright.Route) => {
await interceptor.interceptRoute(route) await interceptor.interceptRoute(page, route)
}) })
}, },
}) })
...@@ -77,12 +77,48 @@ describe('segment cache (basic tests)', () => { ...@@ -77,12 +77,48 @@ describe('segment cache (basic tests)', () => {
) )
}) })
it('navigate to page with lazily-generated (not at build time) static param', async () => {
const interceptor = createRequestInterceptor()
const browser = await interceptor.waitForPrefetches(async () => {
const b = await next.browser('/lazily-generated-params', {
beforePageLoad(page: Playwright.Page) {
page.route('**/*', async (route: Playwright.Route) => {
await interceptor.interceptRoute(page, route)
})
},
})
await b.elementByCss('a')
return b
})
const navigationsLock = interceptor.lockNavigations()
// Navigate to the test page
const link = await browser.elementByCss('a')
await link.click()
// We should be able to render the page with the dynamic param, because
// it is lazily generated
const target = await browser.elementById(
'target-page-with-lazily-generated-param'
)
expect(await target.innerHTML()).toMatchInlineSnapshot(
`"Param: some-param-value"`
)
await navigationsLock.release()
// TODO: Once #73540 lands we can also test that the dynamic nav was skipped
// const navigations = await navigationsLock.release()
// expect(navigations.size).toBe(0)
})
it('prefetch interception route', async () => { it('prefetch interception route', async () => {
const interceptor = createRequestInterceptor() const interceptor = createRequestInterceptor()
const browser = await next.browser('/interception/feed', { const browser = await next.browser('/interception/feed', {
beforePageLoad(page: Page) { beforePageLoad(page: Playwright.Page) {
page.route('**/*', async (route: Route) => { page.route('**/*', async (route: Playwright.Route) => {
await interceptor.interceptRoute(route) await interceptor.interceptRoute(page, route)
}) })
}, },
}) })
...@@ -112,8 +148,11 @@ function createRequestInterceptor() { ...@@ -112,8 +148,11 @@ function createRequestInterceptor() {
// implementation details as much as possible, so the only thing this does // implementation details as much as possible, so the only thing this does
// for now is let you block and release requests from happening based on // for now is let you block and release requests from happening based on
// their type (prefetch requests, navigation requests). // their type (prefetch requests, navigation requests).
let pendingPrefetches: Set<Route> | null = null let pendingPrefetches: Set<Playwright.Route> | null = null
let pendingNavigations: Set<Route> | null = null let pendingNavigations: Set<Playwright.Route> | null = null
let prefetchesPromise: PromiseWithResolvers<void> = null
let lastPrefetchRequest: Playwright.Request | null = null
return { return {
lockNavigations() { lockNavigations() {
...@@ -131,6 +170,7 @@ function createRequestInterceptor() { ...@@ -131,6 +170,7 @@ function createRequestInterceptor() {
for (const route of routes) { for (const route of routes) {
route.continue() route.continue()
} }
return routes
}, },
} }
}, },
...@@ -150,17 +190,75 @@ function createRequestInterceptor() { ...@@ -150,17 +190,75 @@ function createRequestInterceptor() {
for (const route of routes) { for (const route of routes) {
route.continue() route.continue()
} }
return routes
}, },
} }
}, },
async interceptRoute(route: Route) { /**
const requestHeaders = await route.request().allHeaders() * Waits for the next for the next prefetch request, then keeps waiting
* until the prefetch queue is empty (to account for network throttling).
*
* If no prefetches are initiated, this will timeout.
*/
async waitForPrefetches<T>(
scope: () => Promise<T> | T = (): undefined => {}
): Promise<T> {
if (prefetchesPromise === null) {
let resolve
let reject
const promise: Promise<void> = new Promise((res, rej) => {
resolve = res
reject = rej
})
prefetchesPromise = {
resolve,
reject,
promise,
}
}
const result = await scope()
if (prefetchesPromise !== null) {
await prefetchesPromise.promise
}
return result
},
async interceptRoute(page: Playwright.Page, route: Playwright.Route) {
const request = route.request()
const requestHeaders = await request.allHeaders()
if (requestHeaders['RSC'.toLowerCase()]) { if (requestHeaders['RSC'.toLowerCase()]) {
// This is an RSC request. Check if it's a prefetch or a navigation. // This is an RSC request. Check if it's a prefetch or a navigation.
if (requestHeaders['Next-Router-Prefetch'.toLowerCase()]) { if (requestHeaders['Next-Router-Prefetch'.toLowerCase()]) {
// This is a prefetch request. // This is a prefetch request.
if (prefetchesPromise !== null) {
// Wait for the prefetch response to finish, then wait an additional
// async task for additional prefetches to be initiated.
lastPrefetchRequest = request
const waitForMorePrefetches = async () => {
const response = await request.response()
await response.finished()
await page.evaluate(
() =>
// If the prefetch queue is network throttled, the next
// request should be issued within a microtask of the previous
// one finishing.
new Promise<void>((res) => requestIdleCallback(() => res()))
)
if (request === lastPrefetchRequest) {
// No further prefetches were initiated. Assume the prefetch
// queue is now empty.
prefetchesPromise.resolve()
prefetchesPromise = null
lastPrefetchRequest = null
}
}
waitForMorePrefetches().then(
() => {},
() => {}
)
}
if (pendingPrefetches !== null) { if (pendingPrefetches !== null) {
pendingPrefetches.add(route) pendingPrefetches.add(route)
return return
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment