Unverified Commit db0a723d authored by Andrew Clark's avatar Andrew Clark Committed by GitHub

[Segment Cache] Support output: "export" mode (#75671)

Adds support for output: "export" mode to the Segment Cache
implementation. We output an additional `.txt` data file per segment per
page. When the client issues a per-segment request, it appends the
segment path to the end of the page URL, rather than passing it as a
request header.

The segment file output follows this convention:

```
/a/b/c.html
/a/b/c/__next.a.txt         <- corresponds to segment /a
/a/b/c/__next.a.b.txt       <- corresponds to segment /a/b
/a/b/c/__next.a.b.c.txt     <- corresponds to segment /a/b/c

... and so on
```

This scheme is designed so that the server can implement patterns like
protection rules or rewrites using just the original path. i.e. by
blocking access to `/a/b`, you also block access to all of its
associated segment data.

Technically it's possible for the segment files to clash with a nested
segment config. We add a `__next` prefix to make a clash less likely.
It's unlikely this will ever be an issue in practice but if needed we
could make this prefix configurable at build time.
parent e3f0a1ac
...@@ -160,6 +160,20 @@ export async function fetchServerResponse( ...@@ -160,6 +160,20 @@ export async function fetchServerResponse(
: 'low' : 'low'
: 'auto' : 'auto'
if (process.env.NODE_ENV === 'production') {
if (process.env.__NEXT_CONFIG_OUTPUT === 'export') {
// In "output: export" mode, we can't rely on headers to distinguish
// between HTML and RSC requests. Instead, we append an extra prefix
// to the request.
url = new URL(url)
if (url.pathname.endsWith('/')) {
url.pathname += 'index.txt'
} else {
url.pathname += '.txt'
}
}
}
const res = await createFetch( const res = await createFetch(
url, url,
headers, headers,
...@@ -255,16 +269,9 @@ export function createFetch( ...@@ -255,16 +269,9 @@ export function createFetch(
) { ) {
const fetchUrl = new URL(url) const fetchUrl = new URL(url)
if (process.env.NODE_ENV === 'production') { // TODO: In output: "export" mode, the headers do nothing. Omit them (and the
if (process.env.__NEXT_CONFIG_OUTPUT === 'export') { // cache busting search param) from the request so they're
if (fetchUrl.pathname.endsWith('/')) { // maximally cacheable.
fetchUrl.pathname += 'index.txt'
} else {
fetchUrl.pathname += '.txt'
}
}
}
setCacheBustingSearchParam(fetchUrl, headers) setCacheBustingSearchParam(fetchUrl, headers)
if (process.env.__NEXT_TEST_MODE && fetchPriority !== null) { if (process.env.__NEXT_TEST_MODE && fetchPriority !== null) {
......
...@@ -12,11 +12,16 @@ import { existsSync, promises as fs } from 'fs' ...@@ -12,11 +12,16 @@ import { existsSync, promises as fs } from 'fs'
import '../server/require-hook' import '../server/require-hook'
import { dirname, join, resolve, sep } from 'path' import { dirname, join, resolve, sep, relative } from 'path'
import { formatAmpMessages } from '../build/output/index' import { formatAmpMessages } from '../build/output/index'
import type { AmpPageStatus } from '../build/output/index' import type { AmpPageStatus } from '../build/output/index'
import * as Log from '../build/output/log' import * as Log from '../build/output/log'
import { RSC_SUFFIX, SSG_FALLBACK_EXPORT_ERROR } from '../lib/constants' import {
RSC_SEGMENT_SUFFIX,
RSC_SEGMENTS_DIR_SUFFIX,
RSC_SUFFIX,
SSG_FALLBACK_EXPORT_ERROR,
} from '../lib/constants'
import { recursiveCopy } from '../lib/recursive-copy' import { recursiveCopy } from '../lib/recursive-copy'
import { import {
BUILD_ID_FILE, BUILD_ID_FILE,
...@@ -57,6 +62,7 @@ import type { DeepReadonly } from '../shared/lib/deep-readonly' ...@@ -57,6 +62,7 @@ import type { DeepReadonly } from '../shared/lib/deep-readonly'
import { isInterceptionRouteRewrite } from '../lib/generate-interception-routes-rewrites' import { isInterceptionRouteRewrite } from '../lib/generate-interception-routes-rewrites'
import type { ActionManifest } from '../build/webpack/plugins/flight-client-entry-plugin' import type { ActionManifest } from '../build/webpack/plugins/flight-client-entry-plugin'
import { extractInfoFromServerReferenceId } from '../shared/lib/server-reference-info' import { extractInfoFromServerReferenceId } from '../shared/lib/server-reference-info'
import { convertSegmentPathToStaticExportFilename } from '../server/app-render/segment-value-encoding'
export class ExportError extends Error { export class ExportError extends Error {
code = 'NEXT_EXPORT_ERROR' code = 'NEXT_EXPORT_ERROR'
...@@ -684,19 +690,23 @@ async function exportAppImpl( ...@@ -684,19 +690,23 @@ async function exportAppImpl(
// copy prerendered routes to outDir // copy prerendered routes to outDir
if (!options.buildExport && prerenderManifest) { if (!options.buildExport && prerenderManifest) {
await Promise.all( await Promise.all(
Object.keys(prerenderManifest.routes).map(async (route) => { Object.keys(prerenderManifest.routes).map(async (unnormalizedRoute) => {
const { srcRoute } = prerenderManifest!.routes[route] const { srcRoute } = prerenderManifest!.routes[unnormalizedRoute]
const appPageName = mapAppRouteToPage.get(srcRoute || '') const appPageName = mapAppRouteToPage.get(srcRoute || '')
const pageName = appPageName || srcRoute || route const pageName = appPageName || srcRoute || unnormalizedRoute
const isAppPath = Boolean(appPageName) const isAppPath = Boolean(appPageName)
const isAppRouteHandler = appPageName && isAppRouteRoute(appPageName) const isAppRouteHandler = appPageName && isAppRouteRoute(appPageName)
// returning notFound: true from getStaticProps will not // returning notFound: true from getStaticProps will not
// output html/json files during the build // output html/json files during the build
if (prerenderManifest!.notFoundRoutes.includes(route)) { if (prerenderManifest!.notFoundRoutes.includes(unnormalizedRoute)) {
return return
} }
route = normalizePagePath(route) // TODO: This rewrites /index/foo to /index/index/foo. Investigate and
// fix. I presume this was because normalizePagePath was designed for
// some other use case and then reused here for static exports without
// realizing the implications.
const route = normalizePagePath(unnormalizedRoute)
const pagePath = getPagePath(pageName, distDir, undefined, isAppPath) const pagePath = getPagePath(pageName, distDir, undefined, isAppPath)
const distPagesDir = join( const distPagesDir = join(
...@@ -752,6 +762,35 @@ async function exportAppImpl( ...@@ -752,6 +762,35 @@ async function exportAppImpl(
await fs.mkdir(dirname(ampHtmlDest), { recursive: true }) await fs.mkdir(dirname(ampHtmlDest), { recursive: true })
await fs.copyFile(`${orig}.amp.html`, ampHtmlDest) await fs.copyFile(`${orig}.amp.html`, ampHtmlDest)
} }
const segmentsDir = `${orig}${RSC_SEGMENTS_DIR_SUFFIX}`
if (isAppPath && existsSync(segmentsDir)) {
// Output a data file for each of this page's segments
//
// These files are requested by the client router's internal
// prefetcher, not the user directly. So we don't need to account for
// things like trailing slash handling.
//
// To keep the protocol simple, we can use the non-normalized route
// path instead of the normalized one (which, among other things,
// rewrites `/` to `/index`).
const segmentsDirDest = join(outDir, unnormalizedRoute)
const segmentPaths = await collectSegmentPaths(segmentsDir)
await Promise.all(
segmentPaths.map(async (segmentFileSrc) => {
const segmentPath =
'/' + segmentFileSrc.slice(0, -RSC_SEGMENT_SUFFIX.length)
const segmentFilename =
convertSegmentPathToStaticExportFilename(segmentPath)
const segmentFileDest = join(segmentsDirDest, segmentFilename)
await fs.mkdir(dirname(segmentFileDest), { recursive: true })
await fs.copyFile(
join(segmentsDir, segmentFileSrc),
segmentFileDest
)
})
)
}
}) })
) )
} }
...@@ -793,6 +832,40 @@ async function exportAppImpl( ...@@ -793,6 +832,40 @@ async function exportAppImpl(
return collector return collector
} }
async function collectSegmentPaths(segmentsDirectory: string) {
const results: Array<string> = []
await collectSegmentPathsImpl(segmentsDirectory, segmentsDirectory, results)
return results
}
async function collectSegmentPathsImpl(
segmentsDirectory: string,
directory: string,
results: Array<string>
) {
const segmentFiles = await fs.readdir(directory, {
withFileTypes: true,
})
await Promise.all(
segmentFiles.map(async (segmentFile) => {
if (segmentFile.isDirectory()) {
await collectSegmentPathsImpl(
segmentsDirectory,
join(directory, segmentFile.name),
results
)
return
}
if (!segmentFile.name.endsWith(RSC_SEGMENT_SUFFIX)) {
return
}
results.push(
relative(segmentsDirectory, join(directory, segmentFile.name))
)
})
)
}
export default async function exportApp( export default async function exportApp(
dir: string, dir: string,
options: ExportAppOptions, options: ExportAppOptions,
......
...@@ -73,3 +73,9 @@ function encodeToFilesystemAndURLSafeString(value: string) { ...@@ -73,3 +73,9 @@ function encodeToFilesystemAndURLSafeString(value: string) {
.replace(/=+$/, '') // Remove trailing '=' .replace(/=+$/, '') // Remove trailing '='
return '!' + base64url return '!' + base64url
} }
export function convertSegmentPathToStaticExportFilename(
segmentPath: string
): string {
return `__next${segmentPath.replace(/\//g, '.')}.txt`
}
export default function RootLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html lang="en">
<body>{children}</body>
</html>
)
}
import { LinkAccordion } from '../components/link-accordion'
export default function OutputExport() {
return (
<>
<p>
Demonstrates that per-segment prefetching works in{' '}
<code>output: export</code> mode.
</p>
<ul>
<li>
<LinkAccordion href="/target-page">Target</LinkAccordion>
</li>
</ul>
<p>
The following link is rewritten on the server to the same page as the
link above:
</p>
<ul>
<li>
<LinkAccordion href="/rewrite-to-target-page">
Rewrite to target page
</LinkAccordion>
</li>
</ul>
<p>
The following link is redirected on the server to the same page as the
link above:
</p>
<ul>
<li>
<LinkAccordion href="/redirect-to-target-page">
Redirect to target page
</LinkAccordion>
</li>
</ul>
</>
)
}
import Link from 'next/link'
export default function TargetPage() {
return (
<>
<div id="target-page">Target page</div>
<Link href="/">Back to home</Link>
</>
)
}
'use client'
import Link from 'next/link'
import { useState } from 'react'
export function LinkAccordion({ href, children }) {
const [isVisible, setIsVisible] = useState(false)
return (
<>
<input
type="checkbox"
checked={isVisible}
onChange={() => setIsVisible(!isVisible)}
data-link-accordion={href}
/>
{isVisible ? (
<Link href={href}>{children}</Link>
) : (
`${children} (link is hidden)`
)}
</>
)
}
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {
output: 'export',
experimental: {
ppr: false,
dynamicIO: true,
clientSegmentCache: true,
},
}
module.exports = nextConfig
import type * as Playwright from 'playwright'
import webdriver from 'next-webdriver'
import { createRouterAct } from '../router-act'
import { findPort, nextBuild } from 'next-test-utils'
import { isNextStart } from 'e2e-utils'
import { server } from './server.mjs'
describe('segment cache (output: "export")', () => {
if (!isNextStart) {
test('build test should not run during dev test run', () => {})
return
}
// To debug these tests locally, first build the app, then run:
//
// node start.mjs
//
// This will serve the static `/out` directory, and also set up a server-side
// rewrite, which some of the tests below rely on.
let port: number
beforeAll(async () => {
const appDir = __dirname
await nextBuild(appDir, undefined, { cwd: appDir })
port = await findPort()
server.listen(port)
})
afterAll(() => {
server.close()
})
it('basic prefetch in output: "export" mode', async () => {
let act
const browser = await webdriver(port, '/', {
beforePageLoad(p: Playwright.Page) {
act = createRouterAct(p)
},
})
// Initiate a prefetch
await act(
async () => {
const checkbox = await browser.elementByCss(
'[data-link-accordion="/target-page"]'
)
await checkbox.click()
},
{
includes: 'Target page',
}
)
// Navigate to the prefetched target page.
await act(
async () => {
const link = await browser.elementByCss('a[href="/target-page"]')
await link.click()
// The page was prefetched, so we're able to render the target
// page immediately.
const div = await browser.elementById('target-page')
expect(await div.text()).toBe('Target page')
// The target page includes a link back to the home page
await browser.elementByCss('a[href="/"]')
},
{
// Should have prefetched the home page
includes: 'Demonstrates that per-segment prefetching works',
}
)
})
it('prefetch a link to a page that is rewritten server side', async () => {
let act
const browser = await webdriver(port, '/', {
beforePageLoad(p: Playwright.Page) {
act = createRouterAct(p)
},
})
// Initiate a prefetch
await act(
async () => {
const checkbox = await browser.elementByCss(
'[data-link-accordion="/rewrite-to-target-page"]'
)
await checkbox.click()
},
{
includes: 'Target page',
}
)
// Navigate to the prefetched page.
await act(
async () => {
const link = await browser.elementByCss(
'a[href="/rewrite-to-target-page"]'
)
await link.click()
// The page was prefetched, so we're able to render the target
// page immediately.
const div = await browser.elementById('target-page')
expect(await div.text()).toBe('Target page')
// The target page includes a link back to the home page
await browser.elementByCss('a[href="/"]')
},
{
// Should have prefetched the home page
includes: 'Demonstrates that per-segment prefetching works',
}
)
})
it('prefetch a link to a page that is redirected server side', async () => {
let act
const browser = await webdriver(port, '/', {
beforePageLoad(p: Playwright.Page) {
act = createRouterAct(p)
},
})
// Initiate a prefetch
await act(
async () => {
const checkbox = await browser.elementByCss(
'[data-link-accordion="/redirect-to-target-page"]'
)
await checkbox.click()
},
{
includes: 'Target page',
}
)
// Navigate to the prefetched page.
await act(
async () => {
const link = await browser.elementByCss(
'a[href="/redirect-to-target-page"]'
)
await link.click()
// The page was prefetched, so we're able to render the target
// page immediately.
const div = await browser.elementById('target-page')
expect(await div.text()).toBe('Target page')
// The target page includes a link back to the home page
await browser.elementByCss('a[href="/"]')
},
{
// Should have prefetched the home page
includes: 'Demonstrates that per-segment prefetching works',
}
)
})
})
import express from 'express'
import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'
import { createServer } from 'node:http'
const OUT_DIR = join(dirname(fileURLToPath(import.meta.url)), 'out')
const app = express()
// Redirect /redirect-to-target-page/* to /target-page/*
app.get('/redirect-to-target-page/:file?', (req, res) => {
const { file } = req.params
const newUrl = file ? `/target-page/${file}` : '/target-page'
console.log(`Redirecting to ${newUrl}`)
res.redirect(302, newUrl)
})
// Rewrite /rewrite-to-target-page/* to /target-page/*
// NOTE: This intentionally uses `app.use` instead of `app.get` because
// the latter doesn't let you modify the `req.url` property.
app.use((req, res, next) => {
const url = req.originalUrl
if (/^\/rewrite-to-target-page\/?[^/]*$/.test(url)) {
const newUrl = req.originalUrl.replace(
'/rewrite-to-target-page',
'/target-page'
)
console.log(`Rewriting to ${newUrl}`)
req.url = newUrl
}
next()
})
// Serve static files from the out directory
app.use(express.static(OUT_DIR, { extensions: ['html'] }))
export const server = createServer(app)
import { server } from './server.mjs'
const port = 3000
server.listen(3000, () => {
console.log(`Server running at http://localhost:${port}/`)
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment