Unverified Commit 47639cb6 authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

[metadata] initial support of streaming metadata (#74619)

parent 704a24a7
...@@ -359,6 +359,7 @@ async function exportAppImpl( ...@@ -359,6 +359,7 @@ async function exportAppImpl(
clientSegmentCache: nextConfig.experimental.clientSegmentCache ?? false, clientSegmentCache: nextConfig.experimental.clientSegmentCache ?? false,
inlineCss: nextConfig.experimental.inlineCss ?? false, inlineCss: nextConfig.experimental.inlineCss ?? false,
authInterrupts: !!nextConfig.experimental.authInterrupts, authInterrupts: !!nextConfig.experimental.authInterrupts,
streamingMetadata: !!nextConfig.experimental.streamingMetadata,
}, },
reactMaxHeadersLength: nextConfig.reactMaxHeadersLength, reactMaxHeadersLength: nextConfig.reactMaxHeadersLength,
} }
......
'use client'
import { use } from 'react'
import { useServerInsertedHTML } from '../../client/components/navigation'
// This is a SSR-only version that will wait the promise of metadata to resolve
// and
function ServerInsertMetadata({ promise }: { promise: Promise<any> }) {
let metadataToFlush: React.ReactNode = null
// Only inserted once to avoid multi insertion on re-renders
let inserted = false
promise.then((resolvedMetadata) => {
metadataToFlush = resolvedMetadata
})
useServerInsertedHTML(() => {
if (metadataToFlush && !inserted) {
const flushing = metadataToFlush
metadataToFlush = null
return flushing
}
})
inserted = true
return null
}
function BrowserResolvedMetadata({ promise }: { promise: Promise<any> }) {
return use(promise)
}
export function AsyncMetadata({ promise }: { promise: Promise<any> }) {
return (
<>
{typeof window === 'undefined' ? (
<ServerInsertMetadata promise={promise} />
) : (
<BrowserResolvedMetadata promise={promise} />
)}
</>
)
}
...@@ -37,6 +37,7 @@ import { ...@@ -37,6 +37,7 @@ import {
METADATA_BOUNDARY_NAME, METADATA_BOUNDARY_NAME,
VIEWPORT_BOUNDARY_NAME, VIEWPORT_BOUNDARY_NAME,
} from './metadata-constants' } from './metadata-constants'
import { AsyncMetadata } from './async-metadata'
// Use a promise to share the status of the metadata resolving, // Use a promise to share the status of the metadata resolving,
// returning two components `MetadataTree` and `MetadataOutlet` // returning two components `MetadataTree` and `MetadataOutlet`
...@@ -55,6 +56,7 @@ export function createMetadataComponents({ ...@@ -55,6 +56,7 @@ export function createMetadataComponents({
workStore, workStore,
MetadataBoundary, MetadataBoundary,
ViewportBoundary, ViewportBoundary,
serveStreamingMetadata,
}: { }: {
tree: LoaderTree tree: LoaderTree
searchParams: Promise<ParsedUrlQuery> searchParams: Promise<ParsedUrlQuery>
...@@ -66,6 +68,7 @@ export function createMetadataComponents({ ...@@ -66,6 +68,7 @@ export function createMetadataComponents({
workStore: WorkStore workStore: WorkStore
MetadataBoundary: (props: { children: React.ReactNode }) => React.ReactNode MetadataBoundary: (props: { children: React.ReactNode }) => React.ReactNode
ViewportBoundary: (props: { children: React.ReactNode }) => React.ReactNode ViewportBoundary: (props: { children: React.ReactNode }) => React.ReactNode
serveStreamingMetadata: boolean
}): { }): {
MetadataTree: React.ComponentType MetadataTree: React.ComponentType
ViewportTree: React.ComponentType ViewportTree: React.ComponentType
...@@ -94,7 +97,7 @@ export function createMetadataComponents({ ...@@ -94,7 +97,7 @@ export function createMetadataComponents({
) )
} }
async function viewport() { function viewport() {
return getResolvedViewport( return getResolvedViewport(
tree, tree,
searchParams, searchParams,
...@@ -129,7 +132,7 @@ export function createMetadataComponents({ ...@@ -129,7 +132,7 @@ export function createMetadataComponents({
} }
Viewport.displayName = VIEWPORT_BOUNDARY_NAME Viewport.displayName = VIEWPORT_BOUNDARY_NAME
async function metadata() { function metadata() {
return getResolvedMetadata( return getResolvedMetadata(
tree, tree,
searchParams, searchParams,
...@@ -141,7 +144,7 @@ export function createMetadataComponents({ ...@@ -141,7 +144,7 @@ export function createMetadataComponents({
) )
} }
async function Metadata() { async function resolveFinalMetadata() {
try { try {
return await metadata() return await metadata()
} catch (error) { } catch (error) {
...@@ -164,10 +167,21 @@ export function createMetadataComponents({ ...@@ -164,10 +167,21 @@ export function createMetadataComponents({
return null return null
} }
} }
async function Metadata() {
if (serveStreamingMetadata) {
return <AsyncMetadata promise={resolveFinalMetadata()} />
}
return await resolveFinalMetadata()
}
Metadata.displayName = METADATA_BOUNDARY_NAME Metadata.displayName = METADATA_BOUNDARY_NAME
async function getMetadataReady(): Promise<void> { async function getMetadataReady(): Promise<void> {
await metadata() // Only warm up metadata() call when it's blocking metadata,
// otherwise it will be fully managed by AsyncMetadata component.
if (!serveStreamingMetadata) {
await metadata()
}
return undefined return undefined
} }
......
...@@ -314,6 +314,20 @@ function createNotFoundLoaderTree(loaderTree: LoaderTree): LoaderTree { ...@@ -314,6 +314,20 @@ function createNotFoundLoaderTree(loaderTree: LoaderTree): LoaderTree {
] ]
} }
function createSuspenseyMetadata(
Metadata: React.ComponentType<{}>,
serveStreamingMetadata: boolean
): React.ComponentType<{}> {
return () =>
serveStreamingMetadata ? (
<React.Suspense fallback={null}>
<Metadata />
</React.Suspense>
) : (
<Metadata />
)
}
/** /**
* Returns a function that parses the dynamic segment and return the associated value. * Returns a function that parses the dynamic segment and return the associated value.
*/ */
...@@ -466,16 +480,17 @@ async function generateDynamicRSCPayload( ...@@ -466,16 +480,17 @@ async function generateDynamicRSCPayload(
workStore, workStore,
MetadataBoundary, MetadataBoundary,
ViewportBoundary, ViewportBoundary,
serveStreamingMetadata: !!ctx.renderOpts.serveStreamingMetadata,
}) })
const MetadataComponent = () => { const MetadataComponent = createSuspenseyMetadata(() => {
return ( return (
<React.Fragment key={flightDataPathMetadataKey}> <React.Fragment key={flightDataPathMetadataKey}>
{/* Adding requestId as react key to make metadata remount for each render */} {/* Adding requestId as react key to make metadata remount for each render */}
<MetadataTree key={requestId} /> <MetadataTree key={requestId} />
</React.Fragment> </React.Fragment>
) )
} }, !!ctx.renderOpts.serveStreamingMetadata)
flightData = ( flightData = (
await walkTreeWithFlightRouterState({ await walkTreeWithFlightRouterState({
...@@ -484,15 +499,14 @@ async function generateDynamicRSCPayload( ...@@ -484,15 +499,14 @@ async function generateDynamicRSCPayload(
parentParams: {}, parentParams: {},
flightRouterState, flightRouterState,
// For flight, render metadata inside leaf page // For flight, render metadata inside leaf page
rscHead: [ rscHead: (
<React.Fragment key={flightDataPathViewportKey}> <React.Fragment key={flightDataPathViewportKey}>
{/* noindex needs to be blocking */} {/* noindex needs to be blocking */}
<NonIndex ctx={ctx} /> <NonIndex ctx={ctx} />
{/* Adding requestId as react key to make metadata remount for each render */} {/* Adding requestId as react key to make metadata remount for each render */}
<ViewportTree key={requestId} /> <ViewportTree key={requestId} />
</React.Fragment>, </React.Fragment>
null, ),
],
injectedCSS: new Set(), injectedCSS: new Set(),
injectedJS: new Set(), injectedJS: new Set(),
injectedFontPreloadTags: new Set(), injectedFontPreloadTags: new Set(),
...@@ -770,18 +784,19 @@ async function getRSCPayload( ...@@ -770,18 +784,19 @@ async function getRSCPayload(
workStore, workStore,
MetadataBoundary, MetadataBoundary,
ViewportBoundary, ViewportBoundary,
serveStreamingMetadata: !!ctx.renderOpts.serveStreamingMetadata,
}) })
const preloadCallbacks: PreloadCallbacks = [] const preloadCallbacks: PreloadCallbacks = []
function MetadataComponent() { const MetadataComponent = createSuspenseyMetadata(() => {
return ( return (
<React.Fragment key={flightDataPathMetadataKey}> <React.Fragment key={flightDataPathMetadataKey}>
{/* Not add requestId as react key to ensure segment prefetch could result consistently if nothing changed */} {/* Not add requestId as react key to ensure segment prefetch could result consistently if nothing changed */}
<MetadataTree /> <MetadataTree />
</React.Fragment> </React.Fragment>
) )
} }, !!ctx.renderOpts.serveStreamingMetadata)
const seedData = await createComponentTree({ const seedData = await createComponentTree({
ctx, ctx,
...@@ -895,13 +910,17 @@ async function getErrorRSCPayload( ...@@ -895,13 +910,17 @@ async function getErrorRSCPayload(
workStore, workStore,
MetadataBoundary, MetadataBoundary,
ViewportBoundary, ViewportBoundary,
serveStreamingMetadata: !!ctx.renderOpts.serveStreamingMetadata,
}) })
const initialHeadMetadata = ( const ErrorMetadataComponent = createSuspenseyMetadata(
<React.Fragment key={flightDataPathMetadataKey}> () => (
{/* Adding requestId as react key to make metadata remount for each render */} <React.Fragment key={flightDataPathMetadataKey}>
<MetadataTree key={requestId} /> {/* Adding requestId as react key to make metadata remount for each render */}
</React.Fragment> <MetadataTree key={requestId} />
</React.Fragment>
),
!!ctx.renderOpts.serveStreamingMetadata
) )
const initialHeadViewport = ( const initialHeadViewport = (
...@@ -926,7 +945,9 @@ async function getErrorRSCPayload( ...@@ -926,7 +945,9 @@ async function getErrorRSCPayload(
const seedData: CacheNodeSeedData = [ const seedData: CacheNodeSeedData = [
initialTree[0], initialTree[0],
<html id="__next_error__"> <html id="__next_error__">
<head>{initialHeadMetadata}</head> <head>
<ErrorMetadataComponent />
</head>
<body /> <body />
</html>, </html>,
{}, {},
......
...@@ -177,7 +177,7 @@ export interface RenderOptsPartial { ...@@ -177,7 +177,7 @@ export interface RenderOptsPartial {
assetPrefix?: string assetPrefix?: string
crossOrigin?: '' | 'anonymous' | 'use-credentials' | undefined crossOrigin?: '' | 'anonymous' | 'use-credentials' | undefined
nextFontManifest?: DeepReadonly<NextFontManifest> nextFontManifest?: DeepReadonly<NextFontManifest>
isBot?: boolean serveStreamingMetadata?: boolean
incrementalCache?: import('../lib/incremental-cache').IncrementalCache incrementalCache?: import('../lib/incremental-cache').IncrementalCache
cacheLifeProfiles?: { cacheLifeProfiles?: {
[profile: string]: import('../use-cache/cache-life').CacheLife [profile: string]: import('../use-cache/cache-life').CacheLife
...@@ -215,6 +215,7 @@ export interface RenderOptsPartial { ...@@ -215,6 +215,7 @@ export interface RenderOptsPartial {
clientSegmentCache: boolean clientSegmentCache: boolean
inlineCss: boolean inlineCss: boolean
authInterrupts: boolean authInterrupts: boolean
streamingMetadata: boolean
} }
postponed?: string postponed?: string
......
...@@ -83,7 +83,7 @@ import { ...@@ -83,7 +83,7 @@ import {
} from './lib/revalidate' } from './lib/revalidate'
import { execOnce } from '../shared/lib/utils' import { execOnce } from '../shared/lib/utils'
import { isBlockedPage } from './utils' import { isBlockedPage } from './utils'
import { isBot } from '../shared/lib/router/utils/is-bot' import { isBot, isHtmlLimitedBotUA } from '../shared/lib/router/utils/is-bot'
import RenderResult from './render-result' import RenderResult from './render-result'
import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash' import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash'
import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path' import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path'
...@@ -594,6 +594,7 @@ export default abstract class Server< ...@@ -594,6 +594,7 @@ export default abstract class Server<
this.nextConfig.experimental.clientSegmentCache ?? false, this.nextConfig.experimental.clientSegmentCache ?? false,
inlineCss: this.nextConfig.experimental.inlineCss ?? false, inlineCss: this.nextConfig.experimental.inlineCss ?? false,
authInterrupts: !!this.nextConfig.experimental.authInterrupts, authInterrupts: !!this.nextConfig.experimental.authInterrupts,
streamingMetadata: !!this.nextConfig.experimental.streamingMetadata,
}, },
onInstrumentationRequestError: onInstrumentationRequestError:
this.instrumentationOnRequestError.bind(this), this.instrumentationOnRequestError.bind(this),
...@@ -1666,13 +1667,17 @@ export default abstract class Server< ...@@ -1666,13 +1667,17 @@ export default abstract class Server<
'renderOpts' 'renderOpts'
> >
): Promise<void> { ): Promise<void> {
const isBotRequest = isBot(partialContext.req.headers['user-agent'] || '') const ua = partialContext.req.headers['user-agent'] || ''
const isBotRequest = isBot(ua)
const ctx: RequestContext<ServerRequest, ServerResponse> = { const ctx: RequestContext<ServerRequest, ServerResponse> = {
...partialContext, ...partialContext,
renderOpts: { renderOpts: {
...this.renderOpts, ...this.renderOpts,
supportsDynamicResponse: !isBotRequest, supportsDynamicResponse: !isBotRequest,
isBot: !!isBotRequest, serveStreamingMetadata:
this.renderOpts.experimental.streamingMetadata &&
!isHtmlLimitedBotUA(ua),
}, },
} }
const payload = await fn(ctx) const payload = await fn(ctx)
...@@ -2162,7 +2167,8 @@ export default abstract class Server< ...@@ -2162,7 +2167,8 @@ export default abstract class Server<
if ('amp' in query && !query.amp) delete query.amp if ('amp' in query && !query.amp) delete query.amp
if (opts.supportsDynamicResponse === true) { if (opts.supportsDynamicResponse === true) {
const isBotRequest = isBot(req.headers['user-agent'] || '') const ua = req.headers['user-agent'] || ''
const isBotRequest = isBot(ua)
const isSupportedDocument = const isSupportedDocument =
typeof components.Document?.getInitialProps !== 'function' || typeof components.Document?.getInitialProps !== 'function' ||
// The built-in `Document` component also supports dynamic HTML for concurrent mode. // The built-in `Document` component also supports dynamic HTML for concurrent mode.
...@@ -2175,7 +2181,8 @@ export default abstract class Server< ...@@ -2175,7 +2181,8 @@ export default abstract class Server<
// cache if there are no dynamic data requirements // cache if there are no dynamic data requirements
opts.supportsDynamicResponse = opts.supportsDynamicResponse =
!isSSG && !isBotRequest && !query.amp && isSupportedDocument !isSSG && !isBotRequest && !query.amp && isSupportedDocument
opts.isBot = isBotRequest opts.serveStreamingMetadata =
opts.experimental.streamingMetadata && !isHtmlLimitedBotUA(ua)
} }
// In development, we always want to generate dynamic HTML. // In development, we always want to generate dynamic HTML.
......
...@@ -440,6 +440,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() => ...@@ -440,6 +440,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
serverComponentsHmrCache: z.boolean().optional(), serverComponentsHmrCache: z.boolean().optional(),
authInterrupts: z.boolean().optional(), authInterrupts: z.boolean().optional(),
newDevOverlay: z.boolean().optional(), newDevOverlay: z.boolean().optional(),
streamingMetadata: z.boolean().optional(),
}) })
.optional(), .optional(),
exportPathMap: z exportPathMap: z
......
...@@ -567,6 +567,11 @@ export interface ExperimentalConfig { ...@@ -567,6 +567,11 @@ export interface ExperimentalConfig {
* Enables the new dev overlay. * Enables the new dev overlay.
*/ */
newDevOverlay?: boolean newDevOverlay?: boolean
/**
* When enabled will cause async metadata calls to stream rather than block the render.
*/
streamingMetadata?: boolean
} }
export type ExportPathMap = { export type ExportPathMap = {
...@@ -1191,6 +1196,7 @@ export const defaultConfig: NextConfig = { ...@@ -1191,6 +1196,7 @@ export const defaultConfig: NextConfig = {
dynamicIO: false, dynamicIO: false,
inlineCss: false, inlineCss: false,
newDevOverlay: false, newDevOverlay: false,
streamingMetadata: false,
}, },
bundlePagesRouterDependencies: false, bundlePagesRouterDependencies: false,
} }
......
...@@ -265,7 +265,7 @@ export type RenderOptsPartial = { ...@@ -265,7 +265,7 @@ export type RenderOptsPartial = {
domainLocales?: readonly DomainLocale[] domainLocales?: readonly DomainLocale[]
disableOptimizedLoading?: boolean disableOptimizedLoading?: boolean
supportsDynamicResponse: boolean supportsDynamicResponse: boolean
isBot?: boolean serveStreamingMetadata?: boolean
runtime?: ServerRuntime runtime?: ServerRuntime
serverComponents?: boolean serverComponents?: boolean
serverActions?: { serverActions?: {
......
// Bot crawler that will spin up a headless browser and execute JS // Bot crawler that will spin up a headless browser and execute JS
const HEADLESS_BOT_UA_RE = const HEADLESS_BROWSER_BOT_UA_RE =
/Googlebot|Google-PageRenderer|AdsBot-Google|googleweblight|Storebot-Google/i /Googlebot|Google-PageRenderer|AdsBot-Google|googleweblight|Storebot-Google/i
// This regex contains the bots that we need to do a blocking render for and can't safely stream the response // This regex contains the bots that we need to do a blocking render for and can't safely stream the response
...@@ -7,14 +7,14 @@ const HEADLESS_BOT_UA_RE = ...@@ -7,14 +7,14 @@ const HEADLESS_BOT_UA_RE =
const HTML_LIMITED_BOT_UA_RE = const HTML_LIMITED_BOT_UA_RE =
/Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i /Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i
function isHeadlessBrowserBot(userAgent: string) { function isHeadlessBrowserBotUA(userAgent: string) {
return HEADLESS_BOT_UA_RE.test(userAgent) return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
} }
function isHtmlLimitedBot(userAgent: string) { export function isHtmlLimitedBotUA(userAgent: string) {
return HTML_LIMITED_BOT_UA_RE.test(userAgent) return HTML_LIMITED_BOT_UA_RE.test(userAgent)
} }
export function isBot(userAgent: string): boolean { export function isBot(userAgent: string): boolean {
return isHeadlessBrowserBot(userAgent) || isHtmlLimitedBot(userAgent) return isHeadlessBrowserBotUA(userAgent) || isHtmlLimitedBotUA(userAgent)
} }
import Link from 'next/link'
import { ReactNode } from 'react'
export default function Root({ children }: { children: ReactNode }) {
return (
<html>
<body>
<div>
<Link href="/slow" id="to-slow">
{`to /slow`}
</Link>
<br />
<Link href="/" id="to-home">
{`to /`}
</Link>
</div>
{children}
</body>
</html>
)
}
export default function Page() {
return <p>index</p>
}
export async function generateMetadata() {
await new Promise((resolve) => setTimeout(resolve, 3 * 1000))
return {
title: 'index page',
}
}
export const dynamic = 'force-dynamic'
export default function Page() {
return <p>slow</p>
}
export async function generateMetadata() {
await new Promise((resolve) => setTimeout(resolve, 1 * 1000))
return {
title: 'slow page',
description: 'slow page description',
generator: 'next.js',
applicationName: 'test',
referrer: 'origin-when-cross-origin',
keywords: ['next.js', 'react', 'javascript'],
authors: [{ name: 'huozhi' }],
creator: 'huozhi',
publisher: 'vercel',
robots: 'index, follow',
}
}
export const dynamic = 'force-dynamic'
import { nextTestSetup } from 'e2e-utils'
import { retry, createMultiDomMatcher } from 'next-test-utils'
describe('metadata-streaming', () => {
const { next } = nextTestSetup({
files: __dirname,
})
it('should load the page without delayed metadata', async () => {
const $ = await next.render$('/')
expect($('title').length).toBe(0)
})
it('should still load viewport meta tags even if metadata is delayed', async () => {
const $ = await next.render$('/slow')
expect($('meta[name="viewport"]').attr('content')).toBe(
'width=device-width, initial-scale=1'
)
expect($('meta[charset]').attr('charset')).toBe('utf-8')
})
it('should render the metadata in the browser', async () => {
const browser = await next.browser('/')
await retry(async () => {
expect(await browser.elementByCss('title').text()).toBe('index page')
})
})
it('should load the initial html without slow metadata during navigation', async () => {
// navigate from / to /slow, the metadata should be empty first, e.g. no title.
// then the metadata should be loaded after few seconds.
const browser = await next.browser('/')
await browser.elementByCss('#to-slow').click()
await retry(async () => {
expect(await browser.elementByCss('title').text()).toBe('slow page')
const matchMultiDom = createMultiDomMatcher(browser)
await matchMultiDom('meta', 'name', 'content', {
description: 'slow page description',
generator: 'next.js',
'application-name': 'test',
referrer: 'origin-when-cross-origin',
keywords: 'next.js,react,javascript',
author: ['huozhi'],
viewport: 'width=device-width, initial-scale=1',
creator: 'huozhi',
publisher: 'vercel',
robots: 'index, follow',
})
})
})
it('should send the blocking response for html limited bots', async () => {
const $ = await next.render$(
'/',
undefined, // no query
{
headers: {
'user-agent': 'Twitterbot',
},
}
)
expect(await $('title').text()).toBe('index page')
})
it('should send streaming response for headless browser bots', async () => {
const browser = await next.browser('/')
await retry(async () => {
expect(await browser.elementByCss('title').text()).toBe('index page')
})
})
it('should not insert metadata twice or inject into body', async () => {
const browser = await next.browser('/slow')
// each metadata should be inserted only once
expect(await browser.hasElementByCssSelector('body meta')).toBe(false)
expect(await browser.hasElementByCssSelector('body title')).toBe(false)
})
})
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {
experimental: {
streamingMetadata: true,
},
}
module.exports = nextConfig
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment