Unverified Commit d483dfac authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

[metadata] add option of configuring ua of async metadata (#74594)

### What

Introduce an experimental config `experimental.htmlLimitedBots` (type
`Regex`) which can used for controlling the metadata streaming behavior
for different bots, if the bots user agent is matching the regex, it
will serve blocking metadata forcedly. This gives users some control to
opt-out streaming metadata for some clients or bots which cannot handle
it.


Output another manifest `response-config-manfiest.json` containing the
customized UA, which is potentially used for the platform serving
Next.js to control the returned response for certain UA. But it's mostly
for PPR case which we'll tackle later.

Closes NDX-635
Closes NDX-599
parent 36a138a4
...@@ -80,6 +80,7 @@ import { ...@@ -80,6 +80,7 @@ import {
UNDERSCORE_NOT_FOUND_ROUTE_ENTRY, UNDERSCORE_NOT_FOUND_ROUTE_ENTRY,
UNDERSCORE_NOT_FOUND_ROUTE, UNDERSCORE_NOT_FOUND_ROUTE,
DYNAMIC_CSS_MANIFEST, DYNAMIC_CSS_MANIFEST,
RESPONSE_CONFIG_MANIFEST,
} from '../shared/lib/constants' } from '../shared/lib/constants'
import { import {
getSortedRoutes, getSortedRoutes,
...@@ -214,6 +215,7 @@ import { ...@@ -214,6 +215,7 @@ import {
getParsedNodeOptionsWithoutInspect, getParsedNodeOptionsWithoutInspect,
} from '../server/lib/utils' } from '../server/lib/utils'
import { InvariantError } from '../shared/lib/invariant-error' import { InvariantError } from '../shared/lib/invariant-error'
import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot'
type Fallback = null | boolean | string type Fallback = null | boolean | string
...@@ -1307,6 +1309,24 @@ export default async function build( ...@@ -1307,6 +1309,24 @@ export default async function build(
NextBuildContext.clientRouterFilters = clientRouterFilters NextBuildContext.clientRouterFilters = clientRouterFilters
} }
if (config.experimental.streamingMetadata) {
// Write html limited bots config to response-config-manifest
const responseConfigManifestPath = path.join(
distDir,
RESPONSE_CONFIG_MANIFEST
)
const responseConfigManifest: {
version: number
htmlLimitedBots: string
} = {
version: 0,
htmlLimitedBots:
config.experimental.htmlLimitedBots ||
HTML_LIMITED_BOT_UA_RE_STRING,
}
await writeManifest(responseConfigManifestPath, responseConfigManifest)
}
// Ensure commonjs handling is used for files in the distDir (generally .next) // Ensure commonjs handling is used for files in the distDir (generally .next)
// Files outside of the distDir can be "type": "module" // Files outside of the distDir can be "type": "module"
await writeFileUtf8( await writeFileUtf8(
...@@ -2487,6 +2507,9 @@ export default async function build( ...@@ -2487,6 +2507,9 @@ export default async function build(
PRERENDER_MANIFEST, PRERENDER_MANIFEST,
path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST), path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST),
path.join(SERVER_DIRECTORY, MIDDLEWARE_BUILD_MANIFEST + '.js'), path.join(SERVER_DIRECTORY, MIDDLEWARE_BUILD_MANIFEST + '.js'),
...(config.experimental.streamingMetadata
? [RESPONSE_CONFIG_MANIFEST]
: []),
...(!process.env.TURBOPACK ...(!process.env.TURBOPACK
? [ ? [
path.join( path.join(
......
import type { NextConfig, NextConfigComplete } from '../server/config-shared' import type { NextConfigComplete } from '../server/config-shared'
import type { ExperimentalPPRConfig } from '../server/lib/experimental/ppr' import type { ExperimentalPPRConfig } from '../server/lib/experimental/ppr'
import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin' import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin'
import type { AssetBinding } from './webpack/loaders/get-module-build-info' import type { AssetBinding } from './webpack/loaders/get-module-build-info'
...@@ -1466,7 +1466,7 @@ export async function copyTracedFiles( ...@@ -1466,7 +1466,7 @@ export async function copyTracedFiles(
pageKeys: readonly string[], pageKeys: readonly string[],
appPageKeys: readonly string[] | undefined, appPageKeys: readonly string[] | undefined,
tracingRoot: string, tracingRoot: string,
serverConfig: NextConfig, serverConfig: NextConfigComplete,
middlewareManifest: MiddlewareManifest, middlewareManifest: MiddlewareManifest,
hasInstrumentationHook: boolean, hasInstrumentationHook: boolean,
staticPages: Set<string> staticPages: Set<string>
......
...@@ -360,6 +360,7 @@ async function exportAppImpl( ...@@ -360,6 +360,7 @@ async function exportAppImpl(
inlineCss: nextConfig.experimental.inlineCss ?? false, inlineCss: nextConfig.experimental.inlineCss ?? false,
authInterrupts: !!nextConfig.experimental.authInterrupts, authInterrupts: !!nextConfig.experimental.authInterrupts,
streamingMetadata: !!nextConfig.experimental.streamingMetadata, streamingMetadata: !!nextConfig.experimental.streamingMetadata,
htmlLimitedBots: nextConfig.experimental.htmlLimitedBots,
}, },
reactMaxHeadersLength: nextConfig.reactMaxHeadersLength, reactMaxHeadersLength: nextConfig.reactMaxHeadersLength,
} }
......
...@@ -216,6 +216,7 @@ export interface RenderOptsPartial { ...@@ -216,6 +216,7 @@ export interface RenderOptsPartial {
inlineCss: boolean inlineCss: boolean
authInterrupts: boolean authInterrupts: boolean
streamingMetadata: boolean streamingMetadata: boolean
htmlLimitedBots: string | undefined
} }
postponed?: string postponed?: string
......
...@@ -83,7 +83,7 @@ import { ...@@ -83,7 +83,7 @@ import {
} from './lib/revalidate' } from './lib/revalidate'
import { execOnce } from '../shared/lib/utils' import { execOnce } from '../shared/lib/utils'
import { isBlockedPage } from './utils' import { isBlockedPage } from './utils'
import { isBot, isHtmlLimitedBotUA } from '../shared/lib/router/utils/is-bot' import { isBot } from '../shared/lib/router/utils/is-bot'
import RenderResult from './render-result' import RenderResult from './render-result'
import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash' import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash'
import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path' import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path'
...@@ -175,6 +175,7 @@ import type { RouteModule } from './route-modules/route-module' ...@@ -175,6 +175,7 @@ import type { RouteModule } from './route-modules/route-module'
import { FallbackMode, parseFallbackField } from '../lib/fallback' import { FallbackMode, parseFallbackField } from '../lib/fallback'
import { toResponseCacheEntry } from './response-cache/utils' import { toResponseCacheEntry } from './response-cache/utils'
import { scheduleOnNextTick } from '../lib/scheduler' import { scheduleOnNextTick } from '../lib/scheduler'
import { shouldServeStreamingMetadata } from './lib/streaming-metadata'
export type FindComponentsResult = { export type FindComponentsResult = {
components: LoadComponentsReturnType components: LoadComponentsReturnType
...@@ -595,6 +596,7 @@ export default abstract class Server< ...@@ -595,6 +596,7 @@ export default abstract class Server<
inlineCss: this.nextConfig.experimental.inlineCss ?? false, inlineCss: this.nextConfig.experimental.inlineCss ?? false,
authInterrupts: !!this.nextConfig.experimental.authInterrupts, authInterrupts: !!this.nextConfig.experimental.authInterrupts,
streamingMetadata: !!this.nextConfig.experimental.streamingMetadata, streamingMetadata: !!this.nextConfig.experimental.streamingMetadata,
htmlLimitedBots: this.nextConfig.experimental.htmlLimitedBots,
}, },
onInstrumentationRequestError: onInstrumentationRequestError:
this.instrumentationOnRequestError.bind(this), this.instrumentationOnRequestError.bind(this),
...@@ -1675,11 +1677,13 @@ export default abstract class Server< ...@@ -1675,11 +1677,13 @@ export default abstract class Server<
renderOpts: { renderOpts: {
...this.renderOpts, ...this.renderOpts,
supportsDynamicResponse: !isBotRequest, supportsDynamicResponse: !isBotRequest,
serveStreamingMetadata: serveStreamingMetadata: shouldServeStreamingMetadata(
this.renderOpts.experimental.streamingMetadata && ua,
!isHtmlLimitedBotUA(ua), this.renderOpts.experimental
),
}, },
} }
const payload = await fn(ctx) const payload = await fn(ctx)
if (payload === null) { if (payload === null) {
return return
...@@ -2181,8 +2185,6 @@ export default abstract class Server< ...@@ -2181,8 +2185,6 @@ export default abstract class Server<
// cache if there are no dynamic data requirements // cache if there are no dynamic data requirements
opts.supportsDynamicResponse = opts.supportsDynamicResponse =
!isSSG && !isBotRequest && !query.amp && isSupportedDocument !isSSG && !isBotRequest && !query.amp && isSupportedDocument
opts.serveStreamingMetadata =
opts.experimental.streamingMetadata && !isHtmlLimitedBotUA(ua)
} }
// In development, we always want to generate dynamic HTML. // In development, we always want to generate dynamic HTML.
......
...@@ -442,6 +442,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() => ...@@ -442,6 +442,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
authInterrupts: z.boolean().optional(), authInterrupts: z.boolean().optional(),
newDevOverlay: z.boolean().optional(), newDevOverlay: z.boolean().optional(),
streamingMetadata: z.boolean().optional(), streamingMetadata: z.boolean().optional(),
htmlLimitedBots: z.instanceof(RegExp).optional(),
}) })
.optional(), .optional(),
exportPathMap: z exportPathMap: z
......
...@@ -21,6 +21,11 @@ export type NextConfigComplete = Required<NextConfig> & { ...@@ -21,6 +21,11 @@ export type NextConfigComplete = Required<NextConfig> & {
configOrigin?: string configOrigin?: string
configFile?: string configFile?: string
configFileName: string configFileName: string
// override NextConfigComplete.experimental.htmlLimitedBots to string
// because it's not defined in NextConfigComplete.experimental
experimental: Omit<ExperimentalConfig, 'htmlLimitedBots'> & {
htmlLimitedBots: string | undefined
}
} }
export type I18NDomains = readonly DomainLocale[] export type I18NDomains = readonly DomainLocale[]
...@@ -577,6 +582,12 @@ export interface ExperimentalConfig { ...@@ -577,6 +582,12 @@ export interface ExperimentalConfig {
* When enabled will cause async metadata calls to stream rather than block the render. * When enabled will cause async metadata calls to stream rather than block the render.
*/ */
streamingMetadata?: boolean streamingMetadata?: boolean
/**
* User Agent of bots that can handle streaming metadata.
* Besides the default behavior, Next.js act differently on serving metadata to bots based on their capability.
*/
htmlLimitedBots?: RegExp
} }
export type ExportPathMap = { export type ExportPathMap = {
...@@ -1203,6 +1214,7 @@ export const defaultConfig: NextConfig = { ...@@ -1203,6 +1214,7 @@ export const defaultConfig: NextConfig = {
inlineCss: false, inlineCss: false,
newDevOverlay: false, newDevOverlay: false,
streamingMetadata: false, streamingMetadata: false,
htmlLimitedBots: undefined,
}, },
bundlePagesRouterDependencies: false, bundlePagesRouterDependencies: false,
} }
......
...@@ -27,6 +27,7 @@ import { hasNextSupport } from '../server/ci-info' ...@@ -27,6 +27,7 @@ import { hasNextSupport } from '../server/ci-info'
import { transpileConfig } from '../build/next-config-ts/transpile-config' import { transpileConfig } from '../build/next-config-ts/transpile-config'
import { dset } from '../shared/lib/dset' import { dset } from '../shared/lib/dset'
import { normalizeZodErrors } from '../shared/lib/zod' import { normalizeZodErrors } from '../shared/lib/zod'
import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot'
export { normalizeConfig } from './config-shared' export { normalizeConfig } from './config-shared'
export type { DomainLocale, NextConfig } from './config-shared' export type { DomainLocale, NextConfig } from './config-shared'
...@@ -1004,6 +1005,11 @@ function assignDefaults( ...@@ -1004,6 +1005,11 @@ function assignDefaults(
]), ]),
] ]
if (!result.experimental.htmlLimitedBots) {
// @ts-expect-error: override the htmlLimitedBots with default string, type covert: RegExp -> string
result.experimental.htmlLimitedBots = HTML_LIMITED_BOT_UA_RE_STRING
}
return result return result
} }
...@@ -1222,6 +1228,12 @@ export default async function loadConfig( ...@@ -1222,6 +1228,12 @@ export default async function loadConfig(
} }
} }
// serialize the regex config into string
if (userConfig.experimental?.htmlLimitedBots instanceof RegExp) {
userConfig.experimental.htmlLimitedBots =
userConfig.experimental.htmlLimitedBots.source
}
onLoadUserConfig?.(userConfig) onLoadUserConfig?.(userConfig)
const completeConfig = assignDefaults( const completeConfig = assignDefaults(
dir, dir,
......
import { HTML_LIMITED_BOT_UA_RE_STRING } from '../../shared/lib/router/utils/is-bot'
export function shouldServeStreamingMetadata(
userAgent: string,
{
streamingMetadata,
htmlLimitedBots,
}: {
streamingMetadata: boolean
htmlLimitedBots: string | undefined
}
): boolean {
if (!streamingMetadata) {
return false
}
const blockingMetadataUARegex = new RegExp(
htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING,
'i'
)
return !blockingMetadataUARegex.test(userAgent)
}
...@@ -42,6 +42,7 @@ export const EXPORT_DETAIL = 'export-detail.json' ...@@ -42,6 +42,7 @@ export const EXPORT_DETAIL = 'export-detail.json'
export const PRERENDER_MANIFEST = 'prerender-manifest.json' export const PRERENDER_MANIFEST = 'prerender-manifest.json'
export const ROUTES_MANIFEST = 'routes-manifest.json' export const ROUTES_MANIFEST = 'routes-manifest.json'
export const IMAGES_MANIFEST = 'images-manifest.json' export const IMAGES_MANIFEST = 'images-manifest.json'
export const RESPONSE_CONFIG_MANIFEST = 'response-config-manifest.json'
export const SERVER_FILES_MANIFEST = 'required-server-files.json' export const SERVER_FILES_MANIFEST = 'required-server-files.json'
export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json' export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json'
export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json' export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json'
......
...@@ -4,14 +4,19 @@ const HEADLESS_BROWSER_BOT_UA_RE = ...@@ -4,14 +4,19 @@ const HEADLESS_BROWSER_BOT_UA_RE =
// This regex contains the bots that we need to do a blocking render for and can't safely stream the response // This regex contains the bots that we need to do a blocking render for and can't safely stream the response
// due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent. // due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
const HTML_LIMITED_BOT_UA_RE = export const HTML_LIMITED_BOT_UA_RE_STRING =
/Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i 'Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview'
export const HTML_LIMITED_BOT_UA_RE = new RegExp(
HTML_LIMITED_BOT_UA_RE_STRING,
'i'
)
function isHeadlessBrowserBotUA(userAgent: string) { function isHeadlessBrowserBotUA(userAgent: string) {
return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent) return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
} }
export function isHtmlLimitedBotUA(userAgent: string) { function isHtmlLimitedBotUA(userAgent: string) {
return HTML_LIMITED_BOT_UA_RE.test(userAgent) return HTML_LIMITED_BOT_UA_RE.test(userAgent)
} }
......
import { nextTestSetup } from 'e2e-utils'
describe('app-dir - metadata-streaming-customized-rule', () => {
const { next } = nextTestSetup({
files: __dirname,
overrideFiles: {
'next.config.js': `
module.exports = {
experimental: {
streamingMetadata: true,
htmlLimitedBots: /Minibot/i,
}
}
`,
},
})
it('should send the blocking response for html limited bots', async () => {
const $ = await next.render$(
'/',
undefined, // no query
{
headers: {
'user-agent': 'Minibot',
},
}
)
expect(await $('title').text()).toBe('index page')
})
it('should send streaming response for headless browser bots', async () => {
const $ = await next.render$(
'/',
undefined, // no query
{
headers: {
'user-agent': 'Weebot',
},
}
)
expect(await $('title').length).toBe(0)
})
})
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { retry, createMultiDomMatcher } from 'next-test-utils' import { retry, createMultiDomMatcher } from 'next-test-utils'
describe('metadata-streaming', () => { describe('app-dir - metadata-streaming', () => {
const { next } = nextTestSetup({ const { next } = nextTestSetup({
files: __dirname, files: __dirname,
}) })
......
import { ReactNode } from 'react'
export default function Root({ children }: { children: ReactNode }) {
return (
<html>
<body>{children}</body>
</html>
)
}
export default function Page() {
return <p>hello world</p>
}
import { nextTestSetup } from 'e2e-utils'
describe('app-dir - metadata-streaming-config-customized', () => {
const { next } = nextTestSetup({
files: __dirname,
overrideFiles: {
'next.config.js': `
module.exports = {
experimental: {
streamingMetadata: true,
htmlLimitedBots: /MyBot/i,
}
}
`,
},
})
it('should have the default streaming metadata config output in routes-manifest.json', async () => {
const requiredServerFiles = JSON.parse(
await next.readFile('.next/required-server-files.json')
)
expect(requiredServerFiles.files).toContain(
'.next/response-config-manifest.json'
)
expect(
requiredServerFiles.config.experimental.htmlLimitedBots
).toMatchInlineSnapshot(`"MyBot"`)
const responseConfigManifest = JSON.parse(
await next.readFile('.next/response-config-manifest.json')
)
expect(responseConfigManifest).toMatchInlineSnapshot(`
{
"htmlLimitedBots": "MyBot",
"version": 0,
}
`)
})
})
import { nextTestSetup } from 'e2e-utils'
describe('app-dir - metadata-streaming-config', () => {
const { next } = nextTestSetup({
files: __dirname,
})
it('should have the default streaming metadata config output in routes-manifest.json', async () => {
const requiredServerFiles = JSON.parse(
await next.readFile('.next/required-server-files.json')
)
expect(requiredServerFiles.files).toContain(
'.next/response-config-manifest.json'
)
expect(
requiredServerFiles.config.experimental.htmlLimitedBots
).toMatchInlineSnapshot(
`"Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview"`
)
const responseConfigManifest = JSON.parse(
await next.readFile('.next/response-config-manifest.json')
)
expect(responseConfigManifest).toMatchInlineSnapshot(`
{
"htmlLimitedBots": "Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview",
"version": 0,
}
`)
})
})
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {
experimental: {
streamingMetadata: true,
},
}
module.exports = nextConfig
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment