Unverified Commit 6a5d6764 authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

[metadata] clarify the default list behavior (#76450)

parent 81ce3af8
...@@ -1208,6 +1208,7 @@ Some bots, like `Googlebot`, can execute JavaScript and are able to inspect the ...@@ -1208,6 +1208,7 @@ Some bots, like `Googlebot`, can execute JavaScript and are able to inspect the
Next.js automatically detects the user agent of incoming requests to determine whether to serve streaming metadata or fallback to blocking metadata. Next.js automatically detects the user agent of incoming requests to determine whether to serve streaming metadata or fallback to blocking metadata.
If you need to customize this list, you can define them manually using the `htmlLimitedBots` option in `next.config.js`. Next.js will ensure user agents matching this regex receive blocking metadata when requesting your web page. If you need to customize this list, you can define them manually using the `htmlLimitedBots` option in `next.config.js`. Next.js will ensure user agents matching this regex receive blocking metadata when requesting your web page.
Specifying a `htmlLimitedBots` config will override the Next.js' default list, allowing you full control over what user agents should opt into this behavior. This is advanced behavior, and the default should be sufficient for most cases.
```js filename="next.config.js" ```js filename="next.config.js"
module.exports = { module.exports = {
...@@ -1215,6 +1216,8 @@ module.exports = { ...@@ -1215,6 +1216,8 @@ module.exports = {
} }
``` ```
> **Note:** Next.js includes [a default list of HTML limited bots](https://github.com/vercel/next.js/blob/canary/packages/next/src/shared/lib/router/utils/html-bots.ts)
## Version History ## Version History
| Version | Changes | | Version | Changes |
......
...@@ -1071,6 +1071,9 @@ export interface NextConfig extends Record<string, any> { ...@@ -1071,6 +1071,9 @@ export interface NextConfig extends Record<string, any> {
/** /**
* User Agent of bots that can handle streaming metadata. * User Agent of bots that can handle streaming metadata.
* Besides the default behavior, Next.js act differently on serving metadata to bots based on their capability. * Besides the default behavior, Next.js act differently on serving metadata to bots based on their capability.
*
* @default
* /Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i
*/ */
htmlLimitedBots?: RegExp htmlLimitedBots?: RegExp
} }
......
// This regex contains the bots that we need to do a blocking render for and can't safely stream the response
// due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
export const HTML_LIMITED_BOT_UA_RE =
/Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i
import { HTML_LIMITED_BOT_UA_RE } from './html-bots'
// Bot crawler that will spin up a headless browser and execute JS // Bot crawler that will spin up a headless browser and execute JS
const HEADLESS_BROWSER_BOT_UA_RE = const HEADLESS_BROWSER_BOT_UA_RE =
/Googlebot|Google-PageRenderer|AdsBot-Google|googleweblight|Storebot-Google/i /Googlebot|Google-PageRenderer|AdsBot-Google|googleweblight|Storebot-Google/i
// This regex contains the bots that we need to do a blocking render for and can't safely stream the response export const HTML_LIMITED_BOT_UA_RE_STRING = HTML_LIMITED_BOT_UA_RE.source
// due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
export const HTML_LIMITED_BOT_UA_RE_STRING =
'Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview'
export const HTML_LIMITED_BOT_UA_RE = new RegExp( export { HTML_LIMITED_BOT_UA_RE }
HTML_LIMITED_BOT_UA_RE_STRING,
'i'
)
function isDomBotUA(userAgent: string) { function isDomBotUA(userAgent: string) {
return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent) return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment