Unverified Commit 7499b034 authored by Sebastian "Sebbie" Silbermann's avatar Sebastian "Sebbie" Silbermann Committed by GitHub

Cap bot user-agent matching at 1KB (#100604)

No known real UA string we're aware of exceeds ~500 bytes (uap-core:
longest 492 B across 30,873 distinct strings; Matomo DeviceDetector:
longest 482 B across 40,348), so oversized UAs are never bots we need to
identify. This bounds regex work on attacker-controlled input as DoS
defense-in-depth, mainly for user-configured `htmlLimitedBots` patterns
which may be more expensive than the built-in one.

A >1KB UA containing a bot token (e.g. `WhatsApp`) is now treated as a
non-bot (streaming metadata served, `isBot()` false).

Hosting providers likely already cap the `User-Agent` header (or any
header value) e.g. [Vercel caps at
16kb](https://vercel.com/docs/errors/request_header_too_large)

<!-- NEXT_JS_LLM -->

---------
Co-authored-by: 's avatarClaude Code <noreply@anthropic.com>
parent 9ff761f2
import { HTML_LIMITED_BOT_UA_RE_STRING } from '../../shared/lib/router/utils/is-bot'
import {
HTML_LIMITED_BOT_UA_RE_STRING,
isOversizedUserAgent,
} from '../../shared/lib/router/utils/is-bot'
let cachedPattern: string | undefined
let cachedRegex: RegExp | undefined
......@@ -7,13 +10,18 @@ export function shouldServeStreamingMetadata(
userAgent: string,
htmlLimitedBots: string | undefined
): boolean {
// No real UA string exceeds this bound, so oversized UAs are not bots we need
// to identify; skip the (possibly user-configured) regex on them entirely.
if (!userAgent || isOversizedUserAgent(userAgent)) {
return true
}
const pattern = htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING
if (cachedPattern !== pattern) {
cachedPattern = pattern
cachedRegex = new RegExp(pattern, 'i')
}
// Only block metadata for HTML-limited bots
if (userAgent && cachedRegex!.test(userAgent)) {
if (cachedRegex!.test(userAgent)) {
return false
}
return true
......
// This regex contains the bots that we need to do a blocking render for and can't safely stream the response
// due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
// Note: The pattern [\w-]+-Google captures all Google crawlers with "-Google" suffix (e.g., Mediapartners-Google, AdsBot-Google, Storebot-Google)
// Note: The pattern [\w-]-Google captures all Google crawlers with "-Google" suffix (e.g., Mediapartners-Google, AdsBot-Google, Storebot-Google)
// as well as crawlers starting with "Google-" (e.g., Google-PageRenderer, Google-InspectionTool)
export const HTML_LIMITED_BOT_UA_RE =
/[\w-]+-Google|Google-[\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight/i
/[\w-]-Google|Google-[\w-]|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight/i
......@@ -10,12 +10,40 @@ export const HTML_LIMITED_BOT_UA_RE_STRING = HTML_LIMITED_BOT_UA_RE.source
export { HTML_LIMITED_BOT_UA_RE }
// No known real UA string exceeds ~500 bytes; anything larger is not a bot we
// need to identify. Skipping the regex bounds the work done on attacker-controlled
// input.
export const MAX_USER_AGENT_LENGTH = 1024
let didWarnOversizedUserAgent = false
export function isOversizedUserAgent(userAgent: string): boolean {
if (userAgent.length <= MAX_USER_AGENT_LENGTH) {
return false
}
// A 400 response might be more prudent for these requests, but then affected
// bots see no content at all, so we only warn to keep at least some o11y.
// warnOnce is a no-op in production, so dedupe manually to warn in every mode.
if (!didWarnOversizedUserAgent) {
didWarnOversizedUserAgent = true
console.warn(
`Received a User-Agent string longer than ${MAX_USER_AGENT_LENGTH} bytes. Skipping bot detection for this request.`
)
}
return true
}
function isDomBotUA(userAgent: string) {
return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
return (
!isOversizedUserAgent(userAgent) &&
HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
)
}
function isHtmlLimitedBotUA(userAgent: string) {
return HTML_LIMITED_BOT_UA_RE.test(userAgent)
return (
!isOversizedUserAgent(userAgent) && HTML_LIMITED_BOT_UA_RE.test(userAgent)
)
}
export function isBot(userAgent: string): boolean {
......
......@@ -11,7 +11,7 @@ describe('app-dir - metadata-streaming-config', () => {
)
expect(requiredServerFiles.config.htmlLimitedBots).toMatchInlineSnapshot(
`"[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight"`
`"[\\w-]-Google|Google-[\\w-]|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight"`
)
const prerenderManifest = JSON.parse(
......@@ -38,22 +38,22 @@ describe('app-dir - metadata-streaming-config', () => {
"/": {
"key": "user-agent",
"type": "header",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
"value": ".*(?:[\\w-]-Google|Google-[\\w-]|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
"/_global-error": {
"key": "user-agent",
"type": "header",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
"value": ".*(?:[\\w-]-Google|Google-[\\w-]|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
"/_not-found": {
"key": "user-agent",
"type": "header",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
"value": ".*(?:[\\w-]-Google|Google-[\\w-]|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
"/ppr": {
"key": "user-agent",
"type": "header",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
"value": ".*(?:[\\w-]-Google|Google-[\\w-]|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
}
`)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment