Unverified Commit d87a203b authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

Fix HTML-limited bot matching in prerender bypass rules (#96584)

## Summary

- preserve `htmlLimitedBots` substring semantics when generating the PPR
cache-bypass user-agent rule
- allow surrounding user-agent text so bots such as `googleweblight`
correctly bypass the prerender shell
- add deployment coverage confirming blocking metadata for bot
identifiers embedded in full user-agent strings, including custom
`htmlLimitedBots` patterns

<!-- NEXT_JS_LLM -->
parent fca0fea5
......@@ -3315,6 +3315,10 @@ export default async function build(
// pattern.
const htmlLimitedBotsRegexString =
config.htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING
// Route `has` matchers anchor header values. Add surrounding
// wildcards to preserve RegExp.test() substring semantics for
// complete user-agent strings.
const htmlLimitedBotsBypassRegexString = `.*(?:${htmlLimitedBotsRegexString}).*`
// this flag is used to selectively bypass the static cache and invoke the lambda directly
// to enable server actions on static routes
......@@ -3332,7 +3336,7 @@ export default async function build(
{
type: 'header' as const,
key: 'user-agent',
value: htmlLimitedBotsRegexString,
value: htmlLimitedBotsBypassRegexString,
},
]
: []),
......
......@@ -27,6 +27,21 @@ describeCacheComponents('metadata streaming with a custom bot list', () => {
expect($('#dynamic-fallback').length).toBe(0)
})
it('should block metadata for a configured bot within a full user agent', async () => {
const res = await next.fetch('/partial', {
headers: {
'user-agent':
'Mozilla/5.0 (compatible; MyBot; +https://example.com/bot)',
},
})
expect(res.status).toBe(200)
const $ = cheerio.load(await res.text())
expect($('head title').text()).toBe('dynamic title')
expect($('body title').length).toBe(0)
})
it('should serve the PPR shell with streamed metadata to regular user agents', async () => {
const res = await next.fetch('/partial')
......
......@@ -33,6 +33,21 @@ function countSubstring(str: string, substr: string): number {
expect($('#dynamic-content').text()).toBe('dynamic content')
})
it('should block metadata for googleweblight within a full user agent', async () => {
const res = await next.fetch('/partial', {
headers: {
'user-agent':
'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19',
},
})
expect(res.status).toBe(200)
const $ = cheerio.load(await res.text())
expect($('head title').text()).toBe('dynamic title')
expect($('body title').length).toBe(0)
})
it('should block metadata while continuing to stream the body for a default HTML-limited bot', async () => {
const abortController = new AbortController()
let body:
......
......@@ -37,22 +37,22 @@ describe('app-dir - metadata-streaming-config-customized', () => {
"/": {
"key": "user-agent",
"type": "header",
"value": "MyBot",
"value": ".*(?:MyBot).*",
},
"/_global-error": {
"key": "user-agent",
"type": "header",
"value": "MyBot",
"value": ".*(?:MyBot).*",
},
"/_not-found": {
"key": "user-agent",
"type": "header",
"value": "MyBot",
"value": ".*(?:MyBot).*",
},
"/ppr": {
"key": "user-agent",
"type": "header",
"value": "MyBot",
"value": ".*(?:MyBot).*",
},
}
`)
......
......@@ -38,24 +38,33 @@ describe('app-dir - metadata-streaming-config', () => {
"/": {
"key": "user-agent",
"type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
"/_global-error": {
"key": "user-agent",
"type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
"/_not-found": {
"key": "user-agent",
"type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
"/ppr": {
"key": "user-agent",
"type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight",
"value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
},
}
`)
const userAgentPattern = bypassConfigs['/ppr'].value
const fullGoogleWebLightUserAgent =
'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19'
// Route `has` matchers anchor values before matching request headers.
const userAgentMatcher = new RegExp(`^${userAgentPattern}$`, 'i')
expect(userAgentMatcher.test(fullGoogleWebLightUserAgent)).toBe(true)
expect(userAgentMatcher.test('Mozilla/5.0')).toBe(false)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment