Unverified Commit d87a203b authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

Fix HTML-limited bot matching in prerender bypass rules (#96584)

## Summary

- preserve `htmlLimitedBots` substring semantics when generating the PPR
cache-bypass user-agent rule
- allow surrounding user-agent text so bots such as `googleweblight`
correctly bypass the prerender shell
- add deployment coverage confirming blocking metadata for bot
identifiers embedded in full user-agent strings, including custom
`htmlLimitedBots` patterns

<!-- NEXT_JS_LLM -->
parent fca0fea5
...@@ -3315,6 +3315,10 @@ export default async function build( ...@@ -3315,6 +3315,10 @@ export default async function build(
// pattern. // pattern.
const htmlLimitedBotsRegexString = const htmlLimitedBotsRegexString =
config.htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING config.htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING
// Route `has` matchers anchor header values. Add surrounding
// wildcards to preserve RegExp.test() substring semantics for
// complete user-agent strings.
const htmlLimitedBotsBypassRegexString = `.*(?:${htmlLimitedBotsRegexString}).*`
// this flag is used to selectively bypass the static cache and invoke the lambda directly // this flag is used to selectively bypass the static cache and invoke the lambda directly
// to enable server actions on static routes // to enable server actions on static routes
...@@ -3332,7 +3336,7 @@ export default async function build( ...@@ -3332,7 +3336,7 @@ export default async function build(
{ {
type: 'header' as const, type: 'header' as const,
key: 'user-agent', key: 'user-agent',
value: htmlLimitedBotsRegexString, value: htmlLimitedBotsBypassRegexString,
}, },
] ]
: []), : []),
......
...@@ -27,6 +27,21 @@ describeCacheComponents('metadata streaming with a custom bot list', () => { ...@@ -27,6 +27,21 @@ describeCacheComponents('metadata streaming with a custom bot list', () => {
expect($('#dynamic-fallback').length).toBe(0) expect($('#dynamic-fallback').length).toBe(0)
}) })
it('should block metadata for a configured bot within a full user agent', async () => {
const res = await next.fetch('/partial', {
headers: {
'user-agent':
'Mozilla/5.0 (compatible; MyBot; +https://example.com/bot)',
},
})
expect(res.status).toBe(200)
const $ = cheerio.load(await res.text())
expect($('head title').text()).toBe('dynamic title')
expect($('body title').length).toBe(0)
})
it('should serve the PPR shell with streamed metadata to regular user agents', async () => { it('should serve the PPR shell with streamed metadata to regular user agents', async () => {
const res = await next.fetch('/partial') const res = await next.fetch('/partial')
......
...@@ -33,6 +33,21 @@ function countSubstring(str: string, substr: string): number { ...@@ -33,6 +33,21 @@ function countSubstring(str: string, substr: string): number {
expect($('#dynamic-content').text()).toBe('dynamic content') expect($('#dynamic-content').text()).toBe('dynamic content')
}) })
it('should block metadata for googleweblight within a full user agent', async () => {
const res = await next.fetch('/partial', {
headers: {
'user-agent':
'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19',
},
})
expect(res.status).toBe(200)
const $ = cheerio.load(await res.text())
expect($('head title').text()).toBe('dynamic title')
expect($('body title').length).toBe(0)
})
it('should block metadata while continuing to stream the body for a default HTML-limited bot', async () => { it('should block metadata while continuing to stream the body for a default HTML-limited bot', async () => {
const abortController = new AbortController() const abortController = new AbortController()
let body: let body:
......
...@@ -37,22 +37,22 @@ describe('app-dir - metadata-streaming-config-customized', () => { ...@@ -37,22 +37,22 @@ describe('app-dir - metadata-streaming-config-customized', () => {
"/": { "/": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "MyBot", "value": ".*(?:MyBot).*",
}, },
"/_global-error": { "/_global-error": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "MyBot", "value": ".*(?:MyBot).*",
}, },
"/_not-found": { "/_not-found": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "MyBot", "value": ".*(?:MyBot).*",
}, },
"/ppr": { "/ppr": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "MyBot", "value": ".*(?:MyBot).*",
}, },
} }
`) `)
......
...@@ -38,24 +38,33 @@ describe('app-dir - metadata-streaming-config', () => { ...@@ -38,24 +38,33 @@ describe('app-dir - metadata-streaming-config', () => {
"/": { "/": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight", "value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
}, },
"/_global-error": { "/_global-error": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight", "value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
}, },
"/_not-found": { "/_not-found": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight", "value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
}, },
"/ppr": { "/ppr": {
"key": "user-agent", "key": "user-agent",
"type": "header", "type": "header",
"value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight", "value": ".*(?:[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight).*",
}, },
} }
`) `)
const userAgentPattern = bypassConfigs['/ppr'].value
const fullGoogleWebLightUserAgent =
'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19'
// Route `has` matchers anchor values before matching request headers.
const userAgentMatcher = new RegExp(`^${userAgentPattern}$`, 'i')
expect(userAgentMatcher.test(fullGoogleWebLightUserAgent)).toBe(true)
expect(userAgentMatcher.test('Mozilla/5.0')).toBe(false)
}) })
}) })
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment