Unverified Commit d0682022 authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

Separate bots detection utils (#74000)

Co-authored-by: 's avatarZack Tanner <1939140+ztanner@users.noreply.github.com>
parent 6dcfb1fc
const BOT_UA_RE = // Bot crawler that will spin up a headless browser and execute JS
/Googlebot|Mediapartners-Google|AdsBot-Google|googleweblight|Storebot-Google|Google-PageRenderer|Bingbot|BingPreview|Slurp|DuckDuckBot|baiduspider|yandex|sogou|LinkedInBot|bitlybot|tumblr|vkShare|quora link preview|facebookexternalhit|facebookcatalog|Twitterbot|applebot|redditbot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|ia_archiver/i const HEADLESS_BOT_UA_RE =
/Googlebot|Google-PageRenderer|AdsBot-Google|googleweblight|Storebot-Google/i
// This regex contains the bots that we need to do a blocking render for and can't safely stream the response
// due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
const HTML_LIMITED_BOT_UA_RE =
/Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i
function isHeadlessBrowserBot(userAgent: string) {
return HEADLESS_BOT_UA_RE.test(userAgent)
}
function isHtmlLimitedBot(userAgent: string) {
return HTML_LIMITED_BOT_UA_RE.test(userAgent)
}
export function isBot(userAgent: string): boolean { export function isBot(userAgent: string): boolean {
return BOT_UA_RE.test(userAgent) return isHeadlessBrowserBot(userAgent) || isHtmlLimitedBot(userAgent)
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment