Unverified Commit cefd9cf2 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add crawler blocking for fallback: true (#29121)

* Add crawler blocking for fallback: true

* update test

* Update check

* Add note to docs

* use browser agent for non-crawler test
parent e98690c3
...@@ -510,7 +510,7 @@ export default Post ...@@ -510,7 +510,7 @@ export default Post
If `fallback` is `true`, then the behavior of `getStaticProps` changes: If `fallback` is `true`, then the behavior of `getStaticProps` changes:
- The paths returned from `getStaticPaths` will be rendered to HTML at build time by `getStaticProps`. - The paths returned from `getStaticPaths` will be rendered to HTML at build time by `getStaticProps`.
- The paths that have not been generated at build time will **not** result in a 404 page. Instead, Next.js will serve a “fallback” version of the page on the first request to such a path (see [“Fallback pages”](#fallback-pages) below for details). - The paths that have not been generated at build time will **not** result in a 404 page. Instead, Next.js will serve a “fallback” version of the page on the first request to such a path (see [“Fallback pages”](#fallback-pages) below for details). Note: this "fallback" version will not be served for crawlers like Google and instead will render the path in `blocking` mode.
- In the background, Next.js will statically generate the requested path HTML and JSON. This includes running `getStaticProps`. - In the background, Next.js will statically generate the requested path HTML and JSON. This includes running `getStaticProps`.
- When that’s done, the browser receives the JSON for the generated path. This will be used to automatically render the page with the required props. From the user’s perspective, the page will be swapped from the fallback page to the full page. - When that’s done, the browser receives the JSON for the generated path. This will be used to automatically render the page with the required props. From the user’s perspective, the page will be swapped from the fallback page to the full page.
- At the same time, Next.js adds this path to the list of pre-rendered pages. Subsequent requests to the same path will serve the generated page, like other pages pre-rendered at build time. - At the same time, Next.js adds this path to the list of pre-rendered pages. Subsequent requests to the same path will serve the generated page, like other pages pre-rendered at build time.
......
...@@ -1711,10 +1711,17 @@ export default class Server { ...@@ -1711,10 +1711,17 @@ export default class Server {
const isDynamicPathname = isDynamicRoute(pathname) const isDynamicPathname = isDynamicRoute(pathname)
const didRespond = hasResolved || isResSent(res) const didRespond = hasResolved || isResSent(res)
const { staticPaths, fallbackMode } = hasStaticPaths let { staticPaths, fallbackMode } = hasStaticPaths
? await this.getStaticPaths(pathname) ? await this.getStaticPaths(pathname)
: { staticPaths: undefined, fallbackMode: false } : { staticPaths: undefined, fallbackMode: false }
if (
fallbackMode === 'static' &&
isBot(req.headers['user-agent'] || '')
) {
fallbackMode = 'blocking'
}
// When we did not respond from cache, we need to choose to block on // When we did not respond from cache, we need to choose to block on
// rendering or return a skeleton. // rendering or return a skeleton.
// //
......
...@@ -16,7 +16,7 @@ export function cleanAmpPath(pathname: string): string { ...@@ -16,7 +16,7 @@ export function cleanAmpPath(pathname: string): string {
} }
export function isBot(userAgent: string): boolean { export function isBot(userAgent: string): boolean {
return /Googlebot|Mediapartners-Google|AdsBot-Google|googleweblight|Storebot-Google|Bingbot|BingPreview|Slurp|DuckDuckBot|baiduspider|yandex|sogou|LinkedInBot|bitlybot|tumblr|vkShare|quora link preview|facebookexternalhit|facebookcatalog|Twitterbot|applebot|redditbot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i.test( return /Googlebot|Mediapartners-Google|AdsBot-Google|googleweblight|Storebot-Google|Bingbot|BingPreview|Slurp|DuckDuckBot|baiduspider|yandex|sogou|LinkedInBot|bitlybot|tumblr|vkShare|quora link preview|facebookexternalhit|facebookcatalog|Twitterbot|applebot|redditbot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|ia_archiver/i.test(
userAgent userAgent
) )
} }
...@@ -6,7 +6,13 @@ describe('should set-up next', () => { ...@@ -6,7 +6,13 @@ describe('should set-up next', () => {
beforeAll(async () => { beforeAll(async () => {
next = await createNext({ next = await createNext({
files: { 'pages/index.js': 'export default () => "hello" ' }, files: {
'pages/index.js': `
export default function Page() {
return <p>hello world</p>
}
`
},
}) })
}) })
afterAll(() => next.destroy()) afterAll(() => next.destroy())
......
import { createNext } from 'e2e-utils'
import { renderViaHTTP } from 'next-test-utils'
import { NextInstance } from 'test/lib/next-modes/base'
describe('Prerender crawler handling', () => {
let next: NextInstance
beforeAll(async () => {
next = await createNext({
files: {
'pages/index.js': `
export default function Page() {
return <p>index page</p>
}
`,
'pages/blog/[slug].js': `
import {useRouter} from 'next/router'
export default function Page({ slug }) {
const router = useRouter()
if (router.isFallback) {
return 'Loading...'
}
return (
<>
<p id='page'>slug page</p>
<p id='slug'>{slug}</p>
</>
)
}
export async function getStaticProps({ params }) {
return {
props: {
slug: params.slug
}
}
}
export async function getStaticPaths() {
return {
paths: ['/blog/first'],
fallback: true
}
}
`,
},
})
})
afterAll(() => next.destroy())
it('should return prerendered page for correctly', async () => {
const html = await renderViaHTTP(next.url, '/blog/first')
expect(html).not.toContain('Loading...')
expect(html).toContain('first')
expect(html).toContain('slug page')
})
it('should return fallback for non-crawler correctly', async () => {
const html = await renderViaHTTP(next.url, '/blog/test-1', undefined, {
headers: {
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36',
},
})
expect(html).toContain('Loading...')
})
it('should block for crawler correctly', async () => {
const userAgents = [
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
'Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)',
'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
'DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)',
'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
'facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)',
'ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)',
]
for (const userAgent of userAgents) {
console.log('checking', { userAgent })
const uniqueSlug = `test-${Date.now()}${Math.random()}`
const html = await renderViaHTTP(
next.url,
`/blog/${uniqueSlug}`,
undefined,
{
headers: {
'user-agent': userAgent,
},
}
)
expect(html).not.toContain('Loading...')
expect(html).toContain(uniqueSlug)
expect(html).toContain('slug page')
}
})
})
...@@ -24,6 +24,11 @@ const testModeFromFile = testFolderModes.find((mode) => ...@@ -24,6 +24,11 @@ const testModeFromFile = testFolderModes.find((mode) =>
if (testModeFromFile === 'e2e') { if (testModeFromFile === 'e2e') {
const validE2EModes = ['dev', 'start', 'deploy'] const validE2EModes = ['dev', 'start', 'deploy']
if (!process.env.NEXT_TEST_JOB) {
console.warn('Warn: no NEXT_TEST_MODE set, using default of start')
testMode = 'start'
}
assert( assert(
validE2EModes.includes(testMode), validE2EModes.includes(testMode),
`NEXT_TEST_MODE must be one of ${validE2EModes.join( `NEXT_TEST_MODE must be one of ${validE2EModes.join(
......
...@@ -57,7 +57,11 @@ export class NextStartInstance extends NextInstance { ...@@ -57,7 +57,11 @@ export class NextStartInstance extends NextInstance {
}) })
this._buildId = ( this._buildId = (
await fs.readFile( await fs.readFile(
path.join(this.testDir, this.nextConfig.distDir || '.next', 'BUILD_ID'), path.join(
this.testDir,
this.nextConfig?.distDir || '.next',
'BUILD_ID'
),
'utf8' 'utf8'
) )
).trim() ).trim()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment