Unverified Commit 383ec5a6 authored by Shu Ding's avatar Shu Ding Committed by GitHub

Fix static generation and crawler requests (#41735)

Right now the SSG condition is determined _only_ based on
`supportsDynamicHTML`. However that `supportsDynamicHTML` flag can be
affected by bots too.

Here we exclude the `isBot` condition from the SSG flag, and set proper
status and meta tags for static HTML requests (and/or 404 cases).

## Bug

- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)
Co-authored-by: 's avatarJJ Kasper <jj@jjsweb.site>
parent 3a0fc134
...@@ -142,6 +142,7 @@ export type RenderOptsPartial = { ...@@ -142,6 +142,7 @@ export type RenderOptsPartial = {
serverComponents?: boolean serverComponents?: boolean
assetPrefix?: string assetPrefix?: string
fontLoaderManifest?: FontLoaderManifest fontLoaderManifest?: FontLoaderManifest
isBot?: boolean
} }
export type RenderOpts = LoadComponentsReturnType & RenderOptsPartial export type RenderOpts = LoadComponentsReturnType & RenderOptsPartial
...@@ -172,9 +173,12 @@ function createErrorHandler( ...@@ -172,9 +173,12 @@ function createErrorHandler(
* Used for debugging * Used for debugging
*/ */
_source: string, _source: string,
capturedErrors: Error[] capturedErrors: Error[],
allCapturedErrors?: Error[]
) { ) {
return (err: any): string => { return (err: any): string => {
if (allCapturedErrors) allCapturedErrors.push(err)
if ( if (
err.digest === DYNAMIC_ERROR_CODE || err.digest === DYNAMIC_ERROR_CODE ||
err.digest === NOT_FOUND_ERROR_CODE || err.digest === NOT_FOUND_ERROR_CODE ||
...@@ -700,10 +704,12 @@ export async function renderToHTMLOrFlight( ...@@ -700,10 +704,12 @@ export async function renderToHTMLOrFlight(
* These rules help ensure that other existing features like request caching, * These rules help ensure that other existing features like request caching,
* coalescing, and ISR continue working as intended. * coalescing, and ISR continue working as intended.
*/ */
const isStaticGeneration = renderOpts.supportsDynamicHTML !== true const isStaticGeneration =
renderOpts.supportsDynamicHTML !== true && !renderOpts.isBot
const isFlight = req.headers.__rsc__ !== undefined const isFlight = req.headers.__rsc__ !== undefined
const capturedErrors: Error[] = [] const capturedErrors: Error[] = []
const allCapturedErrors: Error[] = []
const serverComponentsErrorHandler = createErrorHandler( const serverComponentsErrorHandler = createErrorHandler(
'serverComponentsRenderer', 'serverComponentsRenderer',
...@@ -715,7 +721,8 @@ export async function renderToHTMLOrFlight( ...@@ -715,7 +721,8 @@ export async function renderToHTMLOrFlight(
) )
const htmlRendererErrorHandler = createErrorHandler( const htmlRendererErrorHandler = createErrorHandler(
'htmlRenderer', 'htmlRenderer',
capturedErrors capturedErrors,
allCapturedErrors
) )
const { const {
...@@ -726,9 +733,11 @@ export async function renderToHTMLOrFlight( ...@@ -726,9 +733,11 @@ export async function renderToHTMLOrFlight(
ComponentMod, ComponentMod,
dev, dev,
fontLoaderManifest, fontLoaderManifest,
supportsDynamicHTML,
} = renderOpts } = renderOpts
patchFetch(ComponentMod) patchFetch(ComponentMod)
const generateStaticHTML = supportsDynamicHTML !== true
const staticGenerationAsyncStorage = ComponentMod.staticGenerationAsyncStorage const staticGenerationAsyncStorage = ComponentMod.staticGenerationAsyncStorage
const requestAsyncStorage = ComponentMod.requestAsyncStorage const requestAsyncStorage = ComponentMod.requestAsyncStorage
...@@ -1505,20 +1514,31 @@ export async function renderToHTMLOrFlight( ...@@ -1505,20 +1514,31 @@ export async function renderToHTMLOrFlight(
}, },
}) })
return await continueFromInitialStream(renderStream, { const result = await continueFromInitialStream(renderStream, {
dataStream: serverComponentsInlinedTransformStream?.readable, dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML: isStaticGeneration, generateStaticHTML: isStaticGeneration || generateStaticHTML,
getServerInsertedHTML, getServerInsertedHTML,
serverInsertedHTMLToHead: true, serverInsertedHTMLToHead: true,
...validateRootLayout, ...validateRootLayout,
}) })
return result
} catch (err: any) { } catch (err: any) {
const shouldNotIndex = err.digest === NOT_FOUND_ERROR_CODE
if (err.digest === NOT_FOUND_ERROR_CODE) {
res.statusCode = 404
}
// TODO-APP: show error overlay in development. `element` should probably be wrapped in AppRouter for this case. // TODO-APP: show error overlay in development. `element` should probably be wrapped in AppRouter for this case.
const renderStream = await renderToInitialStream({ const renderStream = await renderToInitialStream({
ReactDOMServer, ReactDOMServer,
element: ( element: (
<html id="__next_error__"> <html id="__next_error__">
<head></head> <head>
{shouldNotIndex ? (
<meta name="robots" content="noindex" />
) : null}
</head>
<body></body> <body></body>
</html> </html>
), ),
......
...@@ -205,6 +205,7 @@ export default abstract class Server<ServerOptions extends Options = Options> { ...@@ -205,6 +205,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
serverComponents?: boolean serverComponents?: boolean
crossOrigin?: string crossOrigin?: string
supportsDynamicHTML?: boolean supportsDynamicHTML?: boolean
isBot?: boolean
serverComponentManifest?: any serverComponentManifest?: any
serverCSSManifest?: any serverCSSManifest?: any
fontLoaderManifest?: FontLoaderManifest fontLoaderManifest?: FontLoaderManifest
...@@ -868,6 +869,7 @@ export default abstract class Server<ServerOptions extends Options = Options> { ...@@ -868,6 +869,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
renderOpts: { renderOpts: {
...this.renderOpts, ...this.renderOpts,
supportsDynamicHTML: !isBotRequest, supportsDynamicHTML: !isBotRequest,
isBot: !!isBotRequest,
}, },
} as const } as const
const payload = await fn(ctx) const payload = await fn(ctx)
...@@ -1160,6 +1162,7 @@ export default abstract class Server<ServerOptions extends Options = Options> { ...@@ -1160,6 +1162,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
// cache if there are no dynamic data requirements // cache if there are no dynamic data requirements
opts.supportsDynamicHTML = opts.supportsDynamicHTML =
!isSSG && !isBotRequest && !query.amp && isSupportedDocument !isSSG && !isBotRequest && !query.amp && isSupportedDocument
opts.isBot = isBotRequest
} }
const defaultLocale = isSSG const defaultLocale = isSSG
......
...@@ -253,6 +253,7 @@ export type RenderOptsPartial = { ...@@ -253,6 +253,7 @@ export type RenderOptsPartial = {
domainLocales?: DomainLocale[] domainLocales?: DomainLocale[]
disableOptimizedLoading?: boolean disableOptimizedLoading?: boolean
supportsDynamicHTML?: boolean supportsDynamicHTML?: boolean
isBot?: boolean
runtime?: ServerRuntime runtime?: ServerRuntime
serverComponents?: boolean serverComponents?: boolean
customServer?: boolean customServer?: boolean
......
...@@ -1899,6 +1899,24 @@ describe('app dir', () => { ...@@ -1899,6 +1899,24 @@ describe('app dir', () => {
}) })
}) })
describe('bots', () => {
it('should block rendering for bots and return 404 status', async () => {
const res = await fetchViaHTTP(
next.url,
'/not-found/servercomponent',
'',
{
headers: {
'User-Agent': 'Googlebot',
},
}
)
expect(res.status).toBe(404)
expect(await res.text()).toInclude('"noindex"')
})
})
describe('redirect', () => { describe('redirect', () => {
describe('components', () => { describe('components', () => {
it('should redirect in a server component', async () => { it('should redirect in a server component', async () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment