Unverified Commit 383ec5a6 authored by Shu Ding's avatar Shu Ding Committed by GitHub

Fix static generation and crawler requests (#41735)

Right now the SSG condition is determined _only_ based on
`supportsDynamicHTML`. However that `supportsDynamicHTML` flag can be
affected by bots too.

Here we exclude the `isBot` condition from the SSG flag, and set proper
status and meta tags for static HTML requests (and/or 404 cases).

## Bug

- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)
Co-authored-by: 's avatarJJ Kasper <jj@jjsweb.site>
parent 3a0fc134
......@@ -142,6 +142,7 @@ export type RenderOptsPartial = {
serverComponents?: boolean
assetPrefix?: string
fontLoaderManifest?: FontLoaderManifest
isBot?: boolean
}
export type RenderOpts = LoadComponentsReturnType & RenderOptsPartial
......@@ -172,9 +173,12 @@ function createErrorHandler(
* Used for debugging
*/
_source: string,
capturedErrors: Error[]
capturedErrors: Error[],
allCapturedErrors?: Error[]
) {
return (err: any): string => {
if (allCapturedErrors) allCapturedErrors.push(err)
if (
err.digest === DYNAMIC_ERROR_CODE ||
err.digest === NOT_FOUND_ERROR_CODE ||
......@@ -700,10 +704,12 @@ export async function renderToHTMLOrFlight(
* These rules help ensure that other existing features like request caching,
* coalescing, and ISR continue working as intended.
*/
const isStaticGeneration = renderOpts.supportsDynamicHTML !== true
const isStaticGeneration =
renderOpts.supportsDynamicHTML !== true && !renderOpts.isBot
const isFlight = req.headers.__rsc__ !== undefined
const capturedErrors: Error[] = []
const allCapturedErrors: Error[] = []
const serverComponentsErrorHandler = createErrorHandler(
'serverComponentsRenderer',
......@@ -715,7 +721,8 @@ export async function renderToHTMLOrFlight(
)
const htmlRendererErrorHandler = createErrorHandler(
'htmlRenderer',
capturedErrors
capturedErrors,
allCapturedErrors
)
const {
......@@ -726,9 +733,11 @@ export async function renderToHTMLOrFlight(
ComponentMod,
dev,
fontLoaderManifest,
supportsDynamicHTML,
} = renderOpts
patchFetch(ComponentMod)
const generateStaticHTML = supportsDynamicHTML !== true
const staticGenerationAsyncStorage = ComponentMod.staticGenerationAsyncStorage
const requestAsyncStorage = ComponentMod.requestAsyncStorage
......@@ -1505,20 +1514,31 @@ export async function renderToHTMLOrFlight(
},
})
return await continueFromInitialStream(renderStream, {
const result = await continueFromInitialStream(renderStream, {
dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML: isStaticGeneration,
generateStaticHTML: isStaticGeneration || generateStaticHTML,
getServerInsertedHTML,
serverInsertedHTMLToHead: true,
...validateRootLayout,
})
return result
} catch (err: any) {
const shouldNotIndex = err.digest === NOT_FOUND_ERROR_CODE
if (err.digest === NOT_FOUND_ERROR_CODE) {
res.statusCode = 404
}
// TODO-APP: show error overlay in development. `element` should probably be wrapped in AppRouter for this case.
const renderStream = await renderToInitialStream({
ReactDOMServer,
element: (
<html id="__next_error__">
<head></head>
<head>
{shouldNotIndex ? (
<meta name="robots" content="noindex" />
) : null}
</head>
<body></body>
</html>
),
......
......@@ -205,6 +205,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
serverComponents?: boolean
crossOrigin?: string
supportsDynamicHTML?: boolean
isBot?: boolean
serverComponentManifest?: any
serverCSSManifest?: any
fontLoaderManifest?: FontLoaderManifest
......@@ -868,6 +869,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
renderOpts: {
...this.renderOpts,
supportsDynamicHTML: !isBotRequest,
isBot: !!isBotRequest,
},
} as const
const payload = await fn(ctx)
......@@ -1160,6 +1162,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
// cache if there are no dynamic data requirements
opts.supportsDynamicHTML =
!isSSG && !isBotRequest && !query.amp && isSupportedDocument
opts.isBot = isBotRequest
}
const defaultLocale = isSSG
......
......@@ -253,6 +253,7 @@ export type RenderOptsPartial = {
domainLocales?: DomainLocale[]
disableOptimizedLoading?: boolean
supportsDynamicHTML?: boolean
isBot?: boolean
runtime?: ServerRuntime
serverComponents?: boolean
customServer?: boolean
......
......@@ -1899,6 +1899,24 @@ describe('app dir', () => {
})
})
describe('bots', () => {
it('should block rendering for bots and return 404 status', async () => {
const res = await fetchViaHTTP(
next.url,
'/not-found/servercomponent',
'',
{
headers: {
'User-Agent': 'Googlebot',
},
}
)
expect(res.status).toBe(404)
expect(await res.text()).toInclude('"noindex"')
})
})
describe('redirect', () => {
describe('components', () => {
it('should redirect in a server component', async () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment