Unverified Commit 65340b27 authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

feat(metadata): support non-standard directives in robots.ts via `other` (#93206)

### What?

Adds an `other` field to each rule in `MetadataRoute.Robots` so
`app/robots.ts` can emit non-standard per-user-agent directives (e.g.
Seznam `Request-Rate`, Yandex `Clean-param`) alongside the standard
`allow`/`disallow`/`crawlDelay` fields.

```ts
// app/robots.ts
export default function robots(): MetadataRoute.Robots {
  return {
    rules: [
      { userAgent: '*', allow: '/' },
      {
        userAgent: 'SeznamBot',
        allow: '/',
        other: { 'Request-Rate': '10/1m' },
      },
    ],
  }
}
```

emits:

```
User-Agent: *
Allow: /

User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
```

### Why?

The current `resolveRobots` serializer only writes the four standard
rule fields and silently drops everything else, so users who need
non-standard directives (common for Seznam, Yandex, and other crawlers)
have to delete `robots.ts` entirely and hand-maintain a static
`robots.txt`, losing the type-safe, code-driven workflow.

### How?

- **Type**
(`packages/next/src/lib/metadata/types/metadata-interface.ts`): factored
a shared `RobotsRuleBase` containing the existing optional fields plus
`other?: Record<string, string | number | Array<string | number>>`.
`RobotsFile.rules` still enforces `userAgent` as required on the array
form.
- **Serializer**
(`packages/next/src/build/webpack/loaders/metadata/resolve-route-data.ts`):
after `Crawl-delay` and before the trailing blank line, walks
`rule.other` and emits `Key: value` lines preserving casing. Array
values expand into repeated lines; `null`/`undefined` entries are
skipped. Keeps non-standard directives scoped to their `User-Agent`
block.
- **Tests**: added two `resolveRobots` cases covering the `Request-Rate`
reproduction and an array-valued `Clean-param`, plus null/undefined
handling. Existing snapshots are unchanged, proving the change is
additive.
- **Docs**: new "Non-standard directives" section in
`docs/01-app/03-api-reference/03-file-conventions/01-metadata/robots.mdx`
with TS/JS examples and updated `Robots` type reference.

Fixes #89521

<!-- NEXT_JS_LLM_PR -->
parent b2491e30
...@@ -119,6 +119,59 @@ Disallow: / ...@@ -119,6 +119,59 @@ Disallow: /
Sitemap: https://acme.com/sitemap.xml Sitemap: https://acme.com/sitemap.xml
``` ```
### Non-standard directives
Some search engines support directives that aren't part of the [Robots Exclusion Standard](https://en.wikipedia.org/wiki/Robots.txt#Standard), such as `Request-Rate` (Seznam) or `Clean-param` (Yandex). Pass these through the `other` field on a rule. Keys preserve their casing and array values emit one line per entry, scoped to the rule's `User-Agent` block.
```ts filename="app/robots.ts" switcher
import type { MetadataRoute } from 'next'
export default function robots(): MetadataRoute.Robots {
return {
rules: [
{ userAgent: '*', allow: '/' },
{
userAgent: 'SeznamBot',
allow: '/',
other: {
'Request-Rate': '10/1m',
},
},
],
}
}
```
```js filename="app/robots.js" switcher
export default function robots() {
return {
rules: [
{ userAgent: '*', allow: '/' },
{
userAgent: 'SeznamBot',
allow: '/',
other: {
'Request-Rate': '10/1m',
},
},
],
}
}
```
Output:
```txt
User-Agent: *
Allow: /
User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
```
> **Good to know**: Values in `other` are passed through verbatim. Next.js does not validate directive names or values, so refer to the target search engine's documentation for the exact syntax.
### Robots object ### Robots object
```tsx ```tsx
...@@ -129,12 +182,14 @@ type Robots = { ...@@ -129,12 +182,14 @@ type Robots = {
allow?: string | string[] allow?: string | string[]
disallow?: string | string[] disallow?: string | string[]
crawlDelay?: number crawlDelay?: number
other?: Record<string, string | number | Array<string | number>>
} }
| Array<{ | Array<{
userAgent: string | string[] userAgent: string | string[]
allow?: string | string[] allow?: string | string[]
disallow?: string | string[] disallow?: string | string[]
crawlDelay?: number crawlDelay?: number
other?: Record<string, string | number | Array<string | number>>
}> }>
sitemap?: string | string[] sitemap?: string | string[]
host?: string host?: string
...@@ -143,6 +198,7 @@ type Robots = { ...@@ -143,6 +198,7 @@ type Robots = {
## Version History ## Version History
| Version | Changes | | Version | Changes |
| --------- | -------------------- | | --------- | ---------------------------------------------------------- |
| `v13.3.0` | `robots` introduced. | | `v16.3.0` | Added `other` field for non-standard per-agent directives. |
| `v13.3.0` | `robots` introduced. |
...@@ -73,6 +73,72 @@ describe('resolveRouteData', () => { ...@@ -73,6 +73,72 @@ describe('resolveRouteData', () => {
" "
`) `)
}) })
it('should resolve non-standard per-user-agent directives via `other`', () => {
const data: MetadataRoute.Robots = {
rules: [
{
userAgent: '*',
allow: '/',
},
{
userAgent: 'SeznamBot',
allow: '/',
other: {
// https://o-seznam.cz/napoveda/vyhledavani/en/crawling-control/
'Request-Rate': '10/1m',
},
},
{
userAgent: 'Yandex',
allow: '/',
other: {
'Clean-param': ['ref /articles/', 'utm_source /'],
},
},
],
}
expect(resolveRobots(data)).toMatchInlineSnapshot(`
"User-Agent: *
Allow: /
User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
User-Agent: Yandex
Allow: /
Clean-param: ref /articles/
Clean-param: utm_source /
"
`)
})
it('should skip null/undefined entries in `other`', () => {
const data: MetadataRoute.Robots = {
rules: {
userAgent: 'SeznamBot',
allow: '/',
other: {
'Request-Rate': '10/1m',
// @ts-expect-error intentionally testing null handling
'Visit-time': null,
// @ts-expect-error intentionally testing undefined handling
'Clean-param': undefined,
},
},
}
expect(resolveRobots(data)).toMatchInlineSnapshot(`
"User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
"
`)
})
}) })
describe('resolveSitemap', () => { describe('resolveSitemap', () => {
......
...@@ -25,6 +25,16 @@ export function resolveRobots(data: MetadataRoute.Robots): string { ...@@ -25,6 +25,16 @@ export function resolveRobots(data: MetadataRoute.Robots): string {
if (rule.crawlDelay) { if (rule.crawlDelay) {
content += `Crawl-delay: ${rule.crawlDelay}\n` content += `Crawl-delay: ${rule.crawlDelay}\n`
} }
if (rule.other) {
for (const key of Object.keys(rule.other)) {
const value = rule.other[key]
if (value == null) continue
const values = Array.isArray(value) ? value : [value]
for (const v of values) {
content += `${key}: ${v}\n`
}
}
}
content += '\n' content += '\n'
} }
if (data.host) { if (data.host) {
......
...@@ -674,22 +674,40 @@ export type WithStringifiedURLs<T> = T extends URL ...@@ -674,22 +674,40 @@ export type WithStringifiedURLs<T> = T extends URL
*/ */
type ResolvedMetadata = WithStringifiedURLs<ResolvedMetadataWithURLs> type ResolvedMetadata = WithStringifiedURLs<ResolvedMetadataWithURLs>
type RobotsRuleBase = {
allow?: string | string[] | undefined
disallow?: string | string[] | undefined
crawlDelay?: number | undefined
/**
* Non-standard per-user-agent directives passed through verbatim to the
* generated `robots.txt`. Keys preserve their casing and array values emit
* one line per entry.
*
* @example
* ```ts
* // Seznam rate-limiting
* // https://o-seznam.cz/napoveda/vyhledavani/en/crawling-control/
* other: { 'Request-Rate': '10/1m' }
*
* // Yandex Clean-param (multiple values)
* other: { 'Clean-param': ['ref /articles/', 'utm_source /'] }
* ```
*/
other?: Record<string, string | number | Array<string | number>> | undefined
}
type RobotsFile = { type RobotsFile = {
// Apply rules for all // Apply rules for all
rules: rules:
| { | (RobotsRuleBase & {
userAgent?: string | string[] | undefined userAgent?: string | string[] | undefined
allow?: string | string[] | undefined })
disallow?: string | string[] | undefined
crawlDelay?: number | undefined
}
// Apply rules for specific user agents // Apply rules for specific user agents
| Array<{ | Array<
userAgent: string | string[] RobotsRuleBase & {
allow?: string | string[] | undefined userAgent: string | string[]
disallow?: string | string[] | undefined }
crawlDelay?: number | undefined >
}>
sitemap?: string | string[] | undefined sitemap?: string | string[] | undefined
host?: string | undefined host?: string | undefined
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment