Commit eef18ea6 authored by Tim Neutkens's avatar Tim Neutkens

feat(metadata): support non-standard directives in robots.ts via `other`

Adds an `other` field on each rule in `MetadataRoute.Robots` so users can
declare non-standard robots.txt directives (e.g. Seznam `Request-Rate`,
Yandex `Clean-param`) alongside the standard fields. Values are passed
through verbatim and scoped to the rule's User-Agent block. Array values
emit one line per entry.

Fixes #89521
parent de62a966
......@@ -119,6 +119,59 @@ Disallow: /
Sitemap: https://acme.com/sitemap.xml
```
### Non-standard directives
Some search engines support directives that aren't part of the [Robots Exclusion Standard](https://en.wikipedia.org/wiki/Robots.txt#Standard), such as `Request-Rate` (Seznam) or `Clean-param` (Yandex). Pass these through the `other` field on a rule. Keys preserve their casing and array values emit one line per entry, scoped to the rule's `User-Agent` block.
```ts filename="app/robots.ts" switcher
import type { MetadataRoute } from 'next'
export default function robots(): MetadataRoute.Robots {
return {
rules: [
{ userAgent: '*', allow: '/' },
{
userAgent: 'SeznamBot',
allow: '/',
other: {
'Request-Rate': '10/1m',
},
},
],
}
}
```
```js filename="app/robots.js" switcher
export default function robots() {
return {
rules: [
{ userAgent: '*', allow: '/' },
{
userAgent: 'SeznamBot',
allow: '/',
other: {
'Request-Rate': '10/1m',
},
},
],
}
}
```
Output:
```txt
User-Agent: *
Allow: /
User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
```
> **Good to know**: Values in `other` are passed through verbatim. Next.js does not validate directive names or values, so refer to the target search engine's documentation for the exact syntax.
### Robots object
```tsx
......@@ -129,12 +182,14 @@ type Robots = {
allow?: string | string[]
disallow?: string | string[]
crawlDelay?: number
other?: Record<string, string | number | Array<string | number>>
}
| Array<{
userAgent: string | string[]
allow?: string | string[]
disallow?: string | string[]
crawlDelay?: number
other?: Record<string, string | number | Array<string | number>>
}>
sitemap?: string | string[]
host?: string
......@@ -143,6 +198,7 @@ type Robots = {
## Version History
| Version | Changes |
| --------- | -------------------- |
| `v13.3.0` | `robots` introduced. |
| Version | Changes |
| --------- | ---------------------------------------------------------- |
| `v16.3.0` | Added `other` field for non-standard per-agent directives. |
| `v13.3.0` | `robots` introduced. |
......@@ -73,6 +73,72 @@ describe('resolveRouteData', () => {
"
`)
})
it('should resolve non-standard per-user-agent directives via `other`', () => {
const data: MetadataRoute.Robots = {
rules: [
{
userAgent: '*',
allow: '/',
},
{
userAgent: 'SeznamBot',
allow: '/',
other: {
// https://o-seznam.cz/napoveda/vyhledavani/en/crawling-control/
'Request-Rate': '10/1m',
},
},
{
userAgent: 'Yandex',
allow: '/',
other: {
'Clean-param': ['ref /articles/', 'utm_source /'],
},
},
],
}
expect(resolveRobots(data)).toMatchInlineSnapshot(`
"User-Agent: *
Allow: /
User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
User-Agent: Yandex
Allow: /
Clean-param: ref /articles/
Clean-param: utm_source /
"
`)
})
it('should skip null/undefined entries in `other`', () => {
const data: MetadataRoute.Robots = {
rules: {
userAgent: 'SeznamBot',
allow: '/',
other: {
'Request-Rate': '10/1m',
// @ts-expect-error intentionally testing null handling
'Visit-time': null,
// @ts-expect-error intentionally testing undefined handling
'Clean-param': undefined,
},
},
}
expect(resolveRobots(data)).toMatchInlineSnapshot(`
"User-Agent: SeznamBot
Allow: /
Request-Rate: 10/1m
"
`)
})
})
describe('resolveSitemap', () => {
......
......@@ -25,6 +25,16 @@ export function resolveRobots(data: MetadataRoute.Robots): string {
if (rule.crawlDelay) {
content += `Crawl-delay: ${rule.crawlDelay}\n`
}
if (rule.other) {
for (const key of Object.keys(rule.other)) {
const value = rule.other[key]
if (value == null) continue
const values = Array.isArray(value) ? value : [value]
for (const v of values) {
content += `${key}: ${v}\n`
}
}
}
content += '\n'
}
if (data.host) {
......
......@@ -674,22 +674,40 @@ export type WithStringifiedURLs<T> = T extends URL
*/
type ResolvedMetadata = WithStringifiedURLs<ResolvedMetadataWithURLs>
type RobotsRuleBase = {
allow?: string | string[] | undefined
disallow?: string | string[] | undefined
crawlDelay?: number | undefined
/**
* Non-standard per-user-agent directives passed through verbatim to the
* generated `robots.txt`. Keys preserve their casing and array values emit
* one line per entry.
*
* @example
* ```ts
* // Seznam rate-limiting
* // https://o-seznam.cz/napoveda/vyhledavani/en/crawling-control/
* other: { 'Request-Rate': '10/1m' }
*
* // Yandex Clean-param (multiple values)
* other: { 'Clean-param': ['ref /articles/', 'utm_source /'] }
* ```
*/
other?: Record<string, string | number | Array<string | number>> | undefined
}
type RobotsFile = {
// Apply rules for all
rules:
| {
| (RobotsRuleBase & {
userAgent?: string | string[] | undefined
allow?: string | string[] | undefined
disallow?: string | string[] | undefined
crawlDelay?: number | undefined
}
})
// Apply rules for specific user agents
| Array<{
userAgent: string | string[]
allow?: string | string[] | undefined
disallow?: string | string[] | undefined
crawlDelay?: number | undefined
}>
| Array<
RobotsRuleBase & {
userAgent: string | string[]
}
>
sitemap?: string | string[] | undefined
host?: string | undefined
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment