Unverified Commit f55a064c authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

add experimental flag for retrying static generation (#66393)

Transient/recoverable errors can cause page(s) to fail. When building an
app with a lot of static pages, a single page failing to build can be
frustrating, especially if it it'll pass on a retry. This adds an
experimental `staticPageGenerationTimeout` property that can be used to
configure how many times Next.js attempts to build a particular page
before giving up.

I recommend reviewing this PR [without
whitespace](https://github.com/vercel/next.js/pull/66393/files?w=1)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the
PR.
- Read the Docs Contribution Guide to ensure your contribution follows
the docs guidelines:
https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->
parent a0155c79
......@@ -513,6 +513,7 @@ export async function exportAppImpl(
}
const failedExportAttemptsByPage: Map<string, number> = new Map()
const maxAttempts = nextConfig.experimental.staticGenerationRetryCount ?? 1
const results = await Promise.all(
filteredPaths.map(async (path) => {
const pathMap = exportPathMap[path]
......@@ -528,50 +529,98 @@ export async function exportAppImpl(
const pageExportSpan = span.traceChild('export-page')
pageExportSpan.setAttribute('path', path)
const result = await pageExportSpan.traceAsyncFn(async () => {
return await exportPage({
dir,
path,
pathMap,
distDir,
outDir,
pagesDataDir,
renderOpts,
ampValidatorPath: nextConfig.experimental.amp?.validator || undefined,
trailingSlash: nextConfig.trailingSlash,
serverRuntimeConfig,
subFolders,
buildExport: options.buildExport,
optimizeFonts: nextConfig.optimizeFonts as FontConfig,
optimizeCss: nextConfig.experimental.optimizeCss,
disableOptimizedLoading:
nextConfig.experimental.disableOptimizedLoading,
parentSpanId: pageExportSpan.getId(),
httpAgentOptions: nextConfig.httpAgentOptions,
debugOutput: options.debugOutput,
cacheMaxMemorySize: nextConfig.cacheMaxMemorySize,
fetchCache: true,
fetchCacheKeyPrefix: nextConfig.experimental.fetchCacheKeyPrefix,
cacheHandler: nextConfig.cacheHandler,
enableExperimentalReact: needsExperimentalReact(nextConfig),
enabledDirectories,
})
})
if (result && 'error' in result) {
const { page } = exportPathMap[path]
const pageKey = page !== path ? `${page}: ${path}` : path
const currentCount = failedExportAttemptsByPage.get(pageKey) ?? 0
failedExportAttemptsByPage.set(pageKey, currentCount + 1)
if (nextConfig.experimental.prerenderEarlyExit) {
throw new ExportError(
`Export encountered an error on ${path}, exiting the build.`
)
const { page } = exportPathMap[path]
const pageKey = page !== path ? `${page}: ${path}` : path
let result
for (let attempt = 0; attempt < maxAttempts; attempt++) {
try {
result = await pageExportSpan.traceAsyncFn(async () => {
return await exportPage({
dir,
path,
pathMap,
distDir,
outDir,
pagesDataDir,
renderOpts,
ampValidatorPath:
nextConfig.experimental.amp?.validator || undefined,
trailingSlash: nextConfig.trailingSlash,
serverRuntimeConfig,
subFolders,
buildExport: options.buildExport,
optimizeFonts: nextConfig.optimizeFonts as FontConfig,
optimizeCss: nextConfig.experimental.optimizeCss,
disableOptimizedLoading:
nextConfig.experimental.disableOptimizedLoading,
parentSpanId: pageExportSpan.getId(),
httpAgentOptions: nextConfig.httpAgentOptions,
debugOutput: options.debugOutput,
cacheMaxMemorySize: nextConfig.cacheMaxMemorySize,
fetchCache: true,
fetchCacheKeyPrefix: nextConfig.experimental.fetchCacheKeyPrefix,
cacheHandler: nextConfig.cacheHandler,
enableExperimentalReact: needsExperimentalReact(nextConfig),
enabledDirectories,
})
})
// If there was an error in the export, throw it immediately. In the catch block, we might retry the export,
// or immediately fail the build, depending on user configuration. We might also continue on and attempt other pages.
if (result && 'error' in result) {
throw new ExportError()
}
// If the export succeeds, break out of the retry loop
break
} catch (err) {
// The only error that should be caught here is an ExportError, as `exportPage` doesn't throw and instead returns an object with an `error` property.
// This is an overly cautious check to ensure that we don't accidentally catch an unexpected error.
if (!(err instanceof ExportError)) {
throw err
}
const currentCount = failedExportAttemptsByPage.get(pageKey) ?? 0
failedExportAttemptsByPage.set(pageKey, currentCount + 1)
// We've reached the maximum number of attempts
if (attempt >= maxAttempts - 1) {
// Log a message if we've reached the maximum number of attempts.
// We only care to do this if maxAttempts was configured.
if (maxAttempts > 0) {
Log.info(
`Failed to build ${pageKey} after ${maxAttempts} attempts.`
)
}
// If prerenderEarlyExit is enabled, we'll exit the build immediately.
if (nextConfig.experimental.prerenderEarlyExit) {
throw new ExportError(
`Export encountered an error on ${pageKey}, exiting the build.`
)
} else {
// Otherwise, this is a no-op. The build will continue, and a summary of failed pages will be displayed at the end.
}
} else {
// Otherwise, we have more attempts to make. Wait before retrying
Log.info(
`Failed to build ${pageKey} (attempt ${attempt + 1} of ${maxAttempts}). Retrying again shortly.`
)
await new Promise((r) => setTimeout(r, Math.random() * 500))
}
}
}
// if we eventually succeeded, remove the page from the failed attempts map
if (
failedExportAttemptsByPage.has(pageKey) &&
result &&
!('error' in result)
) {
failedExportAttemptsByPage.delete(pageKey)
}
if (progress) progress()
return { result, path }
......
......@@ -435,6 +435,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
})
.optional(),
]),
staticGenerationRetryCount: z.number().int().optional(),
})
.optional(),
exportPathMap: z
......
......@@ -484,6 +484,11 @@ export interface ExperimentalConfig {
* Enables `unstable_after`
*/
after?: boolean
/**
* The number of times to retry static generation (per page) before giving up.
*/
staticGenerationRetryCount?: number
}
export type ExportPathMap = {
......@@ -985,6 +990,7 @@ export const defaultConfig: NextConfig = {
allowDevelopmentBuild: undefined,
reactCompiler: undefined,
after: false,
staticGenerationRetryCount: undefined,
},
bundlePagesRouterDependencies: false,
}
......
export default function Page() {
throw new Error('This page is bad!')
}
module.exports = {
experimental: {
staticGenerationRetryCount: 3,
},
}
export default function Layout({ children }) {
return (
<html>
<body>{children}</body>
</html>
)
}
......@@ -2,10 +2,14 @@ import { nextBuild } from 'next-test-utils'
describe('worker-restart', () => {
it('should properly exhaust all restart attempts and not fail with any worker errors', async () => {
const { stdout, stderr } = await nextBuild(__dirname, [], {
stdout: true,
stderr: true,
})
const { stdout, stderr } = await nextBuild(
__dirname + '/fixtures/timeout',
[],
{
stdout: true,
stderr: true,
}
)
const output = stdout + stderr
expect(output).toContain(
......@@ -24,4 +28,22 @@ describe('worker-restart', () => {
'Error: Farm is ended, no more calls can be done to it'
)
})
it('should support configurable static generation retries', async () => {
const { stdout, stderr } = await nextBuild(
__dirname + '/fixtures/retries',
[],
{
stdout: true,
stderr: true,
}
)
const output = stdout + stderr
expect(output).toContain('This page is bad!')
expect(output).toContain(
'Failed to build /page: / (attempt 2 of 3). Retrying again shortly.'
)
expect(output).toContain('Failed to build /page: / after 3 attempts.')
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment