Unverified Commit 7ece5e6e authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

test: retry vercel deploy on transient pre-deployment network errors (#100070)

## Summary

This retries `vercel deploy` when it fails with a transient network
error before the CLI reports a deployment. It is the third of three
stacked PRs.

**Why.** On v16.5.0-canary.6 attempt 1
([example](https://github.com/vercel/next.js/actions/runs/37957356976/job/113914293394)),
`vercel deploy` printed `Error: fetch failed` and no deployment URL. The
harness failed straight away; the cause it reported was a misleading
`Invalid URL`.

**When it retries.** All of these must hold:
- `vercel deploy` exits non-zero;
- stdout isn't a URL, and the output (with ANSI codes stripped) has no
`Inspect`/`Preview`/`Production` URL line;
- the output has a CLI `Error:` line with a network failure: `fetch
failed`, `ECONNRESET`, `ETIMEDOUT`, `EAI_AGAIN`, `ECONNREFUSED`,
`UND_ERR_SOCKET`, `UND_ERR_CONNECT_TIMEOUT` or `socket hang up`;
- the output doesn't contain an auth or build diagnostic
(`Unauthorized`, `Forbidden`, `Build failed`, `Command "…" exited`, …).

**Limits.**
- At most 3 attempts in total, waiting 5 s and then 10 s between them.
- It only retries while at least 60 s of the setup budget would remain
after the wait.
- Every other failure goes through the existing `throwDeploymentError`
path unchanged, with no extra deploy.

**Accepted risk.** If the connection drops after Vercel accepted the
upload, a retry can leave an orphaned preview deployment (the harness
deploys with `--force`). That's at most 2 extra previews per suite start
in the test project. The retry log line says so and includes the CLI
output.

This PR also fixes a typo in a comment in
`test/e2e/app-dir/resume-data-cache/resume-data-cache.test.ts`. That
change is there so `Test new and changed tests when deployed` runs a
deploy-mode suite against the full stack.

## Verification

- `pnpm jest test/unit/next-deploy-failure`: 43/43 pass.
- **Retries:** retry-then-succeed, giving up after 3 calls, and skipping
the retry when too little budget is left.
- **No retry (exactly one deploy):** auth error, build failure, a
network error after a URL or an `Inspect` line (including ANSI-colored
output), a build diagnostic that mentions `fetch failed`, and
`Unauthorized` next to `fetch failed`.
- prettier, eslint and `pnpm typescript` are clean.
- Not run: a real Vercel deployment (the sandbox has no test
credentials).

<!-- NEXT_JS_LLM -->


<!-- fleet b78d4bc5-dee1-4d80-b647-54229882f21f -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 4bcd07f9
...@@ -364,7 +364,7 @@ function getUniqueRegexMatches(text: string, pattern: RegExp) { ...@@ -364,7 +364,7 @@ function getUniqueRegexMatches(text: string, pattern: RegExp) {
for (const match of text.matchAll( for (const match of text.matchAll(
new RegExp(pattern.source, pattern.flags + 'g') new RegExp(pattern.source, pattern.flags + 'g')
)) { )) {
// `matchAll` returns a match array for each occurence. // `matchAll` returns a match array for each occurrence.
// We don't care about groups, so just take the whole match. // We don't care about groups, so just take the whole match.
uniqueMatches.add(match[0]) uniqueMatches.add(match[0])
} }
......
...@@ -44,22 +44,23 @@ function hasAllBuildLogMarkers(output: string): boolean { ...@@ -44,22 +44,23 @@ function hasAllBuildLogMarkers(output: string): boolean {
const VERCEL_CLI_OWN_LINE = const VERCEL_CLI_OWN_LINE =
/^(?:Vercel CLI \d|Fetching deployment |status\s|>\s|Error:|WARN!|Warning:)/ /^(?:Vercel CLI \d|Fetching deployment |status\s|>\s|Error:|WARN!|Warning:)/
function stripAnsi(output: string): string {
// eslint-disable-next-line no-control-regex -- strip ANSI escape codes
return output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '')
}
/** /**
* Whether `vercel inspect --logs` output contains any build-log lines, rather * Whether `vercel inspect --logs` output contains any build-log lines, rather
* than only the CLI's own header and status lines (which it prints even when * than only the CLI's own header and status lines (which it prints even when
* the log query returns nothing). * the log query returns nothing).
*/ */
function hasBuildLogLines(output: string): boolean { function hasBuildLogLines(output: string): boolean {
return ( return stripAnsi(output)
output .split('\n')
// eslint-disable-next-line no-control-regex -- strip ANSI escape codes .some((line) => {
.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '') const trimmed = line.trim()
.split('\n') return trimmed !== '' && !VERCEL_CLI_OWN_LINE.test(trimmed)
.some((line) => { })
const trimmed = line.trim()
return trimmed !== '' && !VERCEL_CLI_OWN_LINE.test(trimmed)
})
)
} }
/** /**
...@@ -73,6 +74,38 @@ function getSetupHookTimeoutMs(): number { ...@@ -73,6 +74,38 @@ function getSetupHookTimeoutMs(): number {
return (process.platform === 'win32' ? 240 : 120) * 1000 return (process.platform === 'win32' ? 240 : 120) * 1000
} }
// `vercel deploy` retries for network errors that happen before the CLI
// reports a deployment, e.g. `Error: fetch failed` while uploading.
const DEPLOY_RETRY_DELAYS_MS = [5_000, 10_000]
const DEPLOY_RETRY_MIN_REMAINING_MS = 60_000
// An `Error:` line the CLI prints for a failed request.
const TRANSIENT_NETWORK_ERROR_LINE =
/^Error: .*(?:fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|socket hang up)/m
// Diagnostics that mean a retry can't help, even next to a network error.
const NON_RETRYABLE_DEPLOY_ERROR =
/\b(?:Unauthorized|Forbidden|Not authorized|invalid token|Build failed)\b|Command ".*" exited/i
// The `Inspect`/`Preview`/`Production` lines the CLI prints once a deployment
// exists.
const DEPLOYMENT_URL_LINE = /^\s*(?:Inspect|Preview|Production)\s+https?:\/\//m
function isTransientPreUrlDeployFailure(result: {
exitCode: number
stdout: string
stderr: string
}): boolean {
if (result.exitCode === 0) return false
try {
new URL(result.stdout.trim())
return false
} catch {}
const output = stripAnsi(`${result.stdout}\n${result.stderr}`)
return (
!DEPLOYMENT_URL_LINE.test(output) &&
!NON_RETRYABLE_DEPLOY_ERROR.test(output) &&
TRANSIENT_NETWORK_ERROR_LINE.test(output)
)
}
export class NextDeployInstance extends NextInstance { export class NextDeployInstance extends NextInstance {
private _cliOutput: string private _cliOutput: string
private _buildId: string private _buildId: string
...@@ -641,41 +674,53 @@ export class NextDeployInstance extends NextInstance { ...@@ -641,41 +674,53 @@ export class NextDeployInstance extends NextInstance {
additionalEnv.push(`NEXT_ENABLE_ADAPTER=0`) additionalEnv.push(`NEXT_ENABLE_ADAPTER=0`)
} }
const deployment = execa( const deployArgs = [
'vercel', 'deploy',
[ '--build-env',
'deploy', 'NEXT_PRIVATE_TEST_MODE=e2e',
'--build-env', '--build-env',
'NEXT_PRIVATE_TEST_MODE=e2e', 'NEXT_TELEMETRY_DISABLED=1',
'--build-env', '--build-env',
'NEXT_TELEMETRY_DISABLED=1', 'VERCEL_NEXT_BUNDLED_SERVER=1',
'--build-env', // See: https://pnpm.io/npmrc#environment-variables-in-auth-settings
'VERCEL_NEXT_BUNDLED_SERVER=1', ...(this._npmrcAuthFile
// See: https://pnpm.io/npmrc#environment-variables-in-auth-settings ? ['--build-env', `PNPM_CONFIG_NPMRC_AUTH_FILE=${this._npmrcAuthFile}`]
...(this._npmrcAuthFile : []),
? [ ...additionalEnv.flatMap((pair) => ['--env', pair, '--build-env', pair]),
'--build-env', '--force',
`PNPM_CONFIG_NPMRC_AUTH_FILE=${this._npmrcAuthFile}`, ...vercelFlags,
] ]
: []), const deployDeadline =
...additionalEnv.flatMap((pair) => [ deployStartedAt + getSetupHookTimeoutMs() - SETUP_HOOK_MARGIN_MS
'--env', let deployRes: execa.ExecaReturnValue
pair, for (let attempt = 1; ; attempt++) {
'--build-env', const deployment = execa('vercel', deployArgs, {
pair,
]),
'--force',
...vercelFlags,
],
{
cwd: this.testDir, cwd: this.testDir,
env: vercelEnv, env: vercelEnv,
reject: false, reject: false,
})
// Keep showing deployment progress while also retaining failure output.
deployment.stderr?.pipe(process.stderr)
deployRes = await deployment
// Retry only network errors from before the CLI reported a deployment,
// and only while enough of the setup budget remains to build it.
const retryDelayMs = DEPLOY_RETRY_DELAYS_MS[attempt - 1]
if (
retryDelayMs === undefined ||
!isTransientPreUrlDeployFailure(deployRes) ||
deployDeadline - Date.now() - retryDelayMs <
DEPLOY_RETRY_MIN_REMAINING_MS
) {
break
} }
) // The request may have reached Vercel before the connection dropped, so
// Keep showing deployment progress while also retaining failure output. // a retry can leave an orphaned preview deployment behind.
deployment.stderr?.pipe(process.stderr) require('console').log(
const deployRes = await deployment `vercel deploy failed with a transient network error before reporting a deployment (attempt ${attempt}/${DEPLOY_RETRY_DELAYS_MS.length + 1}); retrying in ${retryDelayMs}ms. A deployment may have been created anyway.\n${deployRes.stdout}${deployRes.stderr}`
)
await setTimeout(retryDelayMs)
}
if (deployRes.exitCode !== 0) { if (deployRes.exitCode !== 0) {
await this.throwDeploymentError( await this.throwDeploymentError(
......
...@@ -53,10 +53,13 @@ describe('deployment lifecycle', () => { ...@@ -53,10 +53,13 @@ describe('deployment lifecycle', () => {
let inspectResults: Array<Result | Error | ((call: InspectCall) => Result)> let inspectResults: Array<Result | Error | ((call: InspectCall) => Result)>
let inspectCalls: InspectCall[] let inspectCalls: InspectCall[]
let deployDurationMs: number let deployDurationMs: number
// Results for successive `vercel deploy` calls; `deployResult` once exhausted.
let deployResults: Result[]
beforeEach(() => { beforeEach(() => {
mockClock = null mockClock = null
deployDurationMs = 0 deployDurationMs = 0
deployResults = []
inspectResults = [] inspectResults = []
inspectCalls = [] inspectCalls = []
jest.replaceProperty(process, 'env', { jest.replaceProperty(process, 'env', {
...@@ -116,7 +119,7 @@ describe('deployment lifecycle', () => { ...@@ -116,7 +119,7 @@ describe('deployment lifecycle', () => {
break break
case 'deploy': case 'deploy':
if (mockClock) mockClock.now += deployDurationMs if (mockClock) mockClock.now += deployDurationMs
result = deployResult result = deployResults.shift() ?? deployResult
break break
case 'inspect': { case 'inspect': {
const [, , options] = jest const [, , options] = jest
...@@ -591,4 +594,92 @@ describe('deployment lifecycle', () => { ...@@ -591,4 +594,92 @@ describe('deployment lifecycle', () => {
) )
}) })
}) })
describe('retrying vercel deploy', () => {
const fetchFailed: Result = {
exitCode: 1,
stdout: '',
stderr:
'Vercel CLI 63.1.0\nDeploying vtest314-e2e-tests\nError: fetch failed\n',
}
const deployCalls = () =>
jest.mocked(execa).mock.calls.filter(([, args]) => args?.[0] === 'deploy')
beforeEach(() => {
// Pin the setup budget, whose default depends on the platform.
process.env.NEXT_E2E_TEST_TIMEOUT = '120000'
})
it('retries a network error from before a deployment was reported', async () => {
const clock = installClock()
successfulDeployment()
deployResults = [fetchFailed]
const next = await instance()
await next.start()
expect(next.buildId).toBe('build-id')
expect(deployCalls()).toHaveLength(2)
expect(clock.waits).toEqual([5_000])
})
it('gives up after three attempts', async () => {
const clock = installClock()
deployResult = fetchFailed
const next = await instance()
await expect(next.start()).rejects.toThrow(
'Failed to deploy project: Vercel CLI 63.1.0'
)
expect(deployCalls()).toHaveLength(3)
expect(clock.waits).toEqual([5_000, 10_000])
})
it('does not retry when too little of the setup budget is left', async () => {
installClock(40_000)
deployResult = fetchFailed
const next = await instance()
await expect(next.start()).rejects.toThrow('Failed to deploy project')
expect(deployCalls()).toHaveLength(1)
})
it.each([
['an authentication error', { stdout: '', stderr: 'Unauthorized' }],
[
'a network error after a deployment URL was printed',
{ stdout: deploymentUrl, stderr: 'Error: read ECONNRESET' },
],
[
'a network error after the Inspect line',
{
stdout: '',
stderr:
' Inspect https://vercel.com/team/project/abc\nError: fetch failed',
},
],
['a build failure', { stdout: deploymentUrl, stderr: 'Build failed' }],
[
'a build diagnostic that mentions fetch failed',
{
stdout: '',
stderr: 'Build failed\n TypeError: fetch failed\n at prerender\n',
},
],
[
'an authentication error next to a network error',
{ stdout: '', stderr: 'Error: Unauthorized\nError: fetch failed\n' },
],
[
'a network error after an ANSI-colored Inspect line',
{
stdout: '',
stderr:
'\x1b[2K\x1b[G Inspect \x1b[36mhttps://vercel.com/team/project/abc\x1b[39m\nError: fetch failed',
},
],
])('does not retry %s', async (_, output) => {
installClock()
deployResult = { exitCode: 1, ...output }
const next = await instance()
await expect(next.start()).rejects.toThrow('Failed to deploy project')
expect(deployCalls()).toHaveLength(1)
})
})
}) })
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment