Unverified Commit c00449d5 authored by Hendrik Liebau's avatar Hendrik Liebau Committed by GitHub

[ci] Wait for the preview tarballs before running the deploy tests (#96527)

The "Test new and changed tests when deployed" jobs install Next.js from
the preview tarballs built for the commit under test, but nothing made
them wait for those tarballs to exist. Until #96438 the job was ordered
behind `test-prod` and the other new-test jobs, which delayed it long
enough that the tarballs had usually landed by the time it looked for
them. Now that it starts immediately it frequently fails with "Artifacts
not found for commit ...".

The ordering cannot be expressed as a `needs:` entry, because the
tarballs are published by a different workflow and, for pull requests
from branches in this repository, a different event than the
`build-and-test` run that consumes them. We therefore add a
`wait-for-preview-tarball` job that polls for the tarball instead, and
make both deploy test jobs depend on it. Doing the waiting in one cheap
job keeps the deploy test matrices off their 16-core runners until there
is something for them to install.

The new job is also listed in `tests-pass`. A dependency failure skips
the jobs that need it, and a skipped job does not count as a failure, so
without that entry a tarball that never arrives would have let the
required check pass with the deploy tests never having run.

`test-new-tests-deploy-cache-components` gets the same treatment,
dropping the incidental ordering it still inherited from
`test-cache-components-prod` and picking up the docs-only guard that
`test-new-tests-deploy` already had.
parent 767e0e98
...@@ -721,9 +721,50 @@ jobs: ...@@ -721,9 +721,50 @@ jobs:
timeout_minutes: 120 # Increase the default timeout as tests are intentionally run multiple times to detect flakes timeout_minutes: 120 # Increase the default timeout as tests are intentionally run multiple times to detect flakes
secrets: inherit secrets: inherit
# The deploy tests install Next.js from the preview tarballs for this commit.
# Those are published by `upload-preview-tarballs`, which runs on
# `workflow_run` after `build-and-deploy` finishes, and for non-fork pull
# requests `build-and-deploy` itself runs on the `push` event rather than
# `pull_request`. The tarballs are therefore produced by a different workflow
# and a different event, which `needs` cannot express, so poll for them here
# instead. Keeping the wait in this cheap job means the deploy test matrices
# do not occupy large runners while the tarballs are still being built.
wait-for-preview-tarball:
name: Wait for preview tarball
needs: ['optimize-ci', 'changes']
# `docs-only` and `is-release` mirror the cases where `build-and-deploy`
# resolves its deploy target to `skipped` and never publishes a tarball.
# test-new-tests-if
if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' && needs.changes.outputs.is-release == 'false' }}
# test-new-tests-end-if
runs-on: ubuntu-latest
# Outer bound only. The script gives up first, so a tarball that never
# arrives is reported by it rather than by the runner killing the job.
timeout-minutes: 35
permissions: {}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
scripts/git-info.mjs
scripts/wait-for-preview-tarball.mjs
sparse-checkout-cone-mode: false
fetch-depth: 1
persist-credentials: false
- name: Wait for preview tarball
env:
PREVIEW_BUILDS_BASE_URL: ${{ vars.PREVIEW_BUILDS_BASE_URL }}
PREVIEW_BUILDS_READ_TOKEN: ${{ secrets.PREVIEW_BUILDS_READ_TOKEN }}
run: |
set -euo pipefail
node scripts/wait-for-preview-tarball.mjs \
--preview-builds-base-url "$PREVIEW_BUILDS_BASE_URL" \
--timeout-minutes 30
test-new-tests-deploy: test-new-tests-deploy:
name: Test new and changed tests when deployed name: Test new and changed tests when deployed
needs: ['optimize-ci', 'changes'] needs: ['optimize-ci', 'changes', 'wait-for-preview-tarball']
# test-new-tests-if # test-new-tests-if
if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}
# test-new-tests-end-if # test-new-tests-end-if
...@@ -752,15 +793,9 @@ jobs: ...@@ -752,15 +793,9 @@ jobs:
test-new-tests-deploy-cache-components: test-new-tests-deploy-cache-components:
name: Test new and changed tests when deployed (cache components) name: Test new and changed tests when deployed (cache components)
needs: needs: ['optimize-ci', 'changes', 'wait-for-preview-tarball']
[
'optimize-ci',
'test-cache-components-prod',
'test-new-tests-dev',
'test-new-tests-start',
]
# test-new-tests-if # test-new-tests-if
if: ${{ needs.optimize-ci.outputs.skip == 'false' }} if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}
# test-new-tests-end-if # test-new-tests-end-if
strategy: strategy:
...@@ -1074,6 +1109,7 @@ jobs: ...@@ -1074,6 +1109,7 @@ jobs:
- test-turbopack-dev - test-turbopack-dev
- test-new-tests-dev - test-new-tests-dev
- test-new-tests-start - test-new-tests-start
- wait-for-preview-tarball
- test-new-tests-deploy - test-new-tests-deploy
- test-new-tests-deploy-cache-components - test-new-tests-deploy-cache-components
- test-turbopack-production - test-turbopack-production
......
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
import execa from 'execa' import execa from 'execa'
import yargs from 'yargs' import yargs from 'yargs'
import getChangedTests from './get-changed-tests.mjs' import getChangedTests from './get-changed-tests.mjs'
import {
assertPreviewTarballPublished,
previewTarballUrl,
} from './wait-for-preview-tarball.mjs'
/** /**
* Run tests for added/changed tests in the current branch * Run tests for added/changed tests in the current branch
...@@ -20,8 +24,9 @@ async function main() { ...@@ -20,8 +24,9 @@ async function main() {
const testMode = argv.mode const testMode = argv.mode
const isFlakeDetectionMode = argv['flake-detection'] const isFlakeDetectionMode = argv['flake-detection']
const attempts = isFlakeDetectionMode ? 3 : 1 const attempts = isFlakeDetectionMode ? 3 : 1
const previewBuildsBaseUrl = // Left unset when the workflow passes an empty value, so that
argv['preview-builds-base-url'] || 'https://vercel-packages.vercel.app/next' // wait-for-preview-tarball.mjs owns the default.
const previewBuildsBaseUrl = argv['preview-builds-base-url']
if (testMode && !['dev', 'deploy', 'start'].includes(testMode)) { if (testMode && !['dev', 'deploy', 'start'].includes(testMode)) {
throw new Error( throw new Error(
...@@ -95,50 +100,19 @@ async function main() { ...@@ -95,50 +100,19 @@ async function main() {
// PR number endpoint (which resolves the PR to a SHA on every request). // PR number endpoint (which resolves the PR to a SHA on every request).
const nextTestVersion = const nextTestVersion =
testMode === 'deploy' testMode === 'deploy'
? `${previewBuildsBaseUrl}/commits/${commitSha}/next` ? previewTarballUrl(previewBuildsBaseUrl, commitSha)
: undefined : undefined
const previewBuildsReadToken = process.env.PREVIEW_BUILDS_READ_TOKEN
if (nextTestVersion) { if (nextTestVersion) {
console.log(`Verifying artifacts for commit ${commitSha}`) // The `wait-for-preview-tarball` job in build_and_test.yml does the
// Attempt to fetch the deploy artifacts for the commit // waiting, so this only asserts. It keeps a missing tarball reported as a
// These might take a moment to become available, so we'll retry a few times // missing tarball, rather than as an install failure from run-tests.js
const fetchHeaders = previewBuildsReadToken // resolving NEXT_TEST_VERSION.
? { Authorization: `Bearer ${previewBuildsReadToken}` } await assertPreviewTarballPublished({
: undefined commitSha,
const fetchWithRetry = async (url, retries = 5, timeout = 5000) => { previewBuildsBaseUrl,
for (let i = 0; i < retries; i++) { readToken: process.env.PREVIEW_BUILDS_READ_TOKEN,
const res = await fetch(url, { headers: fetchHeaders }) })
if (res.ok) {
return res
} else if (i < retries - 1) {
console.log(
`Attempt ${i + 1} failed. Retrying in ${timeout / 1000} seconds...`
)
await new Promise((resolve) => setTimeout(resolve, timeout))
} else {
if (res.status === 404) {
throw new Error(
`Artifacts not found for commit ${commitSha}. ` +
`This can happen if the preview builds either failed or didn't succeed yet. ` +
`Once the "Deploy Preview tarball" job has finished, a retry should fix this error.`
)
}
throw new Error(
`Failed to verify artifacts for commit ${commitSha}: ${res.status}`
)
}
}
}
try {
await fetchWithRetry(nextTestVersion)
console.log(`Artifacts verified for commit ${commitSha}`)
} catch (error) {
console.error(error.message)
throw error
}
} }
// We apply the external tests filter before the process.env so that if // We apply the external tests filter before the process.env so that if
......
// @ts-check
import { setTimeout } from 'node:timers/promises'
import { pathToFileURL } from 'node:url'
import { parseArgs } from 'node:util'
import { getGitInfo } from './git-info.mjs'
const DEFAULT_PREVIEW_BUILDS_BASE_URL =
'https://vercel-packages.vercel.app/next'
// Comfortably above the slowest observed build-and-deploy plus
// upload-preview-tarballs, which together have topped out around 27 minutes.
const DEFAULT_TIMEOUT_MINUTES = 30
const POLL_INTERVAL_MS = 15_000
const PROGRESS_LOG_INTERVAL_MS = 60_000
/**
* URL of the `next` preview tarball for a commit. `vercel-packages` answers
* with a redirect to Vercel Blob, which only serves the tarball once
* `upload-preview-tarballs` has published it.
*
* @param {string | undefined} baseUrl
* @param {string} commitSha
* @returns {string}
*/
export function previewTarballUrl(baseUrl, commitSha) {
return `${baseUrl || DEFAULT_PREVIEW_BUILDS_BASE_URL}/commits/${commitSha}/next`
}
/**
* @param {number} milliseconds
* @returns {string}
*/
function formatDuration(milliseconds) {
const totalSeconds = Math.round(milliseconds / 1000)
const minutes = Math.floor(totalSeconds / 60)
const seconds = totalSeconds % 60
return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`
}
/**
* URL of the commit's check runs, where the `build-and-deploy` status is
* visible. Undefined outside of GitHub Actions, which is the only place the
* server and repository are known.
*
* @param {string} commitSha
* @returns {string | undefined}
*/
function commitChecksUrl(commitSha) {
const serverUrl = process.env.GITHUB_SERVER_URL
const repository = process.env.GITHUB_REPOSITORY
if (!serverUrl || !repository) {
return undefined
}
return `${serverUrl}/${repository}/commit/${commitSha}/checks`
}
/**
* Requests the tarball with `HEAD` so polling stays cheap: a `GET` would
* download the whole multi-megabyte tarball on every attempt.
*
* `fetch` follows the redirect, so `response.ok` reflects the blob and not the
* unconditional redirect that `vercel-packages` returns. Anything that inspects
* this URL must follow redirects too, otherwise the redirect itself reads as
* success while the tarball is still missing.
*
* `lastResponse` describes the outcome for the progress and failure messages.
*
* @param {string} url
* @param {Record<string, string> | undefined} headers
* @returns {Promise<{ published: boolean, lastResponse: string }>}
*/
async function probeTarball(url, headers) {
try {
const response = await fetch(url, { method: 'HEAD', headers })
return {
published: response.ok,
lastResponse:
response.status === 404 ? 'not published yet' : `${response.status}`,
}
} catch (error) {
return {
published: false,
lastResponse: `request failed (${error instanceof Error ? error.message : error})`,
}
}
}
/**
* @param {string | undefined} readToken
* @returns {Record<string, string> | undefined}
*/
function requestHeaders(readToken) {
return readToken ? { Authorization: `Bearer ${readToken}` } : undefined
}
/**
* @param {object} options
* @param {string} options.commitSha
* @param {string} options.lastResponse
* @param {number} [options.timeoutMs] Omitted when nothing was waited out.
* @returns {Error}
*/
function notPublishedError({ commitSha, lastResponse, timeoutMs }) {
const checksUrl = commitChecksUrl(commitSha)
return new Error(
`Preview tarball for commit ${commitSha} was not published` +
(timeoutMs === undefined ? '' : ` within ${formatDuration(timeoutMs)}`) +
` (last response: ${lastResponse}). ` +
`The tarball is published by the "upload-preview-tarballs" workflow ` +
`once "build-and-deploy" has completed for this commit, so check ` +
`whether that run failed or is still in progress.` +
(checksUrl ? ` See ${checksUrl}` : '')
)
}
/**
* Checks once whether the `next` preview tarball for `commitSha` is
* downloadable, and throws if it is not. For callers that only need the
* assertion because something else has already done the waiting.
*
* @param {object} options
* @param {string} options.commitSha
* @param {string} [options.previewBuildsBaseUrl]
* @param {string} [options.readToken]
* @returns {Promise<void>}
*/
export async function assertPreviewTarballPublished({
commitSha,
previewBuildsBaseUrl,
readToken,
}) {
const url = previewTarballUrl(previewBuildsBaseUrl, commitSha)
const { published, lastResponse } = await probeTarball(
url,
requestHeaders(readToken)
)
if (!published) {
throw notPublishedError({ commitSha, lastResponse })
}
console.info(`Preview tarball for commit ${commitSha} is available at ${url}`)
}
/**
* Polls until the `next` preview tarball for `commitSha` is downloadable.
* Rejects when `timeoutMs` elapses before that happens. Every response other
* than a success is treated as "not ready", so a transient blob or edge error
* does not end the wait early.
*
* @param {object} options
* @param {string} options.commitSha
* @param {string} [options.previewBuildsBaseUrl]
* @param {number} options.timeoutMs
* @param {string} [options.readToken]
* @param {number} [options.pollIntervalMs]
* @returns {Promise<void>}
*/
export async function waitForPreviewTarball({
commitSha,
previewBuildsBaseUrl,
timeoutMs,
readToken,
pollIntervalMs = POLL_INTERVAL_MS,
}) {
const url = previewTarballUrl(previewBuildsBaseUrl, commitSha)
const headers = requestHeaders(readToken)
const startedAt = Date.now()
const deadline = startedAt + timeoutMs
let lastProgressLogAt = startedAt
console.info(
`Waiting up to ${formatDuration(timeoutMs)} for the preview tarball at ${url}`
)
for (;;) {
const { published, lastResponse } = await probeTarball(url, headers)
const now = Date.now()
if (published) {
console.info(
`Preview tarball for commit ${commitSha} is available after ${formatDuration(now - startedAt)}`
)
return
}
if (now >= deadline) {
throw notPublishedError({ commitSha, lastResponse, timeoutMs })
}
if (now - lastProgressLogAt >= PROGRESS_LOG_INTERVAL_MS) {
console.info(
`Still waiting after ${formatDuration(now - startedAt)} (last response: ${lastResponse})`
)
lastProgressLogAt = now
}
// Capping the sleep at the remaining time keeps the last probe on the
// deadline rather than past it.
await setTimeout(Math.min(pollIntervalMs, deadline - now))
}
}
async function main() {
const { values } = parseArgs({
options: {
'commit-sha': { type: 'string' },
'preview-builds-base-url': { type: 'string' },
'timeout-minutes': { type: 'string' },
},
})
// Resolving the commit through `getGitInfo` rather than from a workflow
// expression keeps this in step with `test-new-tests.mjs`, which reaches the
// same function via `getChangedTests` to build the URL it installs from. The
// two have to agree, otherwise this waits for a tarball the tests never ask
// for.
const commitSha = values['commit-sha'] ?? (await getGitInfo()).commitSha
const rawTimeoutMinutes = values['timeout-minutes']
const timeoutMinutes =
rawTimeoutMinutes === undefined
? DEFAULT_TIMEOUT_MINUTES
: Number(rawTimeoutMinutes)
if (!Number.isFinite(timeoutMinutes) || timeoutMinutes <= 0) {
throw new Error(
`--timeout-minutes must be a positive number but got "${rawTimeoutMinutes}"`
)
}
await waitForPreviewTarball({
commitSha,
previewBuildsBaseUrl: values['preview-builds-base-url'],
timeoutMs: timeoutMinutes * 60_000,
readToken: process.env.PREVIEW_BUILDS_READ_TOKEN,
})
}
// `test-new-tests.mjs` imports this module for its own verification, so the CLI
// only runs when the file is the entry point.
if (
process.argv[1] &&
pathToFileURL(process.argv[1]).href === import.meta.url
) {
main().catch((error) => {
console.error(error instanceof Error ? error.message : error)
process.exit(1)
})
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment