Unverified Commit e464ca39 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Fix pr-status.js: retry on API errors and report timed-out jobs (#92205)

### What?

Fixes two bugs in `scripts/pr-status.js` that caused the script to
silently report zero failures when there were actual CI failures.

### Why?

When running `node scripts/pr-status.js 92080`, the script reported "No
failed jobs found" despite the PR having real failures. Two root causes:

1. **Transient API errors silently swallowed.** `getAllJobs()` had a
bare `catch { break }` that returned an empty array on any GitHub API
error (e.g., HTTP 502). Since the GitHub Actions jobs API for large runs
frequently returns transient 502s, this caused false "no failures"
reports.

2. **`timed_out` and `startup_failure` jobs were invisible.** The script
only checked for `conclusion === 'failure'`, but GitHub uses distinct
conclusion values like `timed_out` (job exceeded timeout) and
`startup_failure` (runner failed to start). These jobs fell through all
filters and were silently omitted from reports.

### How?

**Retry logic in `getAllJobs()`:**
- Retries each paginated API call up to 3 times with 2s/4s backoff
- If all retries fail on the first page, throws an error (no silent
empty results)
- If later pages fail after partial data is collected, warns and returns
what was fetched

**Broader failure detection with `FAILED_CONCLUSIONS`:**
- Added a shared `FAILED_CONCLUSIONS` set: `{'failure', 'timed_out',
'startup_failure'}`
- Used in `getFailedJobs()`, `categorizeJobs()`, and flaky test
detection
- Jobs with non-`failure` conclusions are annotated in the report table
(e.g., "(timed_out)")
- `getFailedJobs()` now also returns the `conclusion` field so
downstream code can distinguish failure types
Co-authored-by: 's avatarTobias Koppers <sokra@users.noreply.github.com>
Co-authored-by: 's avatarClaude <noreply@anthropic.com>
parent 3421858b
...@@ -216,6 +216,8 @@ function getRunMetadata(runId) { ...@@ -216,6 +216,8 @@ function getRunMetadata(runId) {
) )
} }
const FAILED_CONCLUSIONS = new Set(['failure', 'timed_out', 'startup_failure'])
function getFailedJobs(runId) { function getFailedJobs(runId) {
// Fetch all jobs first, then filter for failures in JS. // Fetch all jobs first, then filter for failures in JS.
// We can't use jq filtering during pagination because a page full of // We can't use jq filtering during pagination because a page full of
...@@ -223,8 +225,8 @@ function getFailedJobs(runId) { ...@@ -223,8 +225,8 @@ function getFailedJobs(runId) {
// stop pagination before reaching later pages that contain failures. // stop pagination before reaching later pages that contain failures.
const allJobs = getAllJobs(runId) const allJobs = getAllJobs(runId)
return allJobs return allJobs
.filter((j) => j.conclusion === 'failure') .filter((j) => FAILED_CONCLUSIONS.has(j.conclusion))
.map((j) => ({ id: j.id, name: j.name })) .map((j) => ({ id: j.id, name: j.name, conclusion: j.conclusion }))
} }
function getAllJobs(runId) { function getAllJobs(runId) {
...@@ -235,11 +237,38 @@ function getAllJobs(runId) { ...@@ -235,11 +237,38 @@ function getAllJobs(runId) {
const jqQuery = const jqQuery =
'.jobs[] | {id, name, status, conclusion, started_at, completed_at}' '.jobs[] | {id, name, status, conclusion, started_at, completed_at}'
let output let output
let lastError
// Retry up to 3 times for transient API errors (e.g. HTTP 502)
for (let attempt = 1; attempt <= 3; attempt++) {
try { try {
output = exec( output = exec(
`gh api "repos/vercel/next.js/actions/runs/${runId}/jobs?per_page=100&page=${page}" --jq '${jqQuery}'` `gh api "repos/vercel/next.js/actions/runs/${runId}/jobs?per_page=100&page=${page}" --jq '${jqQuery}'`
) )
} catch { lastError = null
break
} catch (error) {
lastError = error
if (attempt < 3) {
const delay = attempt * 2000
console.error(
`API request failed (attempt ${attempt}/3), retrying in ${delay / 1000}s...`
)
execSync(`sleep ${delay / 1000}`)
}
}
}
if (lastError) {
// If all retries failed on the first page, we have no data at all — throw
// so callers know the fetch failed instead of silently returning [].
if (page === 1) {
throw new Error(
`Failed to fetch jobs for run ${runId} after 3 attempts: ${lastError.message}`
)
}
// For later pages we already have partial data; warn and return what we have
console.error(
`Warning: Failed to fetch page ${page} of jobs after 3 attempts. Returning ${allJobs.length} jobs from previous pages.`
)
break break
} }
...@@ -261,7 +290,7 @@ function getAllJobs(runId) { ...@@ -261,7 +290,7 @@ function getAllJobs(runId) {
function categorizeJobs(jobs) { function categorizeJobs(jobs) {
return { return {
failed: jobs.filter((j) => j.conclusion === 'failure'), failed: jobs.filter((j) => FAILED_CONCLUSIONS.has(j.conclusion)),
inProgress: jobs.filter((j) => j.status === 'in_progress'), inProgress: jobs.filter((j) => j.status === 'in_progress'),
queued: jobs.filter((j) => j.status === 'queued'), queued: jobs.filter((j) => j.status === 'queued'),
succeeded: jobs.filter((j) => j.conclusion === 'success'), succeeded: jobs.filter((j) => j.conclusion === 'success'),
...@@ -634,8 +663,13 @@ function generateIndexMd( ...@@ -634,8 +663,13 @@ function generateIndexMd(
const testsStr = testCount const testsStr = testCount
? `${testCount.failed}/${testCount.total}` ? `${testCount.failed}/${testCount.total}`
: 'N/A' : 'N/A'
const nameStr = escapeMarkdownTableCell(job.name)
const conclusionTag =
job.conclusion && job.conclusion !== 'failure'
? ` (${job.conclusion})`
: ''
lines.push( lines.push(
`| ${job.id} | ${escapeMarkdownTableCell(job.name)} | ${duration} | ${testsStr} | [Details](job-${job.id}.md) |` `| ${job.id} | ${nameStr}${conclusionTag} | ${duration} | ${testsStr} | [Details](job-${job.id}.md) |`
) )
} }
lines.push('') lines.push('')
...@@ -1056,7 +1090,7 @@ async function getFlakyTests(currentBranch, runsToCheck = 5) { ...@@ -1056,7 +1090,7 @@ async function getFlakyTests(currentBranch, runsToCheck = 5) {
) )
// Get recent failed build-and-test runs across ALL branches // Get recent failed build-and-test runs across ALL branches
const jqQuery = `.workflow_runs[] | select(.conclusion == "failure") | {id, head_branch}` const jqQuery = `.workflow_runs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | {id, head_branch}`
let output let output
try { try {
output = exec( output = exec(
...@@ -1094,7 +1128,8 @@ async function getFlakyTests(currentBranch, runsToCheck = 5) { ...@@ -1094,7 +1128,8 @@ async function getFlakyTests(currentBranch, runsToCheck = 5) {
const runJobResults = await Promise.all( const runJobResults = await Promise.all(
allRuns.map(async (run) => { allRuns.map(async (run) => {
try { try {
const jobsJq = '.jobs[] | select(.conclusion == "failure") | {id, name}' const jobsJq =
'.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "startup_failure") | {id, name}'
const jobsOutput = exec( const jobsOutput = exec(
`gh api "repos/vercel/next.js/actions/runs/${run.id}/jobs?per_page=100" --jq '${jobsJq}'` `gh api "repos/vercel/next.js/actions/runs/${run.id}/jobs?per_page=100" --jq '${jobsJq}'`
) )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment