Unverified Commit cd3476c3 authored by Sebastian "Sebbie" Silbermann's avatar Sebastian "Sebbie" Silbermann Committed by GitHub

[ci] Add stats comment with partial stats (#95401)

We used to not post a comment when a part of the bundler matrix got
cancelled. Conceptually, a cancelled job is just like a failed on in the
sense of not being able to produce stats.

Now we always aggregate stats and post stats from the bundlers that did
produce them.

The risk here is a race between a run that was cancelled because another
run superseded it and the run that superseded it. Though practically,
aggregating the stats isn't slower than running the full workflow. Not
impossible (especially given how slow the Webpack job is) but incredibly
rare.
parent f656c597
...@@ -1051,10 +1051,42 @@ ${actionInfo.previewBuildsBaseUrl}/commits/${actionInfo.commitId}/next ...@@ -1051,10 +1051,42 @@ ${actionInfo.previewBuildsBaseUrl}/commits/${actionInfo.commitId}/next
// Hidden marker to identify stats comments (invisible in rendered markdown) // Hidden marker to identify stats comments (invisible in rendered markdown)
const STATS_COMMENT_MARKER = '<!-- __NEXT_STATS_COMMENT__ -->' const STATS_COMMENT_MARKER = '<!-- __NEXT_STATS_COMMENT__ -->'
// Warn when one or more bundlers didn't produce stats because their job failed,
// was cancelled, or timed out. A cancelled job is treated the same as a failed
// one since neither can produce results; we still post whatever we collected and
// call out which bundlers are missing so the numbers below aren't misread as
// complete.
function generateMissingBundlersWarning(
missingBundlers = [],
{ hasResults = true } = {}
) {
if (missingBundlers.length === 0) return ''
const names = missingBundlers.map((b) => `**${b}**`)
const list =
names.length === 1
? names[0]
: `${names.slice(0, -1).join(', ')} and ${names[names.length - 1]}`
const isSingular = names.length === 1
const subject = isSingular ? 'its stats job' : 'their stats jobs'
const reason = isSingular
? 'it failed, was cancelled, or timed out'
: 'they failed, were cancelled, or timed out'
const trailer = hasResults
? ' The results below only cover the bundlers that finished.'
: ' No stats are available for this commit.'
return `> [!WARNING]
> No stats were collected for ${list} because ${subject} did not complete (${reason}).${trailer}
`
}
module.exports = async function addComment( module.exports = async function addComment(
results = [], results = [],
actionInfo, actionInfo,
statsConfig statsConfig,
{ missingBundlers = [] } = {}
) { ) {
// Load historical data // Load historical data
const history = await loadHistory() const history = await loadHistory()
...@@ -1066,6 +1098,10 @@ module.exports = async function addComment( ...@@ -1066,6 +1098,10 @@ module.exports = async function addComment(
: statsConfig.commentHeading || 'Stats from current PR' : statsConfig.commentHeading || 'Stats from current PR'
}\n\n` }\n\n`
comment += generateMissingBundlersWarning(missingBundlers, {
hasResults: results.length > 0,
})
const tableHead = `| | Canary | PR | Change |\n|:--|--:|--:|--:|\n` const tableHead = `| | Canary | PR | Change |\n|:--|--:|--:|--:|\n`
for (let i = 0; i < results.length; i++) { for (let i = 0; i < results.length; i++) {
......
...@@ -13,6 +13,39 @@ const { existsSync } = require('fs') ...@@ -13,6 +13,39 @@ const { existsSync } = require('fs')
const addComment = require('./add-comment') const addComment = require('./add-comment')
const logger = require('./util/logger') const logger = require('./util/logger')
const BUNDLER_DISPLAY_NAMES = {
webpack: 'Webpack',
turbopack: 'Turbopack',
}
function getBundlerDisplayName(bundler) {
return (
BUNDLER_DISPLAY_NAMES[bundler] ||
bundler.charAt(0).toUpperCase() + bundler.slice(1)
)
}
// The bundlers the `stats` matrix is expected to produce results for, provided
// by the workflow via STATS_EXPECTED_BUNDLERS. A bundler whose job failed, was
// cancelled, or timed out never uploads its pr-stats-<bundler>.json artifact, so
// comparing the expected set against the artifacts present tells us which
// bundlers are missing. Returns an empty list when unset (e.g. local runs), in
// which case we don't attempt to report missing bundlers.
function getExpectedBundlers() {
const raw = process.env.STATS_EXPECTED_BUNDLERS
if (!raw) return []
return raw
.split(/[\s,]+/)
.map((b) => b.trim().toLowerCase())
.filter(Boolean)
}
// Extract the bundler slug from a `pr-stats-<bundler>.json` filename.
function getBundlerFromStatsFile(file) {
const match = file.match(/^pr-stats-(.+)\.json$/)
return match ? match[1].toLowerCase() : null
}
async function main() { async function main() {
const resultsDir = process.argv[2] || process.cwd() const resultsDir = process.argv[2] || process.cwd()
...@@ -24,9 +57,55 @@ async function main() { ...@@ -24,9 +57,55 @@ async function main() {
(f) => f.startsWith('pr-stats-') && f.endsWith('.json') (f) => f.startsWith('pr-stats-') && f.endsWith('.json')
) )
// Work out which bundlers are missing so we can report them in the comment.
const expectedBundlers = getExpectedBundlers()
const presentBundlers = statsFiles
.map(getBundlerFromStatsFile)
.filter(Boolean)
const missingBundlers = expectedBundlers.filter(
(bundler) => !presentBundlers.includes(bundler)
)
if (missingBundlers.length > 0) {
logger(
`Missing stats for bundler(s): ${missingBundlers.join(', ')} ` +
`(their job failed, was cancelled, or timed out)`
)
}
if (statsFiles.length === 0) { if (statsFiles.length === 0) {
// This can happen for docs-only changes where stats jobs are skipped // The aggregate step only runs for non-docs changes (docs-only changes skip
logger('No pr-stats-*.json files found - this may be a docs-only change') // it via the DOCS_CHANGE gate), so the absence of any artifact means no
// bundler produced results. Only surface an "all bundlers failed" notice
// when the stats jobs genuinely failed. A 'cancelled' result here means the
// whole run was cancelled (superseded, or every bundler cancelled); we stay
// silent so scripts/pr-ci-comment.mjs posts the cancelled notice instead of
// a misleading failure comment for a run that never really ran.
const statsResult = process.env.STATS_RESULT
if (expectedBundlers.length === 0 || statsResult !== 'failure') {
logger(
`No pr-stats-*.json files found (stats result: ${statsResult || 'unknown'}) - nothing to aggregate`
)
process.exit(0)
}
logger(
`No pr-stats-*.json files found - all bundler jobs ` +
`(${expectedBundlers.join(', ')}) failed`
)
// No successful run means we don't have the actionInfo/statsConfig captured
// inside a pr-stats file, so synthesize the minimum addComment needs to
// render the "all bundlers failed" notice. In CI the rendered comment is
// read from this job's logs and posted by scripts/pr-ci-comment.mjs; the
// direct-post path in addComment is a no-op without a token/endpoint.
await addComment(
[],
{ isRelease: false, commitId: null },
{},
{ missingBundlers: expectedBundlers.map(getBundlerDisplayName) }
)
process.exit(0) process.exit(0)
} }
...@@ -116,7 +195,9 @@ async function main() { ...@@ -116,7 +195,9 @@ async function main() {
) )
// Post the combined comment // Post the combined comment
await addComment(mergedResults, actionInfo, statsConfig) await addComment(mergedResults, actionInfo, statsConfig, {
missingBundlers: missingBundlers.map(getBundlerDisplayName),
})
logger('Aggregation complete') logger('Aggregation complete')
} }
......
...@@ -89,6 +89,7 @@ jobs: ...@@ -89,6 +89,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
# Keep in sync with STATS_EXPECTED_BUNDLERS
bundler: [webpack, turbopack] bundler: [webpack, turbopack]
runs-on: ubuntu-latest-16-core-arm-oss runs-on: ubuntu-latest-16-core-arm-oss
steps: steps:
...@@ -131,7 +132,14 @@ jobs: ...@@ -131,7 +132,14 @@ jobs:
stats-aggregate: stats-aggregate:
name: Aggregate Stats name: Aggregate Stats
needs: stats needs: stats
if: always() && needs.stats.result != 'cancelled' # Always run so we can recover partial results. A single bundler timing out
# cancels its job, which makes `cancelled()` true here and marks the whole
# run "cancelled" even though the other bundler finished, so `!cancelled()`
# would wrongly skip aggregation. Running unconditionally lets us aggregate
# whatever bundlers succeeded; the reason a bundler is missing is reported in
# the comment, and a truly cancelled run (no results at all) is handled by
# the cancelled fallback in scripts/pr-ci-comment.mjs.
if: ${{ always() }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
...@@ -170,3 +178,13 @@ jobs: ...@@ -170,3 +178,13 @@ jobs:
env: env:
KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
# Keep in sync with the `stats` job matrix above. The aggregate uses it
# to detect and report bundlers whose job failed or was cancelled
# (their pr-stats-<bundler>.json artifact will be missing).
STATS_EXPECTED_BUNDLERS: 'webpack turbopack'
# Collapsed matrix result ('success' | 'failure' | 'cancelled' |
# 'skipped'). Used to decide, when no artifacts exist at all, whether
# the bundlers genuinely failed (post an "all failed" notice) or the
# run was cancelled/superseded (stay silent and let pr-ci-comment.mjs
# post the cancelled notice).
STATS_RESULT: ${{ needs.stats.result }}
...@@ -479,23 +479,12 @@ async function handleStatsWorkflow({ github, workflowRun, pr, phase }) { ...@@ -479,23 +479,12 @@ async function handleStatsWorkflow({ github, workflowRun, pr, phase }) {
return return
} }
if (workflowRun.conclusion === 'cancelled') { // Look for a stats block before reacting to the run conclusion. A single
const sha = pr.headSha || workflowRun.head_sha // bundler timing out cancels its job and marks the whole run "cancelled", but
const body = [ // the aggregate still emits stats for the bundlers that finished. Treating a
STATS_COMMENT_MARKER, // cancelled run as "no data" up front would discard that partial comment, so
'## Stats cancelled', // we post whatever the aggregate produced first and only fall back to a
'', // cancelled/skipped notice when there is genuinely no block.
`Commit: ${sha}`,
`[View workflow run](${workflowRun.html_url})`,
'',
].join('\n')
await github.upsertIssueComment(pr.number, STATS_COMMENT_MARKER, body, [
'## Stats from current PR',
])
return
}
const jobs = await github.listJobsForRunAttempt( const jobs = await github.listJobsForRunAttempt(
workflowRun.id, workflowRun.id,
workflowRun.run_attempt || 1 workflowRun.run_attempt || 1
...@@ -503,7 +492,15 @@ async function handleStatsWorkflow({ github, workflowRun, pr, phase }) { ...@@ -503,7 +492,15 @@ async function handleStatsWorkflow({ github, workflowRun, pr, phase }) {
const candidates = jobs.filter((job) => /aggregate stats/i.test(job.name)) const candidates = jobs.filter((job) => /aggregate stats/i.test(job.name))
for (const job of candidates) { for (const job of candidates) {
const logs = await github.downloadJobLogs(job.id) let logs
try {
logs = await github.downloadJobLogs(job.id)
} catch (err) {
// A cancelled or skipped aggregate job may have no retrievable logs.
console.log(`Failed to download logs for job ${job.id}`, err)
continue
}
const stats = extractDelimitedBlock( const stats = extractDelimitedBlock(
logs, logs,
'--stats start--', '--stats start--',
...@@ -531,11 +528,32 @@ async function handleStatsWorkflow({ github, workflowRun, pr, phase }) { ...@@ -531,11 +528,32 @@ async function handleStatsWorkflow({ github, workflowRun, pr, phase }) {
return return
} }
const sha = pr.headSha || workflowRun.head_sha
// No stats block. A cancelled conclusion here means the whole run was
// cancelled (superseded, or every bundler cancelled) rather than an
// individual bundler, since a partial run would have produced a block above.
if (workflowRun.conclusion === 'cancelled') {
console.log('No stats block found and the run was cancelled.')
const body = [
STATS_COMMENT_MARKER,
'## Stats cancelled',
'',
`Commit: ${sha}`,
`[View workflow run](${workflowRun.html_url})`,
'',
].join('\n')
await github.upsertIssueComment(pr.number, STATS_COMMENT_MARKER, body, [
'## Stats from current PR',
])
return
}
console.log( console.log(
'No stats block found in the completed stats workflow. Assuming stats were skipped.' 'No stats block found in the completed stats workflow. Assuming stats were skipped.'
) )
const sha = pr.headSha || workflowRun.head_sha
const body = [ const body = [
STATS_COMMENT_MARKER, STATS_COMMENT_MARKER,
'## Stats skipped', '## Stats skipped',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment