Unverified Commit 7c966537 authored by Matt Mastracci's avatar Matt Mastracci Committed by GitHub

[ci] Cache passing test results so CI retries skip them (#92832)

## What

Cache each passing test file's result in turbo's remote cache during CI runs. On retry attempts (`GITHUB_RUN_ATTEMPT > 1`), skip tests that already passed.

## Why

When `retry_test.yml` calls `rerun-failed-jobs`, the entire test shard re-runs from scratch — 20+ minutes to retry one flaky test. With this change, only the failed tests actually re-run on retry.

## How

- Cache key: `sha256(commit + test file + env fingerprint)` where env fingerprint includes `NEXT_TEST_MODE`, `IS_WEBPACK_TEST`, `IS_TURBOPACK_TEST`, etc.
- Cache value: `{"passed": true, "time": N}` (~50 bytes per test)
- On first attempt: all tests run normally, passes are cached
- On retry: cached passes are skipped, failures re-run. Newly passing tests get cached for further retries
- All caching gated on `CI` + `TURBO_TOKEN` + `GITHUB_SHA` — zero impact on local runs
- Uses existing `scripts/turbo-cache.mjs` client, all cache operations are fire-and-forget with try/catch
parent 4e08dc7b
...@@ -6,6 +6,7 @@ const { existsSync } = require('fs') ...@@ -6,6 +6,7 @@ const { existsSync } = require('fs')
const fsp = require('fs/promises') const fsp = require('fs/promises')
const { createClient } = require('@vercel/kv') const { createClient } = require('@vercel/kv')
const { promisify } = require('util') const { promisify } = require('util')
const { createHash } = require('crypto')
const { Sema } = require('async-sema') const { Sema } = require('async-sema')
const { spawn, exec: execOrig } = require('child_process') const { spawn, exec: execOrig } = require('child_process')
const { createNextInstall } = require('./test/lib/create-next-install') const { createNextInstall } = require('./test/lib/create-next-install')
...@@ -15,6 +16,162 @@ const core = require('@actions/core') ...@@ -15,6 +16,162 @@ const core = require('@actions/core')
const { getTestFilter } = require('./test/get-test-filter') const { getTestFilter } = require('./test/get-test-filter')
const { checkBuildFreshness } = require('./test/lib/check-build-freshness') const { checkBuildFreshness } = require('./test/lib/check-build-freshness')
// --- Test profile and result caching via turbo remote cache ---
// On CI retry attempts, skip tests that already passed on this commit.
class TestProfile {
// Env vars that always affect test behavior (non-NEXT prefixed).
static EXTRA_ENV_VARS = [
'IS_WEBPACK_TEST',
'IS_TURBOPACK_TEST',
'TURBOPACK_DEV',
'TURBOPACK_BUILD',
'BROWSER_NAME',
'DEVICE_NAME',
]
// NEXT_* / __NEXT* vars that are operational, not behavioral.
static IGNORED_VARS = new Set([
'NEXT_TELEMETRY_DISABLED',
'NEXT_TEST_JOB',
'NEXT_JUNIT_TEST_REPORT',
'NEXT_TEST_PREFER_OFFLINE',
'NEXT_CI_RUNNER',
'NEXT_E2E_TEST_TIMEOUT',
'NEXT_TURBOPACK_IO_CONCURRENCY',
])
// All key=value pairs that form the cache identity, sorted by key.
// Computed once at construction from a snapshot of process.env.
entries
// NEXT_*/__NEXT* vars that matched the pattern but were ignored.
ignoredVars
cachingEnabled
#cacheKey
constructor({ group = '', type = '', testPattern = '' } = {}) {
this.cachingEnabled = !!(
process.env.CI &&
process.env.TURBO_TOKEN &&
process.env.GITHUB_SHA &&
!process.env.NEXT_FLAKE_DETECTION &&
!process.env.NEXT_TEST_SKIP_RESULT_CACHE
)
// Snapshot env at construction time so dynamic vars set during the run
// (e.g. NEXT_TEST_STARTER, NEXT_TEST_PKG_PATHS) don't pollute the key.
const env = { ...process.env }
// Collect env vars that affect test behavior
const vars = new Map()
for (const v of TestProfile.EXTRA_ENV_VARS) {
if (env[v]) vars.set(v, env[v])
}
const ignored = []
for (const key of Object.keys(env)) {
if (
(key.startsWith('NEXT_') || key.startsWith('__NEXT')) &&
!key.startsWith('NEXT_SKIP_')
) {
if (TestProfile.IGNORED_VARS.has(key)) {
ignored.push(key)
} else {
vars.set(key, env[key])
}
}
}
this.ignoredVars = ignored.sort()
// Build the single sorted entries list used for hashing, logging, and descriptions.
const branch =
process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME || ''
const map = new Map([
['os', process.platform],
['branch', branch],
['sha', process.env.GITHUB_SHA || ''],
['node', process.versions.node.split('.')[0]],
])
if (group) map.set('group', group)
if (type) map.set('type', type)
if (testPattern) map.set('testPattern', testPattern)
for (const [k, v] of [...vars.entries()].sort((a, b) =>
a[0].localeCompare(b[0])
)) {
map.set(k, v)
}
this.entries = [...map.entries()]
}
get cacheKey() {
if (!this.#cacheKey) {
const hash = createHash('sha256')
hash.update('test-result-v2\0')
for (const [k, v] of this.entries) {
hash.update(`${k}=${v}\0`)
}
this.#cacheKey = hash.digest('hex')
}
return this.#cacheKey
}
get description() {
return this.entries
.map(([k, v]) => (k === 'sha' ? `sha=${v?.slice(0, 10)}` : `${k}=${v}`))
.join(' | ')
}
log() {
console.log(`\nTest profile:`)
for (const [k, v] of this.entries) {
console.log(` ${k}=${k === 'sha' ? v?.slice(0, 10) : v}`)
}
if (this.ignoredVars.length > 0) {
console.log(` Ignored: ${this.ignoredVars.join(', ')}`)
}
console.log(` Caching: ${this.cachingEnabled ? 'enabled' : 'disabled'}`)
console.log('')
}
// --- Cache operations ---
#turboCache = null
async #getTurboCache() {
if (this.#turboCache) return this.#turboCache
if (!this.cachingEnabled) return null
try {
this.#turboCache = await import('./scripts/turbo-cache.mjs')
return this.#turboCache
} catch {
return null
}
}
async loadPassedTests() {
try {
const cache = await this.#getTurboCache()
if (!cache) return new Set()
const data = await cache.get(this.cacheKey)
if (!data) return new Set()
const parsed = JSON.parse(data.toString())
return new Set(parsed.passed || [])
} catch {
return new Set()
}
}
async savePassedTests(passedFiles) {
try {
const cache = await this.#getTurboCache()
if (!cache) return false
const payload = JSON.stringify({ passed: [...passedFiles].sort() })
await cache.put(this.cacheKey, Buffer.from(payload))
return true
} catch {
return false
}
}
}
// Do not rename or format. sync-react script relies on this line. // Do not rename or format. sync-react script relies on this line.
// prettier-ignore // prettier-ignore
const nextjsReactPeerVersion = "19.2.5"; const nextjsReactPeerVersion = "19.2.5";
...@@ -265,6 +422,13 @@ async function main() { ...@@ -265,6 +422,13 @@ async function main() {
process.env.NEXT_TEST_MODE process.env.NEXT_TEST_MODE
) )
const profile = new TestProfile({
group: String(options.group || ''),
type: String(options.type || ''),
testPattern: String(options.testPattern || ''),
})
profile.log()
// Only fetch/update shared timing data during grouped CI runs to avoid // Only fetch/update shared timing data during grouped CI runs to avoid
// individual test runs from polluting the timing data // individual test runs from polluting the timing data
const shouldUseSharedTimings = options.timings && options.group const shouldUseSharedTimings = options.timings && options.group
...@@ -685,7 +849,32 @@ ${ENDGROUP}`) ...@@ -685,7 +849,32 @@ ${ENDGROUP}`)
}) })
}) })
const isRetryAttempt =
process.env.CI && parseInt(process.env.GITHUB_RUN_ATTEMPT || '1', 10) > 1
const passedTestFiles = new Set()
// On retry, load previously-passed tests and filter them out
let cachedPassedTests = new Set()
if (isRetryAttempt) {
cachedPassedTests = await profile.loadPassedTests()
if (cachedPassedTests.size > 0) {
console.log(
`Test result cache: loaded ${cachedPassedTests.size} passed test(s) (${profile.description})`
)
} else {
console.log(`Test result cache: miss (${profile.description})`)
}
}
const runTest = async (/** @type {TestFile} */ test) => { const runTest = async (/** @type {TestFile} */ test) => {
// On CI retry attempts, skip tests that already passed on this commit
if (cachedPassedTests.has(test.file)) {
console.log(
`${test.file} already passed on this commit (cached) — skipping`
)
return
}
let passed = false let passed = false
for (let i = 0; i < numRetries + 1; i++) { for (let i = 0; i < numRetries + 1; i++) {
...@@ -721,6 +910,10 @@ ${ENDGROUP}`) ...@@ -721,6 +910,10 @@ ${ENDGROUP}`)
} }
} }
if (passed) {
passedTestFiles.add(test.file)
}
if (!passed) { if (!passed) {
hadFailures = true hadFailures = true
// "failed to pass within" is a keyword parsed by next-pr-webhook // "failed to pass within" is a keyword parsed by next-pr-webhook
...@@ -810,6 +1003,25 @@ ${ENDGROUP}`) ...@@ -810,6 +1003,25 @@ ${ENDGROUP}`)
} }
} }
// Save all passed tests (from this run + previously cached) as a single cache entry
if (process.env.CI && passedTestFiles.size > 0) {
const allPassed = new Set([...cachedPassedTests, ...passedTestFiles])
const saved = await profile.savePassedTests(allPassed)
if (saved) {
console.log(
`Test result cache: saved ${allPassed.size} passed test(s) (${profile.description})`
)
}
}
if (cachedPassedTests.size > 0) {
const skipped = tests.filter((t) => cachedPassedTests.has(t.file)).length
if (skipped > 0) {
console.log(
`Test result cache: skipped ${skipped} cached, re-ran ${tests.length - skipped}`
)
}
}
if (options.timings) { if (options.timings) {
const curTimings = {} const curTimings = {}
// let junitData = `<testsuites name="jest tests">` // let junitData = `<testsuites name="jest tests">`
......
...@@ -180,6 +180,7 @@ async function main() { ...@@ -180,6 +180,7 @@ async function main() {
NEXT_TEST_MODE: testMode, NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion, NEXT_TEST_VERSION: nextTestVersion,
IS_WEBPACK_TEST: '1', IS_WEBPACK_TEST: '1',
NEXT_TEST_SKIP_RESULT_CACHE: '1',
}, },
}) })
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment