Unverified Commit 108e9652 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

use the action cache for passing tests (#93954)

## Summary

Memoize passing tests across attempts of the same workflow run, so a
re-attempt after a timeout or cancellation can skip what already passed.


## What changed

- `run-tests.js` appends each passing test's filename to
`$NEXT_TEST_PASSED_FILE` as it finishes (sync `write` + `fsync`, fd kept
open across calls). On startup it reads the same file and skips any
entries that match.
- `build_reusable.yml` wraps the test step with `actions/cache/restore`
(before) and `actions/cache/save` (after, with `if: always()`). The save
step runs on success, failure, and cancel — which is the whole point: a
cancelled attempt still gets to preserve its progress for retry.
- Cache key is scoped to
`${input_step_key}-${run_id}-attempt${run_attempt}`, with `restore-keys`
falling back to any earlier attempt of the same run.

The previous implementation wrote a single Turbo-remote-cache blob
*after* `Promise.allSettled` resolved — so a cancellation (like [run
26127371192](https://github.com/vercel/next.js/actions/runs/26127371192/job/76845288867))
lost every passed test from that attempt.

This also removes one dependency on the turbo-remote cache (there are
still two more uses of `scripts/turbo-cache.mjs`)

<!-- NEXT_JS_LLM_PR -->
parent 6e894d05
...@@ -128,6 +128,8 @@ env: ...@@ -128,6 +128,8 @@ env:
KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
NEXT_TEST_JOB: 1 NEXT_TEST_JOB: 1
# Per-job memoization of passed test files.
NEXT_TEST_PASSED_FILE: .next-test-passed.txt
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
VERCEL_TEST_TEAM: vtest314-next-e2e-tests VERCEL_TEST_TEAM: vtest314-next-e2e-tests
VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }} VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
...@@ -371,6 +373,17 @@ jobs: ...@@ -371,6 +373,17 @@ jobs:
if: ${{ inputs.testTimingsArtifact == '' }} if: ${{ inputs.testTimingsArtifact == '' }}
run: pnpm dlx turbo@${TURBO_VERSION} run get-test-timings -- --build ${{ github.sha }} run: pnpm dlx turbo@${TURBO_VERSION} run get-test-timings -- --build ${{ github.sha }}
# Restore the list of tests that already passed in an earlier
# attempt of this same workflow run.
- name: Restore passed-tests cache
if: ${{ inputs.afterBuild && runner.environment == 'github-hosted' }}
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ env.NEXT_TEST_PASSED_FILE }}
key: next-test-passed-${{ steps.var.outputs.input_step_key }}-${{ github.run_id }}-attempt${{ github.run_attempt }}
restore-keys: |
next-test-passed-${{ steps.var.outputs.input_step_key }}-${{ github.run_id }}-
- run: ${{ inputs.afterBuild }} - run: ${{ inputs.afterBuild }}
# defaults.run.shell sets a stronger options (`-leo pipefail`) # defaults.run.shell sets a stronger options (`-leo pipefail`)
# Set this back to github action's weaker defaults: # Set this back to github action's weaker defaults:
...@@ -380,6 +393,15 @@ jobs: ...@@ -380,6 +393,15 @@ jobs:
shell: bash -le {0} shell: bash -le {0}
timeout-minutes: ${{ inputs.timeout_minutes }} timeout-minutes: ${{ inputs.timeout_minutes }}
# Save the passed-tests file so the next attempt (if any) can skip
# them. `always()` makes this run on success, failure, or cancellation.
- name: Save passed-tests cache
if: ${{ always() && inputs.afterBuild && runner.environment == 'github-hosted' }}
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ env.NEXT_TEST_PASSED_FILE }}
key: next-test-passed-${{ steps.var.outputs.input_step_key }}-${{ github.run_id }}-attempt${{ github.run_attempt }}
- name: Upload test result artifacts - name: Upload test result artifacts
if: ${{ inputs.testReportsArtifactPrefix != '' && inputs.afterBuild && always() }} if: ${{ inputs.testReportsArtifactPrefix != '' && inputs.afterBuild && always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
......
...@@ -2,11 +2,10 @@ ...@@ -2,11 +2,10 @@
const path = require('path') const path = require('path')
const _glob = require('glob') const _glob = require('glob')
const { existsSync } = require('fs') const fs = require('fs')
const fsp = require('fs/promises') const fsp = require('fs/promises')
const { createClient } = require('@vercel/kv') const { createClient } = require('@vercel/kv')
const { promisify } = require('util') const { promisify } = require('util')
const { createHash } = require('crypto')
const { Sema } = require('async-sema') const { Sema } = require('async-sema')
const { spawn, exec: execOrig } = require('child_process') const { spawn, exec: execOrig } = require('child_process')
const { createNextInstall } = require('./test/lib/create-next-install') const { createNextInstall } = require('./test/lib/create-next-install')
...@@ -16,8 +15,9 @@ const core = require('@actions/core') ...@@ -16,8 +15,9 @@ const core = require('@actions/core')
const { getTestFilter } = require('./test/get-test-filter') const { getTestFilter } = require('./test/get-test-filter')
const { checkBuildFreshness } = require('./test/lib/check-build-freshness') const { checkBuildFreshness } = require('./test/lib/check-build-freshness')
// --- Test profile and result caching via turbo remote cache --- // --- Test profile and result caching via actions cache ---
// On CI retry attempts, skip tests that already passed on this commit. // On CI retry attempts, skip tests that already passed on this commit.
// The file contains null-byte delimited filenames
class TestProfile { class TestProfile {
// Env vars that always affect test behavior (non-NEXT prefixed). // Env vars that always affect test behavior (non-NEXT prefixed).
...@@ -39,20 +39,21 @@ class TestProfile { ...@@ -39,20 +39,21 @@ class TestProfile {
'NEXT_CI_RUNNER', 'NEXT_CI_RUNNER',
'NEXT_E2E_TEST_TIMEOUT', 'NEXT_E2E_TEST_TIMEOUT',
'NEXT_TURBOPACK_IO_CONCURRENCY', 'NEXT_TURBOPACK_IO_CONCURRENCY',
'NEXT_TEST_PASSED_FILE',
]) ])
// All key=value pairs that form the cache identity, sorted by key. // All key=value pairs identifying this test profile, sorted by key.
// Computed once at construction from a snapshot of process.env. // Used for the diagnostic `log()` output. Computed once at construction
// from a snapshot of process.env. The actual cache key for the workflow
// is `input_step_key` (see `.github/workflows/build_reusable.yml`).
entries entries
// NEXT_*/__NEXT* vars that matched the pattern but were ignored. // NEXT_*/__NEXT* vars that matched the pattern but were ignored.
ignoredVars ignoredVars
cachingEnabled cachingEnabled
#cacheKey
constructor({ group = '', type = '', testPattern = '' } = {}) { constructor({ group = '', type = '', testPattern = '' } = {}) {
this.cachingEnabled = !!( this.cachingEnabled = !!(
process.env.CI && process.env.CI &&
process.env.TURBO_TOKEN &&
process.env.GITHUB_SHA && process.env.GITHUB_SHA &&
!process.env.NEXT_FLAKE_DETECTION && !process.env.NEXT_FLAKE_DETECTION &&
!process.env.NEXT_TEST_SKIP_RESULT_CACHE !process.env.NEXT_TEST_SKIP_RESULT_CACHE
...@@ -102,18 +103,6 @@ class TestProfile { ...@@ -102,18 +103,6 @@ class TestProfile {
this.entries = [...map.entries()] this.entries = [...map.entries()]
} }
get cacheKey() {
if (!this.#cacheKey) {
const hash = createHash('sha256')
hash.update('test-result-v2\0')
for (const [k, v] of this.entries) {
hash.update(`${k}=${v}\0`)
}
this.#cacheKey = hash.digest('hex')
}
return this.#cacheKey
}
get description() { get description() {
return this.entries return this.entries
.map(([k, v]) => (k === 'sha' ? `sha=${v?.slice(0, 10)}` : `${k}=${v}`)) .map(([k, v]) => (k === 'sha' ? `sha=${v?.slice(0, 10)}` : `${k}=${v}`))
...@@ -132,44 +121,28 @@ class TestProfile { ...@@ -132,44 +121,28 @@ class TestProfile {
console.log('') console.log('')
} }
// --- Cache operations --- // Read the passed-tests file (restored from cache by the workflow
// before this script runs). Returns a Set of test filenames. Empty
#turboCache = null // Set on miss / parse error — best-effort by design.
async #getTurboCache() { loadPassedTests() {
if (this.#turboCache) return this.#turboCache if (!this.cachingEnabled) return new Set()
if (!this.cachingEnabled) return null const file = process.env.NEXT_TEST_PASSED_FILE
if (!file) return new Set()
try { try {
this.#turboCache = await import('./scripts/turbo-cache.mjs') const data = fs.readFileSync(file, 'utf8')
return this.#turboCache // Tolerate a partial trailing line from a hard kill mid-append by
} catch { // requiring an explicit '\0' terminator. Lines without it are
return null // dropped.
} const lines = data.split('\0')
return new Set(data.endsWith('\0') ? lines : lines.slice(0, -1))
} catch (err) {
// ENOENT is the normal "no prior attempt" path — silent.
if (err && err.code !== 'ENOENT') {
console.log(`Test result cache: failed to load (${err.message})`)
} }
async loadPassedTests() {
try {
const cache = await this.#getTurboCache()
if (!cache) return new Set()
const data = await cache.get(this.cacheKey)
if (!data) return new Set()
const parsed = JSON.parse(data.toString())
return new Set(parsed.passed || [])
} catch {
return new Set() return new Set()
} }
} }
async savePassedTests(passedFiles) {
try {
const cache = await this.#getTurboCache()
if (!cache) return false
const payload = JSON.stringify({ passed: [...passedFiles].sort() })
await cache.put(this.cacheKey, Buffer.from(payload))
return true
} catch {
return false
}
}
} }
// Do not rename or format. sync-react script relies on this line. // Do not rename or format. sync-react script relies on this line.
...@@ -849,21 +822,46 @@ ${ENDGROUP}`) ...@@ -849,21 +822,46 @@ ${ENDGROUP}`)
const isRetryAttempt = const isRetryAttempt =
process.env.CI && parseInt(process.env.GITHUB_RUN_ATTEMPT || '1', 10) > 1 process.env.CI && parseInt(process.env.GITHUB_RUN_ATTEMPT || '1', 10) > 1
const passedTestFiles = new Set()
// On retry, load previously-passed tests and filter them out // Load tests that already passed (either on this attempt — restored
// from cache by the workflow — or after a timeout/cancel of an earlier
// attempt). The workflow's `actions/cache/restore` step lands the file
// at the path `loadPassedTests` reads.
let cachedPassedTests = new Set() let cachedPassedTests = new Set()
if (isRetryAttempt) { if (process.env.CI) {
cachedPassedTests = await profile.loadPassedTests() cachedPassedTests = await profile.loadPassedTests()
if (cachedPassedTests.size > 0) { if (cachedPassedTests.size > 0) {
console.log( console.log(
`Test result cache: loaded ${cachedPassedTests.size} passed test(s) (${profile.description})` `Test result cache: loaded ${cachedPassedTests.size} passed test(s) (${profile.description})`
) )
} else { } else if (isRetryAttempt) {
console.log(`Test result cache: miss (${profile.description})`) console.log(`Test result cache: miss (${profile.description})`)
} }
} }
// Stream passing test names to this file.
/** @type {number | null} */
let passedTestsFd = null
if (profile.cachingEnabled) {
try {
passedTestsFd = fs.openSync(process.env.NEXT_TEST_PASSED_FILE, 'a')
} catch (err) {
console.log(`Test result cache: open failed (${err.message})`)
}
}
/** @param {string} file */
const recordPassed = (file) => {
if (passedTestsFd === null) return
try {
// Ensure durability of our writes by syncing each one.
// This is a tiny bit of overhead but shouldn't really matter.
fs.writeSync(passedTestsFd, `${file}\0`)
fs.fdatasyncSync(passedTestsFd)
} catch (err) {
console.log(`Test result cache: append failed (${err.message})`)
}
}
const runTest = async (/** @type {TestFile} */ test) => { const runTest = async (/** @type {TestFile} */ test) => {
// On CI retry attempts, skip tests that already passed on this commit // On CI retry attempts, skip tests that already passed on this commit
if (cachedPassedTests.has(test.file)) { if (cachedPassedTests.has(test.file)) {
...@@ -909,7 +907,7 @@ ${ENDGROUP}`) ...@@ -909,7 +907,7 @@ ${ENDGROUP}`)
} }
if (passed) { if (passed) {
passedTestFiles.add(test.file) recordPassed(test.file)
} }
if (!passed) { if (!passed) {
...@@ -989,16 +987,12 @@ ${ENDGROUP}`) ...@@ -989,16 +987,12 @@ ${ENDGROUP}`)
} }
} }
// Save all passed tests (from this run + previously cached) as a single cache entry if (passedTestsFd !== null) {
if (process.env.CI && passedTestFiles.size > 0) { try {
const allPassed = new Set([...cachedPassedTests, ...passedTestFiles]) fs.closeSync(passedTestsFd)
const saved = await profile.savePassedTests(allPassed) } catch {}
if (saved) {
console.log(
`Test result cache: saved ${allPassed.size} passed test(s) (${profile.description})`
)
}
} }
if (cachedPassedTests.size > 0) { if (cachedPassedTests.size > 0) {
const skipped = tests.filter((t) => cachedPassedTests.has(t.file)).length const skipped = tests.filter((t) => cachedPassedTests.has(t.file)).length
if (skipped > 0) { if (skipped > 0) {
...@@ -1047,7 +1041,7 @@ ${ENDGROUP}`) ...@@ -1047,7 +1041,7 @@ ${ENDGROUP}`)
// Clean up stale timings for deleted tests // Clean up stale timings for deleted tests
for (const test of Object.keys(newTimings)) { for (const test of Object.keys(newTimings)) {
if (!existsSync(path.join(__dirname, test))) { if (!fs.existsSync(path.join(__dirname, test))) {
console.log('removing stale timing', test) console.log('removing stale timing', test)
delete newTimings[test] delete newTimings[test]
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment