Unverified Commit 12f20769 authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Add CI for upgrade evals with shared preparation (#99908)

Stacked on #99880.

### Why?

Upgrade evals need CI feedback with reusable preparation and failure
evidence that reviewers can read directly in the job log.

### How?

- Run all twelve Codex/Claude trials in parallel through `pnpm
eval:upgrade --all` on relevant same-repository PR changes and manual
dispatch. Check out the exact PR head to evaluate the reviewed
candidate.
- Prepare candidate packages once per batch. Reuse pnpm/Turbo caches and
expiring credential-free Sandbox tooling snapshots, then share app
preparation between agent forks. Save replacement snapshot IDs even
after a cache hit or eval failure.
- Use repository credentials to pull the existing Vercel project's OIDC
token. Run a pinned Vercel CLI only for authentication, without a root
CLI dependency or GitHub deployment record.
- Fail CI for failed or invalid trials and retain transcripts, delivered
apps and grading evidence for 14 days. Print brief case results, then
collect failed/invalid verdicts and redacted native case logs at the end
with workflow-command parsing disabled.

<!-- NEXT_JS_LLM -->
parent c77c7e10
name: next-upgrade-evals
permissions: {}
on:
workflow_dispatch:
pull_request:
types: [opened, synchronize, reopened]
paths:
- '.github/workflows/next-upgrade-evals.yml'
- 'evals/lib/**'
- 'evals/next-upgrade/**'
- 'patches/@vercel__agent-eval@*.patch'
- 'pnpm-lock.yaml'
- 'package.json'
- 'turbo.json'
- 'packages/next/package.json'
- 'packages/next/turbo.json'
- 'packages/next-codemod/**'
- 'packages/next/src/bin/next.ts'
- 'packages/next/src/cli/next-upgrade.ts'
- 'packages/next/src/cli/next-build.ts'
- 'packages/next/src/cli/next-dev.ts'
- 'packages/next/src/lib/upgrade/**'
- 'packages/next/src/server/config*.ts'
- 'docs/01-app/02-guides/upgrading/**'
- 'docs/02-pages/02-guides/upgrading/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
eval:
name: Run upgrade evals
if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository }}
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
# Evaluate this PR's candidate without merging unreleased base packages.
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 22
package-manager-cache: false
- name: Setup pnpm
id: store
run: |
set -euo pipefail
corepack enable
echo "path=$(pnpm store path)" >> "$GITHUB_OUTPUT"
# Use the same exact-lockfile store as the existing build workflow.
- name: Cache pnpm store
id: store-cache
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
timeout-minutes: 5
with:
path: ${{ steps.store.outputs.path }}
key: pnpm-store-root-v1-${{ hashFiles('pnpm-lock.yaml') }}
- name: Install dependencies
run: pnpm install --frozen-lockfile
# Persist preparation before model execution, so a genuine eval failure
# does not discard the cache needed by the next run.
- name: Save pnpm store
if: ${{ steps.store-cache.outputs.cache-hit != 'true' }}
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
timeout-minutes: 5
with:
path: ${{ steps.store.outputs.path }}
key: pnpm-store-root-v1-${{ hashFiles('pnpm-lock.yaml') }}
- name: Set up Turborepo remote cache
if: ${{ vars.TURBO_TEAM != '' }}
uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0
with:
team: ${{ vars.TURBO_TEAM }}
# Only the package dependency closure is needed. Published SWC bindings
# installed by pnpm execute in Linux Sandboxes; no Rust build is required.
- name: Build eval packages
id: setup
run: |
set -euo pipefail
pnpm exec turbo run build --filter=next... --filter=@next/codemod... --env-mode=loose --remote-cache-timeout=60
echo "key=$(node evals/next-upgrade/run.js --cache-key)" >> "$GITHUB_OUTPUT"
# Cache only a small manifest of credential-free, one-day remote snapshots.
# Daily keys avoid repeatedly restoring IDs after their remote TTL expires.
# Unique entries let replacements supersede stale IDs within the same day.
- name: Restore Sandbox tooling snapshots
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
timeout-minutes: 5
with:
path: evals/next-upgrade/.work/cache/snapshots.json
key: next-upgrade-snapshots-v2-${{ steps.setup.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
next-upgrade-snapshots-v2-${{ steps.setup.outputs.key }}-
# Match deploy CI's global CLI installation, retaining our pinned version.
- name: Install Vercel CLI
run: npm install --global vercel@59.11.0
- name: Pull Vercel OIDC token
env:
VERCEL_PROJECT_ID: ${{ vars.VERCEL_NEXT_UPGRADE_EVALS_PROJECT_ID }}
VERCEL_TEAM_ID: ${{ vars.VERCEL_NEXT_UPGRADE_EVALS_TEAM_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_NEXT_UPGRADE_EVALS_TOKEN }}
run: node evals/next-upgrade/run.js --ci-auth
- name: Run all upgrade evals in parallel
id: evals
env:
NEXT_UPGRADE_EVAL_SNAPSHOT_CACHE: evals/next-upgrade/.work/cache/snapshots.json
TURBO_TOKEN: ''
run: pnpm eval:upgrade --all
- name: Save Sandbox tooling snapshots
if: ${{ always() && !cancelled() && steps.evals.outputs.snapshots-created == 'true' && hashFiles('evals/next-upgrade/.work/cache/snapshots.json') != '' }}
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
timeout-minutes: 5
with:
path: evals/next-upgrade/.work/cache/snapshots.json
key: next-upgrade-snapshots-v2-${{ steps.setup.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }}
- name: Retain transcripts and grading evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: next-upgrade-evals-${{ github.run_id }}-${{ github.run_attempt }}
path: evals/next-upgrade/results/
if-no-files-found: warn
retention-days: 14
...@@ -35,7 +35,8 @@ pnpm eval:upgrade security-same-major --dry ...@@ -35,7 +35,8 @@ pnpm eval:upgrade security-same-major --dry
pnpm eval:upgrade security-same-major pnpm eval:upgrade security-same-major
``` ```
Run each of the six cases once for twelve trials, or select one harness with Run all six cases in parallel with `pnpm eval:upgrade --all` for twelve trials,
or select one harness for individual cases with
`NEXT_UPGRADE_EVAL_EXPERIMENT=codex` or `claude`. List/dry modes do no remote work. `NEXT_UPGRADE_EVAL_EXPERIMENT=codex` or `claude`. List/dry modes do no remote work.
The runner loads root `.env.local` without overwriting inherited values. Only The runner loads root `.env.local` without overwriting inherited values. Only
`VERCEL_OIDC_TOKEN` is required; refresh it before expiry. Never commit or print `VERCEL_OIDC_TOKEN` is required; refresh it before expiry. Never commit or print
...@@ -78,4 +79,6 @@ failures and rerun after correcting an evaluator defect. ...@@ -78,4 +79,6 @@ failures and rerun after correcting an evaluator defect.
CLI/browser tooling and app preparation are shared through credential-free, CLI/browser tooling and app preparation are shared through credential-free,
one-day snapshots. Agent credentials are written only into individual forks. one-day snapshots. Agent credentials are written only into individual forks.
Owned snapshots and temporary fixtures are cleaned up after execution; native Owned snapshots and temporary fixtures are cleaned up after execution; native
cancellation completes capture and teardown before batch cleanup. cancellation completes capture and teardown before runner cleanup. CI retains
tooling snapshot IDs in its cache after uncancelled runs, including eval failures.
CI installs the pinned Vercel CLI only when pulling its OIDC token.
This diff is collapsed.
...@@ -14,51 +14,101 @@ export type Verdict = { ...@@ -14,51 +14,101 @@ export type Verdict = {
// Prepare only trusted tools. Forking this credential-free snapshot saves browser // Prepare only trusted tools. Forking this credential-free snapshot saves browser
// setup time while each delivered app still gets its own clean verifier VM. // setup time while each delivered app still gets its own clean verifier VM.
export async function prepareVerifier() { export async function prepareVerifier(signal: AbortSignal) {
const sandbox = await SandboxManager.create({ const sandbox = await SandboxManager.create({
backend: 'vercel', backend: 'vercel',
runtime: 'node24', runtime: 'node24',
timeout: 600000, timeout: 600000,
}) })
let stopping: Promise<void> | null = null
const stop = () => {
if (!stopping) {
stopping = Promise.resolve().then(() => sandbox.stop())
}
return stopping
}
// Keep setup cancellable while browser or package installation is blocked.
let rejectAbort: (reason: unknown) => void
const aborted = new Promise<never>((_, reject) => {
rejectAbort = reject
})
const cancel = () => {
stop().catch((error) => console.error('Verifier teardown failed:', error))
rejectAbort(signal.reason)
}
signal.addEventListener('abort', cancel, { once: true })
try { try {
signal.throwIfAborted()
sandbox.setWorkingDirectory('/vercel/sandbox/verify') sandbox.setWorkingDirectory('/vercel/sandbox/verify')
await sandbox.writeFiles({ await Promise.race([
'package.json': JSON.stringify({ sandbox.writeFiles({
private: true, 'package.json': JSON.stringify({
type: 'module', private: true,
dependencies: { vitest: '3.1.3', '@playwright/test': '1.51.1' }, type: 'module',
dependencies: { vitest: '3.1.3', '@playwright/test': '1.51.1' },
}),
}), }),
}) aborted,
const install = await sandbox.runCommand('npm', [
'install',
'--no-audit',
'--no-fund',
]) ])
signal.throwIfAborted()
const install = await Promise.race([
sandbox.runCommand('npm', ['install', '--no-audit', '--no-fund']),
aborted,
])
signal.throwIfAborted()
if (install.exitCode !== 0) { if (install.exitCode !== 0) {
throw new Error(`Verifier tooling install failed: ${install.stderr}`) throw new Error(`Verifier tooling install failed: ${install.stderr}`)
} }
await installPlaywright(sandbox) await Promise.race([installPlaywright(sandbox), aborted])
return await sandbox.snapshot({ expiration: 86400000 }) signal.throwIfAborted()
const snapshot = await sandbox.snapshot({ expiration: 86400000 })
if (signal.aborted) {
await snapshot.delete()
signal.throwIfAborted()
}
return snapshot
} finally { } finally {
await sandbox.stop() signal.removeEventListener('abort', cancel)
await stop()
} }
} }
// The verifier receives only exported source and trusted tests. No Gateway key, // The verifier receives only exported source and trusted tests. No Gateway key,
// agent state, cached build, node_modules, or mutable app test code is transferred. // agent state, cached build, node_modules, or mutable app test code is transferred.
let verifierCancelled = false
export async function verifyTree( export async function verifyTree(
tree: DeliveredTree, tree: DeliveredTree,
name: string, name: string,
directory: string directory: string
): Promise<Verdict> { ): Promise<Verdict> {
if (verifierCancelled) {
throw new Error('App verification cancelled')
}
const trusted = process.env.NEXT_UPGRADE_EVAL_TRUSTED_ROOT ?? suite const trusted = process.env.NEXT_UPGRADE_EVAL_TRUSTED_ROOT ?? suite
const sandbox = await SandboxManager.create({ let sandbox: SandboxManager | null = null
backend: 'vercel', const cleanup: { pending: Promise<PromiseSettledResult<void>[]> | null } = {
runtime: 'node24', pending: null,
timeout: 1200000, }
snapshotId: process.env.NEXT_UPGRADE_EVAL_VERIFIER_SNAPSHOT_ID, const cancel = () => {
}) verifierCancelled = true
if (sandbox && !cleanup.pending) {
// Observe cleanup immediately, then report any rejection in finally.
cleanup.pending = Promise.allSettled([sandbox.stop()])
}
}
process.on('SIGINT', cancel)
process.on('SIGTERM', cancel)
try { try {
sandbox = await SandboxManager.create({
backend: 'vercel',
runtime: 'node24',
timeout: 1200000,
snapshotId: process.env.NEXT_UPGRADE_EVAL_VERIFIER_SNAPSHOT_ID,
})
if (verifierCancelled) {
throw new Error('App verification cancelled during VM creation')
}
const files = Object.entries(tree.files).map(([path, data]) => { const files = Object.entries(tree.files).map(([path, data]) => {
if ( if (
path.startsWith('/') || path.startsWith('/') ||
...@@ -150,7 +200,20 @@ export async function verifyTree( ...@@ -150,7 +200,20 @@ export async function verifyTree(
: 'Delivered app failed required checks', : 'Delivered app failed required checks',
} }
} finally { } finally {
await sandbox.stop() try {
if (!cleanup.pending && sandbox) {
cleanup.pending = Promise.allSettled([sandbox.stop()])
}
if (cleanup.pending) {
const [result] = await cleanup.pending
if (result.status === 'rejected') {
throw result.reason
}
}
} finally {
process.off('SIGINT', cancel)
process.off('SIGTERM', cancel)
}
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment