Unverified Commit 0d36c3cd authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Run targeted upgrade evals in CI

Trigger the security upgrade fixtures when their docs, CLI, codemod, shared harness, or fixture inputs change. Select fixture-only changes precisely, prepare the candidate once, run fixture and agent combinations concurrently through agent-eval, authenticate with project OIDC, and retain failure evidence.
parent 3cf1f741
name: next-upgrade-evals
on:
pull_request:
types: [opened, synchronize]
paths:
- '.github/workflows/next-upgrade-evals.yml'
- 'docs/01-app/02-guides/upgrading/**'
- 'docs/02-pages/02-guides/upgrading/**'
- 'evals/lib/**'
- 'evals/next-upgrade/**'
- 'packages/next-codemod/**'
- 'packages/next/src/bin/next.ts'
- 'packages/next/src/cli/next-upgrade.ts'
- 'packages/next/src/lib/upgrade/**'
concurrency:
group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
select-evals:
name: Select evals
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
evals: ${{ steps.select.outputs.evals }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
persist-credentials: false
- name: Select affected evals
id: select
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
node <<'NODE'
const { execFileSync } = require('node:child_process')
const { appendFileSync, readdirSync } = require('node:fs')
const { join } = require('node:path')
const fixturePrefix = 'evals/next-upgrade/evals/'
const fixtureDirectory = join(process.cwd(), fixturePrefix)
const allFixtures = readdirSync(fixtureDirectory, {
withFileTypes: true,
})
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort()
const knownFixtures = new Set(allFixtures)
const changedFixtures = new Set()
const changedFiles = execFileSync(
'git',
[
'diff',
'--name-only',
`${process.env.BASE_SHA}...${process.env.HEAD_SHA}`,
],
{ encoding: 'utf8' }
)
.trim()
.split('\n')
.filter(Boolean)
let runAll = false
for (const file of changedFiles) {
if (
file === '.github/workflows/next-upgrade-evals.yml' ||
file.startsWith('docs/01-app/02-guides/upgrading/') ||
file.startsWith('docs/02-pages/02-guides/upgrading/') ||
file.startsWith('evals/lib/') ||
file.startsWith('packages/next-codemod/') ||
file === 'packages/next/src/bin/next.ts' ||
file === 'packages/next/src/cli/next-upgrade.ts' ||
file.startsWith('packages/next/src/lib/upgrade/') ||
(file.startsWith('evals/next-upgrade/') &&
!file.startsWith(fixturePrefix))
) {
runAll = true
break
}
if (file.startsWith(fixturePrefix)) {
const fixture = file.slice(fixturePrefix.length).split('/')[0]
if (!knownFixtures.has(fixture)) {
runAll = true
break
}
changedFixtures.add(fixture)
}
}
const selected = runAll ? allFixtures : [...changedFixtures].sort()
if (selected.length === 0) {
throw new Error('The workflow was triggered without an affected eval')
}
appendFileSync(
process.env.GITHUB_OUTPUT,
`evals=${JSON.stringify(selected)}\n`
)
console.log(`Selected evals: ${selected.join(', ')}`)
NODE
eval:
name: Run evals
needs: select-evals
if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
runs-on: ubuntu-latest
timeout-minutes: 90
environment: next-upgrade-evals
permissions:
contents: read
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 22
package-manager-cache: false
- name: Setup pnpm
run: |
npm i -g corepack@0.31
corepack enable
- name: Install dependencies
run: pnpm install
- name: Build packages
run: pnpm build-all
- name: Pull Vercel OIDC token
env:
VERCEL_PROJECT_ID: ${{ vars.VERCEL_NEXT_UPGRADE_EVALS_PROJECT_ID }}
VERCEL_TEAM_ID: ${{ vars.VERCEL_NEXT_UPGRADE_EVALS_TEAM_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_NEXT_UPGRADE_EVALS_TOKEN }}
run: |
pnpm dlx vercel@59.11.0 env pull .env.local \
--yes \
--environment=development \
--scope "$VERCEL_TEAM_ID" \
--project "$VERCEL_PROJECT_ID" \
--token "$VERCEL_TOKEN"
node <<'NODE'
const {
appendFileSync,
readFileSync,
rmSync,
} = require('node:fs')
const { parse } = require('dotenv')
const oidc = parse(readFileSync('.env.local')).VERCEL_OIDC_TOKEN
if (!oidc) throw new Error('Vercel did not provide an OIDC token')
console.log(`::add-mask::${oidc}`)
appendFileSync(process.env.GITHUB_ENV, `VERCEL_OIDC_TOKEN=${oidc}\n`)
rmSync('.env.local')
NODE
- name: Run eval
env:
EVALS: ${{ needs.select-evals.outputs.evals }}
run: |
node <<'NODE'
const { spawnSync } = require('node:child_process')
const evals = JSON.parse(process.env.EVALS)
const result = spawnSync('pnpm', ['eval:upgrade', ...evals], {
stdio: 'inherit',
})
if (result.error) throw result.error
if (result.status !== 0) process.exitCode = 1
NODE
- name: Retain transcripts and grading evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: next-upgrade-evals-${{ github.run_id }}-${{ github.run_attempt }}
path: evals/next-upgrade/results/
if-no-files-found: warn
retention-days: 14
.tarballs/
results/
experiments/ci-*.ts
.env
.env.local
......@@ -13,14 +13,15 @@ assertions, judging and result storage belong to `@vercel/agent-eval`.
```sh
pnpm build-all
pnpm eval:upgrade <fixture-name> --dry
NEXT_UPGRADE_EVAL_EXPERIMENT=codex pnpm eval:upgrade <fixture-name>
pnpm eval:upgrade <fixture-name> [fixture-name...] --dry
NEXT_UPGRADE_EVAL_EXPERIMENT=codex pnpm eval:upgrade <fixture-name> [fixture-name...]
```
Omit the experiment filter to run Codex and Claude. `--list` lists fixtures without
packing or making model calls. Run one named fixture at a time. Results use the
framework's normal `results/` layout. Fixtures are added by the feature PRs stacked
above this infrastructure.
packing or making model calls. Named fixtures and their selected harnesses run
concurrently after packages and the shared toolchain are prepared once. Results use
the framework's normal `results/` layout. Fixtures are added by the feature PRs
stacked above this infrastructure.
## Lifecycle
......
......@@ -2,8 +2,11 @@ import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval'
import { setupUpgradeScenario } from '../security/setup'
export async function setupFuture(sandbox: Sandbox) {
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
export async function setupFuture(
sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
const scenarios: Record<string, { source: string; target: string }> = {
'future-cache-components': {
source: '13.5.11',
......@@ -21,19 +24,23 @@ export async function setupFuture(sandbox: Sandbox) {
const scenario = fixture ? scenarios[fixture] : undefined
if (!scenario) throw new Error('Unknown Future Defaults eval case')
await setupUpgradeScenario(sandbox, {
fixturePrefix: 'future-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: undefined,
candidateScripts:
fixture === 'future-cache-components-nudge' ? ['dev'] : undefined,
skillInstructionsPath:
fixture === 'future-cache-components-nudge'
? undefined
: join(
__dirname,
'../../../skills/next-cache-components-adoption/SKILL.md'
),
})
await setupUpgradeScenario(
sandbox,
{
fixturePrefix: 'future-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: undefined,
candidateScripts:
fixture === 'future-cache-components-nudge' ? ['dev'] : undefined,
skillInstructionsPath:
fixture === 'future-cache-components-nudge'
? undefined
: join(
__dirname,
'../../../skills/next-cache-components-adoption/SKILL.md'
),
},
fixture
)
}
......@@ -2,8 +2,11 @@ import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval'
import { setupUpgradeScenario } from '../security/setup'
export async function setupLatest(sandbox: Sandbox) {
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
export async function setupLatest(
sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
const target = '16.3.5'
const scenarios: Record<
string,
......@@ -16,10 +19,14 @@ export async function setupLatest(sandbox: Sandbox) {
const scenario = fixture ? scenarios[fixture] : undefined
if (!scenario) throw new Error('Unknown latest upgrade eval case')
await setupUpgradeScenario(sandbox, {
fixturePrefix: 'latest-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: scenario.installedVersion,
})
await setupUpgradeScenario(
sandbox,
{
fixturePrefix: 'latest-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: scenario.installedVersion,
},
fixture
)
}
......@@ -5,9 +5,10 @@ import { setupLatest } from '../latest/setup'
import { setupFuture } from '../future/setup'
export function upgradeExperiment(
harness: 'codex' | 'claude-code'
harness: 'codex' | 'claude-code',
selectedFixture: string | undefined = undefined
): ExperimentConfig {
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture) throw new Error('Select one upgrade eval case')
const security = fixture.startsWith('security-')
const latest = fixture.startsWith('latest-')
......@@ -25,10 +26,10 @@ export function upgradeExperiment(
timeout: future ? 3600 : 1800,
copyFiles: 'changed',
setup: async (sandbox) => {
const setup = await setupUpgrade(sandbox)
if (security) await setupSecurity(sandbox)
if (latest) await setupLatest(sandbox)
if (future) await setupFuture(sandbox)
const setup = await setupUpgrade(sandbox, fixture)
if (security) await setupSecurity(sandbox, fixture)
if (latest) await setupLatest(sandbox, fixture)
if (future) await setupFuture(sandbox, fixture)
return setup
},
}
......
......@@ -4,7 +4,10 @@ import type { Sandbox } from '@vercel/agent-eval'
export const toolsDirectory = '/tmp/next-upgrade-eval'
export async function setupUpgrade(sandbox: Sandbox) {
export async function setupUpgrade(
sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const run = async (command: string, args: string[]) => {
const result = await sandbox.runCommand(command, args)
if (result.exitCode !== 0)
......@@ -20,7 +23,7 @@ export async function setupUpgrade(sandbox: Sandbox) {
'Run through pnpm eval:upgrade to provide the candidate packages'
)
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture) throw new Error('Select one upgrade eval case')
const fixtureDirectory = join(__dirname, '../evals', fixture)
const baselineFiles = Object.fromEntries(
......
......@@ -8,6 +8,38 @@ const { packPackage } = require('../lib/pack')
const { linkEnvironment } = require('../lib/environment')
const snapshotEnvironmentVariable = 'AGENT_EVAL_SANDBOX_SNAPSHOT_ID'
function createExperimentFiles(directory, fixtures, harness) {
const harnesses = harness
? [[harness, harness === 'claude' ? 'claude-code' : harness]]
: [
['codex', 'codex'],
['claude', 'claude-code'],
]
const files = []
const experiments = []
try {
for (const fixture of fixtures) {
for (const [name, agent] of harnesses) {
const experiment = `ci-${name}-${fixture}`
const file = path.join(directory, `${experiment}.ts`)
if (fs.existsSync(file))
throw new Error(`Generated experiment already exists: ${file}`)
fs.writeFileSync(
file,
`import { upgradeExperiment } from '../lib/experiment'\n` +
`export default upgradeExperiment(${JSON.stringify(agent)}, ${JSON.stringify(fixture)})\n`
)
files.push(file)
experiments.push(experiment)
}
}
} catch (error) {
for (const file of files) fs.rmSync(file, { force: true })
throw error
}
return { experiments, files, harnessCount: harnesses.length }
}
async function createToolchainSnapshot() {
console.log('Preparing shared eval toolchain...')
const token = process.env.VERCEL_TOKEN
......@@ -54,7 +86,9 @@ async function main() {
loadEnvironment({ path: path.join(__dirname, '.env'), override: true })
const args = process.argv.slice(2)
const fixturesDirectory = path.join(__dirname, 'evals')
const { discoverFixtures, loadFixture } = await import('@vercel/agent-eval')
const { discoverFixtures, loadConfig, loadFixture } = await import(
'@vercel/agent-eval'
)
const cases = fs.existsSync(fixturesDirectory)
? discoverFixtures(fixturesDirectory)
: []
......@@ -63,23 +97,39 @@ async function main() {
console.log(cases.join('\n'))
return
}
const selected = args.filter((arg) => !arg.startsWith('--'))
if (selected.length !== 1)
throw new Error(
'Select one upgrade eval fixture; use --list to list fixtures'
)
const fixture = selected[0]
const selected = [...new Set(args.filter((arg) => !arg.startsWith('--')))]
if (selected.length === 0)
throw new Error('Select upgrade eval fixtures; use --list to list fixtures')
const harness = process.env.NEXT_UPGRADE_EVAL_EXPERIMENT
if (harness && !['codex', 'claude'].includes(harness))
throw new Error('Select codex or claude')
if (!cases.includes(fixture))
throw new Error(`Available cases: ${cases.join(', ')}`)
for (const fixture of selected) {
if (!cases.includes(fixture))
throw new Error(`Available cases: ${cases.join(', ')}`)
// Validate using the framework's own fixture rules. Its run command
// otherwise falls back to all fixtures when a filter matches no fixture.
loadFixture(fixturesDirectory, fixture)
}
if (args.some((arg) => arg.startsWith('--') && arg !== '--dry'))
throw new Error('Supported flags: --dry, --list')
// Validate using the framework's own fixture rules. Its run command otherwise
// falls back to all fixtures when an explicit filter matches no valid fixture.
loadFixture(fixturesDirectory, fixture)
if (args.includes('--dry')) return console.log(fixture)
if (args.includes('--dry')) {
let generatedExperiments = []
try {
if (selected.length > 1) {
const generated = createExperimentFiles(
path.join(__dirname, 'experiments'),
selected,
harness
)
generatedExperiments = generated.files
await Promise.all(generated.files.map((file) => loadConfig(file)))
}
console.log(selected.join('\n'))
return
} finally {
for (const file of generatedExperiments) fs.rmSync(file, { force: true })
}
}
for (const [name, entry] of [
['next', 'dist/bin/next'],
['next-codemod', 'bin/next-codemod.js'],
......@@ -102,25 +152,40 @@ async function main() {
}
fs.mkdirSync(path.join(__dirname, 'results'), { recursive: true })
let snapshot
let generatedExperiments = []
try {
if (!env[snapshotEnvironmentVariable]) {
snapshot = await createToolchainSnapshot()
env[snapshotEnvironmentVariable] = snapshot.snapshotId
}
const experiments = harness ? [harness] : ['codex', 'claude']
const generated =
selected.length > 1
? createExperimentFiles(
path.join(__dirname, 'experiments'),
selected,
harness
)
: {
experiments: harness ? [harness] : ['codex', 'claude'],
files: [],
harnessCount: harness ? 1 : 2,
}
generatedExperiments = generated.files
const result = spawnSync(
path.join(root, 'node_modules/.bin/agent-eval'),
['run', ...experiments, '--force', '--ack-failures'],
['run', ...generated.experiments, '--force', '--ack-failures'],
{
cwd: __dirname,
env: {
...env,
NEXT_UPGRADE_EVAL_CASE: fixture,
...(experiments.length > 1
...(selected.length === 1
? { NEXT_UPGRADE_EVAL_CASE: selected[0] }
: {}),
...(generated.harnessCount > 1
? {
AGENT_EVAL_PREPARE_FIXTURE_ONCE: '1',
AGENT_EVAL_PREPARED_FIXTURE_CONSUMERS: String(
experiments.length
generated.harnessCount
),
}
: {}),
......@@ -131,6 +196,7 @@ async function main() {
if (result.error) throw result.error
if (result.status !== 0) process.exitCode = 1
} finally {
for (const file of generatedExperiments) fs.rmSync(file, { force: true })
if (snapshot) {
try {
await snapshot.delete()
......
......@@ -3,8 +3,11 @@ import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval'
import { toolsDirectory } from '../lib/fixture'
export async function setupSecurity(sandbox: Sandbox) {
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
export async function setupSecurity(
sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture?.startsWith('security-'))
throw new Error('Select a security upgrade eval case')
const sameMajorTarget = '15.5.24'
......@@ -56,12 +59,16 @@ export async function setupSecurity(sandbox: Sandbox) {
const scenario = scenarios[fixture]
if (!scenario) throw new Error('Unknown security upgrade eval case')
await setupUpgradeScenario(sandbox, {
fixturePrefix: 'security-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: scenario.installedVersion,
})
await setupUpgradeScenario(
sandbox,
{
fixturePrefix: 'security-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: scenario.installedVersion,
},
fixture
)
}
export async function setupUpgradeScenario(
......@@ -73,7 +80,8 @@ export async function setupUpgradeScenario(
installedVersion: string | undefined
candidateScripts?: string[]
skillInstructionsPath?: string
}
},
selectedFixture: string | undefined = undefined
) {
const run = async (command: string, args: string[]) => {
const result = await sandbox.runCommand(command, args)
......@@ -83,7 +91,7 @@ export async function setupUpgradeScenario(
)
return result.stdout.trim()
}
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture?.startsWith(options.fixturePrefix))
throw new Error(`Select a ${options.fixturePrefix} upgrade eval case`)
const security = `${toolsDirectory}/security`
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment