Unverified Commit 0d36c3cd authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Run targeted upgrade evals in CI

Trigger the security upgrade fixtures when their docs, CLI, codemod, shared harness, or fixture inputs change. Select fixture-only changes precisely, prepare the candidate once, run fixture and agent combinations concurrently through agent-eval, authenticate with project OIDC, and retain failure evidence.
parent 3cf1f741
name: next-upgrade-evals
on:
pull_request:
types: [opened, synchronize]
paths:
- '.github/workflows/next-upgrade-evals.yml'
- 'docs/01-app/02-guides/upgrading/**'
- 'docs/02-pages/02-guides/upgrading/**'
- 'evals/lib/**'
- 'evals/next-upgrade/**'
- 'packages/next-codemod/**'
- 'packages/next/src/bin/next.ts'
- 'packages/next/src/cli/next-upgrade.ts'
- 'packages/next/src/lib/upgrade/**'
concurrency:
group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
select-evals:
name: Select evals
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
evals: ${{ steps.select.outputs.evals }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
persist-credentials: false
- name: Select affected evals
id: select
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
node <<'NODE'
const { execFileSync } = require('node:child_process')
const { appendFileSync, readdirSync } = require('node:fs')
const { join } = require('node:path')
const fixturePrefix = 'evals/next-upgrade/evals/'
const fixtureDirectory = join(process.cwd(), fixturePrefix)
const allFixtures = readdirSync(fixtureDirectory, {
withFileTypes: true,
})
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort()
const knownFixtures = new Set(allFixtures)
const changedFixtures = new Set()
const changedFiles = execFileSync(
'git',
[
'diff',
'--name-only',
`${process.env.BASE_SHA}...${process.env.HEAD_SHA}`,
],
{ encoding: 'utf8' }
)
.trim()
.split('\n')
.filter(Boolean)
let runAll = false
for (const file of changedFiles) {
if (
file === '.github/workflows/next-upgrade-evals.yml' ||
file.startsWith('docs/01-app/02-guides/upgrading/') ||
file.startsWith('docs/02-pages/02-guides/upgrading/') ||
file.startsWith('evals/lib/') ||
file.startsWith('packages/next-codemod/') ||
file === 'packages/next/src/bin/next.ts' ||
file === 'packages/next/src/cli/next-upgrade.ts' ||
file.startsWith('packages/next/src/lib/upgrade/') ||
(file.startsWith('evals/next-upgrade/') &&
!file.startsWith(fixturePrefix))
) {
runAll = true
break
}
if (file.startsWith(fixturePrefix)) {
const fixture = file.slice(fixturePrefix.length).split('/')[0]
if (!knownFixtures.has(fixture)) {
runAll = true
break
}
changedFixtures.add(fixture)
}
}
const selected = runAll ? allFixtures : [...changedFixtures].sort()
if (selected.length === 0) {
throw new Error('The workflow was triggered without an affected eval')
}
appendFileSync(
process.env.GITHUB_OUTPUT,
`evals=${JSON.stringify(selected)}\n`
)
console.log(`Selected evals: ${selected.join(', ')}`)
NODE
eval:
name: Run evals
needs: select-evals
if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
runs-on: ubuntu-latest
timeout-minutes: 90
environment: next-upgrade-evals
permissions:
contents: read
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 22
package-manager-cache: false
- name: Setup pnpm
run: |
npm i -g corepack@0.31
corepack enable
- name: Install dependencies
run: pnpm install
- name: Build packages
run: pnpm build-all
- name: Pull Vercel OIDC token
env:
VERCEL_PROJECT_ID: ${{ vars.VERCEL_NEXT_UPGRADE_EVALS_PROJECT_ID }}
VERCEL_TEAM_ID: ${{ vars.VERCEL_NEXT_UPGRADE_EVALS_TEAM_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_NEXT_UPGRADE_EVALS_TOKEN }}
run: |
pnpm dlx vercel@59.11.0 env pull .env.local \
--yes \
--environment=development \
--scope "$VERCEL_TEAM_ID" \
--project "$VERCEL_PROJECT_ID" \
--token "$VERCEL_TOKEN"
node <<'NODE'
const {
appendFileSync,
readFileSync,
rmSync,
} = require('node:fs')
const { parse } = require('dotenv')
const oidc = parse(readFileSync('.env.local')).VERCEL_OIDC_TOKEN
if (!oidc) throw new Error('Vercel did not provide an OIDC token')
console.log(`::add-mask::${oidc}`)
appendFileSync(process.env.GITHUB_ENV, `VERCEL_OIDC_TOKEN=${oidc}\n`)
rmSync('.env.local')
NODE
- name: Run eval
env:
EVALS: ${{ needs.select-evals.outputs.evals }}
run: |
node <<'NODE'
const { spawnSync } = require('node:child_process')
const evals = JSON.parse(process.env.EVALS)
const result = spawnSync('pnpm', ['eval:upgrade', ...evals], {
stdio: 'inherit',
})
if (result.error) throw result.error
if (result.status !== 0) process.exitCode = 1
NODE
- name: Retain transcripts and grading evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: next-upgrade-evals-${{ github.run_id }}-${{ github.run_attempt }}
path: evals/next-upgrade/results/
if-no-files-found: warn
retention-days: 14
.tarballs/ .tarballs/
results/ results/
experiments/ci-*.ts
.env .env
.env.local .env.local
...@@ -13,14 +13,15 @@ assertions, judging and result storage belong to `@vercel/agent-eval`. ...@@ -13,14 +13,15 @@ assertions, judging and result storage belong to `@vercel/agent-eval`.
```sh ```sh
pnpm build-all pnpm build-all
pnpm eval:upgrade <fixture-name> --dry pnpm eval:upgrade <fixture-name> [fixture-name...] --dry
NEXT_UPGRADE_EVAL_EXPERIMENT=codex pnpm eval:upgrade <fixture-name> NEXT_UPGRADE_EVAL_EXPERIMENT=codex pnpm eval:upgrade <fixture-name> [fixture-name...]
``` ```
Omit the experiment filter to run Codex and Claude. `--list` lists fixtures without Omit the experiment filter to run Codex and Claude. `--list` lists fixtures without
packing or making model calls. Run one named fixture at a time. Results use the packing or making model calls. Named fixtures and their selected harnesses run
framework's normal `results/` layout. Fixtures are added by the feature PRs stacked concurrently after packages and the shared toolchain are prepared once. Results use
above this infrastructure. the framework's normal `results/` layout. Fixtures are added by the feature PRs
stacked above this infrastructure.
## Lifecycle ## Lifecycle
......
...@@ -2,8 +2,11 @@ import { join } from 'node:path' ...@@ -2,8 +2,11 @@ import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval' import type { Sandbox } from '@vercel/agent-eval'
import { setupUpgradeScenario } from '../security/setup' import { setupUpgradeScenario } from '../security/setup'
export async function setupFuture(sandbox: Sandbox) { export async function setupFuture(
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
const scenarios: Record<string, { source: string; target: string }> = { const scenarios: Record<string, { source: string; target: string }> = {
'future-cache-components': { 'future-cache-components': {
source: '13.5.11', source: '13.5.11',
...@@ -21,19 +24,23 @@ export async function setupFuture(sandbox: Sandbox) { ...@@ -21,19 +24,23 @@ export async function setupFuture(sandbox: Sandbox) {
const scenario = fixture ? scenarios[fixture] : undefined const scenario = fixture ? scenarios[fixture] : undefined
if (!scenario) throw new Error('Unknown Future Defaults eval case') if (!scenario) throw new Error('Unknown Future Defaults eval case')
await setupUpgradeScenario(sandbox, { await setupUpgradeScenario(
fixturePrefix: 'future-', sandbox,
assessmentPath: join(__dirname, 'assessment.mjs'), {
assessment: scenario, fixturePrefix: 'future-',
installedVersion: undefined, assessmentPath: join(__dirname, 'assessment.mjs'),
candidateScripts: assessment: scenario,
fixture === 'future-cache-components-nudge' ? ['dev'] : undefined, installedVersion: undefined,
skillInstructionsPath: candidateScripts:
fixture === 'future-cache-components-nudge' fixture === 'future-cache-components-nudge' ? ['dev'] : undefined,
? undefined skillInstructionsPath:
: join( fixture === 'future-cache-components-nudge'
__dirname, ? undefined
'../../../skills/next-cache-components-adoption/SKILL.md' : join(
), __dirname,
}) '../../../skills/next-cache-components-adoption/SKILL.md'
),
},
fixture
)
} }
...@@ -2,8 +2,11 @@ import { join } from 'node:path' ...@@ -2,8 +2,11 @@ import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval' import type { Sandbox } from '@vercel/agent-eval'
import { setupUpgradeScenario } from '../security/setup' import { setupUpgradeScenario } from '../security/setup'
export async function setupLatest(sandbox: Sandbox) { export async function setupLatest(
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
const target = '16.3.5' const target = '16.3.5'
const scenarios: Record< const scenarios: Record<
string, string,
...@@ -16,10 +19,14 @@ export async function setupLatest(sandbox: Sandbox) { ...@@ -16,10 +19,14 @@ export async function setupLatest(sandbox: Sandbox) {
const scenario = fixture ? scenarios[fixture] : undefined const scenario = fixture ? scenarios[fixture] : undefined
if (!scenario) throw new Error('Unknown latest upgrade eval case') if (!scenario) throw new Error('Unknown latest upgrade eval case')
await setupUpgradeScenario(sandbox, { await setupUpgradeScenario(
fixturePrefix: 'latest-', sandbox,
assessmentPath: join(__dirname, 'assessment.mjs'), {
assessment: scenario, fixturePrefix: 'latest-',
installedVersion: scenario.installedVersion, assessmentPath: join(__dirname, 'assessment.mjs'),
}) assessment: scenario,
installedVersion: scenario.installedVersion,
},
fixture
)
} }
...@@ -5,9 +5,10 @@ import { setupLatest } from '../latest/setup' ...@@ -5,9 +5,10 @@ import { setupLatest } from '../latest/setup'
import { setupFuture } from '../future/setup' import { setupFuture } from '../future/setup'
export function upgradeExperiment( export function upgradeExperiment(
harness: 'codex' | 'claude-code' harness: 'codex' | 'claude-code',
selectedFixture: string | undefined = undefined
): ExperimentConfig { ): ExperimentConfig {
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture) throw new Error('Select one upgrade eval case') if (!fixture) throw new Error('Select one upgrade eval case')
const security = fixture.startsWith('security-') const security = fixture.startsWith('security-')
const latest = fixture.startsWith('latest-') const latest = fixture.startsWith('latest-')
...@@ -25,10 +26,10 @@ export function upgradeExperiment( ...@@ -25,10 +26,10 @@ export function upgradeExperiment(
timeout: future ? 3600 : 1800, timeout: future ? 3600 : 1800,
copyFiles: 'changed', copyFiles: 'changed',
setup: async (sandbox) => { setup: async (sandbox) => {
const setup = await setupUpgrade(sandbox) const setup = await setupUpgrade(sandbox, fixture)
if (security) await setupSecurity(sandbox) if (security) await setupSecurity(sandbox, fixture)
if (latest) await setupLatest(sandbox) if (latest) await setupLatest(sandbox, fixture)
if (future) await setupFuture(sandbox) if (future) await setupFuture(sandbox, fixture)
return setup return setup
}, },
} }
......
...@@ -4,7 +4,10 @@ import type { Sandbox } from '@vercel/agent-eval' ...@@ -4,7 +4,10 @@ import type { Sandbox } from '@vercel/agent-eval'
export const toolsDirectory = '/tmp/next-upgrade-eval' export const toolsDirectory = '/tmp/next-upgrade-eval'
export async function setupUpgrade(sandbox: Sandbox) { export async function setupUpgrade(
sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const run = async (command: string, args: string[]) => { const run = async (command: string, args: string[]) => {
const result = await sandbox.runCommand(command, args) const result = await sandbox.runCommand(command, args)
if (result.exitCode !== 0) if (result.exitCode !== 0)
...@@ -20,7 +23,7 @@ export async function setupUpgrade(sandbox: Sandbox) { ...@@ -20,7 +23,7 @@ export async function setupUpgrade(sandbox: Sandbox) {
'Run through pnpm eval:upgrade to provide the candidate packages' 'Run through pnpm eval:upgrade to provide the candidate packages'
) )
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture) throw new Error('Select one upgrade eval case') if (!fixture) throw new Error('Select one upgrade eval case')
const fixtureDirectory = join(__dirname, '../evals', fixture) const fixtureDirectory = join(__dirname, '../evals', fixture)
const baselineFiles = Object.fromEntries( const baselineFiles = Object.fromEntries(
......
...@@ -8,6 +8,38 @@ const { packPackage } = require('../lib/pack') ...@@ -8,6 +8,38 @@ const { packPackage } = require('../lib/pack')
const { linkEnvironment } = require('../lib/environment') const { linkEnvironment } = require('../lib/environment')
const snapshotEnvironmentVariable = 'AGENT_EVAL_SANDBOX_SNAPSHOT_ID' const snapshotEnvironmentVariable = 'AGENT_EVAL_SANDBOX_SNAPSHOT_ID'
function createExperimentFiles(directory, fixtures, harness) {
const harnesses = harness
? [[harness, harness === 'claude' ? 'claude-code' : harness]]
: [
['codex', 'codex'],
['claude', 'claude-code'],
]
const files = []
const experiments = []
try {
for (const fixture of fixtures) {
for (const [name, agent] of harnesses) {
const experiment = `ci-${name}-${fixture}`
const file = path.join(directory, `${experiment}.ts`)
if (fs.existsSync(file))
throw new Error(`Generated experiment already exists: ${file}`)
fs.writeFileSync(
file,
`import { upgradeExperiment } from '../lib/experiment'\n` +
`export default upgradeExperiment(${JSON.stringify(agent)}, ${JSON.stringify(fixture)})\n`
)
files.push(file)
experiments.push(experiment)
}
}
} catch (error) {
for (const file of files) fs.rmSync(file, { force: true })
throw error
}
return { experiments, files, harnessCount: harnesses.length }
}
async function createToolchainSnapshot() { async function createToolchainSnapshot() {
console.log('Preparing shared eval toolchain...') console.log('Preparing shared eval toolchain...')
const token = process.env.VERCEL_TOKEN const token = process.env.VERCEL_TOKEN
...@@ -54,7 +86,9 @@ async function main() { ...@@ -54,7 +86,9 @@ async function main() {
loadEnvironment({ path: path.join(__dirname, '.env'), override: true }) loadEnvironment({ path: path.join(__dirname, '.env'), override: true })
const args = process.argv.slice(2) const args = process.argv.slice(2)
const fixturesDirectory = path.join(__dirname, 'evals') const fixturesDirectory = path.join(__dirname, 'evals')
const { discoverFixtures, loadFixture } = await import('@vercel/agent-eval') const { discoverFixtures, loadConfig, loadFixture } = await import(
'@vercel/agent-eval'
)
const cases = fs.existsSync(fixturesDirectory) const cases = fs.existsSync(fixturesDirectory)
? discoverFixtures(fixturesDirectory) ? discoverFixtures(fixturesDirectory)
: [] : []
...@@ -63,23 +97,39 @@ async function main() { ...@@ -63,23 +97,39 @@ async function main() {
console.log(cases.join('\n')) console.log(cases.join('\n'))
return return
} }
const selected = args.filter((arg) => !arg.startsWith('--')) const selected = [...new Set(args.filter((arg) => !arg.startsWith('--')))]
if (selected.length !== 1) if (selected.length === 0)
throw new Error( throw new Error('Select upgrade eval fixtures; use --list to list fixtures')
'Select one upgrade eval fixture; use --list to list fixtures'
)
const fixture = selected[0]
const harness = process.env.NEXT_UPGRADE_EVAL_EXPERIMENT const harness = process.env.NEXT_UPGRADE_EVAL_EXPERIMENT
if (harness && !['codex', 'claude'].includes(harness)) if (harness && !['codex', 'claude'].includes(harness))
throw new Error('Select codex or claude') throw new Error('Select codex or claude')
if (!cases.includes(fixture)) for (const fixture of selected) {
throw new Error(`Available cases: ${cases.join(', ')}`) if (!cases.includes(fixture))
throw new Error(`Available cases: ${cases.join(', ')}`)
// Validate using the framework's own fixture rules. Its run command
// otherwise falls back to all fixtures when a filter matches no fixture.
loadFixture(fixturesDirectory, fixture)
}
if (args.some((arg) => arg.startsWith('--') && arg !== '--dry')) if (args.some((arg) => arg.startsWith('--') && arg !== '--dry'))
throw new Error('Supported flags: --dry, --list') throw new Error('Supported flags: --dry, --list')
// Validate using the framework's own fixture rules. Its run command otherwise if (args.includes('--dry')) {
// falls back to all fixtures when an explicit filter matches no valid fixture. let generatedExperiments = []
loadFixture(fixturesDirectory, fixture) try {
if (args.includes('--dry')) return console.log(fixture) if (selected.length > 1) {
const generated = createExperimentFiles(
path.join(__dirname, 'experiments'),
selected,
harness
)
generatedExperiments = generated.files
await Promise.all(generated.files.map((file) => loadConfig(file)))
}
console.log(selected.join('\n'))
return
} finally {
for (const file of generatedExperiments) fs.rmSync(file, { force: true })
}
}
for (const [name, entry] of [ for (const [name, entry] of [
['next', 'dist/bin/next'], ['next', 'dist/bin/next'],
['next-codemod', 'bin/next-codemod.js'], ['next-codemod', 'bin/next-codemod.js'],
...@@ -102,25 +152,40 @@ async function main() { ...@@ -102,25 +152,40 @@ async function main() {
} }
fs.mkdirSync(path.join(__dirname, 'results'), { recursive: true }) fs.mkdirSync(path.join(__dirname, 'results'), { recursive: true })
let snapshot let snapshot
let generatedExperiments = []
try { try {
if (!env[snapshotEnvironmentVariable]) { if (!env[snapshotEnvironmentVariable]) {
snapshot = await createToolchainSnapshot() snapshot = await createToolchainSnapshot()
env[snapshotEnvironmentVariable] = snapshot.snapshotId env[snapshotEnvironmentVariable] = snapshot.snapshotId
} }
const experiments = harness ? [harness] : ['codex', 'claude'] const generated =
selected.length > 1
? createExperimentFiles(
path.join(__dirname, 'experiments'),
selected,
harness
)
: {
experiments: harness ? [harness] : ['codex', 'claude'],
files: [],
harnessCount: harness ? 1 : 2,
}
generatedExperiments = generated.files
const result = spawnSync( const result = spawnSync(
path.join(root, 'node_modules/.bin/agent-eval'), path.join(root, 'node_modules/.bin/agent-eval'),
['run', ...experiments, '--force', '--ack-failures'], ['run', ...generated.experiments, '--force', '--ack-failures'],
{ {
cwd: __dirname, cwd: __dirname,
env: { env: {
...env, ...env,
NEXT_UPGRADE_EVAL_CASE: fixture, ...(selected.length === 1
...(experiments.length > 1 ? { NEXT_UPGRADE_EVAL_CASE: selected[0] }
: {}),
...(generated.harnessCount > 1
? { ? {
AGENT_EVAL_PREPARE_FIXTURE_ONCE: '1', AGENT_EVAL_PREPARE_FIXTURE_ONCE: '1',
AGENT_EVAL_PREPARED_FIXTURE_CONSUMERS: String( AGENT_EVAL_PREPARED_FIXTURE_CONSUMERS: String(
experiments.length generated.harnessCount
), ),
} }
: {}), : {}),
...@@ -131,6 +196,7 @@ async function main() { ...@@ -131,6 +196,7 @@ async function main() {
if (result.error) throw result.error if (result.error) throw result.error
if (result.status !== 0) process.exitCode = 1 if (result.status !== 0) process.exitCode = 1
} finally { } finally {
for (const file of generatedExperiments) fs.rmSync(file, { force: true })
if (snapshot) { if (snapshot) {
try { try {
await snapshot.delete() await snapshot.delete()
......
...@@ -3,8 +3,11 @@ import { join } from 'node:path' ...@@ -3,8 +3,11 @@ import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval' import type { Sandbox } from '@vercel/agent-eval'
import { toolsDirectory } from '../lib/fixture' import { toolsDirectory } from '../lib/fixture'
export async function setupSecurity(sandbox: Sandbox) { export async function setupSecurity(
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE sandbox: Sandbox,
selectedFixture: string | undefined = undefined
) {
const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture?.startsWith('security-')) if (!fixture?.startsWith('security-'))
throw new Error('Select a security upgrade eval case') throw new Error('Select a security upgrade eval case')
const sameMajorTarget = '15.5.24' const sameMajorTarget = '15.5.24'
...@@ -56,12 +59,16 @@ export async function setupSecurity(sandbox: Sandbox) { ...@@ -56,12 +59,16 @@ export async function setupSecurity(sandbox: Sandbox) {
const scenario = scenarios[fixture] const scenario = scenarios[fixture]
if (!scenario) throw new Error('Unknown security upgrade eval case') if (!scenario) throw new Error('Unknown security upgrade eval case')
await setupUpgradeScenario(sandbox, { await setupUpgradeScenario(
fixturePrefix: 'security-', sandbox,
assessmentPath: join(__dirname, 'assessment.mjs'), {
assessment: scenario, fixturePrefix: 'security-',
installedVersion: scenario.installedVersion, assessmentPath: join(__dirname, 'assessment.mjs'),
}) assessment: scenario,
installedVersion: scenario.installedVersion,
},
fixture
)
} }
export async function setupUpgradeScenario( export async function setupUpgradeScenario(
...@@ -73,7 +80,8 @@ export async function setupUpgradeScenario( ...@@ -73,7 +80,8 @@ export async function setupUpgradeScenario(
installedVersion: string | undefined installedVersion: string | undefined
candidateScripts?: string[] candidateScripts?: string[]
skillInstructionsPath?: string skillInstructionsPath?: string
} },
selectedFixture: string | undefined = undefined
) { ) {
const run = async (command: string, args: string[]) => { const run = async (command: string, args: string[]) => {
const result = await sandbox.runCommand(command, args) const result = await sandbox.runCommand(command, args)
...@@ -83,7 +91,7 @@ export async function setupUpgradeScenario( ...@@ -83,7 +91,7 @@ export async function setupUpgradeScenario(
) )
return result.stdout.trim() return result.stdout.trim()
} }
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE const fixture = selectedFixture ?? process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture?.startsWith(options.fixturePrefix)) if (!fixture?.startsWith(options.fixturePrefix))
throw new Error(`Select a ${options.fixturePrefix} upgrade eval case`) throw new Error(`Select a ${options.fixturePrefix} upgrade eval case`)
const security = `${toolsDirectory}/security` const security = `${toolsDirectory}/security`
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment