Unverified Commit 97496b20 authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Add "latest" upgrade coverage for `next upgrade --ai` (#98633)

Stacked on #98637.

This PR adds `next upgrade --ai="latest"` flag, which is targeted to
help users leverage agents to upgrade their app to the latest major
version when available. Just like security upgrade, it covers running
codemods and a migration checklist for major-to-major upgrades to
support breaking changes more reliably.
parent 9bb13254
......@@ -13,6 +13,8 @@ as instructed so the original task can continue with a warning. Tell the user
about the finding and ask whether they want to run the upgrade. Continue
independent work while waiting. If approved, complete the upgrade.
For `latest`, the CLI pins npm's latest stable Next.js release as the target.
## 1. Check for duplicates
Before reading another guide or changing files, complete every item:
......
.next/
*.tsbuildinfo
__agent_eval__/
eval-evidence/
node_modules/
next-env.d.ts
!package-lock.json
You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md.
import { expect, test } from 'vitest'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { securityChecks } from './checks/EVAL'
const { target } = JSON.parse(
readFileSync('/tmp/next-upgrade-eval/security/assessment.json', 'utf8')
) as { target: string }
securityChecks(
'13.5.11',
target,
(app) => {
test('completes the async request API migration', () => {
const source = readFileSync(join(app.cwd, 'lib/viewer.ts'), 'utf8')
expect(source).not.toContain('@next-codemod-')
expect(source).not.toContain('UnsafeUnwrapped')
})
test('preserves request identity between visitors', async () => {
await Promise.all(
[
['Alice', 'fr'],
['Bob', 'de'],
['Guest', 'en'],
].map(async ([name, language]) => {
const headers: Record<string, string> = {
'accept-language': language,
}
if (name !== 'Guest') headers.cookie = `member=${name}`
const [page, api] = await Promise.all([
fetch(app.url, { headers }),
fetch(`${app.url}/api/viewer`, { headers }),
])
expect(page.status).toBe(200)
expect(api.status).toBe(200)
const html = await page.text()
expect(html).toContain(`id="member">${name}<`)
expect(html).toContain(`id="language">${language}<`)
expect(await api.json()).toEqual({ name, language })
})
)
})
},
{
changedFiles: [
'app/page.tsx',
'eslint.config.mjs',
'lib/viewer.ts',
'next.config.js',
'package.json',
],
migrationGuides: [14, 15, 16],
upgradeType: 'latest',
}
)
Run `npx next@canary upgrade --ai latest` for this app.
# Next.js 13 member dashboard
The home page greets the member identified by the `member` cookie and displays
the request's `accept-language` header. Missing cookies show Guest. These values
must stay isolated between visitors. `/api/viewer` returns the same request
identity.
Use `npm run lint`, `npm run typecheck`, `npm run build`, and `npm start` to
check the app.
import { viewer } from '../../../lib/viewer'
export async function GET() {
return Response.json(await viewer())
}
export default function Layout({ children }: { children: React.ReactNode }) {
return (
<html>
<body>{children}</body>
</html>
)
}
import { viewer } from '../lib/viewer'
export default function Page() {
const member = viewer()
return (
<main>
<h1>Member dashboard</h1>
<p id="member">{member.name}</p>
<p id="language">{member.language}</p>
</main>
)
}
../../../shared/security-checks.ts
\ No newline at end of file
import { cookies, headers } from 'next/headers'
export function viewer() {
const cookieStore = cookies()
const headerStore = headers()
return {
name: cookieStore.get('member')?.value || 'Guest',
language: headerStore.get('accept-language') || 'en',
}
}
{
"name": "latest-cross-major",
"private": true,
"type": "module",
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"next": "13.5.11",
"react": "18.2.0",
"react-dom": "18.2.0"
},
"devDependencies": {
"@types/node": "20.17.7",
"@types/react": "18.2.79",
"@types/react-dom": "18.2.25",
"eslint": "8.57.1",
"eslint-config-next": "13.5.11",
"typescript": "5.8.3",
"vitest": "3.1.3",
"@vitejs/plugin-react": "4.4.1",
"vite-tsconfig-paths": "5.1.4"
}
}
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"plugins": [{ "name": "next" }]
},
"include": [
"next-env.d.ts",
"app/**/*.ts",
"app/**/*.tsx",
"lib/**/*.ts",
".next/types/**/*.ts"
],
"exclude": ["node_modules"]
}
.next/
*.tsbuildinfo
__agent_eval__/
eval-evidence/
node_modules/
next-env.d.ts
!package-lock.json
You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md.
import { expect, test } from 'vitest'
import { readFileSync } from 'node:fs'
import { securityChecks } from './checks/EVAL'
const { target } = JSON.parse(
readFileSync('/tmp/next-upgrade-eval/security/assessment.json', 'utf8')
) as { target: string }
securityChecks(
'16.2.12',
target,
(app) => {
test('preserves request identity between visitors', async () => {
await Promise.all(
[
['Alice', 'fr'],
['Bob', 'de'],
['Guest', 'en'],
].map(async ([name, language]) => {
const headers: Record<string, string> = {
'accept-language': language,
}
if (name !== 'Guest') headers.cookie = `member=${name}`
const [page, api] = await Promise.all([
fetch(app.url, { headers }),
fetch(`${app.url}/api/viewer`, { headers }),
])
expect(page.status).toBe(200)
expect(api.status).toBe(200)
const html = await page.text()
expect(html).toContain(`id="member">${name}<`)
expect(html).toContain(`id="language">${language}<`)
expect(await api.json()).toEqual({ name, language })
})
)
})
},
{
changedFiles: ['next.config.js', 'package.json'],
migrationGuides: [16],
upgradeType: 'latest',
}
)
Run `npx next@canary upgrade --ai latest` for this app.
# Member dashboard
The home page greets the member identified by the `member` cookie and displays
the request's `accept-language` header. Missing cookies show Guest. These values
must stay isolated between visitors. `/api/viewer` returns the same request
identity.
Use `npm run typecheck`, `npm run build`, and `npm start` to check the app.
import { viewer } from '../../../lib/viewer'
export async function GET() {
return Response.json(await viewer())
}
export default function RootLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html lang="en">
<body>{children}</body>
</html>
)
}
import { viewer } from '../lib/viewer'
export default async function Page() {
const member = await viewer()
return (
<main>
<h1>Member dashboard</h1>
<p id="member">{member.name}</p>
<p id="language">{member.language}</p>
</main>
)
}
../../../shared/security-checks.ts
\ No newline at end of file
import { cookies, headers } from 'next/headers'
export async function viewer() {
const [cookieStore, headerStore] = await Promise.all([cookies(), headers()])
return {
name: cookieStore.get('member')?.value || 'Guest',
language: headerStore.get('accept-language') || 'en',
}
}
{
"name": "latest-same-major",
"private": true,
"type": "module",
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"next": "16.2.12",
"react": "19.1.0",
"react-dom": "19.1.0"
},
"devDependencies": {
"@types/node": "20.17.7",
"@types/react": "19.1.2",
"@types/react-dom": "19.1.2",
"typescript": "5.8.3",
"vitest": "3.1.3",
"@vitejs/plugin-react": "4.4.1",
"vite-tsconfig-paths": "5.1.4"
}
}
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"plugins": [{ "name": "next" }]
},
"include": [
"next-env.d.ts",
"app/**/*.ts",
"app/**/*.tsx",
"lib/**/*.ts",
".next/types/**/*.ts"
],
"exclude": ["node_modules"]
}
import { appendFileSync, readFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
const tools = dirname(dirname(fileURLToPath(import.meta.url)))
const realFetch = globalThis.fetch
const { target } = JSON.parse(
readFileSync(join(tools, 'security/assessment.json'), 'utf8')
)
globalThis.fetch = async (input, init) => {
const url = String(input)
if (url !== 'https://registry.npmjs.org/next/latest') {
return realFetch(input, init)
}
const value = {
version: target,
engines: { node: '>=20.9.0' },
}
appendFileSync(
join(tools, 'assessment.jsonl'),
JSON.stringify({ url, value }) + '\n'
)
return Response.json(value)
}
import { join } from 'node:path'
import type { Sandbox } from '@vercel/agent-eval'
import { setupUpgradeScenario } from '../security/setup'
export async function setupLatest(sandbox: Sandbox) {
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
const target = '16.3.5'
const scenarios: Record<string, { target: string }> = {
'latest-cross-major': { target },
'latest-same-major': { target },
}
const scenario = fixture ? scenarios[fixture] : undefined
if (!scenario) throw new Error('Unknown latest upgrade eval case')
await setupUpgradeScenario(sandbox, {
fixturePrefix: 'latest-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: undefined,
})
}
import type { ExperimentConfig } from '@vercel/agent-eval'
import { setupUpgrade } from './fixture'
import { setupSecurity } from '../security/setup'
import { setupLatest } from '../latest/setup'
export function upgradeExperiment(
harness: 'codex' | 'claude-code'
......@@ -8,6 +9,7 @@ export function upgradeExperiment(
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture) throw new Error('Select one upgrade eval case')
const security = fixture.startsWith('security-')
const latest = fixture.startsWith('latest-')
return {
agent: `vercel-ai-gateway/${harness}`,
......@@ -23,6 +25,7 @@ export function upgradeExperiment(
setup: async (sandbox) => {
const setup = await setupUpgrade(sandbox)
if (security) await setupSecurity(sandbox)
if (latest) await setupLatest(sandbox)
return setup
},
}
......
......@@ -4,14 +4,6 @@ import type { Sandbox } from '@vercel/agent-eval'
import { toolsDirectory } from '../lib/fixture'
export async function setupSecurity(sandbox: Sandbox) {
const run = async (command: string, args: string[]) => {
const result = await sandbox.runCommand(command, args)
if (result.exitCode !== 0)
throw new Error(
`${command} failed during security setup:\n${result.stderr}`
)
return result.stdout.trim()
}
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture?.startsWith('security-'))
throw new Error('Select a security upgrade eval case')
......@@ -63,6 +55,35 @@ export async function setupSecurity(sandbox: Sandbox) {
}
const scenario = scenarios[fixture]
if (!scenario) throw new Error('Unknown security upgrade eval case')
await setupUpgradeScenario(sandbox, {
fixturePrefix: 'security-',
assessmentPath: join(__dirname, 'assessment.mjs'),
assessment: scenario,
installedVersion: scenario.installedVersion,
})
}
export async function setupUpgradeScenario(
sandbox: Sandbox,
options: {
fixturePrefix: string
assessmentPath: string
assessment: object
installedVersion: string | undefined
}
) {
const run = async (command: string, args: string[]) => {
const result = await sandbox.runCommand(command, args)
if (result.exitCode !== 0)
throw new Error(
`${command} failed during security setup:\n${result.stderr}`
)
return result.stdout.trim()
}
const fixture = process.env.NEXT_UPGRADE_EVAL_CASE
if (!fixture?.startsWith(options.fixturePrefix))
throw new Error(`Select a ${options.fixturePrefix} upgrade eval case`)
const security = `${toolsDirectory}/security`
const bin = `${toolsDirectory}/bin`
const repository = 'https://github.com/next-upgrade-eval/fixture.git'
......@@ -73,10 +94,10 @@ export async function setupSecurity(sandbox: Sandbox) {
await run('mkdir', ['-p', security])
await sandbox.writeFiles({
[`${security}/assessment.mjs`]: readFileSync(
join(__dirname, 'assessment.mjs'),
options.assessmentPath,
'utf8'
),
[`${security}/assessment.json`]: JSON.stringify(scenario),
[`${security}/assessment.json`]: JSON.stringify(options.assessment),
[`${security}/provider.mjs`]: readFileSync(
join(__dirname, 'provider.mjs'),
'utf8'
......@@ -93,10 +114,10 @@ export async function setupSecurity(sandbox: Sandbox) {
[`${toolsDirectory}/baseline.json`]: JSON.stringify({ head: baseline }),
})
if (scenario.installedVersion) {
if (options.installedVersion) {
await run('node', [
`${security}/prepare-candidate.mjs`,
scenario.installedVersion,
options.installedVersion,
])
}
......@@ -120,7 +141,7 @@ export async function setupSecurity(sandbox: Sandbox) {
candidateNext: `${toolsDirectory}/next/node_modules/next`,
codemodVersion,
git,
prepareFixture: Boolean(scenario.installedVersion),
prepareFixture: Boolean(options.installedVersion),
remote,
repository,
}),
......
......@@ -463,10 +463,21 @@ export async function runUpgrade(
// https://github.com/codemod-com/codemod/blob/c0cf00d13161a0ec0965b6cc6bc5d54076839cc8/apps/cli/src/flags.ts#L160
// `--allow-dirty` is required because the upgrade above modified package.json
// and the lockfile; the recipe refuses to run on a dirty tree otherwise.
execSync(
`${execCommand} codemod@latest react/19/migration-recipe --no-interactive --allow-dirty`,
{ stdio: 'inherit' }
)
try {
execSync(
`${execCommand} codemod@latest react/19/migration-recipe --no-interactive --allow-dirty`,
{ stdio: 'inherit' }
)
} catch (error) {
// TODO: Remove this fallback once codemod publishes a Linux binary that
// supports the glibc versions used by our upgrade environments.
console.warn(
new Error(
`${pc.yellow('⚠')} The React 19 codemod could not run. Continue the upgrade and review the React 19 migration guide manually.`,
{ cause: error }
)
)
}
}
if (shouldRunReactTypesCodemods) {
......
......@@ -588,7 +588,7 @@ program
.addOption(
new Option(
'--ai, --experimental-ai [type]',
'Upgrade with AI. Defaults to security.'
'Upgrade with AI to security or latest. Defaults to security.'
).conflicts('revision')
)
.action(async (directory, options) => {
......
......@@ -69,17 +69,17 @@ export async function spawnNextUpgrade(
const upgradeType =
typeof options.ai === 'string' ? options.ai : 'security'
if (upgradeType !== 'security') {
if (upgradeType !== 'security' && upgradeType !== 'latest') {
throw new Error(
`Unsupported AI upgrade type ${JSON.stringify(upgradeType)}. Expected "security".`
`Unsupported AI upgrade type ${JSON.stringify(upgradeType)}. Expected "security" or "latest".`
)
}
// Resolve the requested target before preparing an agent session.
const { prepareUpgrade } =
require('../lib/upgrade/prepare-upgrade') as typeof import('../lib/upgrade/prepare-upgrade')
const assessmentSpinner = createSpinner('Checking for security updates')
const result = await prepareUpgrade(baseDir).finally(() =>
const assessmentSpinner = createSpinner('Preparing upgrade')
const result = await prepareUpgrade(baseDir, upgradeType).finally(() =>
assessmentSpinner?.stop()
)
......@@ -89,7 +89,7 @@ export async function spawnNextUpgrade(
}
Log.info(
`Security update: Next.js ${result.installedVersion} → ${result.targetVersion}`
`Upgrade: Next.js ${result.installedVersion} → ${result.targetVersion}`
)
// Use the invoking CLI's guides, even when the app runs an older Next.js.
......@@ -131,17 +131,17 @@ export async function spawnNextUpgrade(
guidesSpinner?.stop()
}
// TODO: Once every eligible security target supports
// `experimental.agenticAutoUpgrade`, ask the agent to enable it after
// verification so future upgrade reminders can use the same policy.
const references = result.references
.map((reference) => `- ${reference}`)
.join('\n')
const reason =
upgradeType === 'security'
? 'the installed version is affected by a published security advisory'
: 'a newer stable Next.js release is available'
// Pass resolved inputs directly; the agent owns repairs and verification.
const prompt = `Read and follow every applicable instruction in ${JSON.stringify(guidePath)} before proceeding.
We're upgrading the app in ${JSON.stringify(baseDir)} from Next.js ${result.installedVersion} to ${result.targetVersion} because the installed version is affected by a published security advisory.
We're upgrading the app in ${JSON.stringify(baseDir)} from Next.js ${result.installedVersion} to ${result.targetVersion} because ${reason}.
References:
${references}`
......@@ -151,7 +151,7 @@ ${references}`
await handoffUpgrade(prompt, baseDir)
} catch (error) {
Log.error(
'Could not prepare the security upgrade:',
'Could not prepare the upgrade:',
error instanceof Error ? error.message : error
)
process.exitCode = 1
......
......@@ -13,8 +13,15 @@ type UpgradePreparation =
}
export async function prepareUpgrade(
directory: string
directory: string,
targetRequest: string = 'security'
): Promise<UpgradePreparation> {
if (targetRequest !== 'security' && targetRequest !== 'latest') {
throw new Error(
`Unsupported AI upgrade type ${JSON.stringify(targetRequest)}. Expected "security" or "latest".`
)
}
// Resolve from the app: the invoking canary is only the upgrade tooling.
const requireFromApp = createRequire(join(directory, 'package.json'))
const { version: installedVersion } = JSON.parse(
......@@ -28,10 +35,55 @@ export async function prepareUpgrade(
// TODO: Handle prereleases
if (semver.prerelease(installedVersion)) {
throw new Error(
'Security upgrades are not available for prerelease versions of Next.js yet.'
'AI upgrades are not available for prerelease versions of Next.js yet.'
)
}
if (targetRequest === 'latest') {
const url = `${NPM_REGISTRY}next/latest`
const { value } = await fetchJSON(url)
const release = value as {
version: string
engines: { node: string | undefined } | undefined
} | null
if (
!release ||
!semver.valid(release.version) ||
semver.prerelease(release.version)
) {
throw new Error('Could not determine the latest stable Next.js version.')
}
if (semver.eq(release.version, installedVersion)) {
return {
status: 'unaffected',
reason: `Next.js ${installedVersion} is already the latest stable release.`,
}
}
if (semver.lt(release.version, installedVersion)) {
return {
status: 'unaffected',
reason: `Next.js ${installedVersion} is newer than the latest stable release ${release.version}.`,
}
}
const nodeRange = release.engines?.node ?? null
if (!nodeRange || !semver.satisfies(process.versions.node, nodeRange)) {
throw new Error(
`Next.js ${release.version} requires Node.js ${nodeRange ?? '(version unavailable)'}. Update Node.js before continuing.`
)
}
return {
status: 'ready',
installedVersion,
targetVersion: release.version,
references: [url],
}
}
const snapshot = await readSecuritySnapshot(installedVersion)
if (!snapshot) {
......@@ -99,7 +151,7 @@ async function fetchJSON(
return { value: await response.json(), headers: response.headers }
} catch (error) {
throw new Error('Could not check for security updates. Please try again.', {
throw new Error('Could not fetch upgrade metadata. Please try again.', {
cause: error,
})
}
......
......@@ -227,7 +227,7 @@ describe('agentic upgrade prompts', () => {
ai: 'security',
})
expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app')
expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', 'security')
const [guidePath, guide] = jest.mocked(writeFile).mock.calls[0]
expect(String(guidePath).replace(/\\+/g, '/')).toBe(
'/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md'
......@@ -271,6 +271,35 @@ describe('agentic upgrade prompts', () => {
ai: true,
})
expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app')
expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', 'security')
})
it('passes the latest target to the existing agent', async () => {
jest.mocked(prepareUpgrade).mockResolvedValue({
status: 'ready',
installedVersion: '16.2.12',
targetVersion: '16.3.5',
references: ['https://registry.npmjs.org/next/latest'],
})
await spawnNextUpgrade('/workspace/app', {
revision: 'latest',
verbose: false,
ai: 'latest',
})
expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', 'latest')
expect(normalizedBootstrapCalls()).toMatchInlineSnapshot(`
[
[
"Read and follow every applicable instruction in "/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md" before proceeding.
We're upgrading the app in "/workspace/app" from Next.js 16.2.12 to 16.3.5 because a newer stable Next.js release is available.
References:
- https://registry.npmjs.org/next/latest",
],
]
`)
})
})
import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises'
import { tmpdir } from 'os'
import { join } from 'path'
import { prepareUpgrade } from 'next/dist/lib/upgrade/prepare-upgrade'
describe('prepare latest upgrade', () => {
const directories: string[] = []
const originalFetch = global.fetch
async function createApp(version: string): Promise<string> {
const directory = await mkdtemp(join(tmpdir(), 'next-latest-upgrade-'))
directories.push(directory)
await mkdir(join(directory, 'node_modules/next'), { recursive: true })
await writeFile(join(directory, 'package.json'), '{}')
await writeFile(
join(directory, 'node_modules/next/package.json'),
JSON.stringify({ version })
)
return directory
}
function mockLatestVersion(version: string) {
global.fetch = jest.fn().mockResolvedValue(
new Response(
JSON.stringify({
version,
engines: { node: '>=18' },
}),
{ status: 200 }
)
)
}
afterEach(async () => {
global.fetch = originalFetch
await Promise.all(
directories
.splice(0)
.map((directory) => rm(directory, { recursive: true, force: true }))
)
})
it('selects the exact latest stable release', async () => {
const directory = await createApp('16.2.1')
mockLatestVersion('17.1.0')
await expect(prepareUpgrade(directory, 'latest')).resolves.toEqual(
expect.objectContaining({
status: 'ready',
installedVersion: '16.2.1',
targetVersion: '17.1.0',
references: ['https://registry.npmjs.org/next/latest'],
})
)
expect(global.fetch).toHaveBeenCalledTimes(1)
expect(jest.mocked(global.fetch).mock.calls[0][0]).toBe(
'https://registry.npmjs.org/next/latest'
)
})
it('does nothing when the app already uses latest', async () => {
const directory = await createApp('17.1.0')
mockLatestVersion('17.1.0')
await expect(prepareUpgrade(directory, 'latest')).resolves.toEqual({
status: 'unaffected',
reason: 'Next.js 17.1.0 is already the latest stable release.',
})
})
it('does not downgrade an app newer than latest', async () => {
const directory = await createApp('18.0.0')
mockLatestVersion('17.1.0')
await expect(prepareUpgrade(directory, 'latest')).resolves.toEqual({
status: 'unaffected',
reason: 'Next.js 18.0.0 is newer than the latest stable release 17.1.0.',
})
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment