Unverified Commit fdf41d6f authored by Jan Kassens's avatar Jan Kassens Committed by GitHub

[ai-upgrade] discover upgrade models and reasoning efforts from agent CLIs (#99736)

The world of AI is moving fast. Instead of hardcoding the available
models, discover them from the CLIs. This is a bit hacky, but probably
more reliable than hardcoding a list that will become outdated fast. The
available models might be different for different accounts.

This involved a bit of code moving as I wanted to have the model
discovery run in parallel to displaying the user the harness selection
to reduce delay in selecting the model.

## Test Plan:
In addition to new tests, also tested interactively with:
```
DEBUG=next:upgrade node -e \
  "require('./packages/next/dist/lib/upgrade/harness').handoffUpgrade('This is a harness test. Do not modify files.', process.cwd(), null).catch(console.error)"
```
parent c3e76ad9
...@@ -10,70 +10,9 @@ import * as Log from '../../build/output/log' ...@@ -10,70 +10,9 @@ import * as Log from '../../build/output/log'
import { getAgentName } from '../../telemetry/agent-name' import { getAgentName } from '../../telemetry/agent-name'
import type { AgentUpgradeHandoffMethod } from '../../telemetry/events/agent-upgrade' import type { AgentUpgradeHandoffMethod } from '../../telemetry/events/agent-upgrade'
import { bold, cyan, dim } from '../picocolors' import { bold, cyan, dim } from '../picocolors'
import { getHarnessModels, type UpgradeModel } from './model-discovery'
import { runChildProcess } from './run-child-process' import { runChildProcess } from './run-child-process'
const CODEX_EFFORTS = [
'low',
'medium',
'high',
'xhigh',
'max',
'ultra',
] as const
const CLAUDE_EFFORTS = ['low', 'medium', 'high', 'max'] as const
const UPGRADE_MODELS = {
codex: [
{ id: 'gpt-5.6-terra', label: 'GPT-5.6-Terra', efforts: CODEX_EFFORTS },
{ id: 'gpt-5.6-sol', label: 'GPT-5.6-Sol', efforts: CODEX_EFFORTS },
{ id: 'gpt-6-astra', label: 'GPT-6-Astra', efforts: CODEX_EFFORTS },
],
claude: [
{
id: 'claude-sonnet-5[1m]',
label: 'Claude Sonnet 5 (1M)',
efforts: CLAUDE_EFFORTS,
},
{ id: 'opus', label: 'Claude Opus (latest)', efforts: CLAUDE_EFFORTS },
{ id: 'fable', label: 'Claude Fable (latest)', efforts: CLAUDE_EFFORTS },
],
} as const
function getCodexModels(path: string) {
const result = spawn.sync(path, ['debug', 'models', '--bundled'], {
encoding: 'utf8',
timeout: 5000,
maxBuffer: 20 * 1024 * 1024,
})
if (result.status !== 0) {
Log.warn(
'Could not read the Codex model catalog; using the CLI defaults.',
result.error ?? (result.stderr?.trim() || `exit status ${result.status}`)
)
return []
}
try {
const catalog = JSON.parse(result.stdout) as {
models?: Array<{ slug: string }>
}
if (!Array.isArray(catalog.models)) {
Log.warn(
'Codex model catalog has an unexpected format; using the CLI defaults.'
)
return []
}
const available = new Set(catalog.models.map(({ slug }) => slug))
return UPGRADE_MODELS.codex.filter(({ id }) => available.has(id))
} catch (error) {
Log.warn(
'Could not parse the Codex model catalog; using the CLI defaults.',
error
)
return []
}
}
const CODEX_APPROVAL_ARGS = [ const CODEX_APPROVAL_ARGS = [
'--sandbox', '--sandbox',
'workspace-write', 'workspace-write',
...@@ -82,7 +21,7 @@ const CODEX_APPROVAL_ARGS = [ ...@@ -82,7 +21,7 @@ const CODEX_APPROVAL_ARGS = [
] as const ] as const
type UpgradeHarness = { type UpgradeHarness = {
name: keyof typeof UPGRADE_MODELS name: 'codex' | 'claude'
path: string path: string
} }
...@@ -386,17 +325,31 @@ export async function handoffUpgrade( ...@@ -386,17 +325,31 @@ export async function handoffUpgrade(
return 'handed_off' return 'handed_off'
} }
const discoveryController = new AbortController()
const modelCatalogs = new Map(
installed.map(({ name, path }) => [
name,
getHarnessModels(name, path, directory, discoveryController.signal),
])
)
const stopDiscovery = async () => {
discoveryController.abort()
await Promise.all(modelCatalogs.values())
}
try {
let stage: 'harness' | 'model' | 'effort' | 'permission' | 'worktree' = let stage: 'harness' | 'model' | 'effort' | 'permission' | 'worktree' =
'harness' 'harness'
let harness: UpgradeHarness | undefined let harness: UpgradeHarness | undefined
let model: { id: string; efforts: readonly string[] } | undefined let model: UpgradeModel | undefined
let effort: string | undefined let effort: string | undefined
let autoPermissionArgs: string[] | null = null let autoPermissionArgs: string[] | null = null
let approvalPermissionArgs: string[] = [] let approvalPermissionArgs: string[] = []
let permissionArgs: string[] = [] let permissionArgs: string[] = []
let useAuto = true let useAuto = true
let useWorktree = true let useWorktree = true
let codexModels: ReturnType<typeof getCodexModels> | undefined const previousModelStage = () =>
model ? (model.efforts.length > 0 ? 'effort' : 'model') : 'harness'
let codexAutoReview: boolean | undefined let codexAutoReview: boolean | undefined
let claudePermissionSupport: let claudePermissionSupport:
| ReturnType<typeof getClaudePermissionSupport> | ReturnType<typeof getClaudePermissionSupport>
...@@ -416,31 +369,33 @@ export async function handoffUpgrade( ...@@ -416,31 +369,33 @@ export async function handoffUpgrade(
harness = choice harness = choice
stage = 'model' stage = 'model'
} else if (stage === 'model') { } else if (stage === 'model') {
const models = const models = await modelCatalogs.get(harness!.name)!
harness!.name === 'codex' if (models === null) {
? (codexModels ??= getCodexModels(harness!.path)) break
: UPGRADE_MODELS.claude }
if (models.length === 0 && harness!.name === 'codex') { if (models.length === 0) {
Log.warn('Could not verify Codex models; using the CLI defaults.')
model = undefined model = undefined
effort = 'default' effort = 'default'
stage = 'effort' stage = 'effort'
continue continue
} }
if (models.length === 0) { const selectedModelId = model?.id
Log.error( const previousModelId = models.some(({ id }) => id === selectedModelId)
'No supported models were found for the selected coding agent.' ? selectedModelId
) : undefined
process.exitCode = 1
return 'failed'
}
const previousModelId = model?.id
const modelId = await chooseOption( const modelId = await chooseOption(
`Which ${getHarnessDisplayName(harness!.name)} model should run the upgrade?`, `Which ${getHarnessDisplayName(harness!.name)} model should run the upgrade?`,
Object.fromEntries(models.map(({ id, label }) => [id, label])), Object.fromEntries(
models.map(({ id, label, description }) => [
id,
description ? `${label} — ${description}` : label,
])
),
Math.max( Math.max(
0, 0,
models.findIndex(({ id }) => id === previousModelId) models.findIndex(({ id, isDefault }) =>
previousModelId ? id === previousModelId : isDefault
)
) )
) )
if (modelId === null) { if (modelId === null) {
...@@ -454,9 +409,12 @@ export async function handoffUpgrade( ...@@ -454,9 +409,12 @@ export async function handoffUpgrade(
if (!model) { if (!model) {
throw new Error(`Unknown upgrade model: ${modelId}`) throw new Error(`Unknown upgrade model: ${modelId}`)
} }
if (!model.efforts.includes(effort ?? 'default')) {
effort = 'default'
}
stage = 'effort' stage = 'effort'
} else if (stage === 'effort') { } else if (stage === 'effort') {
if (model) { if (model && model.efforts.length > 0) {
const efforts = ['default', ...model.efforts] const efforts = ['default', ...model.efforts]
const previousEffortIndex = efforts.indexOf(effort ?? 'default') const previousEffortIndex = efforts.indexOf(effort ?? 'default')
const selectedEffort = await chooseOption( const selectedEffort = await chooseOption(
...@@ -515,7 +473,7 @@ export async function handoffUpgrade( ...@@ -515,7 +473,7 @@ export async function handoffUpgrade(
break break
} }
if (permissionChoice === undefined) { if (permissionChoice === undefined) {
stage = model ? 'effort' : 'harness' stage = previousModelStage()
continue continue
} }
useAuto = permissionChoice === 'yes' useAuto = permissionChoice === 'yes'
...@@ -527,7 +485,7 @@ export async function handoffUpgrade( ...@@ -527,7 +485,7 @@ export async function handoffUpgrade(
break break
} }
if (choice === undefined) { if (choice === undefined) {
stage = autoPermissionArgs ? 'permission' : model ? 'effort' : 'harness' stage = autoPermissionArgs ? 'permission' : previousModelStage()
continue continue
} }
useWorktree = choice useWorktree = choice
...@@ -536,6 +494,7 @@ export async function handoffUpgrade( ...@@ -536,6 +494,7 @@ export async function handoffUpgrade(
) )
// Capture the selected agent before the launch callback runs. // Capture the selected agent before the launch callback runs.
const agentProduct = harness!.name const agentProduct = harness!.name
await stopDiscovery()
try { try {
process.exitCode = await launchHarness( process.exitCode = await launchHarness(
...@@ -559,4 +518,7 @@ export async function handoffUpgrade( ...@@ -559,4 +518,7 @@ export async function handoffUpgrade(
Log.bootstrap(` ${dim('Upgrade cancelled.')}\n`) Log.bootstrap(` ${dim('Upgrade cancelled.')}\n`)
process.exitCode = 1 process.exitCode = 1
return 'cancelled' return 'cancelled'
} finally {
await stopDiscovery()
}
} }
import { randomUUID } from 'crypto'
import spawn from 'next/dist/compiled/cross-spawn'
import createDebug from 'next/dist/compiled/debug'
const debug = createDebug('next:upgrade')
export type UpgradeModel = {
id: string
label: string
description: string
efforts: string[]
isDefault: boolean
}
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === 'object' && !Array.isArray(value)
? (value as Record<string, unknown>)
: {}
}
function string(value: unknown): string {
return typeof value === 'string' ? value : ''
}
function efforts(value: unknown): string[] {
return Array.isArray(value)
? [
...new Set(
value.map(string).filter((level) => level && level !== 'default')
),
]
: []
}
function parseModels(
value: unknown,
parse: (model: Record<string, unknown>) => UpgradeModel | null
): UpgradeModel[] {
if (!Array.isArray(value)) {
throw new Error('unexpected model catalog format')
}
return value
.map((entry) => parse(record(entry)))
.filter((model) => model !== null)
}
type DiscoveryProtocol = {
initialize(): void
receive(message: Record<string, unknown>): void
}
export function getCodexModels(
path: string,
cwd: string,
signal?: AbortSignal
) {
return discoverModels(
'codex',
path,
['app-server'],
cwd,
signal,
(send, complete) => {
const initializeId = randomUUID()
let listId: string | undefined
const models: UpgradeModel[] = []
const cursors = new Set<string>()
const list = (cursor?: string) => {
listId = randomUUID()
send({
id: listId,
method: 'model/list',
params: { includeHidden: false, ...(cursor ? { cursor } : {}) },
})
}
return {
initialize() {
send({
id: initializeId,
method: 'initialize',
params: { clientInfo: { name: 'next_upgrade', version: '1' } },
})
},
receive(message) {
if (
message.id !== initializeId &&
(listId === undefined || message.id !== listId)
)
return
if (message.error) throw new Error('CLI returned a protocol error')
if (message.id === initializeId) {
if (!message.result)
throw new Error('missing initialization result')
send({ method: 'initialized' })
list()
return
}
const response = record(message.result)
models.push(
...parseModels(response.data, (model) => {
const id = string(model.model)
if (!id || model.hidden === true) return null
return {
id,
label: string(model.displayName) || id,
description: string(model.description),
efforts: efforts(
Array.isArray(model.supportedReasoningEfforts)
? model.supportedReasoningEfforts.map(
(level) => record(level).reasoningEffort
)
: undefined
),
isDefault: model.isDefault === true,
}
})
)
const cursor = response.nextCursor
if (cursor !== null && cursor !== undefined) {
if (typeof cursor !== 'string' || !cursor || cursors.has(cursor)) {
throw new Error('invalid pagination cursor')
}
cursors.add(cursor)
list(cursor)
return
}
complete(models)
},
}
}
)
}
export function getClaudeModels(
path: string,
cwd: string,
signal?: AbortSignal
) {
return discoverModels(
'claude',
path,
[
'-p',
'--input-format',
'stream-json',
'--output-format',
'stream-json',
'--verbose',
'--no-session-persistence',
],
cwd,
signal,
(send, complete) => {
const initializeId = randomUUID()
return {
initialize() {
send({
type: 'control_request',
request_id: initializeId,
request: { subtype: 'initialize' },
})
},
receive(message) {
if (message.type !== 'control_response') return
const response = record(message.response)
if (response.request_id !== initializeId) return
if (response.subtype !== 'success')
throw new Error('CLI returned an initialization error')
complete(
parseModels(record(response.response).models, (model) => {
const id = string(model.value)
if (!id) return null
return {
id,
label: string(model.displayName) || id,
description: string(model.description),
efforts:
model.supportsEffort === false
? []
: efforts(model.supportedEffortLevels),
isDefault: id === 'default',
}
})
)
},
}
}
)
}
export function getHarnessModels(
name: 'codex' | 'claude',
path: string,
cwd: string,
signal?: AbortSignal
): Promise<UpgradeModel[] | null> {
return name === 'codex'
? getCodexModels(path, cwd, signal)
: getClaudeModels(path, cwd, signal)
}
// null means discovery was cancelled, rather than an unavailable catalog.
async function discoverModels(
name: string,
path: string,
args: string[],
cwd: string,
signal: AbortSignal | undefined,
createProtocol: (
send: (message: unknown) => void,
complete: (models: UpgradeModel[]) => void
) => DiscoveryProtocol
): Promise<UpgradeModel[] | null> {
if (signal?.aborted) return null
let child: ReturnType<typeof spawn>
try {
child = spawn(path, args, { cwd, stdio: 'pipe' })
} catch {
debug('%s model discovery failed: could not start CLI', name)
return []
}
return new Promise((resolve) => {
let buffer = ''
let bytes = 0
let stopping = false
let result: UpgradeModel[] | null = []
let killTimer: ReturnType<typeof setTimeout> | undefined
const stop = (value: UpgradeModel[] | null, reason?: string) => {
if (stopping) return
stopping = true
result = value
clearTimeout(timeout)
if (reason) debug('%s model discovery failed: %s', name, reason)
child.stdin?.end()
child.kill('SIGTERM')
killTimer = setTimeout(() => child.kill('SIGKILL'), 250)
}
const timeout = setTimeout(() => stop([], 'timed out'), 5000)
const interrupt = () => stop(null)
process.on('SIGINT', interrupt)
process.on('SIGTERM', interrupt)
process.on('SIGHUP', interrupt)
const send = (message: unknown) => {
if (!stopping) child.stdin?.write(JSON.stringify(message) + '\n')
}
const protocol = createProtocol(send, (models) => {
const unique = [
...new Map(models.map((model) => [model.id, model])).values(),
]
stop(unique, unique.length ? undefined : 'no usable models')
})
signal?.addEventListener('abort', interrupt, { once: true })
const countOutput = (chunk: Buffer | string) => {
bytes += Buffer.byteLength(chunk)
if (bytes > 20 * 1024 * 1024) stop([], 'output limit exceeded')
}
child.stdout?.setEncoding('utf8')
child.stdout?.on('data', (chunk: string) => {
countOutput(chunk)
if (stopping) return
buffer += chunk
let newline: number
while (!stopping && (newline = buffer.indexOf('\n')) !== -1) {
const line = buffer.slice(0, newline).trim()
buffer = buffer.slice(newline + 1)
if (!line) continue
try {
protocol.receive(record(JSON.parse(line)))
} catch {
stop([], 'invalid discovery response')
}
}
})
// Drain stderr, but never log raw CLI output (which may contain credentials).
child.stderr?.on('data', countOutput)
child.stdin?.on('error', () =>
stop([], 'could not write discovery request')
)
child.on('error', () => stop([], 'could not start CLI'))
child.once('close', () => {
clearTimeout(timeout)
clearTimeout(killTimer)
process.removeListener('SIGINT', interrupt)
process.removeListener('SIGTERM', interrupt)
process.removeListener('SIGHUP', interrupt)
signal?.removeEventListener('abort', interrupt)
if (!stopping) debug('%s model discovery failed: CLI exited early', name)
resolve(result)
})
child.once('spawn', protocol.initialize)
})
}
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment