Unverified Commit f528c5cc authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Add more metadata to agent upgrade telemetry (#99925)

### Why?

Upgrade outcomes lack version and runtime context, so failures and
missing results cannot be compared by upgrade path or environment.

### How?

Add metadata to the existing run-linked events:

- Run start: the app's starting Next.js version, the invoking CLI's Node
version, and the package manager and its version when available.
- CLI result: the starting and approved target Next.js versions.
- Agent result: the installed app Next.js version when the result is
reported.

Keep existing run IDs, agent detection/selection, and policy attribution
so analyses can group outcomes by version path, Node version, agent,
policy, and package manager.

Reuse the upgrade commands' package-manager selection. Version lookup
failures warn and leave metadata null without stopping the upgrade.
Targets are recorded only after preparation succeeds; copied or printed
prompts may have no identified agent. Analyses should retain unknown
metadata in their denominators.

The receiver and compression schemas must be updated and deployed to
retain the new fields. Historical events cannot gain metadata they never
recorded.

Full CLI execution verification is blocked by missing workspace
dependencies and build outputs.

<!-- NEXT_JS_LLM -->
parent 93a50607
......@@ -10,9 +10,14 @@ import { findDir } from '../lib/find-pages-dir'
import { getProjectDir } from '../lib/get-project-dir'
import { warnMissingReactDependencies } from '../lib/warn-missing-react-dependencies'
import { getNpxCommand } from '../lib/helpers/get-npx-command'
import {
getPkgManager,
type PackageManager,
} from '../lib/helpers/get-pkg-manager'
import { interopDefault } from '../lib/interop-default'
import { dim } from '../lib/picocolors'
import type { UpgradeDocument } from '../lib/upgrade/future-defaults'
import { getInstalledNextVersion } from '../lib/upgrade/prepare-upgrade'
import { runChildProcess } from '../lib/upgrade/run-child-process'
import { getAgentName } from '../telemetry/agent-name'
import {
......@@ -182,12 +187,32 @@ export async function spawnNextUpgrade(
let distDir = '.next'
let configuredPolicy: unknown = null
let configError: unknown = null
let fromVersion: string | null = null
let packageManager: PackageManager | null = null
let packageManagerVersion: string | null = null
try {
baseDir = getProjectDir(directory, false)
warnMissingReactDependencies(baseDir)
const config = await loadAgentUpgradeConfig(baseDir)
distDir = config.distDir || '.next'
configuredPolicy = config.experimental?.agentUpgrade
// Capture the app and runtime before delegation or an agent changes dependencies.
packageManager = getPkgManager(baseDir)
const [manager, version] =
process.env.npm_config_user_agent?.split(' ')[0].split('/') ?? []
if (manager === packageManager && version) {
packageManagerVersion = valid(version)
}
try {
fromVersion = await getInstalledNextVersion(baseDir)
} catch (error) {
Log.warn(
'Could not determine the app version for upgrade telemetry:',
error instanceof Error ? error.message : error
)
}
} catch (error) {
configError = error
}
......@@ -211,6 +236,7 @@ export async function spawnNextUpgrade(
inheritedRunId && !invalidRunId ? inheritedRunId : randomUUID()
let resolvedPolicy: AgentUpgradePolicy | null = null
let targetVersion: string | null = null
let failureStage: 'cli' | 'metadata' | 'guide' | 'handoff' = 'cli'
let cliResultRecorded = false
......@@ -231,6 +257,8 @@ export async function spawnNextUpgrade(
resolvedPolicy,
handoffMethod,
selectedAgentProduct,
fromVersion,
targetVersion,
})
)
}
......@@ -260,6 +288,10 @@ export async function spawnNextUpgrade(
options.agent === 'experimental-future'
? options.agent
: null,
fromVersion,
nodeVersion: process.version,
packageManager,
packageManagerVersion,
})
)
}
......@@ -383,6 +415,10 @@ export async function spawnNextUpgrade(
return
}
// Record the approved target on guide and handoff failures as well as successful delivery.
fromVersion = result.installedVersion
targetVersion = result.targetVersion
const needsVersionUpdate =
result.installedVersion !== result.targetVersion
const crossesMajor =
......@@ -680,6 +716,20 @@ export async function reportAgentUpgradeAgentResult(
distDir: join(process.cwd(), config.distDir || '.next'),
skipNotify: true,
})
await telemetry.record(eventAgentUpgradeAgentResult({ runId, result }))
// Read the upgraded app instead of the pinned reporting CLI; missing dependencies still report a result.
let resultVersion: string | null = null
try {
resultVersion = await getInstalledNextVersion(process.cwd())
} catch (error) {
Log.warn(
'Could not determine the app version for upgrade telemetry:',
error instanceof Error ? error.message : error
)
}
await telemetry.record(
eventAgentUpgradeAgentResult({ runId, result, resultVersion })
)
await telemetry.flush()
}
......@@ -34,18 +34,7 @@ export async function prepareUpgrade(
)
}
// Resolve from the app: the invoking canary is only the upgrade tooling.
const requireFromApp = createRequire(join(directory, 'package.json'))
const installedNext = JSON.parse(
await readFile(requireFromApp.resolve('next/package.json'), 'utf8')
) as {
version: string
}
const installedVersion = installedNext.version
if (!semver.valid(installedVersion)) {
throw new Error('Could not determine the installed Next.js version.')
}
const installedVersion = await getInstalledNextVersion(directory)
const { upgrade } = await getUpgradeAssessment(
installedVersion,
......@@ -77,6 +66,25 @@ export async function prepareUpgrade(
return { ...upgrade, futureDefaults: pendingFutureDefaults }
}
// Resolve from the app for both preparation and reporting; the CLI can run a different version.
export async function getInstalledNextVersion(
directory: string
): Promise<string> {
const requireFromApp = createRequire(join(directory, 'package.json'))
const installedNext = JSON.parse(
await readFile(requireFromApp.resolve('next/package.json'), 'utf8')
) as {
version: string
}
const installedVersion = installedNext.version
if (!semver.valid(installedVersion)) {
throw new Error('Could not determine the installed Next.js version.')
}
return installedVersion
}
export type UpgradeAssessment = {
// null means advisory assessment is unsupported for this release channel.
affected: boolean | null
......
import type { PackageManager } from '../../lib/helpers/get-pkg-manager'
// Shared values keep event payloads consistent across the CLI, nudges, and handoffs.
export type AgentUpgradePolicy = 'security' | 'latest' | 'experimental-future'
......@@ -65,6 +67,10 @@ export function eventAgentUpgradeRunStarted(fields: {
origin: AgentUpgradeOrigin
agentProduct: string | null
requestedPolicy: AgentUpgradePolicy | null
fromVersion: string | null
nodeVersion: string
packageManager: PackageManager | null
packageManagerVersion: string | null
}) {
return event('NEXT_AGENT_UPGRADE_RUN_STARTED', fields)
}
......@@ -76,6 +82,8 @@ export function eventAgentUpgradeCLIResult(fields: {
resolvedPolicy: AgentUpgradePolicy | null
handoffMethod: AgentUpgradeHandoffMethod | null
selectedAgentProduct: string | null
fromVersion: string | null
targetVersion: string | null
}) {
return event('NEXT_AGENT_UPGRADE_CLI_RESULT', fields)
}
......@@ -84,6 +92,7 @@ export function eventAgentUpgradeCLIResult(fields: {
export function eventAgentUpgradeAgentResult(fields: {
runId: string
result: 'success' | 'failure'
resultVersion: string | null
}) {
return event('NEXT_AGENT_UPGRADE_AGENT_RESULT', fields)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment