Unverified Commit f528c5cc authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Add more metadata to agent upgrade telemetry (#99925)

### Why?

Upgrade outcomes lack version and runtime context, so failures and
missing results cannot be compared by upgrade path or environment.

### How?

Add metadata to the existing run-linked events:

- Run start: the app's starting Next.js version, the invoking CLI's Node
version, and the package manager and its version when available.
- CLI result: the starting and approved target Next.js versions.
- Agent result: the installed app Next.js version when the result is
reported.

Keep existing run IDs, agent detection/selection, and policy attribution
so analyses can group outcomes by version path, Node version, agent,
policy, and package manager.

Reuse the upgrade commands' package-manager selection. Version lookup
failures warn and leave metadata null without stopping the upgrade.
Targets are recorded only after preparation succeeds; copied or printed
prompts may have no identified agent. Analyses should retain unknown
metadata in their denominators.

The receiver and compression schemas must be updated and deployed to
retain the new fields. Historical events cannot gain metadata they never
recorded.

Full CLI execution verification is blocked by missing workspace
dependencies and build outputs.

<!-- NEXT_JS_LLM -->
parent 93a50607
...@@ -10,9 +10,14 @@ import { findDir } from '../lib/find-pages-dir' ...@@ -10,9 +10,14 @@ import { findDir } from '../lib/find-pages-dir'
import { getProjectDir } from '../lib/get-project-dir' import { getProjectDir } from '../lib/get-project-dir'
import { warnMissingReactDependencies } from '../lib/warn-missing-react-dependencies' import { warnMissingReactDependencies } from '../lib/warn-missing-react-dependencies'
import { getNpxCommand } from '../lib/helpers/get-npx-command' import { getNpxCommand } from '../lib/helpers/get-npx-command'
import {
getPkgManager,
type PackageManager,
} from '../lib/helpers/get-pkg-manager'
import { interopDefault } from '../lib/interop-default' import { interopDefault } from '../lib/interop-default'
import { dim } from '../lib/picocolors' import { dim } from '../lib/picocolors'
import type { UpgradeDocument } from '../lib/upgrade/future-defaults' import type { UpgradeDocument } from '../lib/upgrade/future-defaults'
import { getInstalledNextVersion } from '../lib/upgrade/prepare-upgrade'
import { runChildProcess } from '../lib/upgrade/run-child-process' import { runChildProcess } from '../lib/upgrade/run-child-process'
import { getAgentName } from '../telemetry/agent-name' import { getAgentName } from '../telemetry/agent-name'
import { import {
...@@ -182,12 +187,32 @@ export async function spawnNextUpgrade( ...@@ -182,12 +187,32 @@ export async function spawnNextUpgrade(
let distDir = '.next' let distDir = '.next'
let configuredPolicy: unknown = null let configuredPolicy: unknown = null
let configError: unknown = null let configError: unknown = null
let fromVersion: string | null = null
let packageManager: PackageManager | null = null
let packageManagerVersion: string | null = null
try { try {
baseDir = getProjectDir(directory, false) baseDir = getProjectDir(directory, false)
warnMissingReactDependencies(baseDir) warnMissingReactDependencies(baseDir)
const config = await loadAgentUpgradeConfig(baseDir) const config = await loadAgentUpgradeConfig(baseDir)
distDir = config.distDir || '.next' distDir = config.distDir || '.next'
configuredPolicy = config.experimental?.agentUpgrade configuredPolicy = config.experimental?.agentUpgrade
// Capture the app and runtime before delegation or an agent changes dependencies.
packageManager = getPkgManager(baseDir)
const [manager, version] =
process.env.npm_config_user_agent?.split(' ')[0].split('/') ?? []
if (manager === packageManager && version) {
packageManagerVersion = valid(version)
}
try {
fromVersion = await getInstalledNextVersion(baseDir)
} catch (error) {
Log.warn(
'Could not determine the app version for upgrade telemetry:',
error instanceof Error ? error.message : error
)
}
} catch (error) { } catch (error) {
configError = error configError = error
} }
...@@ -211,6 +236,7 @@ export async function spawnNextUpgrade( ...@@ -211,6 +236,7 @@ export async function spawnNextUpgrade(
inheritedRunId && !invalidRunId ? inheritedRunId : randomUUID() inheritedRunId && !invalidRunId ? inheritedRunId : randomUUID()
let resolvedPolicy: AgentUpgradePolicy | null = null let resolvedPolicy: AgentUpgradePolicy | null = null
let targetVersion: string | null = null
let failureStage: 'cli' | 'metadata' | 'guide' | 'handoff' = 'cli' let failureStage: 'cli' | 'metadata' | 'guide' | 'handoff' = 'cli'
let cliResultRecorded = false let cliResultRecorded = false
...@@ -231,6 +257,8 @@ export async function spawnNextUpgrade( ...@@ -231,6 +257,8 @@ export async function spawnNextUpgrade(
resolvedPolicy, resolvedPolicy,
handoffMethod, handoffMethod,
selectedAgentProduct, selectedAgentProduct,
fromVersion,
targetVersion,
}) })
) )
} }
...@@ -260,6 +288,10 @@ export async function spawnNextUpgrade( ...@@ -260,6 +288,10 @@ export async function spawnNextUpgrade(
options.agent === 'experimental-future' options.agent === 'experimental-future'
? options.agent ? options.agent
: null, : null,
fromVersion,
nodeVersion: process.version,
packageManager,
packageManagerVersion,
}) })
) )
} }
...@@ -383,6 +415,10 @@ export async function spawnNextUpgrade( ...@@ -383,6 +415,10 @@ export async function spawnNextUpgrade(
return return
} }
// Record the approved target on guide and handoff failures as well as successful delivery.
fromVersion = result.installedVersion
targetVersion = result.targetVersion
const needsVersionUpdate = const needsVersionUpdate =
result.installedVersion !== result.targetVersion result.installedVersion !== result.targetVersion
const crossesMajor = const crossesMajor =
...@@ -680,6 +716,20 @@ export async function reportAgentUpgradeAgentResult( ...@@ -680,6 +716,20 @@ export async function reportAgentUpgradeAgentResult(
distDir: join(process.cwd(), config.distDir || '.next'), distDir: join(process.cwd(), config.distDir || '.next'),
skipNotify: true, skipNotify: true,
}) })
await telemetry.record(eventAgentUpgradeAgentResult({ runId, result }))
// Read the upgraded app instead of the pinned reporting CLI; missing dependencies still report a result.
let resultVersion: string | null = null
try {
resultVersion = await getInstalledNextVersion(process.cwd())
} catch (error) {
Log.warn(
'Could not determine the app version for upgrade telemetry:',
error instanceof Error ? error.message : error
)
}
await telemetry.record(
eventAgentUpgradeAgentResult({ runId, result, resultVersion })
)
await telemetry.flush() await telemetry.flush()
} }
...@@ -34,18 +34,7 @@ export async function prepareUpgrade( ...@@ -34,18 +34,7 @@ export async function prepareUpgrade(
) )
} }
// Resolve from the app: the invoking canary is only the upgrade tooling. const installedVersion = await getInstalledNextVersion(directory)
const requireFromApp = createRequire(join(directory, 'package.json'))
const installedNext = JSON.parse(
await readFile(requireFromApp.resolve('next/package.json'), 'utf8')
) as {
version: string
}
const installedVersion = installedNext.version
if (!semver.valid(installedVersion)) {
throw new Error('Could not determine the installed Next.js version.')
}
const { upgrade } = await getUpgradeAssessment( const { upgrade } = await getUpgradeAssessment(
installedVersion, installedVersion,
...@@ -77,6 +66,25 @@ export async function prepareUpgrade( ...@@ -77,6 +66,25 @@ export async function prepareUpgrade(
return { ...upgrade, futureDefaults: pendingFutureDefaults } return { ...upgrade, futureDefaults: pendingFutureDefaults }
} }
// Resolve from the app for both preparation and reporting; the CLI can run a different version.
export async function getInstalledNextVersion(
directory: string
): Promise<string> {
const requireFromApp = createRequire(join(directory, 'package.json'))
const installedNext = JSON.parse(
await readFile(requireFromApp.resolve('next/package.json'), 'utf8')
) as {
version: string
}
const installedVersion = installedNext.version
if (!semver.valid(installedVersion)) {
throw new Error('Could not determine the installed Next.js version.')
}
return installedVersion
}
export type UpgradeAssessment = { export type UpgradeAssessment = {
// null means advisory assessment is unsupported for this release channel. // null means advisory assessment is unsupported for this release channel.
affected: boolean | null affected: boolean | null
......
import type { PackageManager } from '../../lib/helpers/get-pkg-manager'
// Shared values keep event payloads consistent across the CLI, nudges, and handoffs. // Shared values keep event payloads consistent across the CLI, nudges, and handoffs.
export type AgentUpgradePolicy = 'security' | 'latest' | 'experimental-future' export type AgentUpgradePolicy = 'security' | 'latest' | 'experimental-future'
...@@ -65,6 +67,10 @@ export function eventAgentUpgradeRunStarted(fields: { ...@@ -65,6 +67,10 @@ export function eventAgentUpgradeRunStarted(fields: {
origin: AgentUpgradeOrigin origin: AgentUpgradeOrigin
agentProduct: string | null agentProduct: string | null
requestedPolicy: AgentUpgradePolicy | null requestedPolicy: AgentUpgradePolicy | null
fromVersion: string | null
nodeVersion: string
packageManager: PackageManager | null
packageManagerVersion: string | null
}) { }) {
return event('NEXT_AGENT_UPGRADE_RUN_STARTED', fields) return event('NEXT_AGENT_UPGRADE_RUN_STARTED', fields)
} }
...@@ -76,6 +82,8 @@ export function eventAgentUpgradeCLIResult(fields: { ...@@ -76,6 +82,8 @@ export function eventAgentUpgradeCLIResult(fields: {
resolvedPolicy: AgentUpgradePolicy | null resolvedPolicy: AgentUpgradePolicy | null
handoffMethod: AgentUpgradeHandoffMethod | null handoffMethod: AgentUpgradeHandoffMethod | null
selectedAgentProduct: string | null selectedAgentProduct: string | null
fromVersion: string | null
targetVersion: string | null
}) { }) {
return event('NEXT_AGENT_UPGRADE_CLI_RESULT', fields) return event('NEXT_AGENT_UPGRADE_CLI_RESULT', fields)
} }
...@@ -84,6 +92,7 @@ export function eventAgentUpgradeCLIResult(fields: { ...@@ -84,6 +92,7 @@ export function eventAgentUpgradeCLIResult(fields: {
export function eventAgentUpgradeAgentResult(fields: { export function eventAgentUpgradeAgentResult(fields: {
runId: string runId: string
result: 'success' | 'failure' result: 'success' | 'failure'
resultVersion: string | null
}) { }) {
return event('NEXT_AGENT_UPGRADE_AGENT_RESULT', fields) return event('NEXT_AGENT_UPGRADE_AGENT_RESULT', fields)
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment