Unverified Commit c1fe35e4 authored by Jiwon Choi's avatar Jiwon Choi Committed by GitHub

Measure AI upgrade nudges, handoffs, and completion (#99465)

> [!TIP]
> Recommended to review commit by commit with "Hide whitespace" enabled.

### Why?

Measure how often humans and agents receive upgrade nudges, start AI
upgrades, and finish them.

### How?

Record the configured policy, shown nudges, human decisions, AI upgrade
runs, CLI results, and agent results through existing Next.js telemetry.
Existing telemetry opt-out settings apply.

`next upgrade --ai` prepares an upgrade prompt and hands it to an agent.
The prompt asks the agent to run `next internal report-ai-upgrade
<run-id> success|failure` after completing the upgrade and verification.
The run ID links this report to the original upgrade run.

Completion reporting depends on the agent following that instruction. A
missing report means unknown, not failure. The prompt does not instruct
the agent to hide reporting or require it to mention the telemetry
submission to the user. The reporting command suppresses the normal
telemetry notice.

Closes NAR-938
parent a9c23750
......@@ -590,12 +590,20 @@ program
'Upgrade with AI to security, latest, or experimental-future. Defaults to security.'
).conflicts('revision')
)
// Keep nudge attribution available to agents without exposing it in public help.
.addOption(new Option('--internal-nudge-id <id>').hideHelp())
.action(async (directory, options) => {
const mod = await import('../cli/next-upgrade.js')
await mod.spawnNextUpgrade(directory, {
...options,
ai: options.experimentalAi,
})
await mod.spawnNextUpgrade(
directory,
{
...options,
ai: options.experimentalAi,
},
options.internalNudgeId !== undefined
? { id: options.internalNudgeId, recipient: 'agent' }
: null
)
})
program
......@@ -666,6 +674,17 @@ const internal = program
'Internal debugging commands. Use with caution. Not covered by semver.'
)
// Agents use the pinned CLI to report completion after the upgrade has changed dependencies.
internal
.command('report-ai-upgrade', { hidden: true })
.argument('<run-id>', 'The upgrade run UUID.')
.argument('<result>', 'The agent-reported success or failure result.')
.action((runId: string, result: string) =>
import('../cli/next-upgrade.js').then((mod) =>
mod.reportAIUpgradeAgentResult(runId, result)
)
)
internal
.command('agent-feedback-instructions', { hidden: true })
.option(
......
......@@ -1123,7 +1123,7 @@ export default async function build(
debugBuildPathsPatterns: string[] | undefined,
enabledFeatures: Record<string, unknown> = {},
allowHumanUpgrade = false
): Promise<NudgeKind | 'interrupt' | void> {
): Promise<{ policy: NudgeKind; nudgeId: string | null } | 'interrupt' | void> {
const isCompileMode = experimentalBuildMode === 'compile'
const isGenerateMode = experimentalBuildMode === 'generate'
NextBuildContext.isCompileMode = isCompileMode
......@@ -1202,42 +1202,6 @@ export default async function build(
)
loadedConfig = config
// Reuse the loaded config; ordinary builds do not load upgrade tooling.
if (
config.experimental.agentUpgrade === 'security' ||
config.experimental.agentUpgrade === 'latest' ||
config.experimental.agentUpgrade === 'experimental-future' ||
process.env.__NEXT_AGENT_UPGRADE
) {
const { nudgeUpgrade, getUpgradeContext } =
require('../lib/upgrade/nudge') as typeof import('../lib/upgrade/nudge')
const upgradeContext = getUpgradeContext(config)
if (allowHumanUpgrade) {
// TODO: Do not block the build while prompting for an upgrade.
// Preserve all logs for display after the prompt and stop the build before Update.
const action = await nudgeUpgrade(
dir,
upgradeContext,
'build',
new AbortController().signal
).catch((error) => {
Log.warn(`Could not offer the upgrade: ${String(error)}`)
})
if (action === 'update' && upgradeContext.experimental.agentUpgrade) {
return upgradeContext.experimental.agentUpgrade
}
if (action === 'interrupt') {
return 'interrupt' as const
}
} else {
// Agent checks retain their parallel behavior; humans decide before building.
pendingUpgradeNudge = nudgeUpgrade(dir, upgradeContext, 'build').then(
() => {}
)
void pendingUpgradeNudge.catch(() => {})
}
}
// Resolve selective build paths now that the page extensions are known.
const debugBuildPaths = debugBuildPathsPatterns
? await (async () => {
......@@ -1288,6 +1252,58 @@ export default async function build(
const telemetry = new Telemetry({ distDir })
setGlobal('telemetry', telemetry)
// Reuse the loaded config; ordinary builds do not load upgrade tooling.
if (
config.experimental.agentUpgrade === 'security' ||
config.experimental.agentUpgrade === 'latest' ||
config.experimental.agentUpgrade === 'experimental-future' ||
process.env.__NEXT_AGENT_UPGRADE
) {
const { nudgeUpgrade, getUpgradeContext } =
require('../lib/upgrade/nudge') as typeof import('../lib/upgrade/nudge')
const upgradeContext = getUpgradeContext(config)
if (allowHumanUpgrade) {
// TODO: Do not block the build while prompting for an upgrade.
// Preserve all logs for display after the prompt and stop the build before Update.
let nudgeId: string | null = null
const action = await nudgeUpgrade(
dir,
upgradeContext,
'build',
new AbortController().signal,
null,
{
telemetry,
onNudgeId(id) {
nudgeId = id
},
}
).catch((error) => {
Log.warn(`Could not offer the upgrade: ${String(error)}`)
})
if (action === 'update' && upgradeContext.experimental.agentUpgrade) {
return {
policy: upgradeContext.experimental.agentUpgrade,
nudgeId,
}
}
if (action === 'interrupt') {
return 'interrupt' as const
}
} else {
// Agent checks retain their parallel behavior; humans decide before building.
pendingUpgradeNudge = nudgeUpgrade(
dir,
upgradeContext,
'build',
null,
null,
{ telemetry, onNudgeId: null }
).then(() => {})
void pendingUpgradeNudge.catch(() => {})
}
}
// Install the native bindings early so we can have synchronous access later.
await installBindings(config.experimental?.useWasmBinary)
......
......@@ -157,7 +157,7 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
process.off('SIGTERM', onTerminate)
process.off('SIGINT', onInterrupt)
process.off('SIGHUP', onHangup)
process.exit(await runUpgrade(dir, action))
process.exit(await runUpgrade(dir, action.policy, action.nudgeId))
}
})
.catch((err) => {
......
......@@ -180,7 +180,7 @@ const handleSessionStop = async (
}),
true
)
telemetry.flushDetached('dev', dir)
telemetry.flushDetached({ mode: 'dev', dir, distDir: null, events: null })
} catch (_) {
// errors here aren't actionable so don't add
// noise to the output
......@@ -265,14 +265,29 @@ const nextDev = async (
upgradeInProgress = true
const controller = new AbortController()
upgradeController = controller
// Correlate the parent's rendered menu with the upgrade launched after Update.
const telemetry = new Telemetry({
distDir: path.join(dir, context.distDir),
skipNotify: true,
})
let nudgeId: string | null = null
let action
try {
action = await nudgeUpgrade(
dir,
context,
'dev',
controller.signal,
initialAssessment
initialAssessment,
{
telemetry,
onNudgeId(id) {
nudgeId = id
},
}
)
} catch (error) {
Log.warn(`Could not offer the upgrade: ${String(error)}`)
......@@ -296,7 +311,9 @@ const nextDev = async (
process.off('SIGINT', onInterrupt)
process.off('SIGTERM', onTerminate)
process.off('SIGHUP', onHangup)
process.exit(await runUpgrade(dir, context.experimental.agentUpgrade))
process.exit(
await runUpgrade(dir, context.experimental.agentUpgrade, nudgeId)
)
}
upgradeInProgress = false
process.off('SIGHUP', onHangup)
......
This diff is collapsed.
......@@ -8,6 +8,7 @@ import spawn from 'next/dist/compiled/cross-spawn'
import * as Log from '../../build/output/log'
import { getAgentName } from '../../telemetry/agent-name'
import type { AIUpgradeHandoffMethod } from '../../telemetry/events/ai-upgrade'
import { bold, cyan, dim } from '../picocolors'
import { runChildProcess } from './run-child-process'
......@@ -269,7 +270,10 @@ async function chooseHarness(
return id === 'copy' ? 'copy' : harnesses.find(({ name }) => name === id)
}
function copyUpgradePrompt(prompt: string, noHarness = false): void {
function copyUpgradePrompt(
prompt: string,
noHarness: boolean
): AIUpgradeHandoffMethod {
const commands =
process.platform === 'darwin'
? [['pbcopy']]
......@@ -296,7 +300,7 @@ function copyUpgradePrompt(prompt: string, noHarness = false): void {
? 'No supported coding agent found. The upgrade prompt was copied to your clipboard.'
: 'Upgrade prompt copied. Paste it into your coding agent.'
)
return
return 'copied_prompt'
}
}
......@@ -306,6 +310,7 @@ function copyUpgradePrompt(prompt: string, noHarness = false): void {
: 'Could not access the clipboard. Copy this upgrade prompt:'
)
Log.bootstrap(prompt)
return 'printed_prompt'
}
function launchHarness(
......@@ -314,7 +319,8 @@ function launchHarness(
directory: string,
model: string | null,
effort: string,
permissionArgs: readonly string[]
permissionArgs: readonly string[],
onSpawn: () => void
): Promise<number> {
// Windows shell shims cannot carry literal line breaks in an argument.
if (process.platform === 'win32' && /\.(cmd|bat)$/i.test(harness.path)) {
......@@ -331,34 +337,49 @@ function launchHarness(
}
args.push(...permissionArgs)
args.push(prompt)
return runChildProcess(harness.path, args, {
cwd: directory,
stdio: 'inherit',
})
return runChildProcess(
harness.path,
args,
{
cwd: directory,
stdio: 'inherit',
},
onSpawn
)
}
export async function handoffUpgrade(
prompt: UpgradePrompt,
directory: string
): Promise<void> {
directory: string,
onHandoff:
| ((
method: AIUpgradeHandoffMethod,
selectedAgentProduct: string | null
) => void)
| null
): Promise<'handed_off' | 'cancelled' | 'failed'> {
// Existing agents keep their session and permissions.
if (await getAgentName()) {
const existingAgent = await getAgentName()
if (existingAgent) {
Log.bootstrap(resolvePrompt(prompt, null))
return
onHandoff?.('existing_agent', existingAgent)
return 'handed_off'
}
if (!process.stdin.isTTY || !process.stdout.isTTY) {
Log.info('Copy this upgrade prompt into your coding agent:')
Log.bootstrap(resolvePrompt(prompt, null))
return
onHandoff?.('printed_prompt', null)
return 'handed_off'
}
Log.info(dim('Looking for coding agents...'))
const installed = await findHarnesses()
if (installed.length === 0) {
copyUpgradePrompt(resolvePrompt(prompt, null), true)
return
const method = copyUpgradePrompt(resolvePrompt(prompt, null), true)
onHandoff?.(method, null)
return 'handed_off'
}
let stage: 'harness' | 'model' | 'effort' | 'permission' | 'worktree' =
......@@ -381,8 +402,9 @@ export async function handoffUpgrade(
if (stage === 'harness') {
const choice = await chooseHarness(installed, harness?.name)
if (choice === 'copy') {
copyUpgradePrompt(resolvePrompt(prompt, null))
return
const method = copyUpgradePrompt(resolvePrompt(prompt, null), false)
onHandoff?.(method, null)
return 'handed_off'
}
if (!choice) {
break
......@@ -406,7 +428,7 @@ export async function handoffUpgrade(
'No supported models were found for the selected coding agent.'
)
process.exitCode = 1
return
return 'failed'
}
const previousModelId = model?.id
const modelId = await chooseOption(
......@@ -465,7 +487,7 @@ export async function handoffUpgrade(
if (!approvalMode) {
Log.error('Could not determine a supported Claude approval mode.')
process.exitCode = 1
return
return 'failed'
}
autoPermissionArgs = auto ? ['--permission-mode', 'auto'] : null
approvalPermissionArgs = ['--permission-mode', approvalMode]
......@@ -508,6 +530,9 @@ export async function handoffUpgrade(
Log.bootstrap(
` Continuing with ${cyan(bold(getHarnessDisplayName(harness!.name)))}...\n`
)
// Capture the selected agent before the launch callback runs.
const agentProduct = harness!.name
try {
process.exitCode = await launchHarness(
harness!,
......@@ -515,16 +540,19 @@ export async function handoffUpgrade(
directory,
model?.id ?? null,
effort!,
permissionArgs
permissionArgs,
() => onHandoff?.('launched_agent', agentProduct)
)
} catch {
Log.error(`Could not start ${getHarnessDisplayName(harness!.name)}.`)
process.exitCode = 1
return 'failed'
}
return
return 'handed_off'
}
}
Log.bootstrap(` ${dim('Upgrade cancelled.')}\n`)
process.exitCode = 1
return 'cancelled'
}
......@@ -7,6 +7,12 @@ import { updateInitialEnv } from '@next/env'
import * as Log from '../../build/output/log'
import type { NextConfigComplete } from '../../server/config-shared'
import type { Telemetry } from '../../telemetry/storage'
import {
eventAIUpgradeNudgeDecision,
eventAIUpgradeNudgeShown,
eventAIUpgradePolicyDetected,
} from '../../telemetry/events/ai-upgrade'
import semver from 'next/dist/compiled/semver'
import type { UpgradeAction } from './prompt'
import { getAgentName } from '../../telemetry/agent-name'
......@@ -135,6 +141,7 @@ export type UpgradeContext = Pick<
NextConfigComplete,
'distDir' | 'cacheComponents'
> & {
configuredPolicy: NudgeKind | false | null
experimental: { agentUpgrade: NudgeKind | false }
}
......@@ -155,6 +162,7 @@ export function getUpgradeContext(config: NextConfigComplete): UpgradeContext {
return {
distDir: config.distDir,
cacheComponents: config.cacheComponents,
configuredPolicy: config.experimental.agentUpgrade ?? null,
experimental: {
agentUpgrade:
getRequestedUpgrade() ?? config.experimental.agentUpgrade ?? false,
......@@ -271,7 +279,11 @@ export async function assessUpgrade(
async function nudgeUpgradeForAgent(
options: NudgeOptions,
reminder: UpgradeReminder
reminder: UpgradeReminder,
nudgeId: string,
agentProduct: string,
telemetry: Telemetry | null,
policyEvent: ReturnType<typeof eventAIUpgradePolicyDetected>
): Promise<void> {
let summary: string
let recommendation: string
......@@ -293,7 +305,8 @@ async function nudgeUpgradeForAgent(
recommendation = 'We recommend you adopt these Future Defaults.'
break
}
const command = `next upgrade --ai=${reminder.policy}`
// Pass attribution on the invocation using the same command syntax on every platform.
const command = `next upgrade --ai=${reminder.policy} --internal-nudge-id=${nudgeId}`
const message = `${summary}
**${recommendation}**
......@@ -322,11 +335,37 @@ ${reference ? `Reference: ${reference}` : ''}`
)
}
if (retryAllowed) {
telemetry?.record(policyEvent)
Log.warn(
`${summary} This command is continuing after the upgrade reminder.${reference ? `\nReference: ${reference}` : ''}`
)
return
}
// Queue the full nudge once, then send it outside the command that is about to stop.
if (telemetry) {
try {
if (telemetry.isEnabled || process.env.NEXT_TELEMETRY_DEBUG) {
telemetry.flushDetached({
mode: 'dev',
dir: options.directory,
distDir: resolve(options.directory, options.distDir),
events: [
policyEvent,
eventAIUpgradeNudgeShown({
nudgeId,
recipient: 'agent',
agentProduct,
sourceCommand: options.command,
policy: reminder.policy,
nudgeKind: reminder.kind,
}),
],
})
}
} catch (error) {
Log.warn(`Could not queue upgrade telemetry: ${String(error)}`)
}
}
const error = new Error(message)
error.name =
reminder.kind === 'security' ? 'SecurityFatalError' : 'UpgradeNudgeError'
......@@ -397,7 +436,8 @@ async function getUpgradeDismissal(
async function nudgeUpgradeForHuman(
directory: string,
reminder: UpgradeReminder,
signal: AbortSignal
signal: AbortSignal,
onShown: (() => void) | null
): Promise<UpgradeAction> {
if (signal.aborted) {
return 'skip'
......@@ -425,7 +465,12 @@ async function nudgeUpgradeForHuman(
return 'skip'
}
const { promptUpgrade } = require('./prompt') as typeof import('./prompt')
const action = await promptUpgrade(message, signal, true)
const action = await promptUpgrade({
message,
signal,
canUpdate: true,
onShown,
})
if (signal.aborted) {
return 'skip'
}
......@@ -445,16 +490,22 @@ async function nudgeUpgradeForHuman(
return action
}
export async function runUpgrade(directory: string, policy: NudgeKind) {
export async function runUpgrade(
directory: string,
policy: NudgeKind,
nudgeId: string | null
) {
// The agent's dev/build commands must not trigger this explicit request again.
delete process.env.__NEXT_AGENT_UPGRADE
updateInitialEnv({ __NEXT_AGENT_UPGRADE: undefined })
const { spawnNextUpgrade } = await import('../../cli/next-upgrade.js')
await spawnNextUpgrade(directory, {
revision: 'latest',
verbose: false,
ai: policy,
})
// Human Update actions invoke the CLI directly, so their ID does not need an env var.
await spawnNextUpgrade(
directory,
{ revision: 'latest', verbose: false, ai: policy },
nudgeId ? { id: nudgeId, recipient: 'human' } : null
)
return process.exitCode ?? 0
}
......@@ -466,8 +517,12 @@ export async function nudgeUpgrade(
directory: string,
config: UpgradeContext,
command: 'dev' | 'build',
signal: AbortSignal | null = null,
initialAssessment: Promise<UpgradeReminder | null> | null = null
signal: AbortSignal | null,
initialAssessment: Promise<UpgradeReminder | null> | null,
telemetryOptions: {
telemetry: Telemetry
onNudgeId: ((nudgeId: string) => void) | null
} | null
): Promise<UpgradeAction | void> {
const requested = getRequestedUpgrade()
const policy = requested ?? config.experimental.agentUpgrade
......@@ -478,10 +533,24 @@ export async function nudgeUpgrade(
) {
return
}
// Observe the effective policy even when assessment finds no upgrade to offer.
const telemetry = telemetryOptions?.telemetry ?? null
const policyEvent = eventAIUpgradePolicyDetected({
configuredPolicy: config.configuredPolicy ?? null,
effectivePolicy: policy,
policySource: requested ? 'environment' : 'config',
sourceCommand: command,
})
if (requested && isCI) {
telemetry?.record(policyEvent)
return
}
// An agent's stopped command sends policy and nudge together before synchronous exit.
const agent = await getAgentName()
if (!agent) {
telemetry?.record(policyEvent)
}
const installedVersion = process.env.__NEXT_VERSION || 'unknown'
let stopBefore: NudgeKind | null = null
if (!agent) {
......@@ -502,24 +571,73 @@ export async function nudgeUpgrade(
return
}
}
const reminder = await (stopBefore === null && initialAssessment
? initialAssessment
: assessUpgrade(
directory,
{ ...config, experimental: { agentUpgrade: policy } },
installedVersion,
stopBefore,
requested !== null
))
const reminder = await (
stopBefore === null && initialAssessment
? initialAssessment
: assessUpgrade(
directory,
{ ...config, experimental: { agentUpgrade: policy } },
installedVersion,
stopBefore,
requested !== null
)
).catch((error) => {
if (agent) {
telemetry?.record(policyEvent)
}
throw error
})
if (!reminder || signal?.aborted) {
if (agent) {
telemetry?.record(policyEvent)
}
return
}
// The nudge and any resulting upgrade run share this ID across processes.
const nudgeId = randomUUID()
if (agent) {
await nudgeUpgradeForAgent(
{ directory, distDir: config.distDir, command },
reminder
reminder,
nudgeId,
agent,
telemetry,
policyEvent
)
} else if (signal) {
return nudgeUpgradeForHuman(directory, reminder, signal)
// Count a human nudge only after the menu renders, including its selected action.
let shown = false
const onShown = telemetryOptions
? () => {
shown = true
telemetryOptions.onNudgeId?.(nudgeId)
telemetryOptions.telemetry.record(
eventAIUpgradeNudgeShown({
nudgeId,
recipient: 'human',
agentProduct: null,
sourceCommand: command,
policy: reminder.policy,
nudgeKind: reminder.kind,
})
)
}
: null
const action = await nudgeUpgradeForHuman(
directory,
reminder,
signal,
onShown
)
if (shown && telemetry) {
if (!signal.aborted) {
telemetry.record(eventAIUpgradeNudgeDecision({ nudgeId, action }))
}
await telemetry.flush()
}
return action
}
}
......@@ -11,7 +11,7 @@ import {
} from './future-defaults'
type UpgradePreparation =
| { status: 'unaffected'; reason: string }
| { status: 'unaffected' | 'blocked' | 'unknown'; reason: string }
| {
status: 'ready'
installedVersion: string
......@@ -51,9 +51,6 @@ export async function prepareUpgrade(
installedVersion,
targetRequest
)
if (upgrade.status === 'blocked' || upgrade.status === 'unknown') {
throw new Error(upgrade.reason)
}
if (upgrade.status !== 'ready' || targetRequest !== 'experimental-future') {
return upgrade
}
......@@ -84,10 +81,7 @@ export type UpgradeAssessment = {
// null means advisory assessment is unsupported for this release channel.
affected: boolean | null
reference: string | null
upgrade:
| UpgradePreparation
| { status: 'blocked'; reason: string }
| { status: 'unknown'; reason: string }
upgrade: UpgradePreparation
}
// TODO: Cache assessments briefly by installed version, policy, and resolved
......
......@@ -5,11 +5,17 @@ import { bold, cyan } from '../picocolors'
export type UpgradeAction = 'update' | 'skip' | 'dismiss' | 'interrupt'
export async function promptUpgrade(
message: string,
signal: AbortSignal,
canUpdate: boolean = true
): Promise<UpgradeAction> {
export async function promptUpgrade({
message,
signal,
canUpdate,
onShown,
}: {
message: string
signal: AbortSignal
canUpdate: boolean
onShown: (() => void) | null
}): Promise<UpgradeAction> {
if (signal.aborted) {
return 'skip'
}
......@@ -31,11 +37,18 @@ export async function promptUpgrade(
let cancelled = false
let resized = false
let selectedIndex = 0
let shown = false
const renderValue = (value: string, selected: boolean) => {
if (value === labels[0]) {
// We own this screen. Redraw from the top instead of relying on
// cli-select's one-row-per-choice cursor movement when choices wrap.
terminal.write(`\x1b[H\x1b[2J${heading}`)
// Redraws must not count the same rendered menu as another nudge.
if (!shown) {
shown = true
onShown?.()
}
}
if (selected) {
selectedIndex = labels.indexOf(value)
......
......@@ -6,7 +6,8 @@ import spawn from 'next/dist/compiled/cross-spawn'
export function runChildProcess(
command: string,
args: string[],
options: SpawnOptions
options: SpawnOptions,
onSpawn: (() => void) | null
): Promise<number> {
const child = spawn(command, args, options)
......@@ -24,6 +25,18 @@ export function runChildProcess(
process.removeListener('SIGHUP', onHangup)
}
// Report delivery when the child starts, without waiting for the agent's work to finish.
if (onSpawn) {
child.once('spawn', () => {
try {
onSpawn()
} catch (error) {
cleanup()
reject(error)
}
})
}
child.once('error', (error: Error) => {
cleanup()
reject(error)
......
......@@ -283,7 +283,11 @@ export async function initialize(opts: {
upgradeContext,
'dev',
null,
isCI || forceDevToolsForTesting ? null : assessment
isCI || forceDevToolsForTesting ? null : assessment,
{
telemetry,
onNudgeId: null,
}
).catch((error) => {
const { printAndExit } =
require('./utils') as typeof import('./utils')
......
......@@ -447,7 +447,12 @@ export async function startServer(
// Use flushDetached to avoid blocking process exit
// Each process writes to a unique file (_events_${pid}.json)
// to avoid race conditions with the parent process
telemetry.flushDetached('dev', dir)
telemetry.flushDetached({
mode: 'dev',
dir,
distDir: null,
events: null,
})
}
} catch (_) {
// Ignore telemetry errors during cleanup
......
......@@ -10,26 +10,32 @@ import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
// 1. mode e.g. dev, export, start
// 2. project dir
// 3. events filename (optional, defaults to _events.json)
// 4. resolved dist directory (optional, avoids reloading phase-specific config)
;(async () => {
const args = [...process.argv]
const eventsFile = args.pop()
let dir = args.pop()
const mode = args.pop()
const [mode, inputDir, eventsFile, suppliedDistDir] = process.argv.slice(2)
let dir = inputDir
if (!dir || mode !== 'dev') {
throw new Error(
`Invalid flags should be run as node detached-flush dev ./path-to/project [eventsFile]`
`Invalid flags should be run as node detached-flush dev ./path-to/project [eventsFile] [distDir]`
)
}
dir = getProjectDir(dir)
const config = await loadConfig(PHASE_DEVELOPMENT_SERVER, dir)
const distDir = path.join(dir, config.distDir || '.next')
// Support both old format (no eventsFile arg) and new format (with eventsFile arg)
const eventsPath = path.join(
distDir,
eventsFile && !eventsFile.includes('/') ? eventsFile : '_events.json'
)
// Build nudges pass their resolved output directory to avoid loading dev config again.
const distDir = suppliedDistDir
? path.resolve(suppliedDistDir)
: path.join(
dir,
(await loadConfig(PHASE_DEVELOPMENT_SERVER, dir)).distDir || '.next'
)
// Named batches live in cache so build cleanup cannot remove them before submission.
// Retain the legacy root path for callers without an events filename.
const eventsPath =
eventsFile && !eventsFile.includes('/')
? path.join(distDir, 'cache', eventsFile)
: path.join(distDir, '_events.json')
let events: TelemetryEvent[]
try {
......
// Shared values keep event payloads consistent across the CLI, nudges, and handoffs.
export type AIUpgradePolicy = 'security' | 'latest' | 'experimental-future'
export type AIUpgradeOrigin =
| 'human_manual'
| 'human_nudge'
| 'agent_manual'
| 'agent_nudge'
export type AIUpgradeHandoffMethod =
| 'existing_agent'
| 'launched_agent'
| 'copied_prompt'
| 'printed_prompt'
export type AIUpgradeCLIResult =
| 'no_update_needed'
| 'no_safe_target'
| 'metadata_failure'
| 'guide_failure'
| 'cancelled'
| 'handoff_issued'
| 'handoff_failed'
| 'cli_failure'
// Version every event so consumers can distinguish future schema changes.
function event<T extends object>(eventName: string, fields: T) {
return { eventName, payload: { schemaVersion: 1, ...fields } }
}
// Measure configured reach separately from whether an upgrade reminder is needed.
export function eventAIUpgradePolicyDetected(fields: {
configuredPolicy: AIUpgradePolicy | false | null
effectivePolicy: AIUpgradePolicy
policySource: 'config' | 'environment'
sourceCommand: 'dev' | 'build'
}) {
return event('NEXT_AI_UPGRADE_POLICY_DETECTED', fields)
}
// Correlate a displayed reminder with a later upgrade invocation.
export function eventAIUpgradeNudgeShown(fields: {
nudgeId: string
recipient: 'human' | 'agent'
agentProduct: string | null
sourceCommand: 'dev' | 'build'
policy: AIUpgradePolicy
nudgeKind: AIUpgradePolicy
}) {
return event('NEXT_AI_UPGRADE_NUDGE_SHOWN', fields)
}
// Human menus expose a decision directly; agent decisions are inferred from runs.
export function eventAIUpgradeNudgeDecision(fields: {
nudgeId: string
action: 'update' | 'skip' | 'dismiss' | 'interrupt'
}) {
return event('NEXT_AI_UPGRADE_NUDGE_DECISION', fields)
}
// Count invocations independently of whether preparation or handoff finishes.
export function eventAIUpgradeRunStarted(fields: {
runId: string
nudgeId: string | null
origin: AIUpgradeOrigin
agentProduct: string | null
requestedPolicy: AIUpgradePolicy | null
}) {
return event('NEXT_AI_UPGRADE_RUN_STARTED', fields)
}
// Report the CLI's preparation and prompt delivery, not completion of the agent's work.
export function eventAIUpgradeCLIResult(fields: {
runId: string
result: AIUpgradeCLIResult
resolvedPolicy: AIUpgradePolicy | null
handoffMethod: AIUpgradeHandoffMethod | null
selectedAgentProduct: string | null
}) {
return event('NEXT_AI_UPGRADE_CLI_RESULT', fields)
}
// The agent reports its verified result separately from the CLI's prompt delivery.
export function eventAIUpgradeAgentResult(fields: {
runId: string
result: 'success' | 'failure'
}) {
return event('NEXT_AI_UPGRADE_AGENT_RESULT', fields)
}
......@@ -231,28 +231,43 @@ export class Telemetry {
// writes current events to disk and spawns separate
// detached process to submit the records without blocking
// the main process from exiting
flushDetached = (mode: 'dev', dir: string) => {
const allEvents: TelemetryEvent[] = []
this.queue.forEach((item: any) => {
try {
item._controller?.abort()
allEvents.push(...item._events)
} catch (_) {
// if we fail to abort ignore this event
}
})
flushDetached = ({
mode,
dir,
distDir,
events,
}: {
mode: 'dev'
dir: string
distDir: string | null
events: TelemetryEvent[] | null
}) => {
// Nudges detach only their own events; shutdown flushes the full queue.
const allEvents: TelemetryEvent[] = events ?? []
if (events === null) {
this.queue.forEach((item: any) => {
try {
item._controller?.abort()
allEvents.push(...item._events)
} catch (_) {
// if we fail to abort ignore this event
}
})
}
if (allEvents.length === 0) {
// No events to flush
return
}
fs.mkdirSync(this.distDir, { recursive: true })
// Use unique filename per process to avoid race conditions between parent/child
const eventsFile = `_events_${process.pid}.json`
// Builds preserve cache while cleaning distDir, so detached batches survive cleanup.
const eventsDirectory = path.join(this.distDir, 'cache')
fs.mkdirSync(eventsDirectory, { recursive: true })
// Each flush owns its file so a later shutdown flush cannot replace a nudge batch.
const eventsFile = `_events_${process.pid}_${randomBytes(8).toString('hex')}.json`
fs.writeFileSync(
path.join(this.distDir, eventsFile),
path.join(eventsDirectory, eventsFile),
JSON.stringify(allEvents)
)
......@@ -269,7 +284,13 @@ export class Telemetry {
spawn(
process.execPath,
[require.resolve('./detached-flush'), mode, dir, eventsFile],
[
require.resolve('./detached-flush'),
mode,
dir,
eventsFile,
...(distDir ? [distDir] : []),
],
{
detached: !this.NEXT_TELEMETRY_DEBUG,
windowsHide: true,
......@@ -338,7 +359,9 @@ export class Telemetry {
},
postController.signal
)
// Bound delivery even when callers await record/flush; retries share this deadline.
const timeout = setTimeout(() => postController.abort(), 5000)
res._controller = postController
return res
return res.finally(() => clearTimeout(timeout))
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -51,10 +51,12 @@ describe('upgrade menu', () => {
] as const)('returns %s and restores the terminal', async (action, down) => {
const exits = process.listenerCount('exit')
const resizes = process.stdout.listenerCount('resize')
const pending = promptUpgrade(
'Shared recommendation',
new AbortController().signal
)
const pending = promptUpgrade({
message: 'Shared recommendation',
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
expect(screen.startsWith('\x1b[?1049h')).toBe(true)
for (let i = 0; i < down; i++) {
input.emit('keypress', '', { name: 'down' })
......@@ -82,7 +84,12 @@ describe('upgrade menu', () => {
async (size) => {
process.stdout.columns = size.columns
process.stdout.rows = size.rows
const pending = promptUpgrade(size.message, new AbortController().signal)
const pending = promptUpgrade({
message: size.message,
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
expect(screen).toContain(size.message)
expect(screen).toContain('Upgrade now')
expect(screen).toContain('Skip until next version')
......@@ -102,11 +109,12 @@ describe('upgrade menu', () => {
'preserves selection on resize (canUpdate: %s)',
async (canUpdate) => {
const resizes = process.stdout.listenerCount('resize')
const pending = promptUpgrade(
'Reminder',
new AbortController().signal,
canUpdate
)
const pending = promptUpgrade({
message: 'Reminder',
signal: new AbortController().signal,
canUpdate,
onShown: null,
})
input.emit('keypress', '', { name: 'down' })
const initialScreen = screen
process.stdout.columns = 60
......@@ -132,7 +140,12 @@ describe('upgrade menu', () => {
async (action) => {
const resizes = process.stdout.listenerCount('resize')
const controller = new AbortController()
const pending = promptUpgrade('Reminder', controller.signal)
const pending = promptUpgrade({
message: 'Reminder',
signal: controller.signal,
canUpdate: true,
onShown: null,
})
process.stdout.emit('resize')
if (action === 'abort') {
controller.abort()
......@@ -161,7 +174,12 @@ describe('upgrade menu', () => {
}
return true
})
const pending = promptUpgrade('Reminder', new AbortController().signal)
const pending = promptUpgrade({
message: 'Reminder',
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
await new Promise<void>(queueMicrotask)
expect(screen.split('Reminder')).toHaveLength(3)
input.emit('keypress', '', { name: 'down' })
......@@ -173,7 +191,12 @@ describe('upgrade menu', () => {
const resizes = process.stdout.listenerCount('resize')
const message =
'Complete reminder\n\nReference: https://example.com/advisory'
const pending = promptUpgrade(message, new AbortController().signal)
const pending = promptUpgrade({
message,
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
input.emit('keypress', '', { name: 'down' })
screen = ''
process.stdout.rows = 4
......@@ -189,11 +212,12 @@ describe('upgrade menu', () => {
})
it('only offers Skip and dismissal when no upgrade is available', async () => {
const pending = promptUpgrade(
'No eligible target.',
new AbortController().signal,
false
)
const pending = promptUpgrade({
message: 'No eligible target.',
signal: new AbortController().signal,
canUpdate: false,
onShown: null,
})
expect(screen.includes('Upgrade now')).toBe(false)
expect(screen).toContain('Skip until next version')
input.emit('keypress', '', { name: 'return' })
......@@ -204,7 +228,12 @@ describe('upgrade menu', () => {
const listener = jest.fn()
input.on('keypress', listener)
const controller = new AbortController()
const pending = promptUpgrade('Reminder', controller.signal)
const pending = promptUpgrade({
message: 'Reminder',
signal: controller.signal,
canUpdate: true,
onShown: null,
})
controller.abort()
await expect(pending).resolves.toBe('skip')
expect(listener).toHaveBeenCalledTimes(0)
......@@ -215,7 +244,12 @@ describe('upgrade menu', () => {
it('returns interruption without self-signalling', async () => {
const kill = jest.spyOn(process, 'kill').mockReturnValue(true)
const pending = promptUpgrade('Reminder', new AbortController().signal)
const pending = promptUpgrade({
message: 'Reminder',
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
input.emit('keypress', '', { name: 'c', ctrl: true })
await expect(pending).resolves.toBe('interrupt')
expect(kill).toHaveBeenCalledTimes(0)
......@@ -234,7 +268,12 @@ describe('upgrade menu', () => {
return true
})
await expect(
promptUpgrade('Shared recommendation', new AbortController().signal)
promptUpgrade({
message: 'Shared recommendation',
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
).rejects.toThrow('Rendering failed')
expect(input.setRawMode.mock.calls).toEqual([[true], [false]])
expect(input.listenerCount('keypress')).toBe(0)
......@@ -247,7 +286,12 @@ describe('upgrade menu', () => {
it('preserves an already raw and flowing input stream', async () => {
input.isRaw = true
input.resume()
const pending = promptUpgrade('Reminder', new AbortController().signal)
const pending = promptUpgrade({
message: 'Reminder',
signal: new AbortController().signal,
canUpdate: true,
onShown: null,
})
input.emit('keypress', '', { name: 'escape' })
await expect(pending).resolves.toBe('skip')
expect(input.setRawMode).toHaveBeenLastCalledWith(true)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment