Unverified Commit 98adcb3f authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

telemetry: mcp tool call (#85120)

parent 16760877
......@@ -119,6 +119,7 @@ import { getMcpMiddleware } from '../mcp/get-mcp-middleware'
import { handleErrorStateResponse } from '../mcp/tools/get-errors'
import { handlePageMetadataResponse } from '../mcp/tools/get-page-metadata'
import { setStackFrameResolver } from '../mcp/tools/utils/format-errors'
import { recordMcpTelemetry } from '../mcp/mcp-telemetry-tracker'
import { getFileLogger } from './browser-logs/file-logger'
import type { ServerCacheStatus } from '../../next-devtools/dev-overlay/cache-indicator'
import type { Lockfile } from '../../build/lockfile'
......@@ -1393,6 +1394,9 @@ export async function createHotReloaderTurbopack(
})
},
close() {
// Report MCP telemetry if MCP server is enabled
recordMcpTelemetry(opts.telemetry)
for (const wsClient of [
...clientsWithoutRequestId,
...clientsByRequestId.values(),
......
......@@ -109,6 +109,7 @@ import {
} from './hot-reloader-shared-utils'
import { getMcpMiddleware } from '../mcp/get-mcp-middleware'
import { setStackFrameResolver } from '../mcp/tools/utils/format-errors'
import { recordMcpTelemetry } from '../mcp/mcp-telemetry-tracker'
import { getFileLogger } from './browser-logs/file-logger'
import type { ServerCacheStatus } from '../../next-devtools/dev-overlay/cache-indicator'
import type { Lockfile } from '../../build/lockfile'
......@@ -1821,6 +1822,9 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
}
public close() {
// Report MCP telemetry if MCP server is enabled
recordMcpTelemetry(this.telemetry)
this.webpackHotMiddleware?.close()
}
}
......@@ -419,6 +419,24 @@ export async function startServer(
cleanupListeners?.runAll().catch(console.error),
])
// Flush telemetry if this is a dev server
if (isDev) {
try {
const { traceGlobals } =
require('../../trace/shared') as typeof import('../../trace/shared')
const telemetry = traceGlobals.get('telemetry') as
| InstanceType<
typeof import('../../telemetry/storage').Telemetry
>
| undefined
if (telemetry) {
await telemetry.flush()
}
} catch (_) {
// Ignore telemetry errors during cleanup
}
}
debug('start-server process cleanup finished')
process.exit(0)
})()
......
/**
* @jest-environment node
*/
import { mcpTelemetryTracker } from './mcp-telemetry-tracker'
describe('MCP Telemetry Tracker', () => {
beforeEach(() => {
// Reset tracker state before each test
mcpTelemetryTracker.reset()
})
afterAll(() => {
// Clean up after all tests
mcpTelemetryTracker.reset()
})
it('should start with no usage', () => {
expect(mcpTelemetryTracker.hasUsage()).toBe(false)
expect(mcpTelemetryTracker.getUsages()).toEqual([])
})
it('should track single tool call', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
expect(mcpTelemetryTracker.hasUsage()).toBe(true)
expect(mcpTelemetryTracker.getUsages()).toEqual([
{
featureName: 'mcp/get_errors',
invocationCount: 1,
},
])
})
it('should increment invocation count for repeated calls', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(1)
expect(usages[0]).toEqual({
featureName: 'mcp/get_errors',
invocationCount: 3,
})
})
it('should track multiple different tools', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata')
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // Duplicate
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(3)
// Find each tool in the results
const errorsUsage = usages.find((u) => u.featureName === 'mcp/get_errors')
const logsUsage = usages.find((u) => u.featureName === 'mcp/get_logs')
const pageMetadataUsage = usages.find(
(u) => u.featureName === 'mcp/get_page_metadata'
)
expect(errorsUsage).toEqual({
featureName: 'mcp/get_errors',
invocationCount: 2,
})
expect(logsUsage).toEqual({
featureName: 'mcp/get_logs',
invocationCount: 1,
})
expect(pageMetadataUsage).toEqual({
featureName: 'mcp/get_page_metadata',
invocationCount: 1,
})
})
it('should track all 5 MCP tools', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata')
mcpTelemetryTracker.recordToolCall('mcp/get_project_metadata')
mcpTelemetryTracker.recordToolCall('mcp/get_server_action_by_id')
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(5)
const toolNames = usages.map((u) => u.featureName)
expect(toolNames).toContain('mcp/get_errors')
expect(toolNames).toContain('mcp/get_logs')
expect(toolNames).toContain('mcp/get_page_metadata')
expect(toolNames).toContain('mcp/get_project_metadata')
expect(toolNames).toContain('mcp/get_server_action_by_id')
})
it('should reset tracking', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
expect(mcpTelemetryTracker.hasUsage()).toBe(true)
expect(mcpTelemetryTracker.getUsages()).toHaveLength(2)
mcpTelemetryTracker.reset()
expect(mcpTelemetryTracker.hasUsage()).toBe(false)
expect(mcpTelemetryTracker.getUsages()).toEqual([])
})
it('should maintain accurate counts across multiple operations', () => {
// Simulate realistic usage pattern
mcpTelemetryTracker.recordToolCall('mcp/get_project_metadata') // 1
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata') // 1
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // 1
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // 2
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata') // 2
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // 3
mcpTelemetryTracker.recordToolCall('mcp/get_logs') // 1
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(4)
const counts = new Map(
usages.map((u) => [u.featureName, u.invocationCount])
)
expect(counts.get('mcp/get_errors')).toBe(3)
expect(counts.get('mcp/get_page_metadata')).toBe(2)
expect(counts.get('mcp/get_project_metadata')).toBe(1)
expect(counts.get('mcp/get_logs')).toBe(1)
})
})
/**
* Telemetry tracker for MCP tool call usage.
* Tracks invocation counts for each MCP tool to be reported via telemetry.
*/
import type { McpToolName } from '../../telemetry/events/build'
export interface McpToolUsage {
featureName: McpToolName
invocationCount: number
}
class McpTelemetryTracker {
private usageMap = new Map<McpToolName, number>()
/**
* Record a tool call invocation
*/
recordToolCall(toolName: McpToolName): void {
const current = this.usageMap.get(toolName) || 0
this.usageMap.set(toolName, current + 1)
}
/**
* Get all tool usages as an array
*/
getUsages(): McpToolUsage[] {
return Array.from(this.usageMap.entries()).map(([featureName, count]) => ({
featureName,
invocationCount: count,
}))
}
/**
* Reset all usage tracking
*/
reset(): void {
this.usageMap.clear()
}
/**
* Check if any tools have been called
*/
hasUsage(): boolean {
return this.usageMap.size > 0
}
}
// Singleton instance
export const mcpTelemetryTracker = new McpTelemetryTracker()
/**
* Get MCP tool usage telemetry
*/
export function getMcpTelemetryUsage(): McpToolUsage[] {
return mcpTelemetryTracker.getUsages()
}
/**
* Reset MCP telemetry tracker
*/
export function resetMcpTelemetry(): void {
mcpTelemetryTracker.reset()
}
/**
* Record MCP telemetry usage to the telemetry instance
*/
export function recordMcpTelemetry(telemetry: {
record: (event: any) => void
}): void {
const mcpUsages = getMcpTelemetryUsage()
if (mcpUsages.length === 0) {
return
}
const { eventMcpToolUsage } =
require('../../telemetry/events/build') as typeof import('../../telemetry/events/build')
const events = eventMcpToolUsage(mcpUsages)
for (const event of events) {
telemetry.record(event)
}
}
......@@ -26,6 +26,7 @@ import {
DEFAULT_BROWSER_REQUEST_TIMEOUT_MS,
} from './utils/browser-communication'
import { NextInstanceErrorState } from './next-instance-error-state'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetErrorsTool(
server: McpServer,
......@@ -40,6 +41,9 @@ export function registerGetErrorsTool(
inputSchema: {},
},
async (_request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
try {
const connectionCount = getActiveConnectionCount()
if (connectionCount === 0) {
......
......@@ -7,6 +7,7 @@
import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/server/mcp'
import { stat } from 'fs/promises'
import { join } from 'path'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetLogsTool(server: McpServer, distDir: string) {
server.registerTool(
......@@ -16,6 +17,9 @@ export function registerGetLogsTool(server: McpServer, distDir: string) {
'Get the path to the Next.js development log file. Returns the file path so the agent can read the logs directly.',
},
async () => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
try {
const logFilePath = join(distDir, 'logs', 'next-development.log')
......
......@@ -14,6 +14,7 @@ import type {
SegmentTrieData,
} from '../../../shared/lib/mcp-page-metadata-types'
import type { SegmentTrieNode } from '../../../next-devtools/dev-overlay/segment-explorer-trie'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetPageMetadataTool(
server: McpServer,
......@@ -28,6 +29,9 @@ export function registerGetPageMetadataTool(
inputSchema: {},
},
async (_request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata')
try {
const connectionCount = getActiveConnectionCount()
if (connectionCount === 0) {
......
import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/server/mcp'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetProjectMetadataTool(
server: McpServer,
......@@ -13,6 +14,9 @@ export function registerGetProjectMetadataTool(
inputSchema: {},
},
async (_request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_project_metadata')
try {
if (!projectPath) {
return {
......
......@@ -2,6 +2,7 @@ import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/ser
import { z } from 'next/dist/compiled/zod'
import { promises as fs } from 'fs'
import { join } from 'path'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
const INLINE_ACTION_PREFIX = '$$RSC_SERVER_ACTION_'
......@@ -29,6 +30,9 @@ export function registerGetActionByIdTool(server: McpServer, distDir: string) {
},
},
async (request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_server_action_by_id')
try {
const { actionId } = request
......
......@@ -230,6 +230,32 @@ export function eventPackageUsedInGetServerSideProps(
}))
}
export const EVENT_MCP_TOOL_USAGE = 'NEXT_MCP_TOOL_USAGE'
export type McpToolName =
| 'mcp/get_errors'
| 'mcp/get_logs'
| 'mcp/get_page_metadata'
| 'mcp/get_project_metadata'
| 'mcp/get_server_action_by_id'
export type EventMcpToolUsage = {
toolName: McpToolName
invocationCount: number
}
export function eventMcpToolUsage(
usages: Array<{ featureName: McpToolName; invocationCount: number }>
): Array<{ eventName: string; payload: EventMcpToolUsage }> {
return usages.map(({ featureName, invocationCount }) => ({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: featureName,
invocationCount,
},
}))
}
export const ERROR_THROWN_EVENT = 'NEXT_ERROR_THROWN'
type ErrorThrownEvent = {
eventName: typeof ERROR_THROWN_EVENT
......
export * from './version'
export * from './build'
export * from './plugins'
// Re-export MCP-specific types and functions
export type { McpToolName, EventMcpToolUsage } from './build'
export { EVENT_MCP_TOOL_USAGE, eventMcpToolUsage } from './build'
/**
* @jest-environment node
*/
import { eventMcpToolUsage, EVENT_MCP_TOOL_USAGE } from './build'
import type { McpToolName } from './build'
describe('MCP Telemetry Events', () => {
it('should generate correct telemetry events for single tool', () => {
const usages = [
{
featureName: 'mcp/get_errors' as McpToolName,
invocationCount: 5,
},
]
const events = eventMcpToolUsage(usages)
expect(events).toHaveLength(1)
expect(events[0]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_errors',
invocationCount: 5,
},
})
})
it('should generate correct telemetry events for multiple tools', () => {
const usages = [
{
featureName: 'mcp/get_errors' as McpToolName,
invocationCount: 3,
},
{
featureName: 'mcp/get_logs' as McpToolName,
invocationCount: 1,
},
{
featureName: 'mcp/get_page_metadata' as McpToolName,
invocationCount: 7,
},
]
const events = eventMcpToolUsage(usages)
expect(events).toHaveLength(3)
expect(events[0]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_errors',
invocationCount: 3,
},
})
expect(events[1]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_logs',
invocationCount: 1,
},
})
expect(events[2]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_page_metadata',
invocationCount: 7,
},
})
})
it('should handle all MCP tool types', () => {
const allTools: McpToolName[] = [
'mcp/get_errors',
'mcp/get_logs',
'mcp/get_page_metadata',
'mcp/get_project_metadata',
'mcp/get_server_action_by_id',
]
const usages = allTools.map((tool, index) => ({
featureName: tool,
invocationCount: index + 1,
}))
const events = eventMcpToolUsage(usages)
expect(events).toHaveLength(5)
events.forEach((event, index) => {
expect(event.eventName).toBe(EVENT_MCP_TOOL_USAGE)
expect(event.payload.toolName).toBe(allTools[index])
expect(event.payload.invocationCount).toBe(index + 1)
})
})
it('should handle empty usage array', () => {
const events = eventMcpToolUsage([])
expect(events).toEqual([])
})
it('should use correct event name constant', () => {
expect(EVENT_MCP_TOOL_USAGE).toBe('NEXT_MCP_TOOL_USAGE')
})
it('should transform featureName to toolName in payload', () => {
const usages = [
{
featureName: 'mcp/get_project_metadata' as McpToolName,
invocationCount: 2,
},
]
const events = eventMcpToolUsage(usages)
// Verify the input has 'featureName' but output has 'toolName'
expect(usages[0]).toHaveProperty('featureName')
expect(events[0].payload).toHaveProperty('toolName')
expect(events[0].payload).not.toHaveProperty('featureName')
})
})
......@@ -40,7 +40,6 @@ type EventCliSessionStarted = {
reactCompiler: boolean
reactCompilerCompilationMode: string | null
reactCompilerPanicThreshold: string | null
mcpServer: boolean | null
}
export function eventCliSession(
......@@ -75,7 +74,6 @@ export function eventCliSession(
| 'reactCompiler'
| 'reactCompilerCompilationMode'
| 'reactCompilerPanicThreshold'
| 'mcpServer'
| 'isRspack'
>
): { eventName: string; payload: EventCliSessionStarted }[] {
......@@ -134,11 +132,6 @@ export function eventCliSession(
typeof nextConfig.reactCompiler !== 'boolean'
? (nextConfig.reactCompiler?.panicThreshold ?? null)
: null,
// Default mcpServer to true if not set
mcpServer:
typeof nextConfig.experimental.mcpServer === 'boolean'
? nextConfig.experimental.mcpServer
: true,
}
return [{ eventName: EVENT_VERSION, payload }]
}
import { FileRef, nextTestSetup } from 'e2e-utils'
import { findAllTelemetryEvents } from 'next-test-utils'
import path from 'path'
describe('mcp-server telemetry tracking', () => {
const { next } = nextTestSetup({
files: new FileRef(path.join(__dirname, 'fixtures', 'default-template')),
env: {
NEXT_TELEMETRY_DEBUG: '1',
},
})
async function callMcpTool(
toolName: string,
params: Record<string, any> = {}
) {
const response = await fetch(`${next.url}/_next/mcp`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json, text/event-stream',
},
body: JSON.stringify({
jsonrpc: '2.0',
id: `telemetry-test-${Date.now()}`,
method: 'tools/call',
params: { name: toolName, arguments: params },
}),
})
const text = await response.text()
const match = text.match(/data: ({.*})/s)
if (!match) {
throw new Error(`Failed to parse response for tool ${toolName}`)
}
return JSON.parse(match[1])
}
it('should record MCP tool usage telemetry on server shutdown', async () => {
// Call different MCP tools
await callMcpTool('get_project_metadata')
await callMcpTool('get_logs')
await callMcpTool('get_errors')
// Open browser and call page_metadata
await next.browser('/')
await callMcpTool('get_page_metadata')
// Call some tools multiple times
await callMcpTool('get_project_metadata')
await callMcpTool('get_errors')
// Stop the dev server to trigger telemetry recording
// Use SIGTERM so cleanup handlers can run
await next.stop('SIGTERM')
// Parse telemetry from CLI output
const output = next.cliOutput
const events = findAllTelemetryEvents(output, 'NEXT_MCP_TOOL_USAGE')
// Verify telemetry events were recorded
expect(events.length).toBeGreaterThan(0)
// Check that specific tools were tracked
const toolUsages = new Map(
events.map((e) => [e.toolName, e.invocationCount])
)
// get_project_metadata was called 2 times
expect(toolUsages.get('mcp/get_project_metadata')).toBe(2)
// get_errors was called 2 times
expect(toolUsages.get('mcp/get_errors')).toBe(2)
// get_logs was called 1 time
expect(toolUsages.get('mcp/get_logs')).toBe(1)
// get_page_metadata was called 1 time
expect(toolUsages.get('mcp/get_page_metadata')).toBe(1)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment