Unverified Commit 98adcb3f authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

telemetry: mcp tool call (#85120)

parent 16760877
...@@ -119,6 +119,7 @@ import { getMcpMiddleware } from '../mcp/get-mcp-middleware' ...@@ -119,6 +119,7 @@ import { getMcpMiddleware } from '../mcp/get-mcp-middleware'
import { handleErrorStateResponse } from '../mcp/tools/get-errors' import { handleErrorStateResponse } from '../mcp/tools/get-errors'
import { handlePageMetadataResponse } from '../mcp/tools/get-page-metadata' import { handlePageMetadataResponse } from '../mcp/tools/get-page-metadata'
import { setStackFrameResolver } from '../mcp/tools/utils/format-errors' import { setStackFrameResolver } from '../mcp/tools/utils/format-errors'
import { recordMcpTelemetry } from '../mcp/mcp-telemetry-tracker'
import { getFileLogger } from './browser-logs/file-logger' import { getFileLogger } from './browser-logs/file-logger'
import type { ServerCacheStatus } from '../../next-devtools/dev-overlay/cache-indicator' import type { ServerCacheStatus } from '../../next-devtools/dev-overlay/cache-indicator'
import type { Lockfile } from '../../build/lockfile' import type { Lockfile } from '../../build/lockfile'
...@@ -1393,6 +1394,9 @@ export async function createHotReloaderTurbopack( ...@@ -1393,6 +1394,9 @@ export async function createHotReloaderTurbopack(
}) })
}, },
close() { close() {
// Report MCP telemetry if MCP server is enabled
recordMcpTelemetry(opts.telemetry)
for (const wsClient of [ for (const wsClient of [
...clientsWithoutRequestId, ...clientsWithoutRequestId,
...clientsByRequestId.values(), ...clientsByRequestId.values(),
......
...@@ -109,6 +109,7 @@ import { ...@@ -109,6 +109,7 @@ import {
} from './hot-reloader-shared-utils' } from './hot-reloader-shared-utils'
import { getMcpMiddleware } from '../mcp/get-mcp-middleware' import { getMcpMiddleware } from '../mcp/get-mcp-middleware'
import { setStackFrameResolver } from '../mcp/tools/utils/format-errors' import { setStackFrameResolver } from '../mcp/tools/utils/format-errors'
import { recordMcpTelemetry } from '../mcp/mcp-telemetry-tracker'
import { getFileLogger } from './browser-logs/file-logger' import { getFileLogger } from './browser-logs/file-logger'
import type { ServerCacheStatus } from '../../next-devtools/dev-overlay/cache-indicator' import type { ServerCacheStatus } from '../../next-devtools/dev-overlay/cache-indicator'
import type { Lockfile } from '../../build/lockfile' import type { Lockfile } from '../../build/lockfile'
...@@ -1821,6 +1822,9 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface { ...@@ -1821,6 +1822,9 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
} }
public close() { public close() {
// Report MCP telemetry if MCP server is enabled
recordMcpTelemetry(this.telemetry)
this.webpackHotMiddleware?.close() this.webpackHotMiddleware?.close()
} }
} }
...@@ -419,6 +419,24 @@ export async function startServer( ...@@ -419,6 +419,24 @@ export async function startServer(
cleanupListeners?.runAll().catch(console.error), cleanupListeners?.runAll().catch(console.error),
]) ])
// Flush telemetry if this is a dev server
if (isDev) {
try {
const { traceGlobals } =
require('../../trace/shared') as typeof import('../../trace/shared')
const telemetry = traceGlobals.get('telemetry') as
| InstanceType<
typeof import('../../telemetry/storage').Telemetry
>
| undefined
if (telemetry) {
await telemetry.flush()
}
} catch (_) {
// Ignore telemetry errors during cleanup
}
}
debug('start-server process cleanup finished') debug('start-server process cleanup finished')
process.exit(0) process.exit(0)
})() })()
......
/**
* @jest-environment node
*/
import { mcpTelemetryTracker } from './mcp-telemetry-tracker'
describe('MCP Telemetry Tracker', () => {
beforeEach(() => {
// Reset tracker state before each test
mcpTelemetryTracker.reset()
})
afterAll(() => {
// Clean up after all tests
mcpTelemetryTracker.reset()
})
it('should start with no usage', () => {
expect(mcpTelemetryTracker.hasUsage()).toBe(false)
expect(mcpTelemetryTracker.getUsages()).toEqual([])
})
it('should track single tool call', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
expect(mcpTelemetryTracker.hasUsage()).toBe(true)
expect(mcpTelemetryTracker.getUsages()).toEqual([
{
featureName: 'mcp/get_errors',
invocationCount: 1,
},
])
})
it('should increment invocation count for repeated calls', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(1)
expect(usages[0]).toEqual({
featureName: 'mcp/get_errors',
invocationCount: 3,
})
})
it('should track multiple different tools', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata')
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // Duplicate
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(3)
// Find each tool in the results
const errorsUsage = usages.find((u) => u.featureName === 'mcp/get_errors')
const logsUsage = usages.find((u) => u.featureName === 'mcp/get_logs')
const pageMetadataUsage = usages.find(
(u) => u.featureName === 'mcp/get_page_metadata'
)
expect(errorsUsage).toEqual({
featureName: 'mcp/get_errors',
invocationCount: 2,
})
expect(logsUsage).toEqual({
featureName: 'mcp/get_logs',
invocationCount: 1,
})
expect(pageMetadataUsage).toEqual({
featureName: 'mcp/get_page_metadata',
invocationCount: 1,
})
})
it('should track all 5 MCP tools', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata')
mcpTelemetryTracker.recordToolCall('mcp/get_project_metadata')
mcpTelemetryTracker.recordToolCall('mcp/get_server_action_by_id')
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(5)
const toolNames = usages.map((u) => u.featureName)
expect(toolNames).toContain('mcp/get_errors')
expect(toolNames).toContain('mcp/get_logs')
expect(toolNames).toContain('mcp/get_page_metadata')
expect(toolNames).toContain('mcp/get_project_metadata')
expect(toolNames).toContain('mcp/get_server_action_by_id')
})
it('should reset tracking', () => {
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
expect(mcpTelemetryTracker.hasUsage()).toBe(true)
expect(mcpTelemetryTracker.getUsages()).toHaveLength(2)
mcpTelemetryTracker.reset()
expect(mcpTelemetryTracker.hasUsage()).toBe(false)
expect(mcpTelemetryTracker.getUsages()).toEqual([])
})
it('should maintain accurate counts across multiple operations', () => {
// Simulate realistic usage pattern
mcpTelemetryTracker.recordToolCall('mcp/get_project_metadata') // 1
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata') // 1
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // 1
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // 2
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata') // 2
mcpTelemetryTracker.recordToolCall('mcp/get_errors') // 3
mcpTelemetryTracker.recordToolCall('mcp/get_logs') // 1
const usages = mcpTelemetryTracker.getUsages()
expect(usages).toHaveLength(4)
const counts = new Map(
usages.map((u) => [u.featureName, u.invocationCount])
)
expect(counts.get('mcp/get_errors')).toBe(3)
expect(counts.get('mcp/get_page_metadata')).toBe(2)
expect(counts.get('mcp/get_project_metadata')).toBe(1)
expect(counts.get('mcp/get_logs')).toBe(1)
})
})
/**
* Telemetry tracker for MCP tool call usage.
* Tracks invocation counts for each MCP tool to be reported via telemetry.
*/
import type { McpToolName } from '../../telemetry/events/build'
export interface McpToolUsage {
featureName: McpToolName
invocationCount: number
}
class McpTelemetryTracker {
private usageMap = new Map<McpToolName, number>()
/**
* Record a tool call invocation
*/
recordToolCall(toolName: McpToolName): void {
const current = this.usageMap.get(toolName) || 0
this.usageMap.set(toolName, current + 1)
}
/**
* Get all tool usages as an array
*/
getUsages(): McpToolUsage[] {
return Array.from(this.usageMap.entries()).map(([featureName, count]) => ({
featureName,
invocationCount: count,
}))
}
/**
* Reset all usage tracking
*/
reset(): void {
this.usageMap.clear()
}
/**
* Check if any tools have been called
*/
hasUsage(): boolean {
return this.usageMap.size > 0
}
}
// Singleton instance
export const mcpTelemetryTracker = new McpTelemetryTracker()
/**
* Get MCP tool usage telemetry
*/
export function getMcpTelemetryUsage(): McpToolUsage[] {
return mcpTelemetryTracker.getUsages()
}
/**
* Reset MCP telemetry tracker
*/
export function resetMcpTelemetry(): void {
mcpTelemetryTracker.reset()
}
/**
* Record MCP telemetry usage to the telemetry instance
*/
export function recordMcpTelemetry(telemetry: {
record: (event: any) => void
}): void {
const mcpUsages = getMcpTelemetryUsage()
if (mcpUsages.length === 0) {
return
}
const { eventMcpToolUsage } =
require('../../telemetry/events/build') as typeof import('../../telemetry/events/build')
const events = eventMcpToolUsage(mcpUsages)
for (const event of events) {
telemetry.record(event)
}
}
...@@ -26,6 +26,7 @@ import { ...@@ -26,6 +26,7 @@ import {
DEFAULT_BROWSER_REQUEST_TIMEOUT_MS, DEFAULT_BROWSER_REQUEST_TIMEOUT_MS,
} from './utils/browser-communication' } from './utils/browser-communication'
import { NextInstanceErrorState } from './next-instance-error-state' import { NextInstanceErrorState } from './next-instance-error-state'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetErrorsTool( export function registerGetErrorsTool(
server: McpServer, server: McpServer,
...@@ -40,6 +41,9 @@ export function registerGetErrorsTool( ...@@ -40,6 +41,9 @@ export function registerGetErrorsTool(
inputSchema: {}, inputSchema: {},
}, },
async (_request) => { async (_request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_errors')
try { try {
const connectionCount = getActiveConnectionCount() const connectionCount = getActiveConnectionCount()
if (connectionCount === 0) { if (connectionCount === 0) {
......
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/server/mcp' import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/server/mcp'
import { stat } from 'fs/promises' import { stat } from 'fs/promises'
import { join } from 'path' import { join } from 'path'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetLogsTool(server: McpServer, distDir: string) { export function registerGetLogsTool(server: McpServer, distDir: string) {
server.registerTool( server.registerTool(
...@@ -16,6 +17,9 @@ export function registerGetLogsTool(server: McpServer, distDir: string) { ...@@ -16,6 +17,9 @@ export function registerGetLogsTool(server: McpServer, distDir: string) {
'Get the path to the Next.js development log file. Returns the file path so the agent can read the logs directly.', 'Get the path to the Next.js development log file. Returns the file path so the agent can read the logs directly.',
}, },
async () => { async () => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_logs')
try { try {
const logFilePath = join(distDir, 'logs', 'next-development.log') const logFilePath = join(distDir, 'logs', 'next-development.log')
......
...@@ -14,6 +14,7 @@ import type { ...@@ -14,6 +14,7 @@ import type {
SegmentTrieData, SegmentTrieData,
} from '../../../shared/lib/mcp-page-metadata-types' } from '../../../shared/lib/mcp-page-metadata-types'
import type { SegmentTrieNode } from '../../../next-devtools/dev-overlay/segment-explorer-trie' import type { SegmentTrieNode } from '../../../next-devtools/dev-overlay/segment-explorer-trie'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetPageMetadataTool( export function registerGetPageMetadataTool(
server: McpServer, server: McpServer,
...@@ -28,6 +29,9 @@ export function registerGetPageMetadataTool( ...@@ -28,6 +29,9 @@ export function registerGetPageMetadataTool(
inputSchema: {}, inputSchema: {},
}, },
async (_request) => { async (_request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_page_metadata')
try { try {
const connectionCount = getActiveConnectionCount() const connectionCount = getActiveConnectionCount()
if (connectionCount === 0) { if (connectionCount === 0) {
......
import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/server/mcp' import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/server/mcp'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
export function registerGetProjectMetadataTool( export function registerGetProjectMetadataTool(
server: McpServer, server: McpServer,
...@@ -13,6 +14,9 @@ export function registerGetProjectMetadataTool( ...@@ -13,6 +14,9 @@ export function registerGetProjectMetadataTool(
inputSchema: {}, inputSchema: {},
}, },
async (_request) => { async (_request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_project_metadata')
try { try {
if (!projectPath) { if (!projectPath) {
return { return {
......
...@@ -2,6 +2,7 @@ import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/ser ...@@ -2,6 +2,7 @@ import type { McpServer } from 'next/dist/compiled/@modelcontextprotocol/sdk/ser
import { z } from 'next/dist/compiled/zod' import { z } from 'next/dist/compiled/zod'
import { promises as fs } from 'fs' import { promises as fs } from 'fs'
import { join } from 'path' import { join } from 'path'
import { mcpTelemetryTracker } from '../mcp-telemetry-tracker'
const INLINE_ACTION_PREFIX = '$$RSC_SERVER_ACTION_' const INLINE_ACTION_PREFIX = '$$RSC_SERVER_ACTION_'
...@@ -29,6 +30,9 @@ export function registerGetActionByIdTool(server: McpServer, distDir: string) { ...@@ -29,6 +30,9 @@ export function registerGetActionByIdTool(server: McpServer, distDir: string) {
}, },
}, },
async (request) => { async (request) => {
// Track telemetry
mcpTelemetryTracker.recordToolCall('mcp/get_server_action_by_id')
try { try {
const { actionId } = request const { actionId } = request
......
...@@ -230,6 +230,32 @@ export function eventPackageUsedInGetServerSideProps( ...@@ -230,6 +230,32 @@ export function eventPackageUsedInGetServerSideProps(
})) }))
} }
export const EVENT_MCP_TOOL_USAGE = 'NEXT_MCP_TOOL_USAGE'
export type McpToolName =
| 'mcp/get_errors'
| 'mcp/get_logs'
| 'mcp/get_page_metadata'
| 'mcp/get_project_metadata'
| 'mcp/get_server_action_by_id'
export type EventMcpToolUsage = {
toolName: McpToolName
invocationCount: number
}
export function eventMcpToolUsage(
usages: Array<{ featureName: McpToolName; invocationCount: number }>
): Array<{ eventName: string; payload: EventMcpToolUsage }> {
return usages.map(({ featureName, invocationCount }) => ({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: featureName,
invocationCount,
},
}))
}
export const ERROR_THROWN_EVENT = 'NEXT_ERROR_THROWN' export const ERROR_THROWN_EVENT = 'NEXT_ERROR_THROWN'
type ErrorThrownEvent = { type ErrorThrownEvent = {
eventName: typeof ERROR_THROWN_EVENT eventName: typeof ERROR_THROWN_EVENT
......
export * from './version' export * from './version'
export * from './build' export * from './build'
export * from './plugins' export * from './plugins'
// Re-export MCP-specific types and functions
export type { McpToolName, EventMcpToolUsage } from './build'
export { EVENT_MCP_TOOL_USAGE, eventMcpToolUsage } from './build'
/**
* @jest-environment node
*/
import { eventMcpToolUsage, EVENT_MCP_TOOL_USAGE } from './build'
import type { McpToolName } from './build'
describe('MCP Telemetry Events', () => {
it('should generate correct telemetry events for single tool', () => {
const usages = [
{
featureName: 'mcp/get_errors' as McpToolName,
invocationCount: 5,
},
]
const events = eventMcpToolUsage(usages)
expect(events).toHaveLength(1)
expect(events[0]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_errors',
invocationCount: 5,
},
})
})
it('should generate correct telemetry events for multiple tools', () => {
const usages = [
{
featureName: 'mcp/get_errors' as McpToolName,
invocationCount: 3,
},
{
featureName: 'mcp/get_logs' as McpToolName,
invocationCount: 1,
},
{
featureName: 'mcp/get_page_metadata' as McpToolName,
invocationCount: 7,
},
]
const events = eventMcpToolUsage(usages)
expect(events).toHaveLength(3)
expect(events[0]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_errors',
invocationCount: 3,
},
})
expect(events[1]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_logs',
invocationCount: 1,
},
})
expect(events[2]).toEqual({
eventName: EVENT_MCP_TOOL_USAGE,
payload: {
toolName: 'mcp/get_page_metadata',
invocationCount: 7,
},
})
})
it('should handle all MCP tool types', () => {
const allTools: McpToolName[] = [
'mcp/get_errors',
'mcp/get_logs',
'mcp/get_page_metadata',
'mcp/get_project_metadata',
'mcp/get_server_action_by_id',
]
const usages = allTools.map((tool, index) => ({
featureName: tool,
invocationCount: index + 1,
}))
const events = eventMcpToolUsage(usages)
expect(events).toHaveLength(5)
events.forEach((event, index) => {
expect(event.eventName).toBe(EVENT_MCP_TOOL_USAGE)
expect(event.payload.toolName).toBe(allTools[index])
expect(event.payload.invocationCount).toBe(index + 1)
})
})
it('should handle empty usage array', () => {
const events = eventMcpToolUsage([])
expect(events).toEqual([])
})
it('should use correct event name constant', () => {
expect(EVENT_MCP_TOOL_USAGE).toBe('NEXT_MCP_TOOL_USAGE')
})
it('should transform featureName to toolName in payload', () => {
const usages = [
{
featureName: 'mcp/get_project_metadata' as McpToolName,
invocationCount: 2,
},
]
const events = eventMcpToolUsage(usages)
// Verify the input has 'featureName' but output has 'toolName'
expect(usages[0]).toHaveProperty('featureName')
expect(events[0].payload).toHaveProperty('toolName')
expect(events[0].payload).not.toHaveProperty('featureName')
})
})
...@@ -40,7 +40,6 @@ type EventCliSessionStarted = { ...@@ -40,7 +40,6 @@ type EventCliSessionStarted = {
reactCompiler: boolean reactCompiler: boolean
reactCompilerCompilationMode: string | null reactCompilerCompilationMode: string | null
reactCompilerPanicThreshold: string | null reactCompilerPanicThreshold: string | null
mcpServer: boolean | null
} }
export function eventCliSession( export function eventCliSession(
...@@ -75,7 +74,6 @@ export function eventCliSession( ...@@ -75,7 +74,6 @@ export function eventCliSession(
| 'reactCompiler' | 'reactCompiler'
| 'reactCompilerCompilationMode' | 'reactCompilerCompilationMode'
| 'reactCompilerPanicThreshold' | 'reactCompilerPanicThreshold'
| 'mcpServer'
| 'isRspack' | 'isRspack'
> >
): { eventName: string; payload: EventCliSessionStarted }[] { ): { eventName: string; payload: EventCliSessionStarted }[] {
...@@ -134,11 +132,6 @@ export function eventCliSession( ...@@ -134,11 +132,6 @@ export function eventCliSession(
typeof nextConfig.reactCompiler !== 'boolean' typeof nextConfig.reactCompiler !== 'boolean'
? (nextConfig.reactCompiler?.panicThreshold ?? null) ? (nextConfig.reactCompiler?.panicThreshold ?? null)
: null, : null,
// Default mcpServer to true if not set
mcpServer:
typeof nextConfig.experimental.mcpServer === 'boolean'
? nextConfig.experimental.mcpServer
: true,
} }
return [{ eventName: EVENT_VERSION, payload }] return [{ eventName: EVENT_VERSION, payload }]
} }
import { FileRef, nextTestSetup } from 'e2e-utils'
import { findAllTelemetryEvents } from 'next-test-utils'
import path from 'path'
describe('mcp-server telemetry tracking', () => {
const { next } = nextTestSetup({
files: new FileRef(path.join(__dirname, 'fixtures', 'default-template')),
env: {
NEXT_TELEMETRY_DEBUG: '1',
},
})
async function callMcpTool(
toolName: string,
params: Record<string, any> = {}
) {
const response = await fetch(`${next.url}/_next/mcp`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json, text/event-stream',
},
body: JSON.stringify({
jsonrpc: '2.0',
id: `telemetry-test-${Date.now()}`,
method: 'tools/call',
params: { name: toolName, arguments: params },
}),
})
const text = await response.text()
const match = text.match(/data: ({.*})/s)
if (!match) {
throw new Error(`Failed to parse response for tool ${toolName}`)
}
return JSON.parse(match[1])
}
it('should record MCP tool usage telemetry on server shutdown', async () => {
// Call different MCP tools
await callMcpTool('get_project_metadata')
await callMcpTool('get_logs')
await callMcpTool('get_errors')
// Open browser and call page_metadata
await next.browser('/')
await callMcpTool('get_page_metadata')
// Call some tools multiple times
await callMcpTool('get_project_metadata')
await callMcpTool('get_errors')
// Stop the dev server to trigger telemetry recording
// Use SIGTERM so cleanup handlers can run
await next.stop('SIGTERM')
// Parse telemetry from CLI output
const output = next.cliOutput
const events = findAllTelemetryEvents(output, 'NEXT_MCP_TOOL_USAGE')
// Verify telemetry events were recorded
expect(events.length).toBeGreaterThan(0)
// Check that specific tools were tracked
const toolUsages = new Map(
events.map((e) => [e.toolName, e.invocationCount])
)
// get_project_metadata was called 2 times
expect(toolUsages.get('mcp/get_project_metadata')).toBe(2)
// get_errors was called 2 times
expect(toolUsages.get('mcp/get_errors')).toBe(2)
// get_logs was called 1 time
expect(toolUsages.get('mcp/get_logs')).toBe(1)
// get_page_metadata was called 1 time
expect(toolUsages.get('mcp/get_page_metadata')).toBe(1)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment