Commit a5131d6c authored by Jude Gao's avatar Jude Gao

virtual api route for insights ingestion

parent 1b0083f0
......@@ -20,3 +20,7 @@ export { ImageResponse } from 'next/dist/server/web/spec-extension/image-respons
export type { ImageResponseOptions } from 'next/dist/compiled/@vercel/og/types'
export { after } from 'next/dist/server/after'
export { connection } from 'next/dist/server/request/connection'
export {
handleInsightsReport as unstable_handleInsightsReport,
type InsightsReport as UnstableInsightsReport,
} from 'next/dist/server/lib/insights-handler'
......@@ -13,6 +13,8 @@ const serverExports = {
.URLPattern,
after: require('next/dist/server/after').after,
connection: require('next/dist/server/request/connection').connection,
unstable_handleInsightsReport:
require('next/dist/server/lib/insights-handler').handleInsightsReport,
}
// https://nodejs.org/api/esm.html#commonjs-namespaces
......@@ -28,3 +30,5 @@ exports.userAgent = serverExports.userAgent
exports.URLPattern = serverExports.URLPattern
exports.after = serverExports.after
exports.connection = serverExports.connection
exports.unstable_handleInsightsReport =
serverExports.unstable_handleInsightsReport
......@@ -97,4 +97,5 @@ export const NextBuildContext: Partial<{
debugPrerender: boolean
analyze: boolean
insights: boolean
insightsRouteId: string
}> = {}
......@@ -39,6 +39,7 @@ export interface DefineEnvOptions {
afterFiles: Rewrite[]
fallback: Rewrite[]
}
insightsRouteId?: string
}
interface DefineEnv {
......@@ -111,6 +112,7 @@ export function getDefineEnv({
middlewareMatchers,
omitNonDeterministic,
rewrites,
insightsRouteId,
}: DefineEnvOptions): SerializedDefineEnv {
const nextPublicEnv = getNextPublicEnvironmentVariables()
const nextConfigEnv = getNextConfigEnv(config)
......@@ -349,6 +351,7 @@ export function getDefineEnv({
'process.env.__NEXT_TRANSITION_INDICATOR':
config.experimental.transitionIndicator ?? false,
'process.env.__NEXT_INSIGHTS_BUILD': config.insights ?? false,
'process.env.__NEXT_INSIGHTS_ROUTE': insightsRouteId ?? '',
}
const userDefines = config.compiler?.define ?? {}
......
......@@ -44,6 +44,7 @@ import {
isProxyFile,
isInstrumentationHookFile,
isInstrumentationHookFilename,
isAppBuiltinServerRoute,
} from './utils'
import { getPageStaticInfo } from './analysis/get-page-static-info'
import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep'
......@@ -451,6 +452,7 @@ export async function createPagesMapping({
pagesDir,
appDir,
appDirOnly,
insightsRouteId,
}: {
isDev: boolean
pageExtensions: PageExtensions
......@@ -459,6 +461,7 @@ export async function createPagesMapping({
pagesDir: string | undefined
appDir: string | undefined
appDirOnly: boolean
insightsRouteId?: string
}): Promise<MappedPages> {
const isAppRoute = pagesType === 'app'
......@@ -542,6 +545,12 @@ export async function createPagesMapping({
'next/dist/client/components/builtin/app-error'
),
}),
// Add insights ingest route if enabled
...(insightsRouteId && {
[`/${insightsRouteId}/route`]: require.resolve(
'next/dist/server/insights/insights-ingest-route'
),
}),
...pages,
}
}
......@@ -917,14 +926,19 @@ export async function createEntrypoints(
const isInstrumentation =
isInstrumentationHookFile(page) && pagesType === PAGE_TYPES.ROOT
// Check if this is a built-in server-only route (e.g., insights ingest route)
// These routes should not be compiled by the client compiler
const isBuiltinServerRoute = isAppBuiltinServerRoute(pageFilePath)
runDependingOnPageType({
page,
pageRuntime: staticInfo.runtime,
pageType: pagesType,
onClient: () => {
if (isServerComponent || isInsideAppDir) {
if (isServerComponent || isInsideAppDir || isBuiltinServerRoute) {
// We skip the initial entries for server component pages and let the
// server compiler inject them instead.
// We also skip built-in server routes since they should only run on the server.
} else {
client[clientBundlePath] = getClientEntry({
absolutePagePath,
......
......@@ -79,6 +79,7 @@ import {
FUNCTIONS_CONFIG_MANIFEST,
DYNAMIC_CSS_MANIFEST,
TURBOPACK_CLIENT_MIDDLEWARE_MANIFEST,
INSIGHTS_MANIFEST,
} from '../shared/lib/constants'
import {
UNDERSCORE_NOT_FOUND_ROUTE,
......@@ -541,6 +542,18 @@ async function writePrerenderManifest(
await writeManifest(path.join(distDir, PRERENDER_MANIFEST), manifest)
}
export interface InsightsManifest {
enabled: boolean
routeId: string
}
async function writeInsightsManifest(
distDir: string,
manifest: InsightsManifest
): Promise<void> {
await writeManifest(path.join(distDir, INSIGHTS_MANIFEST), manifest)
}
async function writeClientSsgManifest(
prerenderManifest: DeepReadonly<PrerenderManifest>,
{
......@@ -915,6 +928,10 @@ export default async function build(
NextBuildContext.noMangling = noMangling
NextBuildContext.debugPrerender = debugPrerender
NextBuildContext.insights = insights
// Generate a random route ID for insights builds to avoid collisions with user routes
NextBuildContext.insightsRouteId = insights
? Math.random().toString(36).substring(2, 10)
: ''
await nextBuildSpan.traceAsyncFn(async () => {
// attempt to load global env values so they are available in next.config.js
......@@ -1326,6 +1343,7 @@ export default async function build(
pagesDir,
appDir,
appDirOnly,
insightsRouteId: NextBuildContext.insightsRouteId,
})
)
......@@ -3999,6 +4017,14 @@ export default async function build(
})
}
// Write insights manifest if insights build is enabled
if (insights && NextBuildContext.insightsRouteId) {
await writeInsightsManifest(distDir, {
enabled: true,
routeId: NextBuildContext.insightsRouteId,
})
}
await writeImagesManifest(distDir, config)
await writeManifest(path.join(distDir, EXPORT_MARKER), {
version: 1,
......
......@@ -437,6 +437,7 @@ export function createDefineEnv({
hasRewrites,
middlewareMatchers,
rewrites,
insightsRouteId,
}: Omit<
DefineEnvOptions,
'isClient' | 'isNodeOrEdgeCompilation' | 'isEdgeServer' | 'isNodeServer'
......@@ -463,6 +464,7 @@ export function createDefineEnv({
isNodeServer: variant === 'nodejs',
middlewareMatchers,
rewrites,
insightsRouteId,
})
)
}
......
......@@ -73,6 +73,7 @@ export async function turbopackBuild(): Promise<{
// Implemented separately in Turbopack, doesn't have to be passed here.
middlewareMatchers: undefined,
rewrites,
insightsRouteId: NextBuildContext.insightsRouteId,
}),
buildId,
encryptionKey,
......
......@@ -1436,11 +1436,24 @@ export function isReservedPage(page: string) {
}
export function isAppBuiltinPage(page: string) {
return /next[\\/]dist[\\/](esm[\\/])?client[\\/]components[\\/]builtin[\\/]/.test(
page
return (
// Client-side builtin pages (e.g., not-found, error)
/next[\\/]dist[\\/](esm[\\/])?client[\\/]components[\\/]builtin[\\/]/.test(
page
) ||
// Server-side builtin routes (e.g., insights ingest route)
isAppBuiltinServerRoute(page)
)
}
/**
* Check if a page path is a built-in server-only route (e.g., insights ingest route).
* These routes should not be compiled by the client compiler.
*/
export function isAppBuiltinServerRoute(page: string) {
return /next[\\/]dist[\\/](esm[\\/])?server[\\/]insights[\\/]/.test(page)
}
export function isCustomErrorPage(page: string) {
return page === '/404' || page === '/500'
}
......
......@@ -92,7 +92,7 @@ import {
import { getRspackCore } from '../shared/lib/get-rspack'
import { RspackProfilingPlugin } from './webpack/plugins/rspack-profiling-plugin'
import getWebpackBundler from '../shared/lib/get-webpack-bundler'
import type { NextBuildContext } from './build-context'
import { NextBuildContext } from './build-context'
import type { RootParamsLoaderOpts } from './webpack/loaders/next-root-params-loader'
import type { InvalidImportLoaderOpts } from './webpack/loaders/next-invalid-import-error-loader'
import { defaultOverrides } from '../server/require-hook'
......@@ -2031,6 +2031,7 @@ export default async function getBaseWebpackConfig(
middlewareMatchers,
omitNonDeterministic: isCompileMode,
rewrites,
insightsRouteId: NextBuildContext.insightsRouteId,
})
),
isClient &&
......
......@@ -37,7 +37,15 @@ export async function createAppRouteCode({
// This, when used with the resolver will give us the pathname to the built
// route handler file.
let resolvedPagePath = await resolveAppRoute(routePath)
let resolvedPagePath: string | undefined
// Check if this is an absolute path (built-in route like insights)
if (path.isAbsolute(routePath)) {
resolvedPagePath = routePath
} else {
resolvedPagePath = await resolveAppRoute(routePath)
}
if (!resolvedPagePath) {
throw new Error(
`Invariant: could not resolve page path for ${name} at ${routePath}`
......@@ -47,12 +55,18 @@ export async function createAppRouteCode({
// If this is a metadata route file, then we need to use the metadata-loader
// for the route to ensure that the route is generated.
const fileBaseName = path.parse(resolvedPagePath).name
const appDirRelativePath = resolvedPagePath.slice(appDir.length)
const isMetadataEntryFile = isMetadataRouteFile(
appDirRelativePath,
DEFAULT_METADATA_ROUTE_EXTENSIONS,
true
)
// For built-in routes (absolute paths not under appDir), skip metadata check
const isBuiltinRoute = !resolvedPagePath.startsWith(appDir)
const appDirRelativePath = isBuiltinRoute
? resolvedPagePath
: resolvedPagePath.slice(appDir.length)
const isMetadataEntryFile =
!isBuiltinRoute &&
isMetadataRouteFile(
appDirRelativePath,
DEFAULT_METADATA_ROUTE_EXTENSIONS,
true
)
if (isMetadataEntryFile) {
const { ext } = getFilenameAndExtension(resolvedPagePath)
const isDynamicRouteExtension = pageExtensions.includes(ext)
......
......@@ -19,6 +19,7 @@ import { getModuleBuildInfo } from '../loaders/get-module-build-info'
import { getPageFilePath } from '../../entries'
import { resolveExternal } from '../../handle-externals'
import { isMetadataRouteFile } from '../../../lib/metadata/is-metadata-route'
import { isAppBuiltinServerRoute } from '../../utils'
import { getCompilationSpan } from '../utils'
const PLUGIN_NAME = 'TraceEntryPointsPlugin'
......@@ -236,18 +237,28 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
)
if (entrypoint.name.startsWith('app/') && this.appDir) {
const appDirRelativeEntryPath =
const entryAbsolutePath =
this.buildTraceContext.entriesTrace?.absolutePathByEntryName[
entrypoint.name
]?.replace(this.appDir, '')
]
const appDirRelativeEntryPath = entryAbsolutePath?.replace(
this.appDir,
''
)
const entryIsStaticMetadataRoute =
appDirRelativeEntryPath &&
isMetadataRouteFile(appDirRelativeEntryPath, [], true)
// Check if this is a built-in server route (e.g., insights ingest route)
// These routes don't have client reference manifests.
const entryIsBuiltinServerRoute =
entryAbsolutePath && isAppBuiltinServerRoute(entryAbsolutePath)
// Include the client reference manifest in the trace, but not for
// static metadata routes, for which we don't generate those.
if (!entryIsStaticMetadataRoute) {
// static metadata routes or built-in server routes, for which we don't generate those.
if (!entryIsStaticMetadataRoute && !entryIsBuiltinServerRoute) {
entryFiles.add(
nodePath.join(
outputPath,
......@@ -365,6 +376,11 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
entryModMap.set(absolutePath, entryMod)
entryNameMap.set(absolutePath, name)
absolutePathByEntryName.set(name, absolutePath)
} else if (isAppBuiltinServerRoute(absolutePath)) {
// Built-in server routes (e.g., insights) need to be tracked
// in absolutePathByEntryName so we can skip generating
// client reference manifests for them.
absolutePathByEntryName.set(name, absolutePath)
}
}
......
......@@ -243,7 +243,14 @@ async function sendRawData() {
}
try {
await originalFetch('/__nextjs_insights_ingest', {
// Route ID is injected at build time to avoid collisions with user routes
// The route is created at /{routeId} as a real app route
const routeId = process.env.__NEXT_INSIGHTS_ROUTE
if (!routeId) {
console.warn('[Insights] No insights route configured')
return
}
await originalFetch(`/${routeId}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
......
......@@ -1154,7 +1154,6 @@ export interface NextConfig {
/**
* Enable insights build with source maps and no minification.
* This is set automatically when using `next build --insights`.
* @internal
*/
insights?: boolean
......
import { join } from 'path'
import { NextResponse } from '../web/spec-extension/response'
import type { NextRequest } from '../web/spec-extension/request'
import { handleInsightsReport } from '../lib/insights-handler'
/**
* Insights ingest route handler
* This is a built-in route handler that receives performance insights data
* from the client waterfall detector and processes it server-side.
*
* The handler:
* 1. Receives raw timing data from the client
* 2. Resolves source maps to map bundled code back to source
* 3. Analyzes fetch patterns to detect waterfalls
* 4. Outputs actionable recommendations
*/
export async function POST(request: NextRequest): Promise<NextResponse> {
try {
const body = await request.json()
// Get project directory and dist directory
const projectDir = process.cwd()
const distDir = join(projectDir, '.next')
// Process the insights report (source map resolution + waterfall analysis)
await handleInsightsReport(body, distDir, projectDir)
return NextResponse.json({ ok: true })
} catch (error) {
console.error('[Insights] Error processing report:', error)
return NextResponse.json(
{ error: 'Failed to process insights report' },
{ status: 500 }
)
}
}
// Disable static generation for this route
export const dynamic = 'force-dynamic'
// Force Node.js runtime (required for fs and source-map operations)
export const runtime = 'nodejs'
import fsPromises from 'fs/promises'
import { join, dirname } from 'path'
import { SourceMapConsumer } from 'next/dist/compiled/source-map08'
// Use the pure-JS source-map library (not source-map08 which requires WASM)
// This works better when bundled into app built-in routes
import { SourceMapConsumer } from 'next/dist/compiled/source-map'
type StackFrame = {
functionName: string
......@@ -102,45 +104,41 @@ export async function resolveSourceMapFrame(
const sourceMap = JSON.parse(sourceMapContent)
const consumer = await new SourceMapConsumer(sourceMap)
try {
const original = consumer.originalPositionFor({
line: frame.lineNumber,
column: frame.columnNumber,
})
const original = consumer.originalPositionFor({
line: frame.lineNumber,
column: frame.columnNumber,
})
if (original.source) {
// Convert source paths to absolute file paths
// Uses same patterns as hot-reloader-turbopack.ts and webpack-module-path.ts
let resolvedFileName = original.source
// Turbopack: turbopack:///[project]/path/to/file.tsx -> /absolute/path/to/file.tsx
// Same pattern as hot-reloader-turbopack.ts:162
resolvedFileName = resolvedFileName.replace(
/turbopack:\/\/\/\[project\]/,
projectDir
)
// Webpack: webpack://app/./path -> ./path, webpack://_N_E/./path -> ./path
// Same pattern as webpack-module-path.ts
resolvedFileName = resolvedFileName
.replace(/^webpack-internal:\/\/\/(\([\w-]+\)\/)?/, '')
.replace(/^(webpack:\/\/\/|webpack:\/\/(_N_E\/)?)(\([\w-]+\)\/)?/, '')
// Convert relative paths to absolute
if (resolvedFileName.startsWith('./')) {
resolvedFileName = join(projectDir, resolvedFileName.slice(2))
}
if (original.source) {
// Convert source paths to absolute file paths
// Uses same patterns as hot-reloader-turbopack.ts and webpack-module-path.ts
let resolvedFileName = original.source
return {
functionName: original.name || frame.functionName,
fileName: resolvedFileName,
lineNumber: original.line || frame.lineNumber,
columnNumber: original.column || frame.columnNumber,
raw: frame.raw,
}
// Turbopack: turbopack:///[project]/path/to/file.tsx -> /absolute/path/to/file.tsx
// Same pattern as hot-reloader-turbopack.ts:162
resolvedFileName = resolvedFileName.replace(
/turbopack:\/\/\/\[project\]/,
projectDir
)
// Webpack: webpack://app/./path -> ./path, webpack://_N_E/./path -> ./path
// Same pattern as webpack-module-path.ts
resolvedFileName = resolvedFileName
.replace(/^webpack-internal:\/\/\/(\([\w-]+\)\/)?/, '')
.replace(/^(webpack:\/\/\/|webpack:\/\/(_N_E\/)?)(\([\w-]+\)\/)?/, '')
// Convert relative paths to absolute
if (resolvedFileName.startsWith('./')) {
resolvedFileName = join(projectDir, resolvedFileName.slice(2))
}
return {
functionName: original.name || frame.functionName,
fileName: resolvedFileName,
lineNumber: original.line || frame.lineNumber,
columnNumber: original.column || frame.columnNumber,
raw: frame.raw,
}
} finally {
consumer.destroy()
}
} catch {
// Return original frame on any error
......@@ -217,9 +215,9 @@ export async function handleInsightsReport(
return `${filePath}:${topFrame.lineNumber}`
}
// No fetches - nothing to analyze
// No fetches - no waterfall possible
if (report.fetchEvents.length === 0) {
console.log(`✅ No waterfall detected on ${route} (no client fetches)`)
console.log(`[Insights] ✅ NO WATERFALL on ${route}`)
return
}
......@@ -430,75 +428,55 @@ export async function handleInsightsReport(
}
// Step 4: Determine result and output
const parallelBatches = fetchBatches.filter((b) => b.length > 1)
const hasParallelBatches = parallelBatches.length > 0
// No waterfall detected
if (!treeHasWaterfall) {
if (hasParallelBatches) {
console.log(`✅ No waterfall detected on ${route}`)
parallelBatches.forEach((batch, i) => {
console.log(` Parallel fetch group ${i + 1}:`)
batch.forEach((f) => {
const location = getLocation(f.parsedFrames)
console.log(` - ${f.url}`)
console.log(` at ${location}`)
})
})
} else {
console.log(`✅ No waterfall detected on ${route}`)
}
console.log(`[Insights] ✅ NO WATERFALL on ${route}`)
return
}
// Waterfall detected - detailed output
console.log('')
console.log('═'.repeat(80))
console.log(`🚨 WATERFALL DETECTED on ${route}`)
console.log('═'.repeat(80))
console.log('')
// Recursive function to print tree with proper formatting
const printTreeOutput = (
// Recursive function to build tree string
const buildTreeString = (
node: FetchNode,
depth: number = 0,
prefix: string = ''
) => {
): string => {
let result = ''
if (node.isParallelGroup && node.parallelFetches) {
// Parallel group
if (depth > 0) console.log(`${prefix}↓`)
if (depth > 0) result += `${prefix}↓\n`
node.parallelFetches.forEach((f, idx) => {
const location = getLocation(f.parsedFrames)
const isLast = idx === node.parallelFetches!.length - 1
const bullet = node.parallelFetches!.length > 1 ? '├─' : '└─'
console.log(`${prefix}${isLast ? '└─' : bullet} ${f.url} (${location})`)
result += `${prefix}${isLast ? '└─' : bullet} ${f.url} (${location})\n`
})
} else {
// Single fetch
if (depth > 0) console.log(`${prefix}↓`)
if (depth > 0) result += `${prefix}↓\n`
const location = getLocation(node.fetch.parsedFrames)
console.log(`${prefix}└─ ${node.fetch.url} (${location})`)
result += `${prefix}└─ ${node.fetch.url} (${location})\n`
}
// Print children
// Build children
if (node.children.length > 0) {
const childPrefix = prefix + ' '
node.children.forEach((child) => {
printTreeOutput(child, depth + 1, childPrefix)
result += buildTreeString(child, depth + 1, childPrefix)
})
}
return result
}
// Build tree string
let treeOutput = ''
rootNodes.forEach((root, idx) => {
if (rootNodes.length > 1 && idx > 0) {
console.log('')
console.log('─'.repeat(80))
console.log('')
treeOutput += `\n${'─'.repeat(80)}\n\n`
}
if (rootNodes.length > 1) {
console.log(`Path ${idx + 1}:`)
treeOutput += `Path ${idx + 1}:\n`
}
printTreeOutput(root)
treeOutput += buildTreeString(root)
})
// Calculate severity
......@@ -847,5 +825,10 @@ ${'═'.repeat(80)}
`
}
console.log(aiInsights)
// Build final output with informative first line for Vercel log list view
const serialCount = totalFetches - parallelCount
const firstLine = `[Insights] ${severityEmoji[severity]} WATERFALL DETECTED on ${route} | Severity: ${severity} | Depth: ${maxDepth} | Fetches: ${totalFetches} (${parallelCount} parallel, ${serialCount} serial)`
const fullOutput = `${firstLine}\n${treeOutput}${aiInsights}`
console.log(fullOutput)
}
......@@ -1303,65 +1303,6 @@ export default class NextNodeServer extends BaseServer<
const handler = super.getRequestHandler()
return async (req, res, parsedUrl) => {
const url = req.url || ''
const pathname = url.split('?')[0]
if (
// process.env.__NEXT_INSIGHTS_BUILD && // TODO figure out why DCE sets this to false even in INSIGHTS build?
pathname === '/__nextjs_insights_ingest'
) {
const rawRes =
res instanceof NodeNextResponse ? res.originalResponse : res
const rawReq =
req instanceof NodeNextRequest ? req.originalRequest : req
// Handle POST request with waterfall report
if (rawReq.method === 'POST') {
let body = ''
rawReq.on('data', (chunk: Buffer) => {
body += chunk.toString()
})
rawReq.on('end', async () => {
try {
const report = JSON.parse(body)
const { handleInsightsReport } =
require('./lib/insights-handler') as typeof import('./lib/insights-handler')
handleInsightsReport(
report,
this.distDir,
this.dir,
this.nextConfig.cacheComponents ?? false
)
rawRes.statusCode = 200
rawRes.setHeader('Content-Type', 'application/json')
rawRes.end(JSON.stringify({ ok: true }))
} catch (e) {
console.error('Error processing insights report:', e)
rawRes.statusCode = 400
rawRes.setHeader('Content-Type', 'application/json')
rawRes.end(
JSON.stringify({
ok: false,
error: e instanceof Error ? e.message : 'Invalid JSON',
})
)
}
})
return
}
// Handle GET request (for testing)
rawRes.statusCode = 200
rawRes.setHeader('Content-Type', 'application/json')
rawRes.end(
JSON.stringify({
ok: true,
message: 'Insights ingest endpoint ready',
})
)
return
}
return handler(this.normalizeReq(req), this.normalizeRes(res), parsedUrl)
}
}
......
......@@ -136,6 +136,8 @@ export const INTERCEPTION_ROUTE_REWRITE_MANIFEST =
'interception-route-rewrite-manifest'
// server/dynamic-css-manifest.js
export const DYNAMIC_CSS_MANIFEST = 'dynamic-css-manifest'
// insights-manifest.json (written during insights builds)
export const INSIGHTS_MANIFEST = 'insights-manifest.json'
// static/runtime/main.js
export const CLIENT_STATIC_FILES_RUNTIME_MAIN = `main`
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment