Unverified Commit 9316dc0a authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add default trace format that is exported automatically (#28461)

Co-authored-by: 's avatarSteven <steven@ceriously.com>
parent 706547e0
...@@ -40,7 +40,7 @@ const nextBabelLoaderOuter = function nextBabelLoaderOuter( ...@@ -40,7 +40,7 @@ const nextBabelLoaderOuter = function nextBabelLoaderOuter(
) { ) {
const callback = this.async() const callback = this.async()
const loaderSpan = trace('next-babel-turbo-loader', this.currentTraceSpan?.id) const loaderSpan = this.currentTraceSpan.traceChild('next-babel-turbo-loader')
loaderSpan loaderSpan
.traceAsyncFn(() => .traceAsyncFn(() =>
nextBabelLoader.call(this, loaderSpan, inputSource, inputSourceMap) nextBabelLoader.call(this, loaderSpan, inputSource, inputSourceMap)
......
...@@ -2,7 +2,7 @@ import { loader } from 'next/dist/compiled/webpack/webpack' ...@@ -2,7 +2,7 @@ import { loader } from 'next/dist/compiled/webpack/webpack'
import { Span } from '../../../telemetry/trace' import { Span } from '../../../telemetry/trace'
export interface NextJsLoaderContext extends loader.LoaderContext { export interface NextJsLoaderContext extends loader.LoaderContext {
currentTraceSpan?: Span currentTraceSpan: Span
} }
export interface NextBabelLoaderOptions { export interface NextBabelLoaderOptions {
......
...@@ -130,11 +130,13 @@ export default async function build( ...@@ -130,11 +130,13 @@ export default async function build(
const config: NextConfigComplete = await nextBuildSpan const config: NextConfigComplete = await nextBuildSpan
.traceChild('load-next-config') .traceChild('load-next-config')
.traceAsyncFn(() => loadConfig(PHASE_PRODUCTION_BUILD, dir, conf)) .traceAsyncFn(() => loadConfig(PHASE_PRODUCTION_BUILD, dir, conf))
const distDir = path.join(dir, config.distDir)
setGlobal('distDir', distDir)
const { target } = config const { target } = config
const buildId: string = await nextBuildSpan const buildId: string = await nextBuildSpan
.traceChild('generate-buildid') .traceChild('generate-buildid')
.traceAsyncFn(() => generateBuildId(config.generateBuildId, nanoid)) .traceAsyncFn(() => generateBuildId(config.generateBuildId, nanoid))
const distDir = path.join(dir, config.distDir)
const customRoutes: CustomRoutes = await nextBuildSpan const customRoutes: CustomRoutes = await nextBuildSpan
.traceChild('load-custom-routes') .traceChild('load-custom-routes')
......
...@@ -240,7 +240,7 @@ export default async function getBaseWebpackConfig( ...@@ -240,7 +240,7 @@ export default async function getBaseWebpackConfig(
entrypoints: WebpackEntrypoints entrypoints: WebpackEntrypoints
rewrites: CustomRoutes['rewrites'] rewrites: CustomRoutes['rewrites']
isDevFallback?: boolean isDevFallback?: boolean
runWebpackSpan?: Span runWebpackSpan: Span
} }
): Promise<webpack.Configuration> { ): Promise<webpack.Configuration> {
const hasRewrites = const hasRewrites =
......
...@@ -11,7 +11,6 @@ import { ...@@ -11,7 +11,6 @@ import {
ROUTES_MANIFEST, ROUTES_MANIFEST,
REACT_LOADABLE_MANIFEST, REACT_LOADABLE_MANIFEST,
} from '../../../../shared/lib/constants' } from '../../../../shared/lib/constants'
import { trace } from '../../../../telemetry/trace'
export type ServerlessLoaderQuery = { export type ServerlessLoaderQuery = {
page: string page: string
...@@ -34,8 +33,6 @@ export type ServerlessLoaderQuery = { ...@@ -34,8 +33,6 @@ export type ServerlessLoaderQuery = {
} }
const nextServerlessLoader: webpack.loader.Loader = function () { const nextServerlessLoader: webpack.loader.Loader = function () {
const loaderSpan = trace('next-serverless-loader')
return loaderSpan.traceFn(() => {
const { const {
distDir, distDir,
absolutePagePath, absolutePagePath,
...@@ -58,10 +55,10 @@ const nextServerlessLoader: webpack.loader.Loader = function () { ...@@ -58,10 +55,10 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query
const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/') const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/')
const reactLoadableManifest = join( const reactLoadableManifest = join(distDir, REACT_LOADABLE_MANIFEST).replace(
distDir, /\\/g,
REACT_LOADABLE_MANIFEST '/'
).replace(/\\/g, '/') )
const routesManifest = join(distDir, ROUTES_MANIFEST).replace(/\\/g, '/') const routesManifest = join(distDir, ROUTES_MANIFEST).replace(/\\/g, '/')
const escapedBuildId = escapeRegexp(buildId) const escapedBuildId = escapeRegexp(buildId)
...@@ -207,7 +204,6 @@ const nextServerlessLoader: webpack.loader.Loader = function () { ...@@ -207,7 +204,6 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
export { renderReqToHTML, render } export { renderReqToHTML, render }
` `
} }
})
} }
export default nextServerlessLoader export default nextServerlessLoader
...@@ -123,7 +123,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) { ...@@ -123,7 +123,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) {
} }
export default function swcLoader(inputSource, inputSourceMap) { export default function swcLoader(inputSource, inputSourceMap) {
const loaderSpan = trace('next-swc-loader', this.currentTraceSpan?.id) const loaderSpan = this.currentTraceSpan.traceChild('next-swc-loader')
const callback = this.async() const callback = this.async()
loaderSpan loaderSpan
.traceAsyncFn(() => .traceAsyncFn(() =>
......
...@@ -118,7 +118,7 @@ export default class BuildStatsPlugin { ...@@ -118,7 +118,7 @@ export default class BuildStatsPlugin {
async (stats, callback) => { async (stats, callback) => {
const compilerSpan = spans.get(compiler) const compilerSpan = spans.get(compiler)
try { try {
const writeStatsSpan = trace('NextJsBuildStats', compilerSpan?.id) const writeStatsSpan = compilerSpan!.traceChild('NextJsBuildStats')
await writeStatsSpan.traceAsyncFn(() => { await writeStatsSpan.traceAsyncFn(() => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const statsJson = reduceSize( const statsJson = reduceSize(
......
...@@ -71,9 +71,8 @@ export class CssMinimizerPlugin { ...@@ -71,9 +71,8 @@ export class CssMinimizerPlugin {
}, },
async (assets: any) => { async (assets: any) => {
const compilerSpan = spans.get(compiler) const compilerSpan = spans.get(compiler)
const cssMinimizerSpan = trace( const cssMinimizerSpan = compilerSpan!.traceChild(
'css-minimizer-plugin', 'css-minimizer-plugin'
compilerSpan?.id
) )
cssMinimizerSpan.setAttribute('webpackVersion', 5) cssMinimizerSpan.setAttribute('webpackVersion', 5)
...@@ -83,7 +82,7 @@ export class CssMinimizerPlugin { ...@@ -83,7 +82,7 @@ export class CssMinimizerPlugin {
files files
.filter((file) => CSS_REGEX.test(file)) .filter((file) => CSS_REGEX.test(file))
.map(async (file) => { .map(async (file) => {
const assetSpan = trace('minify-css', cssMinimizerSpan.id) const assetSpan = cssMinimizerSpan.traceChild('minify-css')
assetSpan.setAttribute('file', file) assetSpan.setAttribute('file', file)
return assetSpan.traceAsyncFn(async () => { return assetSpan.traceAsyncFn(async () => {
......
...@@ -15,9 +15,9 @@ function getNormalModuleLoaderHook(compilation: any) { ...@@ -15,9 +15,9 @@ function getNormalModuleLoaderHook(compilation: any) {
export class ProfilingPlugin { export class ProfilingPlugin {
compiler: any compiler: any
runWebpackSpan: Span | undefined runWebpackSpan: Span
constructor({ runWebpackSpan }: { runWebpackSpan: Span | undefined }) { constructor({ runWebpackSpan }: { runWebpackSpan: Span }) {
this.runWebpackSpan = runWebpackSpan this.runWebpackSpan = runWebpackSpan
} }
apply(compiler: any) { apply(compiler: any) {
...@@ -35,14 +35,17 @@ export class ProfilingPlugin { ...@@ -35,14 +35,17 @@ export class ProfilingPlugin {
attrs, attrs,
onSetSpan, onSetSpan,
}: { }: {
parentSpan?: () => Span | undefined parentSpan?: () => Span
attrs?: any attrs?: any
onSetSpan?: (span: Span) => void onSetSpan?: (span: Span) => void
} = {} } = {}
) { ) {
let span: Span | undefined let span: Span | undefined
startHook.tap(pluginName, () => { startHook.tap(pluginName, () => {
span = trace(spanName, parentSpan?.()?.id, attrs ? attrs() : attrs) span = parentSpan
? parentSpan().traceChild(spanName, attrs ? attrs() : attrs)
: trace(spanName, undefined, attrs ? attrs() : attrs)
onSetSpan?.(span) onSetSpan?.(span)
}) })
stopHook.tap(pluginName, () => { stopHook.tap(pluginName, () => {
...@@ -103,10 +106,16 @@ export class ProfilingPlugin { ...@@ -103,10 +106,16 @@ export class ProfilingPlugin {
const issuerModule = compilation?.moduleGraph?.getIssuer(module) const issuerModule = compilation?.moduleGraph?.getIssuer(module)
const span = trace( let span: Span
`build-module${moduleType ? `-${moduleType}` : ''}`,
issuerModule ? spans.get(issuerModule)?.id : compilerSpan.id const spanName = `build-module${moduleType ? `-${moduleType}` : ''}`
) const issuerSpan: Span | undefined =
issuerModule && spans.get(issuerModule)
if (issuerSpan) {
span = issuerSpan.traceChild(spanName)
} else {
span = compilerSpan.traceChild(spanName)
}
span.setAttribute('name', module.userRequest) span.setAttribute('name', module.userRequest)
spans.set(module, span) spans.set(module, span)
}) })
......
...@@ -27,6 +27,7 @@ import { difference } from '../../build/utils' ...@@ -27,6 +27,7 @@ import { difference } from '../../build/utils'
import { NextConfigComplete } from '../config-shared' import { NextConfigComplete } from '../config-shared'
import { CustomRoutes } from '../../lib/load-custom-routes' import { CustomRoutes } from '../../lib/load-custom-routes'
import { DecodeError } from '../../shared/lib/utils' import { DecodeError } from '../../shared/lib/utils'
import { Span, trace } from '../../telemetry/trace'
export async function renderScriptError( export async function renderScriptError(
res: ServerResponse, res: ServerResponse,
...@@ -143,6 +144,7 @@ export default class HotReloader { ...@@ -143,6 +144,7 @@ export default class HotReloader {
private watcher: any private watcher: any
private rewrites: CustomRoutes['rewrites'] private rewrites: CustomRoutes['rewrites']
private fallbackWatcher: any private fallbackWatcher: any
private hotReloaderSpan: Span
public isWebpack5: any public isWebpack5: any
constructor( constructor(
...@@ -174,6 +176,7 @@ export default class HotReloader { ...@@ -174,6 +176,7 @@ export default class HotReloader {
this.previewProps = previewProps this.previewProps = previewProps
this.rewrites = rewrites this.rewrites = rewrites
this.isWebpack5 = isWebpack5 this.isWebpack5 = isWebpack5
this.hotReloaderSpan = trace('hot-reloader')
} }
public async run( public async run(
...@@ -283,6 +286,7 @@ export default class HotReloader { ...@@ -283,6 +286,7 @@ export default class HotReloader {
pagesDir: this.pagesDir, pagesDir: this.pagesDir,
rewrites: this.rewrites, rewrites: this.rewrites,
entrypoints: entrypoints.client, entrypoints: entrypoints.client,
runWebpackSpan: this.hotReloaderSpan,
}), }),
getBaseWebpackConfig(this.dir, { getBaseWebpackConfig(this.dir, {
dev: true, dev: true,
...@@ -292,6 +296,7 @@ export default class HotReloader { ...@@ -292,6 +296,7 @@ export default class HotReloader {
pagesDir: this.pagesDir, pagesDir: this.pagesDir,
rewrites: this.rewrites, rewrites: this.rewrites,
entrypoints: entrypoints.server, entrypoints: entrypoints.server,
runWebpackSpan: this.hotReloaderSpan,
}), }),
]) ])
} }
...@@ -300,6 +305,7 @@ export default class HotReloader { ...@@ -300,6 +305,7 @@ export default class HotReloader {
if (this.fallbackWatcher) return if (this.fallbackWatcher) return
const fallbackConfig = await getBaseWebpackConfig(this.dir, { const fallbackConfig = await getBaseWebpackConfig(this.dir, {
runWebpackSpan: this.hotReloaderSpan,
dev: true, dev: true,
isServer: false, isServer: false,
config: this.config, config: this.config,
......
...@@ -3,6 +3,7 @@ import reportToConsole from './to-console' ...@@ -3,6 +3,7 @@ import reportToConsole from './to-console'
import reportToZipkin from './to-zipkin' import reportToZipkin from './to-zipkin'
import reportToJaeger from './to-jaeger' import reportToJaeger from './to-jaeger'
import reportToTelemetry from './to-telemetry' import reportToTelemetry from './to-telemetry'
import reportToJson from './to-json'
type Reporter = { type Reporter = {
flushAll: () => Promise<void> | void flushAll: () => Promise<void> | void
...@@ -16,6 +17,31 @@ type Reporter = { ...@@ -16,6 +17,31 @@ type Reporter = {
) => void ) => void
} }
class MultiReporter implements Reporter {
private reporters: Reporter[] = []
constructor(reporters: Reporter[]) {
this.reporters = reporters
}
async flushAll() {
await Promise.all(this.reporters.map((reporter) => reporter.flushAll()))
}
report(
spanName: string,
duration: number,
timestamp: number,
id: SpanId,
parentId?: SpanId,
attrs?: Object
) {
this.reporters.forEach((reporter) =>
reporter.report(spanName, duration, timestamp, id, parentId, attrs)
)
}
}
const target = const target =
process.env.TRACE_TARGET && process.env.TRACE_TARGET in TARGET process.env.TRACE_TARGET && process.env.TRACE_TARGET in TARGET
? TARGET[process.env.TRACE_TARGET as TARGET] ? TARGET[process.env.TRACE_TARGET as TARGET]
...@@ -27,14 +53,17 @@ if (process.env.TRACE_TARGET && !target) { ...@@ -27,14 +53,17 @@ if (process.env.TRACE_TARGET && !target) {
) )
} }
export let reporter: Reporter let traceTargetReporter: Reporter
if (target === TARGET.CONSOLE) { if (target === TARGET.CONSOLE) {
reporter = reportToConsole traceTargetReporter = reportToConsole
} else if (target === TARGET.ZIPKIN) { } else if (target === TARGET.ZIPKIN) {
reporter = reportToZipkin traceTargetReporter = reportToZipkin
} else if (target === TARGET.JAEGER) { } else if (target === TARGET.JAEGER) {
reporter = reportToJaeger traceTargetReporter = reportToJaeger
} else { } else {
reporter = reportToTelemetry traceTargetReporter = reportToTelemetry
} }
// JSON is always reported to allow for diagnostics
export const reporter = new MultiReporter([reportToJson, traceTargetReporter])
...@@ -5,7 +5,7 @@ import * as Log from '../../../build/output/log' ...@@ -5,7 +5,7 @@ import * as Log from '../../../build/output/log'
// Jaeger uses Zipkin's reporting // Jaeger uses Zipkin's reporting
import { batcher } from './to-zipkin' import { batcher } from './to-zipkin'
let traceId = process.env.TRACE_ID let traceId: string
let batch: ReturnType<typeof batcher> | undefined let batch: ReturnType<typeof batcher> | undefined
const localEndpoint = { const localEndpoint = {
...@@ -33,7 +33,7 @@ const reportToLocalHost = ( ...@@ -33,7 +33,7 @@ const reportToLocalHost = (
attrs?: Object attrs?: Object
) => { ) => {
if (!traceId) { if (!traceId) {
traceId = process.env.TRACE_ID = randomBytes(8).toString('hex') traceId = process.env.TRACE_ID || randomBytes(8).toString('hex')
logWebUrl() logWebUrl()
} }
......
import { randomBytes } from 'crypto'
import { batcher } from './to-zipkin'
import { traceGlobals } from '../shared'
import fs from 'fs'
import path from 'path'
let writeStream: fs.WriteStream
let traceId: string
let batch: ReturnType<typeof batcher> | undefined
const reportToLocalHost = (
name: string,
duration: number,
timestamp: number,
id: string,
parentId?: string,
attrs?: Object
) => {
const distDir = traceGlobals.get('distDir')
if (!distDir) {
return
}
if (!traceId) {
traceId = process.env.TRACE_ID || randomBytes(8).toString('hex')
}
if (!batch) {
batch = batcher(async (events) => {
if (!writeStream) {
const tracesDir = path.join(distDir, 'traces')
await fs.promises.mkdir(tracesDir, { recursive: true })
const file = path.join(distDir, 'trace')
writeStream = fs.createWriteStream(file, {
flags: 'a',
encoding: 'utf8',
})
}
const eventsJson = JSON.stringify(events)
try {
await new Promise<void>((resolve, reject) => {
writeStream.write(eventsJson + '\n', 'utf8', (err) => {
err ? reject(err) : resolve()
})
})
} catch (err) {
console.log(err)
}
})
}
batch.report({
traceId,
parentId,
name,
id,
timestamp,
duration,
tags: attrs,
})
}
export default {
flushAll: () =>
batch
? batch.flushAll().then(() => {
writeStream.end('', 'utf8')
})
: undefined,
report: reportToLocalHost,
}
...@@ -3,7 +3,7 @@ import { randomBytes } from 'crypto' ...@@ -3,7 +3,7 @@ import { randomBytes } from 'crypto'
import fetch from 'node-fetch' import fetch from 'node-fetch'
import * as Log from '../../../build/output/log' import * as Log from '../../../build/output/log'
let traceId = process.env.TRACE_ID let traceId: string
let batch: ReturnType<typeof batcher> | undefined let batch: ReturnType<typeof batcher> | undefined
const localEndpoint = { const localEndpoint = {
...@@ -21,7 +21,7 @@ type Event = { ...@@ -21,7 +21,7 @@ type Event = {
id: string id: string
timestamp: number timestamp: number
duration: number duration: number
localEndpoint: typeof localEndpoint localEndpoint?: typeof localEndpoint
tags?: Object tags?: Object
} }
...@@ -42,8 +42,9 @@ export function batcher(reportEvents: (evts: Event[]) => Promise<void>) { ...@@ -42,8 +42,9 @@ export function batcher(reportEvents: (evts: Event[]) => Promise<void>) {
events.push(event) events.push(event)
if (events.length > 100) { if (events.length > 100) {
const report = reportEvents(events.slice()) const evts = events.slice()
events.length = 0 events.length = 0
const report = reportEvents(evts)
queue.add(report) queue.add(report)
report.then(() => queue.delete(report)) report.then(() => queue.delete(report))
} }
...@@ -60,7 +61,7 @@ const reportToLocalHost = ( ...@@ -60,7 +61,7 @@ const reportToLocalHost = (
attrs?: Object attrs?: Object
) => { ) => {
if (!traceId) { if (!traceId) {
traceId = process.env.TRACE_ID = randomBytes(8).toString('hex') traceId = process.env.TRACE_ID || randomBytes(8).toString('hex')
Log.info( Log.info(
`Zipkin trace will be available on ${zipkinUrl}/zipkin/traces/${traceId}` `Zipkin trace will be available on ${zipkinUrl}/zipkin/traces/${traceId}`
) )
......
import fs from 'fs'
import eventStream from 'event-stream'
import retry from 'async-retry'
import fetch from 'node-fetch'
const file = fs.createReadStream(process.argv[2])
const localEndpoint = {
serviceName: 'nextjs',
ipv4: '127.0.0.1',
port: 9411,
}
// Jaeger supports Zipkin's reporting API
const zipkinUrl = `http://${localEndpoint.ipv4}:${localEndpoint.port}`
const jaegerWebUiUrl = `http://${localEndpoint.ipv4}:16686`
const zipkinAPI = `${zipkinUrl}/api/v2/spans`
let loggedUrl = false
function logWebUrl(traceId) {
console.log(
`Jaeger trace will be available on ${jaegerWebUiUrl}/trace/${traceId}`
)
}
file.pipe(eventStream.split()).pipe(
eventStream.map((data, cb) => {
if (data === '') {
return cb(null, '')
}
const eventsJson = JSON.parse(data).map((item) => {
item.localEndpoint = localEndpoint
return item
})
if (!loggedUrl) {
logWebUrl(eventsJson[0].traceId)
loggedUrl = true
}
retry(
() =>
// Send events to zipkin
fetch(zipkinAPI, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(eventsJson),
}),
{ minTimeout: 500, retries: 3, factor: 1 }
)
.then(async (res) => {
if (res.status !== 202) {
console.log({
status: res.status,
body: await res.text(),
events: eventsJson,
})
}
cb(null, '')
})
.catch((err) => {
console.log(err)
cb(null, '')
})
})
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment