Unverified Commit 9316dc0a authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add default trace format that is exported automatically (#28461)

Co-authored-by: 's avatarSteven <steven@ceriously.com>
parent 706547e0
......@@ -40,7 +40,7 @@ const nextBabelLoaderOuter = function nextBabelLoaderOuter(
) {
const callback = this.async()
const loaderSpan = trace('next-babel-turbo-loader', this.currentTraceSpan?.id)
const loaderSpan = this.currentTraceSpan.traceChild('next-babel-turbo-loader')
loaderSpan
.traceAsyncFn(() =>
nextBabelLoader.call(this, loaderSpan, inputSource, inputSourceMap)
......
......@@ -2,7 +2,7 @@ import { loader } from 'next/dist/compiled/webpack/webpack'
import { Span } from '../../../telemetry/trace'
export interface NextJsLoaderContext extends loader.LoaderContext {
currentTraceSpan?: Span
currentTraceSpan: Span
}
export interface NextBabelLoaderOptions {
......
......@@ -130,11 +130,13 @@ export default async function build(
const config: NextConfigComplete = await nextBuildSpan
.traceChild('load-next-config')
.traceAsyncFn(() => loadConfig(PHASE_PRODUCTION_BUILD, dir, conf))
const distDir = path.join(dir, config.distDir)
setGlobal('distDir', distDir)
const { target } = config
const buildId: string = await nextBuildSpan
.traceChild('generate-buildid')
.traceAsyncFn(() => generateBuildId(config.generateBuildId, nanoid))
const distDir = path.join(dir, config.distDir)
const customRoutes: CustomRoutes = await nextBuildSpan
.traceChild('load-custom-routes')
......
......@@ -240,7 +240,7 @@ export default async function getBaseWebpackConfig(
entrypoints: WebpackEntrypoints
rewrites: CustomRoutes['rewrites']
isDevFallback?: boolean
runWebpackSpan?: Span
runWebpackSpan: Span
}
): Promise<webpack.Configuration> {
const hasRewrites =
......
......@@ -11,7 +11,6 @@ import {
ROUTES_MANIFEST,
REACT_LOADABLE_MANIFEST,
} from '../../../../shared/lib/constants'
import { trace } from '../../../../telemetry/trace'
export type ServerlessLoaderQuery = {
page: string
......@@ -34,63 +33,61 @@ export type ServerlessLoaderQuery = {
}
const nextServerlessLoader: webpack.loader.Loader = function () {
const loaderSpan = trace('next-serverless-loader')
return loaderSpan.traceFn(() => {
const {
distDir,
absolutePagePath,
page,
buildId,
canonicalBase,
assetPrefix,
absoluteAppPath,
absoluteDocumentPath,
absoluteErrorPath,
absolute404Path,
generateEtags,
poweredByHeader,
basePath,
runtimeConfig,
previewProps,
loadedEnvFiles,
i18n,
}: ServerlessLoaderQuery =
typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query
const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/')
const reactLoadableManifest = join(
distDir,
REACT_LOADABLE_MANIFEST
).replace(/\\/g, '/')
const routesManifest = join(distDir, ROUTES_MANIFEST).replace(/\\/g, '/')
const escapedBuildId = escapeRegexp(buildId)
const pageIsDynamicRoute = isDynamicRoute(page)
const encodedPreviewProps = devalue(
JSON.parse(previewProps) as __ApiPreviewProps
)
const envLoading = `
const {
distDir,
absolutePagePath,
page,
buildId,
canonicalBase,
assetPrefix,
absoluteAppPath,
absoluteDocumentPath,
absoluteErrorPath,
absolute404Path,
generateEtags,
poweredByHeader,
basePath,
runtimeConfig,
previewProps,
loadedEnvFiles,
i18n,
}: ServerlessLoaderQuery =
typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query
const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/')
const reactLoadableManifest = join(distDir, REACT_LOADABLE_MANIFEST).replace(
/\\/g,
'/'
)
const routesManifest = join(distDir, ROUTES_MANIFEST).replace(/\\/g, '/')
const escapedBuildId = escapeRegexp(buildId)
const pageIsDynamicRoute = isDynamicRoute(page)
const encodedPreviewProps = devalue(
JSON.parse(previewProps) as __ApiPreviewProps
)
const envLoading = `
const { processEnv } = require('@next/env')
processEnv(${Buffer.from(loadedEnvFiles, 'base64').toString()})
`
const runtimeConfigImports = runtimeConfig
? `
const runtimeConfigImports = runtimeConfig
? `
const { setConfig } = require('next/config')
`
: ''
: ''
const runtimeConfigSetter = runtimeConfig
? `
const runtimeConfigSetter = runtimeConfig
? `
const runtimeConfig = ${runtimeConfig}
setConfig(runtimeConfig)
`
: 'const runtimeConfig = {}'
: 'const runtimeConfig = {}'
if (page.match(API_ROUTE)) {
return `
if (page.match(API_ROUTE)) {
return `
${envLoading}
${runtimeConfigImports}
${
......@@ -125,8 +122,8 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
})
export default apiHandler
`
} else {
return `
} else {
return `
import 'next/dist/server/node-polyfill-fetch'
import routesManifest from '${routesManifest}'
import buildManifest from '${buildManifest}'
......@@ -206,8 +203,7 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
})
export { renderReqToHTML, render }
`
}
})
}
}
export default nextServerlessLoader
......@@ -123,7 +123,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) {
}
export default function swcLoader(inputSource, inputSourceMap) {
const loaderSpan = trace('next-swc-loader', this.currentTraceSpan?.id)
const loaderSpan = this.currentTraceSpan.traceChild('next-swc-loader')
const callback = this.async()
loaderSpan
.traceAsyncFn(() =>
......
......@@ -118,7 +118,7 @@ export default class BuildStatsPlugin {
async (stats, callback) => {
const compilerSpan = spans.get(compiler)
try {
const writeStatsSpan = trace('NextJsBuildStats', compilerSpan?.id)
const writeStatsSpan = compilerSpan!.traceChild('NextJsBuildStats')
await writeStatsSpan.traceAsyncFn(() => {
return new Promise((resolve, reject) => {
const statsJson = reduceSize(
......
......@@ -71,9 +71,8 @@ export class CssMinimizerPlugin {
},
async (assets: any) => {
const compilerSpan = spans.get(compiler)
const cssMinimizerSpan = trace(
'css-minimizer-plugin',
compilerSpan?.id
const cssMinimizerSpan = compilerSpan!.traceChild(
'css-minimizer-plugin'
)
cssMinimizerSpan.setAttribute('webpackVersion', 5)
......@@ -83,7 +82,7 @@ export class CssMinimizerPlugin {
files
.filter((file) => CSS_REGEX.test(file))
.map(async (file) => {
const assetSpan = trace('minify-css', cssMinimizerSpan.id)
const assetSpan = cssMinimizerSpan.traceChild('minify-css')
assetSpan.setAttribute('file', file)
return assetSpan.traceAsyncFn(async () => {
......
......@@ -15,9 +15,9 @@ function getNormalModuleLoaderHook(compilation: any) {
export class ProfilingPlugin {
compiler: any
runWebpackSpan: Span | undefined
runWebpackSpan: Span
constructor({ runWebpackSpan }: { runWebpackSpan: Span | undefined }) {
constructor({ runWebpackSpan }: { runWebpackSpan: Span }) {
this.runWebpackSpan = runWebpackSpan
}
apply(compiler: any) {
......@@ -35,14 +35,17 @@ export class ProfilingPlugin {
attrs,
onSetSpan,
}: {
parentSpan?: () => Span | undefined
parentSpan?: () => Span
attrs?: any
onSetSpan?: (span: Span) => void
} = {}
) {
let span: Span | undefined
startHook.tap(pluginName, () => {
span = trace(spanName, parentSpan?.()?.id, attrs ? attrs() : attrs)
span = parentSpan
? parentSpan().traceChild(spanName, attrs ? attrs() : attrs)
: trace(spanName, undefined, attrs ? attrs() : attrs)
onSetSpan?.(span)
})
stopHook.tap(pluginName, () => {
......@@ -103,10 +106,16 @@ export class ProfilingPlugin {
const issuerModule = compilation?.moduleGraph?.getIssuer(module)
const span = trace(
`build-module${moduleType ? `-${moduleType}` : ''}`,
issuerModule ? spans.get(issuerModule)?.id : compilerSpan.id
)
let span: Span
const spanName = `build-module${moduleType ? `-${moduleType}` : ''}`
const issuerSpan: Span | undefined =
issuerModule && spans.get(issuerModule)
if (issuerSpan) {
span = issuerSpan.traceChild(spanName)
} else {
span = compilerSpan.traceChild(spanName)
}
span.setAttribute('name', module.userRequest)
spans.set(module, span)
})
......
......@@ -27,6 +27,7 @@ import { difference } from '../../build/utils'
import { NextConfigComplete } from '../config-shared'
import { CustomRoutes } from '../../lib/load-custom-routes'
import { DecodeError } from '../../shared/lib/utils'
import { Span, trace } from '../../telemetry/trace'
export async function renderScriptError(
res: ServerResponse,
......@@ -143,6 +144,7 @@ export default class HotReloader {
private watcher: any
private rewrites: CustomRoutes['rewrites']
private fallbackWatcher: any
private hotReloaderSpan: Span
public isWebpack5: any
constructor(
......@@ -174,6 +176,7 @@ export default class HotReloader {
this.previewProps = previewProps
this.rewrites = rewrites
this.isWebpack5 = isWebpack5
this.hotReloaderSpan = trace('hot-reloader')
}
public async run(
......@@ -283,6 +286,7 @@ export default class HotReloader {
pagesDir: this.pagesDir,
rewrites: this.rewrites,
entrypoints: entrypoints.client,
runWebpackSpan: this.hotReloaderSpan,
}),
getBaseWebpackConfig(this.dir, {
dev: true,
......@@ -292,6 +296,7 @@ export default class HotReloader {
pagesDir: this.pagesDir,
rewrites: this.rewrites,
entrypoints: entrypoints.server,
runWebpackSpan: this.hotReloaderSpan,
}),
])
}
......@@ -300,6 +305,7 @@ export default class HotReloader {
if (this.fallbackWatcher) return
const fallbackConfig = await getBaseWebpackConfig(this.dir, {
runWebpackSpan: this.hotReloaderSpan,
dev: true,
isServer: false,
config: this.config,
......
......@@ -3,6 +3,7 @@ import reportToConsole from './to-console'
import reportToZipkin from './to-zipkin'
import reportToJaeger from './to-jaeger'
import reportToTelemetry from './to-telemetry'
import reportToJson from './to-json'
type Reporter = {
flushAll: () => Promise<void> | void
......@@ -16,6 +17,31 @@ type Reporter = {
) => void
}
class MultiReporter implements Reporter {
private reporters: Reporter[] = []
constructor(reporters: Reporter[]) {
this.reporters = reporters
}
async flushAll() {
await Promise.all(this.reporters.map((reporter) => reporter.flushAll()))
}
report(
spanName: string,
duration: number,
timestamp: number,
id: SpanId,
parentId?: SpanId,
attrs?: Object
) {
this.reporters.forEach((reporter) =>
reporter.report(spanName, duration, timestamp, id, parentId, attrs)
)
}
}
const target =
process.env.TRACE_TARGET && process.env.TRACE_TARGET in TARGET
? TARGET[process.env.TRACE_TARGET as TARGET]
......@@ -27,14 +53,17 @@ if (process.env.TRACE_TARGET && !target) {
)
}
export let reporter: Reporter
let traceTargetReporter: Reporter
if (target === TARGET.CONSOLE) {
reporter = reportToConsole
traceTargetReporter = reportToConsole
} else if (target === TARGET.ZIPKIN) {
reporter = reportToZipkin
traceTargetReporter = reportToZipkin
} else if (target === TARGET.JAEGER) {
reporter = reportToJaeger
traceTargetReporter = reportToJaeger
} else {
reporter = reportToTelemetry
traceTargetReporter = reportToTelemetry
}
// JSON is always reported to allow for diagnostics
export const reporter = new MultiReporter([reportToJson, traceTargetReporter])
......@@ -5,7 +5,7 @@ import * as Log from '../../../build/output/log'
// Jaeger uses Zipkin's reporting
import { batcher } from './to-zipkin'
let traceId = process.env.TRACE_ID
let traceId: string
let batch: ReturnType<typeof batcher> | undefined
const localEndpoint = {
......@@ -33,7 +33,7 @@ const reportToLocalHost = (
attrs?: Object
) => {
if (!traceId) {
traceId = process.env.TRACE_ID = randomBytes(8).toString('hex')
traceId = process.env.TRACE_ID || randomBytes(8).toString('hex')
logWebUrl()
}
......
import { randomBytes } from 'crypto'
import { batcher } from './to-zipkin'
import { traceGlobals } from '../shared'
import fs from 'fs'
import path from 'path'
let writeStream: fs.WriteStream
let traceId: string
let batch: ReturnType<typeof batcher> | undefined
const reportToLocalHost = (
name: string,
duration: number,
timestamp: number,
id: string,
parentId?: string,
attrs?: Object
) => {
const distDir = traceGlobals.get('distDir')
if (!distDir) {
return
}
if (!traceId) {
traceId = process.env.TRACE_ID || randomBytes(8).toString('hex')
}
if (!batch) {
batch = batcher(async (events) => {
if (!writeStream) {
const tracesDir = path.join(distDir, 'traces')
await fs.promises.mkdir(tracesDir, { recursive: true })
const file = path.join(distDir, 'trace')
writeStream = fs.createWriteStream(file, {
flags: 'a',
encoding: 'utf8',
})
}
const eventsJson = JSON.stringify(events)
try {
await new Promise<void>((resolve, reject) => {
writeStream.write(eventsJson + '\n', 'utf8', (err) => {
err ? reject(err) : resolve()
})
})
} catch (err) {
console.log(err)
}
})
}
batch.report({
traceId,
parentId,
name,
id,
timestamp,
duration,
tags: attrs,
})
}
export default {
flushAll: () =>
batch
? batch.flushAll().then(() => {
writeStream.end('', 'utf8')
})
: undefined,
report: reportToLocalHost,
}
......@@ -3,7 +3,7 @@ import { randomBytes } from 'crypto'
import fetch from 'node-fetch'
import * as Log from '../../../build/output/log'
let traceId = process.env.TRACE_ID
let traceId: string
let batch: ReturnType<typeof batcher> | undefined
const localEndpoint = {
......@@ -21,7 +21,7 @@ type Event = {
id: string
timestamp: number
duration: number
localEndpoint: typeof localEndpoint
localEndpoint?: typeof localEndpoint
tags?: Object
}
......@@ -42,8 +42,9 @@ export function batcher(reportEvents: (evts: Event[]) => Promise<void>) {
events.push(event)
if (events.length > 100) {
const report = reportEvents(events.slice())
const evts = events.slice()
events.length = 0
const report = reportEvents(evts)
queue.add(report)
report.then(() => queue.delete(report))
}
......@@ -60,7 +61,7 @@ const reportToLocalHost = (
attrs?: Object
) => {
if (!traceId) {
traceId = process.env.TRACE_ID = randomBytes(8).toString('hex')
traceId = process.env.TRACE_ID || randomBytes(8).toString('hex')
Log.info(
`Zipkin trace will be available on ${zipkinUrl}/zipkin/traces/${traceId}`
)
......
import fs from 'fs'
import eventStream from 'event-stream'
import retry from 'async-retry'
import fetch from 'node-fetch'
const file = fs.createReadStream(process.argv[2])
const localEndpoint = {
serviceName: 'nextjs',
ipv4: '127.0.0.1',
port: 9411,
}
// Jaeger supports Zipkin's reporting API
const zipkinUrl = `http://${localEndpoint.ipv4}:${localEndpoint.port}`
const jaegerWebUiUrl = `http://${localEndpoint.ipv4}:16686`
const zipkinAPI = `${zipkinUrl}/api/v2/spans`
let loggedUrl = false
function logWebUrl(traceId) {
console.log(
`Jaeger trace will be available on ${jaegerWebUiUrl}/trace/${traceId}`
)
}
file.pipe(eventStream.split()).pipe(
eventStream.map((data, cb) => {
if (data === '') {
return cb(null, '')
}
const eventsJson = JSON.parse(data).map((item) => {
item.localEndpoint = localEndpoint
return item
})
if (!loggedUrl) {
logWebUrl(eventsJson[0].traceId)
loggedUrl = true
}
retry(
() =>
// Send events to zipkin
fetch(zipkinAPI, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(eventsJson),
}),
{ minTimeout: 500, retries: 3, factor: 1 }
)
.then(async (res) => {
if (res.status !== 202) {
console.log({
status: res.status,
body: await res.text(),
events: eventsJson,
})
}
cb(null, '')
})
.catch((err) => {
console.log(err)
cb(null, '')
})
})
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment