Unverified Commit bd6a431b authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add updated flush handling on sigterm (#44614)

parent 02f97ab0
...@@ -60,9 +60,10 @@ const handleSessionStop = async () => { ...@@ -60,9 +60,10 @@ const handleSessionStop = async () => {
durationMilliseconds: Date.now() - sessionStarted, durationMilliseconds: Date.now() - sessionStarted,
pagesDir, pagesDir,
appDir, appDir,
}) }),
true
) )
await telemetry.flush() telemetry.flushDetached('dev', dir)
} catch (err) { } catch (err) {
console.error(err) console.error(err)
} }
......
import fs from 'fs'
import path from 'path'
import { Telemetry, TelemetryEvent } from './storage'
import loadConfig from '../server/config'
import { getProjectDir } from '../lib/get-project-dir'
import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
// this process should be started with following arg order
// 1. mode e.g. dev, export, start
// 2. project dir
;(async () => {
const args = [...process.argv]
let dir = args.pop()
const mode = args.pop()
if (!dir || mode !== 'dev') {
throw new Error(
`Invalid flags should be run as node detached-flush dev ./path-to/project`
)
}
dir = getProjectDir(dir)
const config = await loadConfig(PHASE_DEVELOPMENT_SERVER, dir)
const distDir = path.join(dir, config.distDir || '.next')
const eventsPath = path.join(distDir, '_events.json')
let events: TelemetryEvent[]
try {
events = JSON.parse(fs.readFileSync(eventsPath, 'utf8'))
} catch (err: any) {
if (err.code === 'ENOENT') {
// no events to process we can exit now
process.exit(0)
}
throw err
}
const telemetry = new Telemetry({ distDir })
await telemetry.record(events)
await telemetry.flush()
// finished flushing events clean-up/exit
fs.unlinkSync(eventsPath)
process.exit(0)
})()
import retry from 'next/dist/compiled/async-retry' import retry from 'next/dist/compiled/async-retry'
import fetch from 'next/dist/compiled/node-fetch' import fetch from 'next/dist/compiled/node-fetch'
export function _postPayload(endpoint: string, body: object) { export function _postPayload(endpoint: string, body: object, signal?: any) {
return ( return (
retry( retry(
() => () =>
...@@ -10,6 +10,7 @@ export function _postPayload(endpoint: string, body: object) { ...@@ -10,6 +10,7 @@ export function _postPayload(endpoint: string, body: object) {
body: JSON.stringify(body), body: JSON.stringify(body),
headers: { 'content-type': 'application/json' }, headers: { 'content-type': 'application/json' },
timeout: 5000, timeout: 5000,
signal,
}).then((res) => { }).then((res) => {
if (!res.ok) { if (!res.ok) {
const err = new Error(res.statusText) const err = new Error(res.statusText)
......
...@@ -8,6 +8,9 @@ import { getAnonymousMeta } from './anonymous-meta' ...@@ -8,6 +8,9 @@ import { getAnonymousMeta } from './anonymous-meta'
import * as ciEnvironment from './ci-info' import * as ciEnvironment from './ci-info'
import { _postPayload } from './post-payload' import { _postPayload } from './post-payload'
import { getRawProjectId } from './project-id' import { getRawProjectId } from './project-id'
import { AbortController } from 'next/dist/compiled/@edge-runtime/primitives/abort-controller'
import fs from 'fs'
import spawn from 'next/dist/compiled/cross-spawn'
// This is the key that stores whether or not telemetry is enabled or disabled. // This is the key that stores whether or not telemetry is enabled or disabled.
const TELEMETRY_KEY_ENABLED = 'telemetry.enabled' const TELEMETRY_KEY_ENABLED = 'telemetry.enabled'
...@@ -26,7 +29,7 @@ const TELEMETRY_KEY_ID = `telemetry.anonymousId` ...@@ -26,7 +29,7 @@ const TELEMETRY_KEY_ID = `telemetry.anonymousId`
// See the `oneWayHash` function. // See the `oneWayHash` function.
const TELEMETRY_KEY_SALT = `telemetry.salt` const TELEMETRY_KEY_SALT = `telemetry.salt`
type TelemetryEvent = { eventName: string; payload: object } export type TelemetryEvent = { eventName: string; payload: object }
type EventContext = { type EventContext = {
anonymousId: string anonymousId: string
projectId: string projectId: string
...@@ -57,6 +60,7 @@ function getStorageDirectory(distDir: string): string | undefined { ...@@ -57,6 +60,7 @@ function getStorageDirectory(distDir: string): string | undefined {
export class Telemetry { export class Telemetry {
private conf: Conf<any> | null private conf: Conf<any> | null
private distDir: string
private sessionId: string private sessionId: string
private rawProjectId: string private rawProjectId: string
private NEXT_TELEMETRY_DISABLED: any private NEXT_TELEMETRY_DISABLED: any
...@@ -69,6 +73,7 @@ export class Telemetry { ...@@ -69,6 +73,7 @@ export class Telemetry {
const { NEXT_TELEMETRY_DISABLED, NEXT_TELEMETRY_DEBUG } = process.env const { NEXT_TELEMETRY_DISABLED, NEXT_TELEMETRY_DEBUG } = process.env
this.NEXT_TELEMETRY_DISABLED = NEXT_TELEMETRY_DISABLED this.NEXT_TELEMETRY_DISABLED = NEXT_TELEMETRY_DISABLED
this.NEXT_TELEMETRY_DEBUG = NEXT_TELEMETRY_DEBUG this.NEXT_TELEMETRY_DEBUG = NEXT_TELEMETRY_DEBUG
this.distDir = distDir
const storageDirectory = getStorageDirectory(distDir) const storageDirectory = getStorageDirectory(distDir)
try { try {
...@@ -177,15 +182,23 @@ export class Telemetry { ...@@ -177,15 +182,23 @@ export class Telemetry {
} }
record = ( record = (
_events: TelemetryEvent | TelemetryEvent[] _events: TelemetryEvent | TelemetryEvent[],
deferred?: boolean
): Promise<RecordObject> => { ): Promise<RecordObject> => {
const _this = this const prom = (
// pseudo try-catch deferred
async function wrapper() { ? // if we know we are going to immediately call
return await _this.submitRecord(_events) // flushDetached we can skip starting the initial
} // submitRecord which will then be cancelled
new Promise((resolve) =>
const prom = wrapper() resolve({
isFulfilled: true,
isRejected: false,
value: _events,
})
)
: this.submitRecord(_events)
)
.then((value) => ({ .then((value) => ({
isFulfilled: true, isFulfilled: true,
isRejected: false, isRejected: false,
...@@ -203,6 +216,8 @@ export class Telemetry { ...@@ -203,6 +216,8 @@ export class Telemetry {
return res return res
}) })
;(prom as any)._events = Array.isArray(_events) ? _events : [_events]
;(prom as any)._controller = (prom as any)._controller
// Track this `Promise` so we can flush pending events // Track this `Promise` so we can flush pending events
this.queue.add(prom) this.queue.add(prom)
...@@ -211,6 +226,35 @@ export class Telemetry { ...@@ -211,6 +226,35 @@ export class Telemetry {
flush = async () => Promise.all(this.queue).catch(() => null) flush = async () => Promise.all(this.queue).catch(() => null)
// writes current events to disk and spawns separate
// detached process to submit the records without blocking
// the main process from exiting
flushDetached = (mode: 'dev', dir: string) => {
const allEvents: TelemetryEvent[] = []
this.queue.forEach((item: any) => {
try {
item._controller?.abort()
allEvents.push(...item._events)
} catch (_) {
// if we fail to abort ignore this event
}
})
fs.writeFileSync(
path.join(this.distDir, '_events.json'),
JSON.stringify(allEvents)
)
spawn('node', [require.resolve('./deteched-flush'), mode, dir], {
detached: !this.NEXT_TELEMETRY_DEBUG,
...(this.NEXT_TELEMETRY_DEBUG
? {
stdio: 'inherit',
}
: {}),
})
}
private submitRecord = ( private submitRecord = (
_events: TelemetryEvent | TelemetryEvent[] _events: TelemetryEvent | TelemetryEvent[]
): Promise<any> => { ): Promise<any> => {
...@@ -248,13 +292,20 @@ export class Telemetry { ...@@ -248,13 +292,20 @@ export class Telemetry {
sessionId: this.sessionId, sessionId: this.sessionId,
} }
const meta: EventMeta = getAnonymousMeta() const meta: EventMeta = getAnonymousMeta()
return _postPayload(`https://telemetry.nextjs.org/api/v1/record`, { const postController = new AbortController()
context, const res = _postPayload(
meta, `https://telemetry.nextjs.org/api/v1/record`,
events: events.map(({ eventName, payload }) => ({ {
eventName, context,
fields: payload, meta,
})) as Array<EventBatchShape>, events: events.map(({ eventName, payload }) => ({
}) eventName,
fields: payload,
})) as Array<EventBatchShape>,
},
postController.signal
)
res._controller = postController
return res
} }
} }
...@@ -480,6 +480,10 @@ describe('Telemetry CLI', () => { ...@@ -480,6 +480,10 @@ describe('Telemetry CLI', () => {
expect(event1).toMatch(/"pagesDir": true/) expect(event1).toMatch(/"pagesDir": true/)
expect(event1).toMatch(/"turboFlag": true/) expect(event1).toMatch(/"turboFlag": true/)
expect(await fs.pathExists(path.join(appDir, '.next/_events.json'))).toBe(
false
)
} finally { } finally {
await teardown() await teardown()
} }
...@@ -516,6 +520,10 @@ describe('Telemetry CLI', () => { ...@@ -516,6 +520,10 @@ describe('Telemetry CLI', () => {
expect(event1).toMatch(/"turboFlag": false/) expect(event1).toMatch(/"turboFlag": false/)
expect(event1).toMatch(/"pagesDir": true/) expect(event1).toMatch(/"pagesDir": true/)
expect(event1).toMatch(/"appDir": true/) expect(event1).toMatch(/"appDir": true/)
expect(await fs.pathExists(path.join(appDir, '.next/_events.json'))).toBe(
false
)
} finally { } finally {
await teardown() await teardown()
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment