Unverified Commit 774b78a0 authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Stats Action: Add ready in time (#88283)

## What?

Measures the time to `Ready in X ms` in the PR stats action so that
improvements/regressions on it are highlighted on each PR.

---------
Co-authored-by: 's avatarVercel <vercel[bot]@users.noreply.github.com>
parent 25046dfa
...@@ -69,6 +69,11 @@ const METRIC_GROUPS = { ...@@ -69,6 +69,11 @@ const METRIC_GROUPS = {
key: 'nextDevColdListenDurationTurbo', key: 'nextDevColdListenDurationTurbo',
description: 'Time until TCP port accepts connections', description: 'Time until TCP port accepts connections',
}, },
{
label: 'Cold (Ready in log)',
key: 'nextDevColdReadyInDurationTurbo',
description: 'Time until "Ready in X" log message is printed',
},
{ {
label: 'Cold (First Request)', label: 'Cold (First Request)',
key: 'nextDevColdReadyDurationTurbo', key: 'nextDevColdReadyDurationTurbo',
...@@ -79,6 +84,11 @@ const METRIC_GROUPS = { ...@@ -79,6 +84,11 @@ const METRIC_GROUPS = {
key: 'nextDevWarmListenDurationTurbo', key: 'nextDevWarmListenDurationTurbo',
description: 'Time until TCP port accepts connections (cached)', description: 'Time until TCP port accepts connections (cached)',
}, },
{
label: 'Warm (Ready in log)',
key: 'nextDevWarmReadyInDurationTurbo',
description: 'Time until "Ready in X" log message is printed (cached)',
},
{ {
label: 'Warm (First Request)', label: 'Warm (First Request)',
key: 'nextDevWarmReadyDurationTurbo', key: 'nextDevWarmReadyDurationTurbo',
...@@ -98,6 +108,11 @@ const METRIC_GROUPS = { ...@@ -98,6 +108,11 @@ const METRIC_GROUPS = {
key: 'nextDevColdListenDurationWebpack', key: 'nextDevColdListenDurationWebpack',
description: 'Time until TCP port accepts connections', description: 'Time until TCP port accepts connections',
}, },
{
label: 'Cold (Ready in log)',
key: 'nextDevColdReadyInDurationWebpack',
description: 'Time until "Ready in X" log message is printed',
},
{ {
label: 'Cold (First Request)', label: 'Cold (First Request)',
key: 'nextDevColdReadyDurationWebpack', key: 'nextDevColdReadyDurationWebpack',
...@@ -108,6 +123,11 @@ const METRIC_GROUPS = { ...@@ -108,6 +123,11 @@ const METRIC_GROUPS = {
key: 'nextDevWarmListenDurationWebpack', key: 'nextDevWarmListenDurationWebpack',
description: 'Time until TCP port accepts connections (cached)', description: 'Time until TCP port accepts connections (cached)',
}, },
{
label: 'Warm (Ready in log)',
key: 'nextDevWarmReadyInDurationWebpack',
description: 'Time until "Ready in X" log message is printed (cached)',
},
{ {
label: 'Warm (First Request)', label: 'Warm (First Request)',
key: 'nextDevWarmReadyDurationWebpack', key: 'nextDevWarmReadyDurationWebpack',
......
...@@ -15,7 +15,9 @@ const { statsAppDir, diffingDir, benchTitle } = require('../constants') ...@@ -15,7 +15,9 @@ const { statsAppDir, diffingDir, benchTitle } = require('../constants')
const { calcStats } = require('../util/stats') const { calcStats } = require('../util/stats')
// Number of iterations for timing benchmarks to get stable median // Number of iterations for timing benchmarks to get stable median
const BENCHMARK_ITERATIONS = 9 const BENCHMARK_ITERATIONS = process.env.BENCHMARK_ITERATIONS
? parseInt(process.env.BENCHMARK_ITERATIONS)
: 9
// Check if a port is accepting TCP connections // Check if a port is accepting TCP connections
function checkPort(port, timeout = 100) { function checkPort(port, timeout = 100) {
...@@ -89,9 +91,19 @@ async function benchmarkDevBoot(appDevCommand, curDir, port, cleanBuild) { ...@@ -89,9 +91,19 @@ async function benchmarkDevBoot(appDevCommand, curDir, port, cleanBuild) {
exited = true exited = true
}) })
let readyInTime = null
let stdoutBuffer = ''
// Capture output for debugging // Capture output for debugging
devChild.stdout.on('data', (data) => process.stdout.write(data)) devChild.stdout.on('data', (data) => {
devChild.stderr.on('data', (data) => process.stderr.write(data)) stdoutBuffer += data.toString()
if (readyInTime === null && stdoutBuffer.includes('Ready in')) {
readyInTime = Date.now() - startTime
}
process.stdout.write(data)
})
devChild.stderr.on('data', (data) => {
process.stderr.write(data)
})
// Measure time to port listening (TCP level) // Measure time to port listening (TCP level)
const listenTime = await waitForPort(port, 60000) const listenTime = await waitForPort(port, 60000)
...@@ -115,6 +127,7 @@ async function benchmarkDevBoot(appDevCommand, curDir, port, cleanBuild) { ...@@ -115,6 +127,7 @@ async function benchmarkDevBoot(appDevCommand, curDir, port, cleanBuild) {
return { return {
listenTime, listenTime,
readyInTime,
readyTime, readyTime,
} }
} }
...@@ -128,6 +141,7 @@ async function benchmarkDevBootWithIterations( ...@@ -128,6 +141,7 @@ async function benchmarkDevBootWithIterations(
label label
) { ) {
const listenTimes = [] const listenTimes = []
const readyInTimes = []
const readyTimes = [] const readyTimes = []
for (let i = 0; i < BENCHMARK_ITERATIONS; i++) { for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
...@@ -145,6 +159,12 @@ async function benchmarkDevBootWithIterations( ...@@ -145,6 +159,12 @@ async function benchmarkDevBootWithIterations(
listenTimes.push(result.listenTime) listenTimes.push(result.listenTime)
logger(` Boot: ${result.listenTime}ms`) logger(` Boot: ${result.listenTime}ms`)
} }
if (result.readyInTime !== null) {
readyInTimes.push(result.readyInTime)
logger(` Ready in: ${result.readyInTime}ms`)
}
if (result.readyTime !== null) { if (result.readyTime !== null) {
readyTimes.push(result.readyTime) readyTimes.push(result.readyTime)
logger(` Ready: ${result.readyTime}ms`) logger(` Ready: ${result.readyTime}ms`)
...@@ -155,6 +175,7 @@ async function benchmarkDevBootWithIterations( ...@@ -155,6 +175,7 @@ async function benchmarkDevBootWithIterations(
} }
const listenStats = calcStats(listenTimes) const listenStats = calcStats(listenTimes)
const readyInStats = calcStats(readyInTimes)
const readyStats = calcStats(readyTimes) const readyStats = calcStats(readyTimes)
// Log detailed stats for debugging // Log detailed stats for debugging
...@@ -163,6 +184,11 @@ async function benchmarkDevBootWithIterations( ...@@ -163,6 +184,11 @@ async function benchmarkDevBootWithIterations(
` ${label} Boot: median=${listenStats.median}ms, range=${listenStats.min}-${listenStats.max}ms, CV=${listenStats.cv}%` ` ${label} Boot: median=${listenStats.median}ms, range=${listenStats.min}-${listenStats.max}ms, CV=${listenStats.cv}%`
) )
} }
if (readyInStats) {
logger(
` ${label} Ready in: median=${readyInStats.median}ms, range=${readyInStats.min}-${readyInStats.max}ms, CV=${readyInStats.cv}%`
)
}
if (readyStats) { if (readyStats) {
logger( logger(
` ${label} Ready: median=${readyStats.median}ms, range=${readyStats.min}-${readyStats.max}ms, CV=${readyStats.cv}%` ` ${label} Ready: median=${readyStats.median}ms, range=${readyStats.min}-${readyStats.max}ms, CV=${readyStats.cv}%`
...@@ -171,6 +197,7 @@ async function benchmarkDevBootWithIterations( ...@@ -171,6 +197,7 @@ async function benchmarkDevBootWithIterations(
return { return {
listenTime: listenStats?.median ?? null, listenTime: listenStats?.median ?? null,
readyInTime: readyInStats?.median ?? null,
readyTime: readyStats?.median ?? null, readyTime: readyStats?.median ?? null,
} }
} }
...@@ -430,6 +457,10 @@ module.exports = async function collectStats( ...@@ -430,6 +457,10 @@ module.exports = async function collectStats(
orderedStats['General'][`nextDevColdListenDuration${bundler.suffix}`] = orderedStats['General'][`nextDevColdListenDuration${bundler.suffix}`] =
coldResult.listenTime coldResult.listenTime
} }
if (coldResult.readyInTime !== null) {
orderedStats['General'][`nextDevColdReadyInDuration${bundler.suffix}`] =
coldResult.readyInTime
}
if (coldResult.readyTime !== null) { if (coldResult.readyTime !== null) {
orderedStats['General'][`nextDevColdReadyDuration${bundler.suffix}`] = orderedStats['General'][`nextDevColdReadyDuration${bundler.suffix}`] =
coldResult.readyTime coldResult.readyTime
...@@ -485,6 +516,13 @@ module.exports = async function collectStats( ...@@ -485,6 +516,13 @@ module.exports = async function collectStats(
`nextDevWarmListenDuration${bundler.suffix}` `nextDevWarmListenDuration${bundler.suffix}`
] = warmResult.listenTime ] = warmResult.listenTime
} }
if (warmResult.readyInTime !== null) {
orderedStats['General'][
`nextDevWarmReadyInDuration${bundler.suffix}`
] = warmResult.readyInTime
}
if (warmResult.readyTime !== null) { if (warmResult.readyTime !== null) {
orderedStats['General'][`nextDevWarmReadyDuration${bundler.suffix}`] = orderedStats['General'][`nextDevWarmReadyDuration${bundler.suffix}`] =
warmResult.readyTime warmResult.readyTime
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment