Unverified Commit be6fcb25 authored by Jimmy Lai's avatar Jimmy Lai Committed by GitHub

perf: improve stats thresholds to reduce CI noise (#88158)

## Summary

Add per-metric significance thresholds that account for variance.

**Time metrics** (high variance from CI):
- Dev boot: <100ms AND <15%, OR <3%
- Build times: <500ms AND <5%, OR <2%

**Size metrics** (deterministic):
- node_modules: <10KB AND <1%, OR <0.01%
- Bundle sizes: <2KB AND <1%, OR <0.1%

## Problem

Current thresholds flag insignificant changes as regressions:
- 3.45 KB change on 457 MB node_modules (0.0008%) was flagged as a
regression

## Solution

Use per-metric thresholds with OR logic for very small percentage
changes:
- Time metrics need more generous thresholds due to CI variance
- Size metrics can be tighter since they're deterministic

## Test plan

- Verify that 3.45 KB on 457 MB (0%) is now marked as insignificant
- Verify that real regressions (>0.1% for sizes, >3% for times) are
still flagged

---

Stack:
- #88157
- **→ This PR**
parent 43eead23
...@@ -162,6 +162,38 @@ const METRIC_GROUPS = { ...@@ -162,6 +162,38 @@ const METRIC_GROUPS = {
}, },
} }
// Per-metric significance thresholds
// A change is insignificant if:
// - (absoluteDiff < absoluteMin AND percentDiff < percentMin), OR
// - percentDiff < percentOnly
//
// Time metrics have HIGH VARIANCE (CPU, I/O, cache) - need generous thresholds
// Size metrics are DETERMINISTIC - can be tighter
const METRIC_THRESHOLDS = {
// Dev boot times (~300-400ms): high variance from CI
// <100ms AND <15%, OR <3%
ms: { absoluteMin: 100, percentMin: 15, percentOnly: 3 },
// Build times (~13s): high variance, longer duration
// <500ms AND <5%, OR <2%
buildDurationTurbo: { absoluteMin: 500, percentMin: 5, percentOnly: 2 },
buildDurationCachedTurbo: { absoluteMin: 500, percentMin: 5, percentOnly: 2 },
buildDurationWebpack: { absoluteMin: 500, percentMin: 5, percentOnly: 2 },
buildDurationCachedWebpack: {
absoluteMin: 500,
percentMin: 5,
percentOnly: 2,
},
// node_modules (~450MB): deterministic, huge baseline
// <10KB AND <1%, OR <0.01%
nodeModulesSize: { absoluteMin: 10240, percentMin: 1, percentOnly: 0.01 },
// Bundle sizes (KB-MB): deterministic
// <2KB AND <1%, OR <0.1%
bytes: { absoluteMin: 2048, percentMin: 1, percentOnly: 0.1 },
}
// ============================================================================ // ============================================================================
// Historical Data (Vercel KV) // Historical Data (Vercel KV)
// ============================================================================ // ============================================================================
...@@ -229,7 +261,7 @@ function getMetricLabel(key) { ...@@ -229,7 +261,7 @@ function getMetricLabel(key) {
return METRIC_LABELS[key] || shortenLabel(key) return METRIC_LABELS[key] || shortenLabel(key)
} }
function formatChange(mainVal, diffVal, type = 'bytes') { function formatChange(mainVal, diffVal, type = 'bytes', metricKey = null) {
if (typeof mainVal !== 'number' || typeof diffVal !== 'number') { if (typeof mainVal !== 'number' || typeof diffVal !== 'number') {
return { text: '-', significant: false, improved: false, regression: false } return { text: '-', significant: false, improved: false, regression: false }
} }
...@@ -237,19 +269,22 @@ function formatChange(mainVal, diffVal, type = 'bytes') { ...@@ -237,19 +269,22 @@ function formatChange(mainVal, diffVal, type = 'bytes') {
const diff = diffVal - mainVal const diff = diffVal - mainVal
const percentChange = mainVal > 0 ? (diff / mainVal) * 100 : 0 const percentChange = mainVal > 0 ? (diff / mainVal) * 100 : 0
// Thresholds: filter CI noise while catching real regressions // Get threshold config: prefer metric-specific, then type-based, then default to bytes
// For time: const threshold =
// - (<50ms AND <10%) = insignificant for short ops (dev boot ~300ms) METRIC_THRESHOLDS[metricKey] ||
// - OR <2% = insignificant for long ops (builds ~13s where 70ms is noise) METRIC_THRESHOLDS[type] ||
// For size: <1KB AND <1% = not worth mentioning METRIC_THRESHOLDS.bytes
// If mainVal is 0 and diff is non-zero, always consider it significant
// A change is insignificant if:
// - (absoluteDiff < absoluteMin AND percentDiff < percentMin), OR
// - percentDiff < percentOnly (definitely noise regardless of absolute)
// Exception: if mainVal is 0 and diff is non-zero, always significant
const isInsignificant = const isInsignificant =
mainVal === 0 && diff !== 0 mainVal === 0 && diff !== 0
? false ? false
: type === 'ms' : (Math.abs(diff) < threshold.absoluteMin &&
? (Math.abs(diff) < 50 && Math.abs(percentChange) < 10) || Math.abs(percentChange) < threshold.percentMin) ||
Math.abs(percentChange) < 2 Math.abs(percentChange) < threshold.percentOnly
: Math.abs(diff) < 1024 && Math.abs(percentChange) < 1
if (isInsignificant) { if (isInsignificant) {
return { text: '✓', significant: false, improved: false, regression: false } return { text: '✓', significant: false, improved: false, regression: false }
...@@ -378,7 +413,7 @@ function generateChangeSummary(mainStats, diffStats, history) { ...@@ -378,7 +413,7 @@ function generateChangeSummary(mainStats, diffStats, history) {
const mainVal = mainGeneral[key] const mainVal = mainGeneral[key]
const diffVal = diffGeneral[key] const diffVal = diffGeneral[key]
const type = getMetricType(key) const type = getMetricType(key)
const change = formatChange(mainVal, diffVal, type) const change = formatChange(mainVal, diffVal, type, key)
if (change.significant) { if (change.significant) {
const histValues = getHistoricalValues(history, key) const histValues = getHistoricalValues(history, key)
...@@ -483,7 +518,7 @@ function generateMetricsTable( ...@@ -483,7 +518,7 @@ function generateMetricsTable(
const metricType = metricDef.type || 'ms' const metricType = metricDef.type || 'ms'
const mainStr = prettify(mainVal, metricType) const mainStr = prettify(mainVal, metricType)
const diffStr = prettify(diffVal, metricType) const diffStr = prettify(diffVal, metricType)
const change = formatChange(mainVal, diffVal, metricType) const change = formatChange(mainVal, diffVal, metricType, metricDef.key)
const histValues = getHistoricalValues(history, metricDef.key) const histValues = getHistoricalValues(history, metricDef.key)
const sparkline = generateTrendBar(histValues) const sparkline = generateTrendBar(histValues)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment