Unverified Commit 2c752da3 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

devlow: use commander for cli argument parsing (#93860)

This has devlow use `commander` to parse its arguments.

There's a breaking change here to make it fit better into `commander`'s
patterns without extra code. It also makes things more explicit:

Arbitrary variant filtering can no longer be done with
`--variantname=value`. Now that there are more flags, these share the
namespace with these, so instead we use `--filter variantname=value`, or
`-F variantname=value`.
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent adba1e3f
......@@ -381,12 +381,12 @@ jobs:
fail-fast: false
matrix:
mode:
- '--turbopack=false'
- '--turbopack=true'
- '-F turbopack=false'
- '-F turbopack=true'
selector:
- '--scenario=heavy-npm-deps-dev --page=homepage'
- '--scenario=heavy-npm-deps-build --page=homepage'
- '--scenario=heavy-npm-deps-build-turbo-cache-enabled --page=homepage'
- '--scenario=heavy-npm-deps-dev -F page=homepage'
- '--scenario=heavy-npm-deps-build -F page=homepage'
- '--scenario=heavy-npm-deps-build-turbo-cache-enabled -F page=homepage'
permissions:
contents: read
id-token: write
......
......@@ -10,19 +10,36 @@ npm install devlow-bench
## Usage
```bash
Usage: devlow-bench [options] <scenario files>
## Selecting scenarios
--scenario=<filter>, -s=<filter> Only run the scenario with the given name
--interactive, -i Select scenarios and variants interactively
--<prop>=<value> Filter by any variant property defined in scenarios
## Output
--json=<path>, -j=<path> Write the results to the given path as JSON
--console Print the results to the console
--datadog[=<hostname>] Upload the results to Datadog
(requires DATADOG_API_KEY environment variables)
## Help
--help, -h, -? Show this help
```text
Usage: devlow-bench [options] [command]
Run developer-workflow benchmarks.
Options:
-h, --help display help for command
Commands:
run [options] [scenarios...] Run scenario files and report measurements.
compare <baseline> <current> Compare two snapshot CSVs side-by-side.
help [command] display help for command
```
`run` is the default command, so scenario paths can still be passed without
writing `run` explicitly. Its options include:
```text
-s, --scenario <filter> Only run scenarios whose name matches the filter (repeatable).
-F, --filter <pair> Filter variants by property: key=value (repeatable).
-i, --interactive Select scenarios and variants interactively.
--n <number> Run each variant N times.
--warmup <number> Discard the first N runs before sampling.
--snapshot <path> Override the snapshot CSV path.
--compare Print a comparison table after the run.
--baseline <path> Select a comparison baseline; implies --compare.
-j, --json <path> Write results as JSON.
--no-console Suppress console output.
--datadog [host] Upload results to Datadog.
--snowflake [batchUri] Upload results to Snowflake.
```
## Scenarios
......
......@@ -44,6 +44,7 @@
},
"dependencies": {
"@datadog/datadog-api-client": "^1.13.0",
"commander": "^14.0.3",
"inquirer": "^9.2.7",
"jstat": "^1.9.6",
"minimist": "^1.2.8",
......
import minimist from 'minimist'
import { setCurrentScenarios } from './describe.js'
import { Command } from 'commander'
import { join } from 'path'
import { pathToFileURL } from 'url'
import { groupRows, printComparison } from './compare.js'
import { setCurrentScenarios } from './describe.js'
import { type Scenario, type ScenarioVariant, runScenarios } from './index.js'
import compose from './interfaces/compose.js'
import { groupRows, printComparison } from './compare.js'
import { readSnapshot, resolveCompareTarget } from './snapshot.js'
import { pathToFileURL } from 'url'
const SUBCOMMANDS = new Set(['run', 'compare'])
interface RunOptions {
scenario?: string[]
filter?: string[]
interactive?: boolean
n?: number
warmup?: number
// Path override for the always-on snapshot CSV.
// undefined means the default path of .devlow-bench/snapshots/<ts>.csv
snapshot?: string
compare?: boolean
baseline?: string
json?: string
console?: boolean
datadog?: string | boolean
snowflake?: string | boolean
}
;(async () => {
// Subcommand dispatch. `devlow-bench run [opts] scenario.mjs` and
// `devlow-bench compare <a.csv> <b.csv>` are the explicit forms. A bare
// `devlow-bench <script>` falls through to legacy run-mode for back-compat.
const argv = process.argv.slice(2)
const first = argv[0]
const sub = first && SUBCOMMANDS.has(first) ? first : 'run'
const rest = first && SUBCOMMANDS.has(first) ? argv.slice(1) : argv
const program = new Command()
.name('devlow-bench')
.description('Run developer-workflow benchmarks.')
.showHelpAfterError()
if (sub === 'compare') {
await runCompareSubcommand(rest)
return
}
await runRunSubcommand(rest)
program
.command('run', { isDefault: true })
.description('Run scenario files and report measurements.')
.argument('[scenarios...]', 'Scenario module paths to load.')
.option(
'-s, --scenario <filter>',
'Only run scenarios whose name matches the filter (repeatable).',
(v: string, prev: string[] | undefined) => (prev ?? []).concat(v),
[]
)
.option(
'-F, --filter <pair>',
'Filter variants by property: -F key=value (repeatable; same key repeated = OR).',
(v: string, prev: string[] | undefined) => (prev ?? []).concat(v),
[]
)
.option('-i, --interactive', 'Select scenarios and variants interactively.')
.option(
'--n <number>',
'Run each variant N times; reports mean/p50/p90 per metric. Default: 1.',
(v: string) => Number(v)
)
.option(
'--warmup <number>',
'Discard the first N runs of each variant before sampling. Default: 0. Do NOT enable when measuring cold-start metrics.',
(v: string) => Number(v)
)
.option(
'--snapshot <path>',
'Override the snapshot CSV path. Default: ./.devlow-bench/snapshots/<ts>.csv. Snapshots are always written.'
)
.option(
'--compare',
'Print a comparison table at end of run. Baseline = newest snapshot, unless --baseline overrides.'
)
.option(
'--baseline <path>',
'Explicit baseline (file or directory). Implies --compare.'
)
.option('-j, --json <path>', 'Write the results to the given path as JSON.')
.option('--no-console', 'Suppress console output.')
.option(
'--datadog [host]',
'Upload the results to Datadog (requires DATADOG_API_KEY).'
)
.option(
'--snowflake [batchUri]',
'Upload the results to Snowflake (requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID).'
)
.action(runRun)
program
.command('compare')
.description(
'Compare two snapshot CSVs side-by-side, with p50/p90/p99 plus a Mann–Whitney U p-value per metric.'
)
.argument('<baseline>', 'Baseline snapshot CSV path.')
.argument('<current>', 'Current snapshot CSV path.')
.action(runCompare)
await program.parseAsync()
})().catch((e) => {
console.error(e.stack)
process.exit(1)
})
async function runCompareSubcommand(argv: string[]): Promise<void> {
const args = minimist(argv, {
alias: { '?': 'help', h: 'help' },
})
if (args.help || args._.length === 0) {
console.log(`Usage: devlow-bench compare <baseline.csv> <current.csv>
Reads both snapshot CSVs and prints a side-by-side comparison table
with p50/p90/p99 plus a Mann–Whitney U p-value per metric.`)
if (args.help) return
}
if (args._.length !== 2) {
console.error(
`devlow-bench compare: expected 2 positional CSV paths, got ${args._.length}.`
)
console.error('Usage: devlow-bench compare <baseline.csv> <current.csv>')
process.exit(1)
}
const [baselinePath, currentPath] = args._
async function runCompare(
baselinePath: string,
currentPath: string
): Promise<void> {
const [baseRows, curRows] = await Promise.all([
readSnapshot(baselinePath),
readSnapshot(currentPath),
......@@ -56,99 +110,43 @@ async function runCompareSubcommand(argv: string[]): Promise<void> {
})
}
async function runRunSubcommand(argv: string[]): Promise<void> {
const knownArgs = new Set([
'scenario',
's',
'json',
'j',
'console',
'datadog',
'snowflake',
'interactive',
'i',
'n',
'warmup',
'snapshot',
'compare',
'baseline',
'help',
'h',
'?',
'_',
])
const args = minimist(argv, {
alias: {
s: 'scenario',
j: 'json',
i: 'interactive',
'?': 'help',
h: 'help',
},
// `compare` is strictly a boolean toggle now (use --baseline=<path> for
// the value form). Declaring it here prevents `--compare scenario.mjs`
// from consuming the scenario as the flag value.
boolean: ['compare'],
})
if (args.help || (Object.keys(args).length === 1 && args._.length === 0)) {
console.log(`Usage: devlow-bench [run] [options] <scenario files>
devlow-bench compare <baseline.csv> <current.csv>
## Selecting scenarios
--scenario=<filter>, -s=<filter> Only run the scenario with the given name
--interactive, -i Select scenarios and variants interactively
--<prop>=<value> Filter by any variant property defined in scenarios
## Repeated sampling
--n=<number> Run each variant N times. Reports mean/p50/p90 per metric.
Default: 1.
--warmup=<number> Discard the first N runs of each variant before sampling.
Default: 0. Do NOT enable when measuring cold-start metrics.
## Snapshots & comparison
--snapshot=<path> Override the snapshot CSV path.
Default: ./.devlow-bench/snapshots/<ts>.csv. Snapshots are always written.
--compare Print a comparison table at end of run. Baseline = newest snapshot,
unless --baseline overrides.
--baseline=<path> Explicit baseline (file or directory). Implies --compare.
## Output
--json=<path>, -j=<path> Write the results to the given path as JSON
--console Print the results to the console
--datadog[=<hostname>] Upload the results to Datadog
(requires DATADOG_API_KEY environment variables)
--snowflake[=<batch-uri>] Upload the results to Snowflake
(requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID and environment variables)
## Help
--help, -h, -? Show this help`)
if (args.help) return
async function runRun(
scenarioPaths: string[],
opts: RunOptions
): Promise<void> {
const propFilters: [string, string][] = []
for (const pair of opts.filter ?? []) {
const eq = pair.indexOf('=')
if (eq === -1) {
console.error(`devlow-bench: invalid -F ${pair} (expected key=value).`)
process.exit(1)
}
propFilters.push([pair.slice(0, eq), pair.slice(eq + 1)])
}
const scenarios: Scenario[] = []
setCurrentScenarios(scenarios)
for (const path of args._) {
for (const path of scenarioPaths) {
await import(pathToFileURL(join(process.cwd(), path)).toString())
}
setCurrentScenarios(null)
const cliIface = {
filterScenarios: async (allScenarios: Scenario[]) => {
if (args.scenario) {
const filters = [].concat(args.scenario)
return allScenarios.filter((s) =>
filters.some((filter) => s.name.includes(filter))
)
}
return allScenarios
const filters = opts.scenario
if (!filters || filters.length === 0) return allScenarios
return allScenarios.filter((s) => filters.some((f) => s.name.includes(f)))
},
filterScenarioVariants: async (variants: ScenarioVariant[]) => {
const propEntries = Object.entries(args).filter(
([key]) => !knownArgs.has(key)
)
if (propEntries.length === 0) return variants
for (const [key, value] of propEntries) {
const values = (Array.isArray(value) ? value : [value]).map((v) =>
v.toString()
)
if (propFilters.length === 0) return variants
// Group multiple -F key=value with the same key into an OR set.
const byKey = new Map<string, string[]>()
for (const [k, v] of propFilters) {
const existing = byKey.get(k)
if (existing) existing.push(v)
else byKey.set(k, [v])
}
for (const [key, values] of byKey) {
variants = variants.filter((variant) => {
const prop = variant.props[key]
if (typeof prop === 'undefined') return false
......@@ -159,26 +157,26 @@ async function runRunSubcommand(argv: string[]): Promise<void> {
return variants
},
}
// Validation (clamp to non-negative integers, default n=1/warmup=0) is
// delegated to runScenarios — see runner.ts.
const n = typeof args.n === 'number' ? args.n : 1
const warmup = typeof args.warmup === 'number' ? args.warmup : 0
const n = typeof opts.n === 'number' && Number.isFinite(opts.n) ? opts.n : 1
const warmup =
typeof opts.warmup === 'number' && Number.isFinite(opts.warmup)
? opts.warmup
: 0
// Snapshot is always on. `--snapshot=<path>` overrides the default path.
const snapshotPath =
typeof args.snapshot === 'string' ? args.snapshot : undefined
// Snapshot is always on. --snapshot=<path> overrides the default path.
const snapshotIface = (await import('./interfaces/snapshot.js')).default({
path: snapshotPath,
path: opts.snapshot,
})
// Comparison: enabled by either --compare (boolean) or --baseline=<path>.
// The baseline is the value of --baseline if given, else "auto" (newest
// snapshot in the default snapshot dir).
// Comparison: enabled by --compare or by giving an explicit --baseline.
const compareEnabled =
args.compare === true || typeof args.baseline === 'string'
opts.compare === true || typeof opts.baseline === 'string'
let compareIface: any = null
if (compareEnabled) {
const baselineArg = typeof args.baseline === 'string' ? args.baseline : true
const baselineArg = typeof opts.baseline === 'string' ? opts.baseline : true
const baselinePath = await resolveCompareTarget(
baselineArg,
snapshotIface.resolvedPath
......@@ -194,22 +192,22 @@ async function runRunSubcommand(argv: string[]): Promise<void> {
).default({ baselinePath })
}
let ifaces = [
const ifaces = [
cliIface,
args.interactive && (await import('./interfaces/interactive.js')).default(),
args.json &&
(await import('./interfaces/json.js')).default(args.json, { n }),
args.datadog &&
opts.interactive && (await import('./interfaces/interactive.js')).default(),
opts.json &&
(await import('./interfaces/json.js')).default(opts.json, { n }),
opts.datadog &&
(await import('./interfaces/datadog.js')).default(
typeof args.datadog === 'string' ? { host: args.datadog } : undefined
typeof opts.datadog === 'string' ? { host: opts.datadog } : undefined
),
args.snowflake &&
opts.snowflake &&
(await import('./interfaces/snowflake.js')).default(
typeof args.snowflake === 'string'
? { gatewayUri: args.snowflake }
typeof opts.snowflake === 'string'
? { gatewayUri: opts.snowflake }
: undefined
),
args.console !== false &&
opts.console !== false &&
(await import('./interfaces/console.js')).default({ n }),
compareIface,
snapshotIface,
......
......@@ -980,6 +980,9 @@ importers:
'@datadog/datadog-api-client':
specifier: ^1.13.0
version: 1.25.0(encoding@0.1.13)
commander:
specifier: ^14.0.3
version: 14.0.3
inquirer:
specifier: ^9.2.7
version: 9.2.23
......@@ -9093,6 +9096,10 @@ packages:
resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==}
engines: {node: '>=18'}
commander@14.0.3:
resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}
engines: {node: '>=20'}
commander@2.20.0:
resolution: {integrity: sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==}
......@@ -26665,6 +26672,8 @@ snapshots:
commander@12.1.0: {}
commander@14.0.3: {}
commander@2.20.0: {}
commander@3.0.2: {}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment