Unverified Commit 2c752da3 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

devlow: use commander for cli argument parsing (#93860)

This has devlow use `commander` to parse its arguments.

There's a breaking change here to make it fit better into `commander`'s
patterns without extra code. It also makes things more explicit:

Arbitrary variant filtering can no longer be done with
`--variantname=value`. Now that there are more flags, these share the
namespace with these, so instead we use `--filter variantname=value`, or
`-F variantname=value`.
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent adba1e3f
...@@ -381,12 +381,12 @@ jobs: ...@@ -381,12 +381,12 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
mode: mode:
- '--turbopack=false' - '-F turbopack=false'
- '--turbopack=true' - '-F turbopack=true'
selector: selector:
- '--scenario=heavy-npm-deps-dev --page=homepage' - '--scenario=heavy-npm-deps-dev -F page=homepage'
- '--scenario=heavy-npm-deps-build --page=homepage' - '--scenario=heavy-npm-deps-build -F page=homepage'
- '--scenario=heavy-npm-deps-build-turbo-cache-enabled --page=homepage' - '--scenario=heavy-npm-deps-build-turbo-cache-enabled -F page=homepage'
permissions: permissions:
contents: read contents: read
id-token: write id-token: write
......
...@@ -10,19 +10,36 @@ npm install devlow-bench ...@@ -10,19 +10,36 @@ npm install devlow-bench
## Usage ## Usage
```bash ```text
Usage: devlow-bench [options] <scenario files> Usage: devlow-bench [options] [command]
## Selecting scenarios
--scenario=<filter>, -s=<filter> Only run the scenario with the given name Run developer-workflow benchmarks.
--interactive, -i Select scenarios and variants interactively
--<prop>=<value> Filter by any variant property defined in scenarios Options:
## Output -h, --help display help for command
--json=<path>, -j=<path> Write the results to the given path as JSON
--console Print the results to the console Commands:
--datadog[=<hostname>] Upload the results to Datadog run [options] [scenarios...] Run scenario files and report measurements.
(requires DATADOG_API_KEY environment variables) compare <baseline> <current> Compare two snapshot CSVs side-by-side.
## Help help [command] display help for command
--help, -h, -? Show this help ```
`run` is the default command, so scenario paths can still be passed without
writing `run` explicitly. Its options include:
```text
-s, --scenario <filter> Only run scenarios whose name matches the filter (repeatable).
-F, --filter <pair> Filter variants by property: key=value (repeatable).
-i, --interactive Select scenarios and variants interactively.
--n <number> Run each variant N times.
--warmup <number> Discard the first N runs before sampling.
--snapshot <path> Override the snapshot CSV path.
--compare Print a comparison table after the run.
--baseline <path> Select a comparison baseline; implies --compare.
-j, --json <path> Write results as JSON.
--no-console Suppress console output.
--datadog [host] Upload results to Datadog.
--snowflake [batchUri] Upload results to Snowflake.
``` ```
## Scenarios ## Scenarios
......
...@@ -44,6 +44,7 @@ ...@@ -44,6 +44,7 @@
}, },
"dependencies": { "dependencies": {
"@datadog/datadog-api-client": "^1.13.0", "@datadog/datadog-api-client": "^1.13.0",
"commander": "^14.0.3",
"inquirer": "^9.2.7", "inquirer": "^9.2.7",
"jstat": "^1.9.6", "jstat": "^1.9.6",
"minimist": "^1.2.8", "minimist": "^1.2.8",
......
import minimist from 'minimist' import { Command } from 'commander'
import { setCurrentScenarios } from './describe.js'
import { join } from 'path' import { join } from 'path'
import { pathToFileURL } from 'url'
import { groupRows, printComparison } from './compare.js'
import { setCurrentScenarios } from './describe.js'
import { type Scenario, type ScenarioVariant, runScenarios } from './index.js' import { type Scenario, type ScenarioVariant, runScenarios } from './index.js'
import compose from './interfaces/compose.js' import compose from './interfaces/compose.js'
import { groupRows, printComparison } from './compare.js'
import { readSnapshot, resolveCompareTarget } from './snapshot.js' import { readSnapshot, resolveCompareTarget } from './snapshot.js'
import { pathToFileURL } from 'url'
const SUBCOMMANDS = new Set(['run', 'compare']) interface RunOptions {
scenario?: string[]
filter?: string[]
interactive?: boolean
n?: number
warmup?: number
// Path override for the always-on snapshot CSV.
// undefined means the default path of .devlow-bench/snapshots/<ts>.csv
snapshot?: string
compare?: boolean
baseline?: string
json?: string
console?: boolean
datadog?: string | boolean
snowflake?: string | boolean
}
;(async () => { ;(async () => {
// Subcommand dispatch. `devlow-bench run [opts] scenario.mjs` and const program = new Command()
// `devlow-bench compare <a.csv> <b.csv>` are the explicit forms. A bare .name('devlow-bench')
// `devlow-bench <script>` falls through to legacy run-mode for back-compat. .description('Run developer-workflow benchmarks.')
const argv = process.argv.slice(2) .showHelpAfterError()
const first = argv[0]
const sub = first && SUBCOMMANDS.has(first) ? first : 'run'
const rest = first && SUBCOMMANDS.has(first) ? argv.slice(1) : argv
if (sub === 'compare') { program
await runCompareSubcommand(rest) .command('run', { isDefault: true })
return .description('Run scenario files and report measurements.')
} .argument('[scenarios...]', 'Scenario module paths to load.')
await runRunSubcommand(rest) .option(
'-s, --scenario <filter>',
'Only run scenarios whose name matches the filter (repeatable).',
(v: string, prev: string[] | undefined) => (prev ?? []).concat(v),
[]
)
.option(
'-F, --filter <pair>',
'Filter variants by property: -F key=value (repeatable; same key repeated = OR).',
(v: string, prev: string[] | undefined) => (prev ?? []).concat(v),
[]
)
.option('-i, --interactive', 'Select scenarios and variants interactively.')
.option(
'--n <number>',
'Run each variant N times; reports mean/p50/p90 per metric. Default: 1.',
(v: string) => Number(v)
)
.option(
'--warmup <number>',
'Discard the first N runs of each variant before sampling. Default: 0. Do NOT enable when measuring cold-start metrics.',
(v: string) => Number(v)
)
.option(
'--snapshot <path>',
'Override the snapshot CSV path. Default: ./.devlow-bench/snapshots/<ts>.csv. Snapshots are always written.'
)
.option(
'--compare',
'Print a comparison table at end of run. Baseline = newest snapshot, unless --baseline overrides.'
)
.option(
'--baseline <path>',
'Explicit baseline (file or directory). Implies --compare.'
)
.option('-j, --json <path>', 'Write the results to the given path as JSON.')
.option('--no-console', 'Suppress console output.')
.option(
'--datadog [host]',
'Upload the results to Datadog (requires DATADOG_API_KEY).'
)
.option(
'--snowflake [batchUri]',
'Upload the results to Snowflake (requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID).'
)
.action(runRun)
program
.command('compare')
.description(
'Compare two snapshot CSVs side-by-side, with p50/p90/p99 plus a Mann–Whitney U p-value per metric.'
)
.argument('<baseline>', 'Baseline snapshot CSV path.')
.argument('<current>', 'Current snapshot CSV path.')
.action(runCompare)
await program.parseAsync()
})().catch((e) => { })().catch((e) => {
console.error(e.stack) console.error(e.stack)
process.exit(1) process.exit(1)
}) })
async function runCompareSubcommand(argv: string[]): Promise<void> { async function runCompare(
const args = minimist(argv, { baselinePath: string,
alias: { '?': 'help', h: 'help' }, currentPath: string
}) ): Promise<void> {
if (args.help || args._.length === 0) {
console.log(`Usage: devlow-bench compare <baseline.csv> <current.csv>
Reads both snapshot CSVs and prints a side-by-side comparison table
with p50/p90/p99 plus a Mann–Whitney U p-value per metric.`)
if (args.help) return
}
if (args._.length !== 2) {
console.error(
`devlow-bench compare: expected 2 positional CSV paths, got ${args._.length}.`
)
console.error('Usage: devlow-bench compare <baseline.csv> <current.csv>')
process.exit(1)
}
const [baselinePath, currentPath] = args._
const [baseRows, curRows] = await Promise.all([ const [baseRows, curRows] = await Promise.all([
readSnapshot(baselinePath), readSnapshot(baselinePath),
readSnapshot(currentPath), readSnapshot(currentPath),
...@@ -56,99 +110,43 @@ async function runCompareSubcommand(argv: string[]): Promise<void> { ...@@ -56,99 +110,43 @@ async function runCompareSubcommand(argv: string[]): Promise<void> {
}) })
} }
async function runRunSubcommand(argv: string[]): Promise<void> { async function runRun(
const knownArgs = new Set([ scenarioPaths: string[],
'scenario', opts: RunOptions
's', ): Promise<void> {
'json', const propFilters: [string, string][] = []
'j', for (const pair of opts.filter ?? []) {
'console', const eq = pair.indexOf('=')
'datadog', if (eq === -1) {
'snowflake', console.error(`devlow-bench: invalid -F ${pair} (expected key=value).`)
'interactive', process.exit(1)
'i', }
'n', propFilters.push([pair.slice(0, eq), pair.slice(eq + 1)])
'warmup',
'snapshot',
'compare',
'baseline',
'help',
'h',
'?',
'_',
])
const args = minimist(argv, {
alias: {
s: 'scenario',
j: 'json',
i: 'interactive',
'?': 'help',
h: 'help',
},
// `compare` is strictly a boolean toggle now (use --baseline=<path> for
// the value form). Declaring it here prevents `--compare scenario.mjs`
// from consuming the scenario as the flag value.
boolean: ['compare'],
})
if (args.help || (Object.keys(args).length === 1 && args._.length === 0)) {
console.log(`Usage: devlow-bench [run] [options] <scenario files>
devlow-bench compare <baseline.csv> <current.csv>
## Selecting scenarios
--scenario=<filter>, -s=<filter> Only run the scenario with the given name
--interactive, -i Select scenarios and variants interactively
--<prop>=<value> Filter by any variant property defined in scenarios
## Repeated sampling
--n=<number> Run each variant N times. Reports mean/p50/p90 per metric.
Default: 1.
--warmup=<number> Discard the first N runs of each variant before sampling.
Default: 0. Do NOT enable when measuring cold-start metrics.
## Snapshots & comparison
--snapshot=<path> Override the snapshot CSV path.
Default: ./.devlow-bench/snapshots/<ts>.csv. Snapshots are always written.
--compare Print a comparison table at end of run. Baseline = newest snapshot,
unless --baseline overrides.
--baseline=<path> Explicit baseline (file or directory). Implies --compare.
## Output
--json=<path>, -j=<path> Write the results to the given path as JSON
--console Print the results to the console
--datadog[=<hostname>] Upload the results to Datadog
(requires DATADOG_API_KEY environment variables)
--snowflake[=<batch-uri>] Upload the results to Snowflake
(requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID and environment variables)
## Help
--help, -h, -? Show this help`)
if (args.help) return
} }
const scenarios: Scenario[] = [] const scenarios: Scenario[] = []
setCurrentScenarios(scenarios) setCurrentScenarios(scenarios)
for (const path of scenarioPaths) {
for (const path of args._) {
await import(pathToFileURL(join(process.cwd(), path)).toString()) await import(pathToFileURL(join(process.cwd(), path)).toString())
} }
setCurrentScenarios(null) setCurrentScenarios(null)
const cliIface = { const cliIface = {
filterScenarios: async (allScenarios: Scenario[]) => { filterScenarios: async (allScenarios: Scenario[]) => {
if (args.scenario) { const filters = opts.scenario
const filters = [].concat(args.scenario) if (!filters || filters.length === 0) return allScenarios
return allScenarios.filter((s) => return allScenarios.filter((s) => filters.some((f) => s.name.includes(f)))
filters.some((filter) => s.name.includes(filter))
)
}
return allScenarios
}, },
filterScenarioVariants: async (variants: ScenarioVariant[]) => { filterScenarioVariants: async (variants: ScenarioVariant[]) => {
const propEntries = Object.entries(args).filter( if (propFilters.length === 0) return variants
([key]) => !knownArgs.has(key) // Group multiple -F key=value with the same key into an OR set.
) const byKey = new Map<string, string[]>()
if (propEntries.length === 0) return variants for (const [k, v] of propFilters) {
for (const [key, value] of propEntries) { const existing = byKey.get(k)
const values = (Array.isArray(value) ? value : [value]).map((v) => if (existing) existing.push(v)
v.toString() else byKey.set(k, [v])
) }
for (const [key, values] of byKey) {
variants = variants.filter((variant) => { variants = variants.filter((variant) => {
const prop = variant.props[key] const prop = variant.props[key]
if (typeof prop === 'undefined') return false if (typeof prop === 'undefined') return false
...@@ -159,26 +157,26 @@ async function runRunSubcommand(argv: string[]): Promise<void> { ...@@ -159,26 +157,26 @@ async function runRunSubcommand(argv: string[]): Promise<void> {
return variants return variants
}, },
} }
// Validation (clamp to non-negative integers, default n=1/warmup=0) is // Validation (clamp to non-negative integers, default n=1/warmup=0) is
// delegated to runScenarios — see runner.ts. // delegated to runScenarios — see runner.ts.
const n = typeof args.n === 'number' ? args.n : 1 const n = typeof opts.n === 'number' && Number.isFinite(opts.n) ? opts.n : 1
const warmup = typeof args.warmup === 'number' ? args.warmup : 0 const warmup =
typeof opts.warmup === 'number' && Number.isFinite(opts.warmup)
? opts.warmup
: 0
// Snapshot is always on. `--snapshot=<path>` overrides the default path. // Snapshot is always on. --snapshot=<path> overrides the default path.
const snapshotPath =
typeof args.snapshot === 'string' ? args.snapshot : undefined
const snapshotIface = (await import('./interfaces/snapshot.js')).default({ const snapshotIface = (await import('./interfaces/snapshot.js')).default({
path: snapshotPath, path: opts.snapshot,
}) })
// Comparison: enabled by either --compare (boolean) or --baseline=<path>. // Comparison: enabled by --compare or by giving an explicit --baseline.
// The baseline is the value of --baseline if given, else "auto" (newest
// snapshot in the default snapshot dir).
const compareEnabled = const compareEnabled =
args.compare === true || typeof args.baseline === 'string' opts.compare === true || typeof opts.baseline === 'string'
let compareIface: any = null let compareIface: any = null
if (compareEnabled) { if (compareEnabled) {
const baselineArg = typeof args.baseline === 'string' ? args.baseline : true const baselineArg = typeof opts.baseline === 'string' ? opts.baseline : true
const baselinePath = await resolveCompareTarget( const baselinePath = await resolveCompareTarget(
baselineArg, baselineArg,
snapshotIface.resolvedPath snapshotIface.resolvedPath
...@@ -194,22 +192,22 @@ async function runRunSubcommand(argv: string[]): Promise<void> { ...@@ -194,22 +192,22 @@ async function runRunSubcommand(argv: string[]): Promise<void> {
).default({ baselinePath }) ).default({ baselinePath })
} }
let ifaces = [ const ifaces = [
cliIface, cliIface,
args.interactive && (await import('./interfaces/interactive.js')).default(), opts.interactive && (await import('./interfaces/interactive.js')).default(),
args.json && opts.json &&
(await import('./interfaces/json.js')).default(args.json, { n }), (await import('./interfaces/json.js')).default(opts.json, { n }),
args.datadog && opts.datadog &&
(await import('./interfaces/datadog.js')).default( (await import('./interfaces/datadog.js')).default(
typeof args.datadog === 'string' ? { host: args.datadog } : undefined typeof opts.datadog === 'string' ? { host: opts.datadog } : undefined
), ),
args.snowflake && opts.snowflake &&
(await import('./interfaces/snowflake.js')).default( (await import('./interfaces/snowflake.js')).default(
typeof args.snowflake === 'string' typeof opts.snowflake === 'string'
? { gatewayUri: args.snowflake } ? { gatewayUri: opts.snowflake }
: undefined : undefined
), ),
args.console !== false && opts.console !== false &&
(await import('./interfaces/console.js')).default({ n }), (await import('./interfaces/console.js')).default({ n }),
compareIface, compareIface,
snapshotIface, snapshotIface,
......
...@@ -980,6 +980,9 @@ importers: ...@@ -980,6 +980,9 @@ importers:
'@datadog/datadog-api-client': '@datadog/datadog-api-client':
specifier: ^1.13.0 specifier: ^1.13.0
version: 1.25.0(encoding@0.1.13) version: 1.25.0(encoding@0.1.13)
commander:
specifier: ^14.0.3
version: 14.0.3
inquirer: inquirer:
specifier: ^9.2.7 specifier: ^9.2.7
version: 9.2.23 version: 9.2.23
...@@ -9093,6 +9096,10 @@ packages: ...@@ -9093,6 +9096,10 @@ packages:
resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==}
engines: {node: '>=18'} engines: {node: '>=18'}
commander@14.0.3:
resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}
engines: {node: '>=20'}
commander@2.20.0: commander@2.20.0:
resolution: {integrity: sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==} resolution: {integrity: sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==}
...@@ -26665,6 +26672,8 @@ snapshots: ...@@ -26665,6 +26672,8 @@ snapshots:
commander@12.1.0: {} commander@12.1.0: {}
commander@14.0.3: {}
commander@2.20.0: {} commander@2.20.0: {}
commander@3.0.2: {} commander@3.0.2: {}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment