Unverified Commit e6dfc19f authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-tasks-backend: add CPU-scaling capture script for priority_contention bench

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 32b2c910
...@@ -154,6 +154,8 @@ async fn set_and_read( ...@@ -154,6 +154,8 @@ async fn set_and_read(
Ok(elapsed) Ok(elapsed)
} }
/// To capture how this benchmark scales with the number of CPUs on a machine, run
/// `priority_contention_scaling.py` next to this file; it writes the results to a CSV file.
pub fn priority_contention(c: &mut Criterion) { pub fn priority_contention(c: &mut Criterion) {
let mut group = c.benchmark_group("turbo_tasks_backend_priority_contention"); let mut group = c.benchmark_group("turbo_tasks_backend_priority_contention");
group.sample_size(20); group.sample_size(20);
......
#!/usr/bin/env python3
"""Capture CPU-scaling data for the `priority_contention` benchmark on this machine.
Builds the `turbo-tasks-backend` bench executable once (with the committed bench settings) and
runs the `priority_contention` benchmark group repeatedly, each time restricted to a different
number of logical CPUs. Tokio and turbo-tasks size their default worker pools from the CPUs the
process may run on (CPU quotas, e.g. of containers, can limit them further), so this sweeps the
number of workers. The script only captures data: there is no before/after comparison and no
plotting (Criterion runs with `--noplot`). To compare commits, run it once per commit.
Linux only (uses `os.sched_setaffinity`, `/proc/stat` and `/sys/devices/system/cpu`). Only the
Python 3 standard library is required.
Usage (from anywhere in the repository):
python3 turbopack/crates/turbo-tasks-backend/benches/priority_contention_scaling.py
python3 .../priority_contention_scaling.py --passes 1 --cpu-counts 1,4,16 --output out.csv
python3 .../priority_contention_scaling.py --dry-run
CPU selection: the CPUs available to this process (`sched_getaffinity`) are ordered one logical
CPU per physical core first (by package, core and CPU id), then their SMT siblings. A run with
`n` CPUs uses the first `n` of that order. The default counts are 1, 2, 4, ... below the number
of available CPUs, plus that number. The topology is the one the OS reports; in a VM it may not
match the host.
CSV columns (one row per pass x CPU count x benchmark variant):
pass, cpus, cpu_list, variant,
median_ns, median_ci_lower_ns, median_ci_upper_ns, mean_ns, ci_confidence_level,
steal_pct, git_commit, git_dirty, timestamp_utc
`cpus` is the number of allowed logical CPUs (not a measured worker count). `median_*` and
`mean_ns` are Criterion's estimates of the time per iteration. `steal_pct` is the share of steal
time in all CPU time of the whole machine (all CPUs, `/proc/stat`) during the run; it is empty
when unavailable. High steal (e.g. on shared VMs) makes the results unreliable. Rows are
written as soon as a run finishes, so an interrupted capture keeps its completed runs.
"""
from __future__ import annotations
import argparse
import csv
import datetime
import json
import math
import os
import pathlib
import signal
import subprocess
import sys
from typing import Iterable, Optional, Sequence
GROUP = "turbo_tasks_backend_priority_contention"
EXPECTED_VARIANTS = 4
OVERRIDE_ENV_VARS = ("TOKIO_WORKER_THREADS", "TURBO_TASKS_AVAILABLE_PARALLELISM")
CSV_COLUMNS = [
"pass",
"cpus",
"cpu_list",
"variant",
"median_ns",
"median_ci_lower_ns",
"median_ci_upper_ns",
"mean_ns",
"ci_confidence_level",
"steal_pct",
"git_commit",
"git_dirty",
"timestamp_utc",
]
REPO_ROOT = pathlib.Path(__file__).resolve().parents[4]
class ScalingError(Exception):
pass
def cpu_counts(available: int) -> list[int]:
"""1, 2, 4, ... below `available`, then `available`."""
counts = []
n = 1
while n < available:
counts.append(n)
n *= 2
counts.append(available)
return counts
def parse_cpu_counts(value: str, available: int) -> list[int]:
"""Parses a comma-separated list of CPU counts (deduplicated and sorted)."""
counts = set()
for part in value.split(","):
part = part.strip()
try:
count = int(part)
except ValueError:
raise ScalingError(f"invalid CPU count {part!r}") from None
if not 1 <= count <= available:
raise ScalingError(f"CPU count {count} is not between 1 and {available}")
counts.add(count)
return sorted(counts)
def order_cpus(cpus: Iterable[int], sysfs: pathlib.Path) -> tuple[list[int], bool]:
"""Orders CPUs one per physical core first, then the SMT siblings.
Returns the order and whether the topology could be read (otherwise numeric order)."""
cpus = sorted(cpus)
keys = {}
for cpu in cpus:
topology = sysfs / f"cpu{cpu}" / "topology"
try:
package = int((topology / "physical_package_id").read_text().strip())
core = int((topology / "core_id").read_text().strip())
except (OSError, ValueError):
return cpus, False
keys[cpu] = (package, core)
first, siblings, seen = [], [], set()
for cpu in sorted(cpus, key=lambda c: (keys[c], c)):
if keys[cpu] in seen:
siblings.append(cpu)
else:
seen.add(keys[cpu])
first.append(cpu)
siblings.sort(key=lambda c: (keys[c], c))
return first + siblings, True
def find_bench_executable(cargo_json_lines: Iterable[str]) -> str:
"""Finds the `mod` bench executable of turbo-tasks-backend in Cargo's JSON messages."""
for line in cargo_json_lines:
line = line.strip()
if not line.startswith("{"):
continue
try:
message = json.loads(line)
except json.JSONDecodeError:
continue
if message.get("reason") != "compiler-artifact" or not message.get("executable"):
continue
target = message.get("target", {})
package_id = message.get("package_id", "")
if (
target.get("name") == "mod"
and "bench" in target.get("kind", [])
and "turbo-tasks-backend" in package_id
):
return message["executable"]
raise ScalingError("cargo did not report a turbo-tasks-backend `mod` bench executable")
def build_bench() -> str:
command = [
"cargo",
"bench",
"-p",
"turbo-tasks-backend",
"--bench",
"mod",
"--no-run",
"--message-format=json-render-diagnostics",
]
print("building:", " ".join(command), file=sys.stderr)
# Cargo's progress goes to stderr; compiler diagnostics are JSON messages on stdout.
try:
result = subprocess.run(command, cwd=REPO_ROOT, stdout=subprocess.PIPE, text=True)
except OSError as error:
raise ScalingError(f"could not run cargo: {error}") from None
if result.returncode != 0:
for diagnostic in rendered_diagnostics(result.stdout.splitlines()):
print(diagnostic, file=sys.stderr, end="" if diagnostic.endswith("\n") else "\n")
raise ScalingError(f"cargo bench --no-run failed with exit code {result.returncode}")
return find_bench_executable(result.stdout.splitlines())
def rendered_diagnostics(cargo_json_lines: Iterable[str]) -> list[str]:
"""The rendered compiler diagnostics in Cargo's JSON messages."""
diagnostics = []
for line in cargo_json_lines:
try:
message = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(message, dict) and message.get("reason") == "compiler-message":
rendered = (message.get("message") or {}).get("rendered")
if rendered:
diagnostics.append(rendered)
return diagnostics
def read_estimates(criterion_home: pathlib.Path) -> dict[str, dict]:
"""Reads `<group>/<kind>/<size>/new/estimates.json` for every variant of the group."""
group_dir = criterion_home / GROUP
files = sorted(group_dir.glob("*/*/new/estimates.json"))
if len(files) != EXPECTED_VARIANTS:
raise ScalingError(
f"expected {EXPECTED_VARIANTS} variants in {group_dir}, found {len(files)}"
)
estimates = {}
for path in files:
variant = "/".join(path.relative_to(group_dir).parts[:2])
try:
data = json.loads(path.read_text())
median = data["median"]
interval = median["confidence_interval"]
values = {
"median_ns": float(median["point_estimate"]),
"median_ci_lower_ns": float(interval["lower_bound"]),
"median_ci_upper_ns": float(interval["upper_bound"]),
"mean_ns": float(data["mean"]["point_estimate"]),
"ci_confidence_level": float(interval["confidence_level"]),
}
except (OSError, ValueError, KeyError, TypeError) as error:
raise ScalingError(f"invalid estimates in {path}: {error}") from None
problem = validate_estimate(values)
if problem:
raise ScalingError(f"invalid estimates in {path}: {problem}")
estimates[variant] = values
return estimates
def validate_estimate(values: dict) -> Optional[str]:
"""Returns why the estimate values are implausible, or None."""
if not all(math.isfinite(value) for value in values.values()):
return "non-finite value"
if min(values["median_ns"], values["median_ci_lower_ns"], values["mean_ns"]) <= 0:
return "non-positive time"
if not values["median_ci_lower_ns"] <= values["median_ns"] <= values["median_ci_upper_ns"]:
return "median outside its confidence interval"
if not 0 < values["ci_confidence_level"] < 1:
return "confidence level not between 0 and 1"
return None
def read_cpu_ticks(proc_stat: pathlib.Path = pathlib.Path("/proc/stat")) -> Optional[tuple[int, int]]:
"""(total ticks, steal ticks) of the aggregate `cpu` line, or None when unavailable."""
try:
for line in proc_stat.read_text().splitlines():
fields = line.split()
if fields and fields[0] == "cpu":
# user nice system idle iowait irq softirq steal [guest guest_nice]
values = [int(v) for v in fields[1:9]]
if len(values) < 8:
return None
return sum(values), values[7]
except (OSError, ValueError):
return None
return None
def steal_pct(before: Optional[tuple[int, int]], after: Optional[tuple[int, int]]) -> Optional[float]:
if before is None or after is None:
return None
total = after[0] - before[0]
steal = after[1] - before[1]
if total <= 0 or not 0 <= steal <= total:
return None
return 100.0 * steal / total
def git_info() -> tuple[str, str]:
def git(*args: str) -> Optional[str]:
try:
result = subprocess.run(
["git", *args], cwd=REPO_ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True
)
except OSError:
return None
return result.stdout if result.returncode == 0 else None
commit = (git("rev-parse", "HEAD") or "").strip()
status = git("status", "--porcelain")
dirty = "" if status is None else ("true" if status.strip() else "false")
return commit, dirty
def check_environment(env: dict) -> None:
set_vars = [name for name in OVERRIDE_ENV_VARS if name in env]
if set_vars:
raise ScalingError(
f"{', '.join(set_vars)} is set; the worker count would not follow the CPU restriction"
)
def open_csv(path: pathlib.Path, force: bool):
"""Creates the CSV file and writes the header. Without `force`, an existing file is an error."""
path.parent.mkdir(parents=True, exist_ok=True)
try:
handle = path.open("w" if force else "x", newline="")
except FileExistsError:
raise ScalingError(f"{path} already exists (use --force to overwrite)") from None
except OSError as error:
raise ScalingError(f"could not create {path}: {error}") from None
writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS)
writer.writeheader()
handle.flush()
return handle, writer
def run_bench(executable: str, cpus: Sequence[int], criterion_home: pathlib.Path, log_path: pathlib.Path) -> None:
"""Runs the benchmark group restricted to `cpus`, with its output in `log_path`."""
env = dict(os.environ, CRITERION_HOME=str(criterion_home))
with log_path.open("w") as log:
try:
process = subprocess.Popen(
[executable, "--bench", "priority_contention", "--noplot"],
env=env,
stdout=log,
stderr=subprocess.STDOUT,
preexec_fn=lambda: os.sched_setaffinity(0, cpus),
)
except (OSError, subprocess.SubprocessError) as error:
raise ScalingError(f"could not start {executable}: {error}") from None
try:
returncode = process.wait()
except KeyboardInterrupt:
process.send_signal(signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
raise
if returncode != 0:
raise ScalingError(f"benchmark exited with code {returncode}, see {log_path}")
def parse_args(argv: Sequence[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--passes", type=int, default=3, help="number of passes (default: 3)")
parser.add_argument(
"--output",
type=pathlib.Path,
default=pathlib.Path("priority_contention_scaling.csv"),
help="CSV output path (default: ./priority_contention_scaling.csv)",
)
parser.add_argument(
"--work-dir",
type=pathlib.Path,
help="directory for Criterion data and logs (default: <output>.d)",
)
parser.add_argument("--cpu-counts", help="comma-separated CPU counts instead of 1,2,4,...,N")
parser.add_argument("--bench-binary", help="use this bench executable instead of building")
parser.add_argument("--dry-run", action="store_true", help="print the plan, build/run nothing")
parser.add_argument(
"--force", action="store_true", help="overwrite an existing output, reuse an existing work dir"
)
args = parser.parse_args(argv)
if args.passes < 1:
parser.error("--passes must be at least 1")
return args
def main(argv: Sequence[str]) -> int:
args = parse_args(argv)
if not hasattr(os, "sched_setaffinity"):
raise ScalingError("this script needs Linux (os.sched_setaffinity)")
check_environment(os.environ)
order, topology_ok = order_cpus(os.sched_getaffinity(0), pathlib.Path("/sys/devices/system/cpu"))
if not topology_ok:
print("warning: CPU topology unavailable, using numeric CPU order", file=sys.stderr)
if args.cpu_counts is not None:
counts = parse_cpu_counts(args.cpu_counts, len(order))
else:
counts = cpu_counts(len(order))
print(f"CPU order: {' '.join(map(str, order))}")
print(f"CPU counts: {','.join(map(str, counts))}")
if args.dry_run:
for count in counts:
print(f" {count}: {' '.join(map(str, order[:count]))}")
return 0
work_dir = args.work_dir or args.output.with_name(args.output.name + ".d")
if args.output.exists() and not args.force:
raise ScalingError(f"{args.output} already exists (use --force to overwrite)")
if args.output.is_dir():
raise ScalingError(f"{args.output} is a directory")
if work_dir.resolve() == args.output.resolve():
raise ScalingError("--work-dir must differ from --output")
if work_dir.exists() and not (args.force and work_dir.is_dir()):
raise ScalingError(f"{work_dir} already exists (use --force to reuse a directory)")
executable = args.bench_binary or build_bench()
commit, dirty = git_info()
handle, writer = open_csv(args.output, args.force)
# Unique per invocation, so reusing the work dir with --force never mixes runs.
run_id = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
run_id += f"-{os.getpid()}"
try:
for pass_index in range(1, args.passes + 1):
for count in counts:
cpus = order[:count]
run_dir = work_dir / run_id / f"pass{pass_index}-cpus{count}"
try:
run_dir.mkdir(parents=True)
except OSError as error:
raise ScalingError(f"could not create {run_dir}: {error}") from None
print(f"pass {pass_index}/{args.passes}, {count} CPUs ...", file=sys.stderr)
before = read_cpu_ticks()
run_bench(executable, cpus, run_dir / "criterion", run_dir / "bench.log")
steal = steal_pct(before, read_cpu_ticks())
timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
for variant, values in sorted(read_estimates(run_dir / "criterion").items()):
writer.writerow(
{
"pass": pass_index,
"cpus": count,
"cpu_list": " ".join(map(str, cpus)),
"variant": variant,
**values,
"steal_pct": "" if steal is None else f"{steal:.3f}",
"git_commit": commit,
"git_dirty": dirty,
"timestamp_utc": timestamp,
}
)
handle.flush()
finally:
handle.close()
print(f"wrote {args.output}", file=sys.stderr)
return 0
if __name__ == "__main__":
try:
sys.exit(main(sys.argv[1:]))
except ScalingError as error:
print(f"error: {error}", file=sys.stderr)
sys.exit(1)
except KeyboardInterrupt:
print("interrupted", file=sys.stderr)
sys.exit(130)
"""Tests for priority_contention_scaling.py. Run: python3 -m unittest priority_contention_scaling_test"""
import contextlib
import io
import json
import os
import pathlib
import sys
import tempfile
import unittest
from unittest import mock
import priority_contention_scaling as scaling
def write_topology(root: pathlib.Path, cpu: int, package: int, core: int) -> None:
topology = root / f"cpu{cpu}" / "topology"
topology.mkdir(parents=True)
(topology / "physical_package_id").write_text(f"{package}\n")
(topology / "core_id").write_text(f"{core}\n")
def write_estimates(root: pathlib.Path, variant: str, median: float, content=None) -> None:
path = root / scaling.GROUP / variant / "new" / "estimates.json"
path.parent.mkdir(parents=True)
if content is None:
interval = {"lower_bound": median - 1, "upper_bound": median + 1, "confidence_level": 0.95}
content = json.dumps(
{
"median": {"point_estimate": median, "confidence_interval": interval},
"mean": {"point_estimate": median + 0.5, "confidence_interval": interval},
}
)
path.write_text(content)
VARIANTS = ["warm/32x2048", "initial/32x2048", "warm/128x512", "initial/128x512"]
class CpuCountsTest(unittest.TestCase):
def test_default_counts(self):
self.assertEqual(scaling.cpu_counts(1), [1])
self.assertEqual(scaling.cpu_counts(6), [1, 2, 4, 6])
self.assertEqual(scaling.cpu_counts(32), [1, 2, 4, 8, 16, 32])
def test_explicit_counts(self):
self.assertEqual(scaling.parse_cpu_counts("4, 1,4,2", 8), [1, 2, 4])
for bad in ["0", "9", "x", "1,,2", ""]:
with self.assertRaises(scaling.ScalingError):
scaling.parse_cpu_counts(bad, 8)
class OrderCpusTest(unittest.TestCase):
def test_cores_first_then_siblings(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
# package 0: cores 0,1 with siblings (0,4) (1,5); package 1: core 0 with (2,6), core 1 with (3,7)
for cpu, package, core in [(0, 0, 0), (4, 0, 0), (1, 0, 1), (5, 0, 1), (2, 1, 0), (6, 1, 0), (3, 1, 1), (7, 1, 1)]:
write_topology(root, cpu, package, core)
order, ok = scaling.order_cpus(range(8), root)
self.assertTrue(ok)
self.assertEqual(order, [0, 1, 2, 3, 4, 5, 6, 7])
def test_sparse_mask_and_interleaved_siblings(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for cpu in range(8):
write_topology(root, cpu, 0, cpu // 2) # siblings (0,1) (2,3) ...
order, ok = scaling.order_cpus({1, 2, 3, 6}, root)
self.assertTrue(ok)
self.assertEqual(order, [1, 2, 6, 3])
def test_fallback_to_numeric_order(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
write_topology(root, 0, 0, 0)
order, ok = scaling.order_cpus({3, 0, 1}, root)
self.assertFalse(ok)
self.assertEqual(order, [0, 1, 3])
class CargoJsonTest(unittest.TestCase):
def test_finds_bench_executable(self):
lines = [
"Compiling something",
json.dumps({"reason": "compiler-artifact", "package_id": "path+file:///x/turbo-tasks#0.1.0",
"target": {"name": "turbo_tasks", "kind": ["lib"]}, "executable": None}),
json.dumps({"reason": "compiler-artifact", "package_id": "path+file:///x/turbo-tasks-backend#0.1.0",
"target": {"name": "mod", "kind": ["bench"]}, "executable": "/t/mod-abc"}),
json.dumps({"reason": "build-finished", "success": True}),
]
self.assertEqual(scaling.find_bench_executable(lines), "/t/mod-abc")
def test_missing_executable(self):
with self.assertRaises(scaling.ScalingError):
scaling.find_bench_executable([json.dumps({"reason": "build-finished"})])
def test_rendered_diagnostics(self):
lines = [
"not json",
json.dumps({"reason": "compiler-message", "message": {"rendered": "error[E0425]: x\n"}}),
json.dumps({"reason": "compiler-message", "message": {"rendered": None}}),
json.dumps({"reason": "build-finished", "success": False}),
]
self.assertEqual(scaling.rendered_diagnostics(lines), ["error[E0425]: x\n"])
class EstimatesTest(unittest.TestCase):
def test_reads_all_variants(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for i, variant in enumerate(VARIANTS):
write_estimates(root, variant, 1000.0 * (i + 1))
estimates = scaling.read_estimates(root)
self.assertEqual(sorted(estimates), sorted(VARIANTS))
warm = estimates["warm/32x2048"]
self.assertEqual(warm["median_ns"], 1000.0)
self.assertEqual(warm["median_ci_lower_ns"], 999.0)
self.assertEqual(warm["mean_ns"], 1000.5)
self.assertEqual(warm["ci_confidence_level"], 0.95)
def test_missing_variant(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for variant in VARIANTS[:3]:
write_estimates(root, variant, 1.0)
with self.assertRaises(scaling.ScalingError):
scaling.read_estimates(root)
def test_invalid_json(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for variant in VARIANTS[:3]:
write_estimates(root, variant, 1.0)
write_estimates(root, VARIANTS[3], 1.0, content="{not json")
with self.assertRaises(scaling.ScalingError):
scaling.read_estimates(root)
def test_implausible_values(self):
good = {
"median_ns": 10.0,
"median_ci_lower_ns": 9.0,
"median_ci_upper_ns": 11.0,
"mean_ns": 10.5,
"ci_confidence_level": 0.95,
}
self.assertIsNone(scaling.validate_estimate(good))
for key, value in [
("median_ns", float("nan")),
("mean_ns", float("inf")),
("median_ci_lower_ns", 0.0),
("median_ci_upper_ns", 9.5),
("ci_confidence_level", 1.0),
]:
self.assertIsNotNone(scaling.validate_estimate(dict(good, **{key: value})), key)
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for variant in VARIANTS[:3]:
write_estimates(root, variant, 1000.0)
write_estimates(root, VARIANTS[3], -5.0)
with self.assertRaises(scaling.ScalingError):
scaling.read_estimates(root)
class StealAndEnvTest(unittest.TestCase):
def test_steal(self):
with tempfile.TemporaryDirectory() as tmp:
stat = pathlib.Path(tmp) / "stat"
stat.write_text("cpu 10 0 10 70 0 0 0 10 0 0\ncpu0 1 0 1 7 0 0 0 1 0 0\n")
before = scaling.read_cpu_ticks(stat)
self.assertEqual(before, (100, 10))
stat.write_text("cpu 20 0 20 140 0 0 0 20 0 0\n")
self.assertAlmostEqual(scaling.steal_pct(before, scaling.read_cpu_ticks(stat)), 10.0)
self.assertIsNone(scaling.read_cpu_ticks(pathlib.Path("/nonexistent/stat")))
self.assertIsNone(scaling.steal_pct(None, (1, 1)))
self.assertIsNone(scaling.steal_pct((100, 10), (200, 5)))
def test_override_env_refused(self):
scaling.check_environment({})
for name in scaling.OVERRIDE_ENV_VARS:
for value in ["4", ""]:
with self.assertRaises(scaling.ScalingError):
scaling.check_environment({name: value})
class CsvTest(unittest.TestCase):
def test_header_and_refusal(self):
with tempfile.TemporaryDirectory() as tmp:
path = pathlib.Path(tmp) / "out.csv"
handle, writer = scaling.open_csv(path, force=False)
writer.writerow({column: "x" for column in scaling.CSV_COLUMNS})
handle.close()
lines = path.read_text().splitlines()
self.assertEqual(lines[0], ",".join(scaling.CSV_COLUMNS))
self.assertEqual(len(lines), 2)
with self.assertRaises(scaling.ScalingError):
scaling.open_csv(path, force=False)
handle, _ = scaling.open_csv(path, force=True)
handle.close()
self.assertEqual(len(path.read_text().splitlines()), 1)
# A fake bench executable: writes estimates for all variants into $CRITERION_HOME, or fails
# with exit code 3 when it may run on more CPUs than $FAKE_MAX_CPUS.
FAKE_BENCH = f"""#!{sys.executable}
import json, os, pathlib, sys
assert sys.argv[1:] == ["--bench", "priority_contention", "--noplot"], sys.argv
if len(os.sched_getaffinity(0)) > int(os.environ["FAKE_MAX_CPUS"]):
sys.exit(3)
root = pathlib.Path(os.environ["CRITERION_HOME"]) / {scaling.GROUP!r}
for variant in {VARIANTS!r}:
path = root / variant / "new" / "estimates.json"
path.parent.mkdir(parents=True)
interval = {{"lower_bound": 9.0, "upper_bound": 11.0, "confidence_level": 0.95}}
path.write_text(json.dumps({{
"median": {{"point_estimate": 10.0, "confidence_interval": interval}},
"mean": {{"point_estimate": 10.0, "confidence_interval": interval}},
}}))
"""
@unittest.skipUnless(hasattr(os, "sched_getaffinity"), "needs Linux")
class MainTest(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.tmp = pathlib.Path(tmp.name)
self.bench = self.tmp / "fake-bench"
self.bench.write_text(FAKE_BENCH)
self.bench.chmod(0o755)
env = mock.patch.dict(os.environ, {"FAKE_MAX_CPUS": "1"})
env.start()
self.addCleanup(env.stop)
for name in scaling.OVERRIDE_ENV_VARS:
os.environ.pop(name, None)
def run_main(self, *args):
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
return scaling.main(list(args))
def test_dry_run_builds_nothing(self):
with mock.patch.object(scaling, "build_bench") as build, mock.patch.object(
scaling, "run_bench"
) as run:
self.assertEqual(self.run_main("--dry-run", "--output", str(self.tmp / "o.csv")), 0)
build.assert_not_called()
run.assert_not_called()
self.assertFalse((self.tmp / "o.csv").exists())
def test_rows_of_completed_runs_survive_a_failure(self):
if len(os.sched_getaffinity(0)) < 2:
self.skipTest("needs 2 CPUs")
output = self.tmp / "o.csv"
with self.assertRaises(scaling.ScalingError):
self.run_main(
"--passes", "1", "--cpu-counts", "1,2", "--output", str(output),
"--bench-binary", str(self.bench),
)
lines = output.read_text().splitlines()
self.assertEqual(len(lines), 1 + len(VARIANTS))
self.assertTrue(all(line.startswith("1,1,") for line in lines[1:]))
def test_full_run_and_refusals(self):
os.environ["FAKE_MAX_CPUS"] = "1000"
output = self.tmp / "o.csv"
args = ["--passes", "2", "--cpu-counts", "1", "--output", str(output),
"--bench-binary", str(self.bench)]
self.assertEqual(self.run_main(*args), 0)
self.assertEqual(len(output.read_text().splitlines()), 1 + 2 * len(VARIANTS))
with self.assertRaises(scaling.ScalingError):
self.run_main(*args)
# --force overwrites the CSV and reuses the work dir with a new run directory
self.assertEqual(self.run_main(*args, "--force"), 0)
self.assertEqual(len(list((self.tmp / "o.csv.d").iterdir())), 2)
with self.assertRaises(scaling.ScalingError):
self.run_main("--cpu-counts", "", "--output", str(self.tmp / "empty.csv"),
"--bench-binary", str(self.bench))
self.assertFalse((self.tmp / "empty.csv").exists())
with self.assertRaises(scaling.ScalingError):
self.run_main("--cpu-counts", "1", "--output", str(self.tmp / "same"),
"--work-dir", str(self.tmp / "same"), "--bench-binary", str(self.bench))
def test_missing_executable(self):
with self.assertRaises(scaling.ScalingError):
self.run_main("--cpu-counts", "1", "--output", str(self.tmp / "o.csv"),
"--bench-binary", str(self.tmp / "missing"))
if __name__ == "__main__":
unittest.main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment