Unverified Commit 73de88f3 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-tasks-backend: add realistic variants to priority_contention bench

Add `realistic_warm` and `realistic_initial` variants (same node budgets as
the uniform stress variants) that model the invalidation contention of a warm
build with some changed files:

- partial invalidation with cutoff: one input state per column, 1/16 of the
  inputs change per warm iteration, and imports are read through a rarely
  changing summary, so most re-executed nodes keep their value
- irregular shape: 0-5 hash-chosen imports per node
- hierarchical fan-in: 16 differently sized groups, each read by its own root
  task, with imports mostly within a group plus a shared group
- a fixed amount of per-node work

Every warm iteration is checked against an incrementally updated reference.
`PRIORITY_CONTENTION_DIAGNOSTICS=1` runs an untimed check that records actual
executions and produced values and asserts that the invalidation is partial and
cut off.

The scaling script now expects exactly the 8 variants of the group.
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 93e77030
...@@ -9,6 +9,7 @@ pub(crate) mod collectibles; ...@@ -9,6 +9,7 @@ pub(crate) mod collectibles;
pub(crate) mod gc; pub(crate) mod gc;
pub(crate) mod overhead; pub(crate) mod overhead;
pub(crate) mod priority_contention; pub(crate) mod priority_contention;
pub(crate) mod priority_contention_realistic;
pub(crate) mod scope_stress; pub(crate) mod scope_stress;
pub(crate) mod state_fan_out; pub(crate) mod state_fan_out;
pub(crate) mod stress; pub(crate) mod stress;
......
...@@ -13,6 +13,12 @@ ...@@ -13,6 +13,12 @@
//! - `initial`: each iteration builds the graph from scratch in a fresh `TurboTasks` instance, so //! - `initial`: each iteration builds the graph from scratch in a fresh `TurboTasks` instance, so
//! all tasks are scheduled with the initial priority, like in an initial build. //! all tasks are scheduled with the initial priority, like in an initial build.
//! //!
//! These two are stress variants: every warm iteration changes every node, and the bulk of the
//! aggregation updates go to one root task. The `realistic_warm` and `realistic_initial` variants
//! (see `priority_contention_realistic.rs`) use the same node budgets with partial invalidation and
//! cutoff, an irregular shape, hierarchical fan-in and per-task work, to model the invalidation
//! contention of a warm build with some changed files.
//!
//! ```bash //! ```bash
//! cargo bench -p turbo-tasks-backend --bench mod -- priority_contention //! cargo bench -p turbo-tasks-backend --bench mod -- priority_contention
//! ``` //! ```
...@@ -27,8 +33,10 @@ use criterion::{BenchmarkId, Criterion, Throughput}; ...@@ -27,8 +33,10 @@ use criterion::{BenchmarkId, Criterion, Throughput};
use turbo_tasks::{OperationVc, ResolvedVc, State, TryJoinIterExt, TurboTasks, Vc}; use turbo_tasks::{OperationVc, ResolvedVc, State, TryJoinIterExt, TurboTasks, Vc};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use crate::priority_contention_realistic;
/// `(depth, width)` of the benchmarked graphs. /// `(depth, width)` of the benchmarked graphs.
const SIZES: [(u32, u32); 2] = [(32, 2048), (128, 512)]; pub(crate) const SIZES: [(u32, u32); 2] = [(32, 2048), (128, 512)];
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
struct Input(State<u64>); struct Input(State<u64>);
...@@ -45,7 +53,7 @@ fn second_parent(index: u32, width: u32) -> u32 { ...@@ -45,7 +53,7 @@ fn second_parent(index: u32, width: u32) -> u32 {
/// Combines the values of a node's parents. Mixing (instead of e.g. adding) keeps a change of the /// Combines the values of a node's parents. Mixing (instead of e.g. adding) keeps a change of the
/// state from cancelling out in deep graphs, so every node changes whenever the state changes. /// state from cancelling out in deep graphs, so every node changes whenever the state changes.
fn mix(a: u64, b: u64) -> u64 { pub(crate) fn mix(a: u64, b: u64) -> u64 {
(a ^ b.rotate_left(29)).wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ a.rotate_right(17) (a ^ b.rotate_left(29)).wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ a.rotate_right(17)
} }
...@@ -116,7 +124,7 @@ fn assert_every_node_changes(previous: u64, value: u64, depth: u32, width: u32) ...@@ -116,7 +124,7 @@ fn assert_every_node_changes(previous: u64, value: u64, depth: u32, width: u32)
} }
} }
fn create_tt() -> Arc<TurboTasks<TurboTasksBackend>> { pub(crate) fn create_tt() -> Arc<TurboTasks<TurboTasksBackend>> {
TurboTasks::new(TurboTasksBackend::new( TurboTasks::new(TurboTasksBackend::new(
BackendOptions { BackendOptions {
storage_mode: None, storage_mode: None,
...@@ -157,6 +165,10 @@ async fn set_and_read( ...@@ -157,6 +165,10 @@ async fn set_and_read(
/// To capture how this benchmark scales with the number of CPUs on a machine, run /// To capture how this benchmark scales with the number of CPUs on a machine, run
/// `priority_contention_scaling.py` next to this file; it writes the results to a CSV file. /// `priority_contention_scaling.py` next to this file; it writes the results to a CSV file.
pub fn priority_contention(c: &mut Criterion) { pub fn priority_contention(c: &mut Criterion) {
if priority_contention_realistic::diagnostics_enabled() {
priority_contention_realistic::run_diagnostics();
return;
}
let mut group = c.benchmark_group("turbo_tasks_backend_priority_contention"); let mut group = c.benchmark_group("turbo_tasks_backend_priority_contention");
group.sample_size(20); group.sample_size(20);
group.measurement_time(Duration::from_secs(10)); group.measurement_time(Duration::from_secs(10));
...@@ -222,5 +234,7 @@ pub fn priority_contention(c: &mut Criterion) { ...@@ -222,5 +234,7 @@ pub fn priority_contention(c: &mut Criterion) {
}); });
} }
priority_contention_realistic::bench(&mut group);
group.finish(); group.finish();
} }
...@@ -57,7 +57,12 @@ import sys ...@@ -57,7 +57,12 @@ import sys
from typing import Iterable, Optional, Sequence from typing import Iterable, Optional, Sequence
GROUP = "turbo_tasks_backend_priority_contention" GROUP = "turbo_tasks_backend_priority_contention"
EXPECTED_VARIANTS = 4 # The variants of the benchmark group, as `<kind>/<size>`.
EXPECTED_VARIANTS = tuple(
f"{kind}/{size}"
for size in ("32x2048", "128x512")
for kind in ("warm", "initial", "realistic_warm", "realistic_initial")
)
OVERRIDE_ENV_VARS = ("TOKIO_WORKER_THREADS", "TURBO_TASKS_AVAILABLE_PARALLELISM") OVERRIDE_ENV_VARS = ("TOKIO_WORKER_THREADS", "TURBO_TASKS_AVAILABLE_PARALLELISM")
CSV_COLUMNS = [ CSV_COLUMNS = [
"pass", "pass",
...@@ -199,9 +204,13 @@ def read_estimates(criterion_home: pathlib.Path) -> dict[str, dict]: ...@@ -199,9 +204,13 @@ def read_estimates(criterion_home: pathlib.Path) -> dict[str, dict]:
"""Reads `<group>/<kind>/<size>/new/estimates.json` for every variant of the group.""" """Reads `<group>/<kind>/<size>/new/estimates.json` for every variant of the group."""
group_dir = criterion_home / GROUP group_dir = criterion_home / GROUP
files = sorted(group_dir.glob("*/*/new/estimates.json")) files = sorted(group_dir.glob("*/*/new/estimates.json"))
if len(files) != EXPECTED_VARIANTS: found = {"/".join(path.relative_to(group_dir).parts[:2]) for path in files}
if found != set(EXPECTED_VARIANTS):
missing = sorted(set(EXPECTED_VARIANTS) - found)
unexpected = sorted(found - set(EXPECTED_VARIANTS))
raise ScalingError( raise ScalingError(
f"expected {EXPECTED_VARIANTS} variants in {group_dir}, found {len(files)}" f"expected the {len(EXPECTED_VARIANTS)} variants {', '.join(EXPECTED_VARIANTS)} in "
f"{group_dir}; missing: {missing or 'none'}, unexpected: {unexpected or 'none'}"
) )
estimates = {} estimates = {}
for path in files: for path in files:
......
...@@ -34,7 +34,16 @@ def write_estimates(root: pathlib.Path, variant: str, median: float, content=Non ...@@ -34,7 +34,16 @@ def write_estimates(root: pathlib.Path, variant: str, median: float, content=Non
path.write_text(content) path.write_text(content)
VARIANTS = ["warm/32x2048", "initial/32x2048", "warm/128x512", "initial/128x512"] VARIANTS = [
"warm/32x2048",
"initial/32x2048",
"realistic_warm/32x2048",
"realistic_initial/32x2048",
"warm/128x512",
"initial/128x512",
"realistic_warm/128x512",
"realistic_initial/128x512",
]
class CpuCountsTest(unittest.TestCase): class CpuCountsTest(unittest.TestCase):
...@@ -119,21 +128,33 @@ class EstimatesTest(unittest.TestCase): ...@@ -119,21 +128,33 @@ class EstimatesTest(unittest.TestCase):
self.assertEqual(warm["mean_ns"], 1000.5) self.assertEqual(warm["mean_ns"], 1000.5)
self.assertEqual(warm["ci_confidence_level"], 0.95) self.assertEqual(warm["ci_confidence_level"], 0.95)
def test_expected_variants(self):
self.assertEqual(sorted(scaling.EXPECTED_VARIANTS), sorted(VARIANTS))
def test_missing_variant(self): def test_missing_variant(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp) root = pathlib.Path(tmp)
for variant in VARIANTS[:3]: for variant in VARIANTS[:-1]:
write_estimates(root, variant, 1.0) write_estimates(root, variant, 1.0)
with self.assertRaises(scaling.ScalingError): with self.assertRaisesRegex(scaling.ScalingError, "missing: \\['realistic_initial/128x512'\\]"):
scaling.read_estimates(root)
def test_unexpected_variant(self):
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for variant in VARIANTS:
write_estimates(root, variant, 1.0)
write_estimates(root, "other/32x2048", 1.0)
with self.assertRaisesRegex(scaling.ScalingError, "unexpected: \\['other/32x2048'\\]"):
scaling.read_estimates(root) scaling.read_estimates(root)
def test_invalid_json(self): def test_invalid_json(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp) root = pathlib.Path(tmp)
for variant in VARIANTS[:3]: for variant in VARIANTS[:-1]:
write_estimates(root, variant, 1.0) write_estimates(root, variant, 1.0)
write_estimates(root, VARIANTS[3], 1.0, content="{not json") write_estimates(root, VARIANTS[-1], 1.0, content="{not json")
with self.assertRaises(scaling.ScalingError): with self.assertRaisesRegex(scaling.ScalingError, "invalid estimates"):
scaling.read_estimates(root) scaling.read_estimates(root)
def test_implausible_values(self): def test_implausible_values(self):
...@@ -155,10 +176,10 @@ class EstimatesTest(unittest.TestCase): ...@@ -155,10 +176,10 @@ class EstimatesTest(unittest.TestCase):
self.assertIsNotNone(scaling.validate_estimate(dict(good, **{key: value})), key) self.assertIsNotNone(scaling.validate_estimate(dict(good, **{key: value})), key)
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp) root = pathlib.Path(tmp)
for variant in VARIANTS[:3]: for variant in VARIANTS[:-1]:
write_estimates(root, variant, 1000.0) write_estimates(root, variant, 1000.0)
write_estimates(root, VARIANTS[3], -5.0) write_estimates(root, VARIANTS[-1], -5.0)
with self.assertRaises(scaling.ScalingError): with self.assertRaisesRegex(scaling.ScalingError, "invalid estimates"):
scaling.read_estimates(root) scaling.read_estimates(root)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment