Unverified Commit cbc5ea55 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Exclude blocking spans from trace work accounting (#99685)

## Summary

Stacked above #99254. Allow `blocking = true` on elapsed-duration events
and entered tracing spans, using the same ignore-self-time accounting as
existing `blocking` spans.

Waiting durations otherwise inflate concurrency and CPU totals even
after the tasks yield—for example, a pending fetch emits overlapping
request and attempt durations. Blocking spans now keep their elapsed
ranges and metadata, but contribute no own CPU time, corrected time or
concurrency, and do not affect other work's correction factor. Counted
descendants remain included.

Classify network and Node.js pool-acquisition waits as blocking, while
keeping Node.js startup, initialization and evaluation counted as
external work. Missing/false attributes preserve legacy accounting. No
trace format or viewer payload changes.

Existing name-exempt `thread`/`blocking` spans also retain their full
elapsed end now, rather than dropping it along with self-time.
Attributes are classified at creation, not retroactively via record
updates. External-work durations remain elapsed approximations, not
OS-measured CPU time.

## Verification

- 85 trace-server/trace-utils tests passed, including new reader
regressions for flags, retained ranges, counted children, correction
factors and live cache invalidation; affected Rust producers and native
bindings compile.
- Rebuilt trace server queried over WebSocket: the 40-wait fixture
changes from concurrency 80 to 0 during the wait when its duration
events carry `blocking = true`. Root CPU totals drop from 8002ms to the
2ms entered work; waiting leaf retains its 100ms elapsed range with zero
CPU/corrected duration.
- No new benchmark screenshot: actual raw-reader tests and the
rebuilt-server fixture verify the changed accounting.

<!-- NEXT_JS_LLM -->


<!-- fleet 0c297c2c-8e55-4012-be8a-752ff8175d1b -->
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent e51db393
...@@ -185,12 +185,17 @@ impl FetchClientConfig { ...@@ -185,12 +185,17 @@ impl FetchClientConfig {
} }
let response = { let response = {
let _span = duration_span!("fetch request", url = url_ref); let _span = duration_span!("fetch request", blocking = true, url = url_ref);
let mut attempt = 0; let mut attempt = 0;
loop { loop {
let request = builder.try_clone().expect("request should be cloneable"); let request = builder.try_clone().expect("request should be cloneable");
let result = { let result = {
let _span = duration_span!("fetch attempt", url = url_ref, attempt); let _span = duration_span!(
"fetch attempt",
blocking = true,
url = url_ref,
attempt
);
request.send().await.and_then(|r| r.error_for_status()) request.send().await.and_then(|r| r.error_for_status())
}; };
match result { match result {
...@@ -213,7 +218,7 @@ impl FetchClientConfig { ...@@ -213,7 +218,7 @@ impl FetchClientConfig {
let max_age = parse_cache_control(response.headers()); let max_age = parse_cache_control(response.headers());
let body = { let body = {
let _span = duration_span!("fetch response", url = url_ref); let _span = duration_span!("fetch response", blocking = true, url = url_ref);
response.bytes().await? response.bytes().await?
} }
.to_vec(); .to_vec();
......
...@@ -30,6 +30,14 @@ impl<F: FnOnce(u64)> Drop for DurationSpanGuard<F> { ...@@ -30,6 +30,14 @@ impl<F: FnOnce(u64)> Drop for DurationSpanGuard<F> {
/// It will trace a duration and not the time the cpu is doing actual work. This /// It will trace a duration and not the time the cpu is doing actual work. This
/// way it can be used to trace non-cpu-time or time that is spend in other /// way it can be used to trace non-cpu-time or time that is spend in other
/// processes. /// processes.
///
/// Set `blocking = true` for waiting (for example, network I/O or a pool slot).
/// The trace server preserves the elapsed range but excludes its own duration
/// from CPU time, concurrency and corrected-time accounting, like an entered
/// span with the same attribute. Counted child work is unaffected.
/// Use `blocking = false` for work performed by another thread or process;
/// omitting the attribute also retains this counted behavior for compatibility.
/// Such durations approximate external work, not measured process CPU time.
#[macro_export] #[macro_export]
macro_rules! duration_span { macro_rules! duration_span {
($name:literal) => { ($name:literal) => {
......
...@@ -557,7 +557,7 @@ async fn pull_operation<T: EvaluateContext>( ...@@ -557,7 +557,7 @@ async fn pull_operation<T: EvaluateContext>(
evaluate_context: &T, evaluate_context: &T,
state: &mut T::State, state: &mut T::State,
) -> Result<Option<String>> { ) -> Result<Option<String>> {
let _guard = duration_span!("Node.js evaluation"); let _guard = duration_span!("Node.js evaluation", blocking = false);
loop { loop {
let recv_result = operation.recv().await; let recv_result = operation.recv().await;
......
...@@ -311,7 +311,7 @@ impl NodeJsPoolProcess { ...@@ -311,7 +311,7 @@ impl NodeJsPoolProcess {
shared_stderr: SharedOutputSet, shared_stderr: SharedOutputSet,
debug: bool, debug: bool,
) -> Result<Self> { ) -> Result<Self> {
let guard = duration_span!("Node.js process startup"); let guard = duration_span!("Node.js process startup", blocking = false);
let listener = TcpListener::bind("127.0.0.1:0") let listener = TcpListener::bind("127.0.0.1:0")
.await .await
.context("binding to a port")?; .context("binding to a port")?;
...@@ -437,7 +437,7 @@ impl NodeJsPoolProcess { ...@@ -437,7 +437,7 @@ impl NodeJsPoolProcess {
drop(guard); drop(guard);
let guard = duration_span!("Node.js initialization"); let guard = duration_span!("Node.js initialization", blocking = false);
let ready_signal = process.recv().await?; let ready_signal = process.recv().await?;
if !ready_signal.is_empty() { if !ready_signal.is_empty() {
...@@ -739,7 +739,7 @@ impl EvaluateOperation for ChildProcessPool { ...@@ -739,7 +739,7 @@ impl EvaluateOperation for ChildProcessPool {
// Acquire a running process (handles concurrency limits, boots up the process) // Acquire a running process (handles concurrency limits, boots up the process)
let operation = { let operation = {
let _guard = duration_span!("Node.js operation"); let _guard = duration_span!("Node.js operation", blocking = true);
let (process, permits) = self.acquire_process().await?; let (process, permits) = self.acquire_process().await?;
ChildProcessOperation { ChildProcessOperation {
process: Some(process), process: Some(process),
......
...@@ -220,7 +220,7 @@ impl WorkerThreadPool { ...@@ -220,7 +220,7 @@ impl WorkerThreadPool {
impl EvaluateOperation for WorkerThreadPool { impl EvaluateOperation for WorkerThreadPool {
async fn operation(&self) -> Result<Box<dyn Operation>> { async fn operation(&self) -> Result<Box<dyn Operation>> {
let operation = { let operation = {
let _guard = duration_span!("Node.js operation"); let _guard = duration_span!("Node.js operation", blocking = true);
let worker_options = self.worker_options.clone(); let worker_options = self.worker_options.clone();
let task_id = OPERATION_TASK_ID.fetch_add(1, Ordering::Release); let task_id = OPERATION_TASK_ID.fetch_add(1, Ordering::Release);
......
use std::sync::Arc; use std::sync::Arc;
use turbopack_trace_utils::tracing::TraceRow; use turbopack_trace_utils::tracing::{TraceRow, TraceValue};
use crate::{ use crate::{
QueryOptions, SortMode, query_spans, QueryOptions, SortMode, query_spans,
...@@ -76,3 +76,312 @@ fn reads_active_workers_and_exposes_them_in_span_queries() { ...@@ -76,3 +76,312 @@ fn reads_active_workers_and_exposes_them_in_span_queries() {
assert_eq!(result.spans[0].memory_samples, vec![(1000, 1234, 7, 2)]); assert_eq!(result.spans[0].memory_samples, vec![(1000, 1234, 7, 2)]);
} }
} }
fn ingest(format: &mut TurbopackFormat, rows: &[TraceRow<'_>]) {
let bytes: Vec<_> = rows
.iter()
.flat_map(|row| postcard::to_stdvec(row).unwrap())
.collect();
assert_eq!(
format
.read(&bytes, &mut TurbopackFormat::create_reused())
.unwrap(),
bytes.len()
);
}
fn start(
id: u64,
parent: Option<u64>,
ts: u64,
name: &'static str,
blocking: Option<bool>,
) -> TraceRow<'static> {
TraceRow::Start {
ts,
id,
parent,
name: name.into(),
target: "test".into(),
values: blocking
.map(|b| vec![("blocking".into(), TraceValue::Bool(b))])
.unwrap_or_default(),
}
}
fn duration_event(
ts: u64,
duration: u64,
name: &'static str,
blocking: Option<bool>,
) -> TraceRow<'static> {
let mut values = vec![
("name".into(), TraceValue::String(name.into())),
("duration".into(), TraceValue::UInt(duration)),
];
if let Some(blocking) = blocking {
values.push(("blocking".into(), TraceValue::Bool(blocking)));
}
TraceRow::Event {
ts,
parent: Some(1),
values,
}
}
#[test]
fn blocking_duration_events_keep_ranges_without_inflating_work() {
let store = Arc::new(StoreContainer::new());
let mut format = TurbopackFormat::new(store.clone());
ingest(
&mut format,
&[
start(1, None, 0, "root", None),
TraceRow::Enter {
ts: 0,
id: 1,
thread_id: 1,
},
TraceRow::Exit {
ts: 10,
id: 1,
thread_id: 1,
},
duration_event(100, 90, "fetch request", Some(true)),
duration_event(100, 90, "fetch attempt", Some(true)),
TraceRow::Enter {
ts: 100,
id: 1,
thread_id: 1,
},
TraceRow::Exit {
ts: 110,
id: 1,
thread_id: 1,
},
TraceRow::End { ts: 110, id: 1 },
],
);
let store = store.read();
let root = store.root_spans().next().unwrap();
assert_eq!(root.total_time(), Timestamp::from_micros(20));
assert_eq!(root.corrected_total_time(), Timestamp::from_micros(20));
assert_eq!(root.end(), Timestamp::from_micros(110));
assert_eq!(
store
.concurrency_samples_for_range(Timestamp::from_micros(10), Timestamp::from_micros(100)),
vec![0.0; 200]
);
let mut count = 0;
for event in root.children() {
count += 1;
assert_eq!(event.start(), Timestamp::from_micros(10));
assert_eq!(event.end(), Timestamp::from_micros(100));
assert_eq!(event.total_time(), Timestamp::ZERO);
assert_eq!(event.corrected_total_time(), Timestamp::ZERO);
assert!(event.is_complete());
assert!(
event
.args()
.any(|(key, value)| key.as_str() == "blocking" && value.as_str() == "true")
);
}
assert_eq!(count, 2);
}
#[test]
fn blocking_events_do_not_change_counted_work_correction() {
let store = Arc::new(StoreContainer::new());
let mut format = TurbopackFormat::new(store.clone());
ingest(
&mut format,
&[
start(1, None, 0, "root", None),
TraceRow::Enter {
ts: 0,
id: 1,
thread_id: 1,
},
duration_event(100, 100, "waiting", Some(true)),
TraceRow::Exit {
ts: 100,
id: 1,
thread_id: 1,
},
TraceRow::End { ts: 100, id: 1 },
],
);
let store = store.read();
let root = store.root_spans().next().unwrap();
assert_eq!(root.self_time(), Timestamp::from_micros(100));
assert_eq!(root.corrected_self_time(), Timestamp::from_micros(100));
assert_eq!(root.total_time(), Timestamp::from_micros(100));
assert_eq!(root.corrected_total_time(), Timestamp::from_micros(100));
assert_eq!(
store.concurrency_samples_for_range(Timestamp::ZERO, Timestamp::from_micros(100)),
vec![1.0; 200]
);
}
#[test]
fn non_blocking_and_legacy_duration_events_still_count() {
let store = Arc::new(StoreContainer::new());
let mut format = TurbopackFormat::new(store.clone());
ingest(
&mut format,
&[
start(1, None, 0, "root", None),
TraceRow::Enter {
ts: 0,
id: 1,
thread_id: 1,
},
duration_event(80, 50, "external work", Some(false)),
duration_event(80, 50, "legacy external work", None),
TraceRow::Exit {
ts: 100,
id: 1,
thread_id: 1,
},
TraceRow::End { ts: 100, id: 1 },
],
);
let store = store.read();
let root = store.root_spans().next().unwrap();
assert_eq!(root.total_time(), Timestamp::from_micros(200));
assert!(root.corrected_self_time() < root.self_time());
assert_eq!(
store.concurrency_samples_for_range(Timestamp::from_micros(30), Timestamp::from_micros(80)),
vec![3.0; 200]
);
for event in root.children() {
assert_eq!(event.total_time(), Timestamp::from_micros(50));
assert!(event.corrected_total_time() > Timestamp::ZERO);
}
}
#[test]
fn blocking_entered_spans_ignore_own_work_but_count_children() {
for (name, blocking, ignored) in [
("flagged", Some(true), true),
("regular", Some(false), false),
("legacy", None, false),
("blocking", Some(false), true),
("thread", None, true),
] {
let store = Arc::new(StoreContainer::new());
let mut format = TurbopackFormat::new(store.clone());
ingest(
&mut format,
&[
start(1, None, 0, name, blocking),
TraceRow::Enter {
ts: 0,
id: 1,
thread_id: 1,
},
start(2, Some(1), 10, "child work", None),
TraceRow::Enter {
ts: 10,
id: 2,
thread_id: 1,
},
TraceRow::Exit {
ts: 20,
id: 2,
thread_id: 1,
},
TraceRow::End { ts: 20, id: 2 },
TraceRow::Exit {
ts: 30,
id: 1,
thread_id: 1,
},
TraceRow::End { ts: 30, id: 1 },
],
);
let store = store.read();
let parent = store.root_spans().next().unwrap();
assert_eq!(parent.end(), Timestamp::from_micros(30), "{name}");
let own_time = if ignored { 0 } else { 20 };
assert_eq!(
parent.self_time(),
Timestamp::from_micros(own_time),
"{name}"
);
assert_eq!(
parent.total_time(),
Timestamp::from_micros(own_time + 10),
"{name}"
);
assert_eq!(parent.corrected_total_time(), parent.total_time(), "{name}");
assert_eq!(
store.concurrency_samples_for_range(Timestamp::ZERO, Timestamp::from_micros(10)),
vec![if ignored { 0.0 } else { 1.0 }; 200],
"{name}"
);
assert_eq!(
store.concurrency_samples_for_range(
Timestamp::from_micros(10),
Timestamp::from_micros(20)
),
vec![1.0; 200],
"{name}"
);
}
}
#[test]
fn blocking_elapsed_ranges_invalidate_cached_ends() {
let store = Arc::new(StoreContainer::new());
let mut format = TurbopackFormat::new(store.clone());
ingest(
&mut format,
&[
start(1, None, 0, "parent", None),
start(2, Some(1), 0, "waiting", Some(true)),
TraceRow::Enter {
ts: 0,
id: 2,
thread_id: 1,
},
TraceRow::Exit {
ts: 10,
id: 2,
thread_id: 1,
},
],
);
{
let store = store.read();
let parent = store.root_spans().next().unwrap();
let span = parent.children().next().unwrap();
assert_eq!(span.end(), Timestamp::from_micros(10));
assert_eq!(parent.end(), Timestamp::from_micros(10));
assert_eq!(span.corrected_total_time(), Timestamp::ZERO);
}
ingest(
&mut format,
&[
TraceRow::Enter {
ts: 20,
id: 2,
thread_id: 1,
},
TraceRow::Exit {
ts: 30,
id: 2,
thread_id: 1,
},
TraceRow::End { ts: 30, id: 2 },
],
);
let store = store.read();
let parent = store.root_spans().next().unwrap();
let span = parent.children().next().unwrap();
assert_eq!(span.end(), Timestamp::from_micros(30));
assert_eq!(parent.end(), Timestamp::from_micros(30));
assert_eq!(span.total_time(), Timestamp::ZERO);
assert_eq!(span.corrected_total_time(), Timestamp::ZERO);
assert!(span.is_complete());
}
...@@ -63,6 +63,9 @@ pub struct SpanTotals { ...@@ -63,6 +63,9 @@ pub struct SpanTotals {
#[derive(Default)] #[derive(Default)]
pub struct SpanTimeData { pub struct SpanTimeData {
// These values won't change after creation: // These values won't change after creation:
/// Excludes own work from CPU, concurrency and corrected-time accounting,
/// but not the elapsed range or counted descendants. Set for `blocking =
/// true` attributes and the existing `thread`/`blocking` wrapper names.
pub ignore_self_time: bool, pub ignore_self_time: bool,
// This might change during writing: // This might change during writing:
......
...@@ -248,7 +248,7 @@ impl<'a> SpanRef<'a> { ...@@ -248,7 +248,7 @@ impl<'a> SpanRef<'a> {
None None
}) })
.sum(); .sum();
if self.children().next().is_none() { if !self.time_data().ignore_self_time && self.children().next().is_none() {
self_time = max(self_time, Timestamp::from_value(1)); self_time = max(self_time, Timestamp::from_value(1));
} }
self_time self_time
......
...@@ -113,7 +113,11 @@ impl Store { ...@@ -113,7 +113,11 @@ impl Store {
outdated_spans: &mut FxHashSet<SpanIndex>, outdated_spans: &mut FxHashSet<SpanIndex>,
) -> SpanIndex { ) -> SpanIndex {
let id = SpanIndex::new(self.spans.len()).unwrap(); let id = SpanIndex::new(self.spans.len()).unwrap();
let ignore_self_time = &name == "thread" || &name == "blocking"; let ignore_self_time = &name == "thread"
|| &name == "blocking"
|| args
.iter()
.any(|(key, value)| key.as_str() == "blocking" && value.as_str() == "true");
self.spans.push(Span { self.spans.push(Span {
parent, parent,
depth: 0, depth: 0,
...@@ -216,12 +220,14 @@ impl Store { ...@@ -216,12 +220,14 @@ impl Store {
let event = SpanEvent::self_time(start, end); let event = SpanEvent::self_time(start, end);
let span = &mut self.spans[span_index.get()]; let span = &mut self.spans[span_index.get()];
let time_data = &mut span.time_data; let time_data = &mut span.time_data;
// Waiting intervals still extend the elapsed range, but contribute no
// work and must not affect other intervals' concurrency correction.
outdated_spans.insert(span_index);
time_data.self_end = max(time_data.self_end, end);
if time_data.ignore_self_time { if time_data.ignore_self_time {
return; return;
} }
outdated_spans.insert(span_index);
time_data.self_time += end - start; time_data.self_time += end - start;
time_data.self_end = max(time_data.self_end, end);
if let Some(event) = event { if let Some(event) = event {
span.events.push(event); span.events.push(event);
self.insert_self_time(start, end, span_index, outdated_spans); self.insert_self_time(start, end, span_index, outdated_spans);
...@@ -235,6 +241,13 @@ impl Store { ...@@ -235,6 +241,13 @@ impl Store {
total_time: Timestamp, total_time: Timestamp,
outdated_spans: &mut FxHashSet<SpanIndex>, outdated_spans: &mut FxHashSet<SpanIndex>,
) { ) {
if self.spans[span_index.get()].time_data.ignore_self_time {
let span = &mut self.spans[span_index.get()];
span.start = start_time;
span.time_data.self_end = start_time + total_time;
outdated_spans.insert(span_index);
return;
}
let span = SpanRef { let span = SpanRef {
span: &self.spans[span_index.get()], span: &self.spans[span_index.get()],
store: self, store: self,
...@@ -535,6 +548,54 @@ impl Store { ...@@ -535,6 +548,54 @@ impl Store {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn blocking_total_time_preserves_ranges_and_counted_children() {
let mut store = Store::new();
let mut outdated = FxHashSet::default();
let parent = store.add_span(
None,
Timestamp::from_micros(5),
rcstr!("test"),
rcstr!("waiting"),
vec![(rcstr!("blocking"), rcstr!("true"))].into(),
&mut outdated,
);
let child = store.add_span(
Some(parent),
Timestamp::from_micros(10),
rcstr!("test"),
rcstr!("external work"),
SpanArgs::new(),
&mut outdated,
);
store.set_total_time(
child,
Timestamp::from_micros(10),
Timestamp::from_micros(10),
&mut outdated,
);
store.set_total_time(
parent,
Timestamp::from_micros(5),
Timestamp::from_micros(25),
&mut outdated,
);
store.invalidate_outdated_spans(&outdated);
let parent = store.root_spans().next().unwrap();
assert_eq!(parent.start(), Timestamp::from_micros(5));
assert_eq!(parent.end(), Timestamp::from_micros(30));
assert_eq!(parent.self_time(), Timestamp::ZERO);
assert_eq!(parent.total_time(), Timestamp::from_micros(10));
assert_eq!(parent.corrected_total_time(), Timestamp::from_micros(10));
assert_eq!(
store.concurrency_samples_for_range(
Timestamp::from_micros(5),
Timestamp::from_micros(10)
),
vec![0.0; 200]
);
}
#[test] #[test]
fn concurrency_samples_are_empty_without_a_self_time_tree() { fn concurrency_samples_are_empty_without_a_self_time_tree() {
let mut store = Store::new(); let mut store = Store::new();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment