Unverified Commit 8dee7acb authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack-trace-server] optimize loading (#93264)

Land a few optimizations to the trace server

* Change `SpanEvent` so it is 32 bytes instead of 40 bytes by triggering a `niche` optimization
* Change `args` and `events` to be a `smallvec` with inline size 1
  * for `args` it is size <=1 ~31% of the time
  * for `events` it is size <=1 69% of the time
* Compute min/max timestamps in a single pass instead of 2 when inserting into the selftimetree
* Bundle dynamically computed 'total' fields behind a single OnceLock
   * saves 40 bytes per span due to `Oncelock` overheads
* Inline SpanTimeData and SpanNames into Span
   * We get little benefit from deferring the allocations and by inlining we save time and improve memory locality.
     * post load SpanTimeData is allocated for 94% of spans, but after loading `trace.nextjs.org` it is 100%
     * post load SpanNames is allocated for 0% of spans, but after loading it is 96.2% of spans
* Remove the `inner` `OnceLocks` from `SpanNames` we can just allocate these all together


Measuring with one 10gb trace file I see loading times progress from 75.7s (33G of ram)  to 60.5s (19.5G of ram).  With loading times hitting >200mb/s occasionally
parent 470cf12a
......@@ -10809,6 +10809,7 @@ dependencies = [
"rustc-hash 2.1.1",
"serde",
"serde_json",
"smallvec",
"tungstenite 0.21.0",
"turbo-rcstr",
"turbo-tasks-malloc",
......
......@@ -28,6 +28,7 @@ rustc-demangle = "0.1"
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
smallvec = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-tasks-malloc = { workspace = true, default-features = false }
tungstenite = { version = "0.21.0" }
......
use std::{cell::UnsafeCell, ops::Deref, sync::Once};
use smallvec::SmallVec;
/// Backing storage for `LazySortedVec`. Inlines a single element so that
/// the common case (e.g. a leaf span with one self-time event) does not
/// pay a heap allocation. With one inline slot plus the workspace's
/// `union` smallvec feature, this is the same size as a `Vec`.
type Inner<T> = SmallVec<[T; 1]>;
pub struct LazySortedVec<T> {
vec: UnsafeCell<Vec<T>>,
vec: UnsafeCell<Inner<T>>,
once: Once,
}
......@@ -11,7 +19,7 @@ unsafe impl<T> Sync for LazySortedVec<T> where T: Sync {}
impl<T> LazySortedVec<T> {
pub fn new() -> Self {
Self {
vec: UnsafeCell::new(Vec::new()),
vec: UnsafeCell::new(SmallVec::new()),
once: Once::new(),
}
}
......@@ -21,8 +29,8 @@ impl<T> LazySortedVec<T> {
self.vec.get_mut().push(value);
}
pub fn retain_unordered(&mut self, f: impl FnMut(&T) -> bool) {
self.vec.get_mut().retain(f);
pub fn retain_unordered(&mut self, mut f: impl FnMut(&T) -> bool) {
self.vec.get_mut().retain(|t| f(t));
}
pub fn iter_mut_unordered(&mut self) -> std::slice::IterMut<'_, T> {
......@@ -31,7 +39,7 @@ impl<T> LazySortedVec<T> {
}
impl<T: Ord> Deref for LazySortedVec<T> {
type Target = Vec<T>;
type Target = [T];
fn deref(&self) -> &Self::Target {
let ptr = self.vec.get();
......@@ -44,7 +52,7 @@ impl<T: Ord> Deref for LazySortedVec<T> {
// SAFETY: Returning this reference is safe because the lifetime guarantees that there is no
// `&mut self` that could cause a simultaneous access to the `vec`, and the `Once`
// guarantees that the sorting is complete before we return the reference.
unsafe { &*ptr }
unsafe { &*ptr }.as_slice()
}
}
......@@ -54,10 +62,19 @@ impl<T> Default for LazySortedVec<T> {
}
}
impl<T> From<Inner<T>> for LazySortedVec<T> {
fn from(vec: Inner<T>) -> Self {
Self {
vec: UnsafeCell::new(vec),
once: Once::new(),
}
}
}
impl<T> From<Vec<T>> for LazySortedVec<T> {
fn from(vec: Vec<T>) -> Self {
Self {
vec: UnsafeCell::new(vec),
vec: UnsafeCell::new(SmallVec::from_vec(vec)),
once: Once::new(),
}
}
......
......@@ -7,7 +7,12 @@ use rustc_hash::{FxHashMap, FxHashSet};
use turbo_rcstr::{RcStr, rcstr};
use super::TraceFormat;
use crate::{FxIndexMap, span::SpanIndex, store_container::StoreContainer, timestamp::Timestamp};
use crate::{
FxIndexMap,
span::{SpanArgs, SpanIndex},
store_container::StoreContainer,
timestamp::Timestamp,
};
#[derive(Debug, Clone, Copy)]
struct TraceNode {
......@@ -276,7 +281,7 @@ impl TraceFormat for HeaptrackFormat {
self.last_timestamp,
RcStr::default(),
rcstr!("recursion"),
Vec::new(),
SpanArgs::new(),
&mut outdated_spans,
);
store.complete_span(span_index);
......@@ -321,7 +326,7 @@ impl TraceFormat for HeaptrackFormat {
} else {
"unknown".to_string()
};
let mut args = Vec::new();
let mut args = SpanArgs::new();
for Frame {
function_index,
file_index,
......
......@@ -90,8 +90,12 @@ impl<T> SelfTimeTree<T> {
fn distribute_entries(&mut self) {
if self.children.is_none() {
let start = self.entries.iter().min_by_key(|e| e.start).unwrap().start;
let end = self.entries.iter().max_by_key(|e| e.end).unwrap().end;
let (start, end) = self
.entries
.iter()
.fold((Timestamp::MAX, Timestamp::ZERO), |(lo, hi), e| {
(lo.min(e.start), hi.max(e.end))
});
let middle = (start + end) / 2;
// Pre-allocate half the split threshold: after distributing, each child
// typically receives ~SPLIT_COUNT / 2 entries.
......@@ -235,7 +239,6 @@ impl<T> SelfTimeTree<T> {
}
pub fn lookup_range_corrected_time(&self, start: Timestamp, end: Timestamp) -> Timestamp {
let mut factor_times_1000 = 0u64;
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum Change {
Start,
......@@ -253,7 +256,18 @@ impl<T> SelfTimeTree<T> {
changes.push((e, Change::End));
}
});
// Fast path: every overlapping interval fully contains `[start, end]`, so the
// count is constant over the whole window. Skip the sort and the sweep.
if changes.is_empty() {
if current_count == 0 {
return Timestamp::ZERO;
}
return Timestamp::from_value(*(end - start) / current_count);
}
changes.sort_unstable();
let mut factor_times_1000 = 0u64;
let mut current_ts = start;
for (ts, change) in changes {
if current_ts < ts {
......@@ -443,4 +457,53 @@ mod tests {
print_tree(&tree, 0);
assert_balanced(&tree);
}
#[test]
fn test_corrected_time_no_overlap() {
// No intervals at all — corrected time of an empty range is zero.
let tree: SelfTimeTree<u32> = SelfTimeTree::new();
let r = tree
.lookup_range_corrected_time(Timestamp::from_micros(0), Timestamp::from_micros(100));
assert_eq!(r, Timestamp::ZERO);
}
#[test]
fn test_corrected_time_single_interval() {
// One interval matching the query window exactly: correction factor 1, returns full
// duration.
let mut tree = SelfTimeTree::new();
tree.insert(Timestamp::from_micros(0), Timestamp::from_micros(100), 0u32);
let r = tree
.lookup_range_corrected_time(Timestamp::from_micros(0), Timestamp::from_micros(100));
assert_eq!(r, Timestamp::from_micros(100));
}
#[test]
fn test_corrected_time_fast_path_full_containment() {
// Two intervals each fully contain [10, 20]: count is 2 throughout, answer = 10/2 = 5us.
let mut tree = SelfTimeTree::new();
tree.insert(Timestamp::from_micros(0), Timestamp::from_micros(100), 0u32);
tree.insert(Timestamp::from_micros(5), Timestamp::from_micros(50), 1u32);
let r = tree
.lookup_range_corrected_time(Timestamp::from_micros(10), Timestamp::from_micros(20));
assert_eq!(r, Timestamp::from_micros(5));
}
#[test]
fn test_corrected_time_partial_overlap() {
// [0, 100] is the query window. Intervals:
// A: [0, 100] (covers whole window)
// B: [30, 70] (fully inside, contributes corrections during [30, 70])
// Expected:
// [0, 30): count=1, time=30, corrected = 30 / 1 = 30
// [30, 70): count=2, time=40, corrected = 40 / 2 = 20
// [70, 100): count=1, time=30, corrected = 30 / 1 = 30
// Total: 80us
let mut tree = SelfTimeTree::new();
tree.insert(Timestamp::from_micros(0), Timestamp::from_micros(100), 0u32);
tree.insert(Timestamp::from_micros(30), Timestamp::from_micros(70), 1u32);
let r = tree
.lookup_range_corrected_time(Timestamp::from_micros(0), Timestamp::from_micros(100));
assert_eq!(r, Timestamp::from_micros(80));
}
}
use std::{
num::NonZeroUsize,
num::{NonZeroU64, NonZeroUsize},
sync::{Arc, OnceLock},
};
use hashbrown::HashMap;
use smallvec::SmallVec;
use turbo_rcstr::RcStr;
use crate::{lazy_sorted_vec::LazySortedVec, timestamp::Timestamp};
pub type SpanIndex = NonZeroUsize;
/// Storage for `Span::args` ~32% of spans have <=1 arg (typically just the
/// `name` key for `turbo_tasks::function` spans), so inlining one entry
/// avoids a heap allocation in this common case.
pub type SpanArgs = SmallVec<[(RcStr, RcStr); 1]>;
pub struct Span {
// These values won't change after creation:
pub parent: Option<SpanIndex>,
......@@ -17,10 +23,12 @@ pub struct Span {
pub start: Timestamp,
pub category: RcStr,
pub name: RcStr,
pub args: Vec<(RcStr, RcStr)>,
pub args: SpanArgs,
// This might change during writing:
/// The list of events sorted by start time
/// The list of events sorted by start time. Backed by a SmallVec so leaf
/// spans (~69%, typically just one self-time event) don't pay a heap
/// allocation.
pub events: LazySortedVec<SpanEvent>,
pub is_complete: bool,
......@@ -30,18 +38,26 @@ pub struct Span {
pub self_deallocations: u64,
pub self_deallocation_count: u64,
// These values are computed when accessed (and maybe deleted during writing):
pub max_depth: OnceLock<u32>,
pub total_allocations: OnceLock<u64>,
pub total_deallocations: OnceLock<u64>,
pub total_persistent_allocations: OnceLock<u64>,
pub total_span_count: OnceLock<u64>,
pub total_allocation_count: OnceLock<u64>,
// More nested fields, but memory lazily allocated
pub time_data: OnceLock<Box<SpanTimeData>>,
// These values are computed when accessed (and maybe deleted during writing).
// Bundling the subtree totals into a single OnceLock pays a small cost on
// partial reads in exchange for a much-reduced lock count per Span.
pub totals: OnceLock<SpanTotals>,
pub time_data: SpanTimeData,
pub extra: OnceLock<Box<SpanExtra>>,
pub names: OnceLock<Box<SpanNames>>,
/// Lazy first-touch via `OnceLock`, but inline rather than boxed: ~96% of
/// spans get names populated after browsing, never invalidated, so the box
/// indirection is pure overhead.
pub names: OnceLock<SpanNames>,
}
#[derive(Default)]
pub struct SpanTotals {
pub max_depth: u32,
pub allocations: u64,
pub deallocations: u64,
pub persistent_allocations: u64,
pub allocation_count: u64,
pub span_count: u64,
}
#[derive(Default)]
......@@ -69,56 +85,127 @@ pub struct SpanExtra {
pub search_index: OnceLock<HashMap<RcStr, Vec<SpanIndex>>>,
}
#[derive(Default)]
#[derive(Clone)]
pub struct SpanName {
pub category: RcStr,
pub title: RcStr,
}
pub struct SpanNames {
// These values are computed when accessed (and maybe deleted during writing):
pub nice_name: OnceLock<(RcStr, RcStr)>,
pub group_name: OnceLock<(RcStr, RcStr)>,
pub nice_name: SpanName,
pub group_name: SpanName,
}
impl Span {
pub fn time_data(&self) -> &SpanTimeData {
self.time_data.get_or_init(|| {
Box::new(SpanTimeData {
self_end: self.start,
ignore_self_time: &self.name == "thread" || &self.name == "blocking",
..Default::default()
})
})
}
pub fn time_data_mut(&mut self) -> &mut SpanTimeData {
self.time_data();
self.time_data.get_mut().unwrap()
}
pub fn extra(&self) -> &SpanExtra {
self.extra.get_or_init(Default::default)
}
pub fn names(&self) -> &SpanNames {
self.names.get_or_init(Default::default)
self.names.get_or_init(|| self.compute_names())
}
fn compute_names(&self) -> SpanNames {
// Classify the span. `turbo_tasks::function` and the resolve-call spans
// get special-cased rendering when they carry a `name` arg; everything
// else is rendered generically.
enum Kind {
Function,
Resolve,
Other,
}
let kind = match self.name.as_str() {
"turbo_tasks::function" => Kind::Function,
"turbo_tasks::resolve_call" | "turbo_tasks::resolve_trait_call" => Kind::Resolve,
_ => Kind::Other,
};
let arg_name = self.args.iter().find(|&(k, _)| k == "name").map(|(_, v)| v);
// Generic fallback used by both names whenever no special case applies.
let generic = || SpanName {
category: self.category.clone(),
title: self.name.clone(),
};
// Each arm constructs the full `SpanNames` so the relationship between
// `nice_name` and `group_name` is visible at a glance. The `Some(n)`
// rows handle the "this span carries a `name` arg" case; the `None`
// arm falls back to the generic shape for both names — including for
// function/resolve spans, which (in practice) always carry a name arg,
// so the fallback is mostly defensive.
match (kind, arg_name) {
(Kind::Function, Some(n)) => {
let pretty = SpanName {
category: self.name.clone(),
title: n.clone(),
};
SpanNames {
nice_name: pretty.clone(),
group_name: pretty,
}
}
(Kind::Resolve, Some(n)) => SpanNames {
nice_name: SpanName {
category: self.name.clone(),
title: format!("*{n}").into(),
},
group_name: SpanName {
category: self.category.clone(),
title: format!("{} *{n}", self.name).into(),
},
},
(Kind::Other, Some(n)) => SpanNames {
nice_name: SpanName {
category: self.category.clone(),
title: format!("{} {n}", self.name).into(),
},
group_name: generic(),
},
(_, None) => SpanNames {
nice_name: generic(),
group_name: generic(),
},
}
}
}
/// Stores `duration` as `NonZeroU64` so the variant has a niche; combined with
/// `Child`'s `NonZeroUsize` index, this lets the compiler pack `SpanEvent`
/// without a separate discriminant byte (saving 8 bytes per event vs. an
/// `end: Timestamp` layout). Callers must filter zero-duration self-time
/// events before constructing — see [`SpanEvent::self_time`].
pub struct SpanEventSelfTime {
pub start: Timestamp,
pub end: Timestamp,
pub duration: NonZeroU64,
pub corrected_self_time: OnceLock<Timestamp>,
}
impl SpanEventSelfTime {
pub fn end(&self) -> Timestamp {
Timestamp::from_value(*self.start + self.duration.get())
}
}
pub enum SpanEvent {
SelfTime(SpanEventSelfTime),
Child { start: Timestamp, index: SpanIndex },
}
// 32 bytes = 8 (start) + 8 (duration) + 16 (OnceLock<Timestamp>) for the
// SelfTime variant; the Child variant fits in 16 and uses the niche, so no
// extra discriminant byte is needed.
const _: () = assert!(std::mem::size_of::<SpanEvent>() == 32);
impl SpanEvent {
pub fn self_time(start: Timestamp, end: Timestamp) -> Self {
Self::SelfTime(SpanEventSelfTime {
/// Constructs a `SelfTime` event from start and end timestamps. Returns `None`
/// if `end <= start` (zero or negative duration).
pub fn self_time(start: Timestamp, end: Timestamp) -> Option<Self> {
let duration = NonZeroU64::new(*end.saturating_sub(start))?;
Some(SpanEvent::SelfTime(SpanEventSelfTime {
start,
end,
duration,
corrected_self_time: OnceLock::new(),
})
}))
}
pub fn start(&self) -> Timestamp {
......@@ -150,7 +237,7 @@ impl Ord for SpanEvent {
.then_with(|| match (self, other) {
(SpanEvent::SelfTime(_), SpanEvent::Child { .. }) => std::cmp::Ordering::Less,
(SpanEvent::Child { .. }, SpanEvent::SelfTime(_)) => std::cmp::Ordering::Greater,
(SpanEvent::SelfTime(a), SpanEvent::SelfTime(b)) => a.end.cmp(&b.end),
(SpanEvent::SelfTime(a), SpanEvent::SelfTime(b)) => a.duration.cmp(&b.duration),
(
SpanEvent::Child { start: _, index: a },
SpanEvent::Child { start: _, index: b },
......@@ -233,3 +320,38 @@ impl SpanBottomUp {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn span_event_self_time_filters_zero_duration() {
let t = Timestamp::from_micros(100);
assert!(SpanEvent::self_time(t, t).is_none());
// end < start should also return None (saturating_sub clamps to 0).
assert!(SpanEvent::self_time(t, Timestamp::from_micros(50)).is_none());
}
#[test]
fn span_event_self_time_constructs_positive_duration() {
let start = Timestamp::from_micros(100);
let end = Timestamp::from_micros(150);
let event = SpanEvent::self_time(start, end).unwrap();
match event {
SpanEvent::SelfTime(self_time) => {
assert_eq!(self_time.start, start);
assert_eq!(self_time.duration.get(), *end - *start);
assert_eq!(self_time.end(), end);
}
SpanEvent::Child { .. } => panic!("expected SelfTime"),
}
}
#[test]
fn span_event_size_is_packed() {
// Backstop for the const assert; if this fails the const assert above
// would also fail, but having a test gives a clearer error message.
assert_eq!(std::mem::size_of::<SpanEvent>(), 32);
}
}
......@@ -10,7 +10,7 @@ use turbo_rcstr::{RcStr, rcstr};
use crate::{
self_time_tree::SelfTimeTree,
span::{Span, SpanEvent, SpanIndex},
span::{Span, SpanArgs, SpanEvent, SpanIndex, SpanTimeData},
span_ref::SpanRef,
timestamp::Timestamp,
};
......@@ -45,20 +45,18 @@ fn new_root_span() -> Span {
start: Timestamp::MAX,
category: RcStr::default(),
name: rcstr!("(root)"),
args: vec![],
args: SpanArgs::new(),
events: Default::default(),
is_complete: true,
max_depth: OnceLock::new(),
self_allocations: 0,
self_allocation_count: 0,
self_deallocations: 0,
self_deallocation_count: 0,
total_allocations: OnceLock::new(),
total_deallocations: OnceLock::new(),
total_persistent_allocations: OnceLock::new(),
total_allocation_count: OnceLock::new(),
total_span_count: OnceLock::new(),
time_data: OnceLock::new(),
totals: OnceLock::new(),
time_data: SpanTimeData {
self_end: Timestamp::MAX,
..Default::default()
},
extra: OnceLock::new(),
names: OnceLock::new(),
}
......@@ -109,10 +107,11 @@ impl Store {
start: Timestamp,
category: RcStr,
name: RcStr,
args: Vec<(RcStr, RcStr)>,
args: SpanArgs,
outdated_spans: &mut FxHashSet<SpanIndex>,
) -> SpanIndex {
let id = SpanIndex::new(self.spans.len()).unwrap();
let ignore_self_time = &name == "thread" || &name == "blocking";
self.spans.push(Span {
parent,
depth: 0,
......@@ -120,19 +119,18 @@ impl Store {
category,
name,
args,
events: vec![].into(),
events: Default::default(),
is_complete: false,
max_depth: OnceLock::new(),
self_allocations: 0,
self_allocation_count: 0,
self_deallocations: 0,
self_deallocation_count: 0,
total_allocations: OnceLock::new(),
total_deallocations: OnceLock::new(),
total_persistent_allocations: OnceLock::new(),
total_allocation_count: OnceLock::new(),
total_span_count: OnceLock::new(),
time_data: OnceLock::new(),
totals: OnceLock::new(),
time_data: SpanTimeData {
self_end: start,
ignore_self_time,
..Default::default()
},
extra: OnceLock::new(),
names: OnceLock::new(),
});
......@@ -213,16 +211,19 @@ impl Store {
end: Timestamp,
outdated_spans: &mut FxHashSet<SpanIndex>,
) {
let event = SpanEvent::self_time(start, end);
let span = &mut self.spans[span_index.get()];
let time_data = span.time_data_mut();
let time_data = &mut span.time_data;
if time_data.ignore_self_time {
return;
}
outdated_spans.insert(span_index);
time_data.self_time += end - start;
time_data.self_end = max(time_data.self_end, end);
span.events.push(SpanEvent::self_time(start, end));
self.insert_self_time(start, end, span_index, outdated_spans);
if let Some(event) = event {
span.events.push(event);
self.insert_self_time(start, end, span_index, outdated_spans);
}
}
pub fn set_total_time(
......@@ -239,7 +240,7 @@ impl Store {
};
let mut children = span
.children()
.map(|c| (c.span.start, c.span.time_data().self_end, c.index()))
.map(|c| (c.span.start, c.span.time_data.self_end, c.index()))
.collect::<Vec<_>>();
children.sort();
let self_end = start_time + total_time;
......@@ -249,14 +250,18 @@ impl Store {
for (start, end, index) in children {
if start > current {
if start > self_end {
events.push(SpanEvent::self_time(current, self_end));
self.insert_self_time(current, self_end, span_index, outdated_spans);
self_time += self_end - current;
if let Some(event) = SpanEvent::self_time(current, self_end) {
events.push(event);
self.insert_self_time(current, self_end, span_index, outdated_spans);
self_time += self_end - current;
}
break;
}
events.push(SpanEvent::self_time(current, start));
self.insert_self_time(current, start, span_index, outdated_spans);
self_time += start - current;
if let Some(event) = SpanEvent::self_time(current, start) {
events.push(event);
self.insert_self_time(current, start, span_index, outdated_spans);
self_time += start - current;
}
}
events.push(SpanEvent::Child { start, index });
current = max(current, end);
......@@ -264,20 +269,16 @@ impl Store {
current -= start_time;
if current < total_time {
self_time += total_time - current;
events.push(SpanEvent::self_time(
current + start_time,
start_time + total_time,
));
self.insert_self_time(
current + start_time,
start_time + total_time,
span_index,
outdated_spans,
);
let st = current + start_time;
let en = start_time + total_time;
if let Some(event) = SpanEvent::self_time(st, en) {
events.push(event);
self.insert_self_time(st, en, span_index, outdated_spans);
}
}
let span = &mut self.spans[span_index.get()];
outdated_spans.insert(span_index);
let time_data = span.time_data_mut();
let time_data = &mut span.time_data;
time_data.self_time = self_time;
time_data.self_end = self_end;
span.events = events.into();
......@@ -385,22 +386,16 @@ impl Store {
pub fn invalidate_outdated_spans(&mut self, outdated_spans: &FxHashSet<SpanId>) {
fn invalidate_span(span: &mut Span) {
if let Some(time_data) = span.time_data.get_mut() {
time_data.end.take();
time_data.total_time.take();
time_data.corrected_self_time.take();
time_data.corrected_total_time.take();
}
span.time_data.end.take();
span.time_data.total_time.take();
span.time_data.corrected_self_time.take();
span.time_data.corrected_total_time.take();
for event in span.events.iter_mut_unordered() {
if let SpanEvent::SelfTime(self_time) = event {
self_time.corrected_self_time.take();
}
}
span.total_allocations.take();
span.total_deallocations.take();
span.total_persistent_allocations.take();
span.total_allocation_count.take();
span.total_span_count.take();
span.totals.take();
span.extra.take();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment