Unverified Commit dfa14fbd authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] use thread local trace collecting (#73615)

### What?

Collect trace events in a thread local buffer first and then send a batch of events to the writer thread.

But there are some challenges to that:

Events might be out of order in the trace file. Usually that should be fine since they have a timestamp, but

* span ids might be reused faster than all events are send to the file. So we assign our own globally unique ids instead.
* the trace server need to be able to process events out of order. So we queue up events when they are received for spans that are not created yet.

And the thread local buffer might also have a problem when the thread is idle or hanging. This would cause events to never be written to the file.
To fix that the writer thread will steal the outstanding events from the thread local buffers when it is idle for a second. This ensures that all events are eventually written, even in case of hanging threads.
parent e0eb1656
......@@ -1546,12 +1546,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.16"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if",
]
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crossterm"
......@@ -9262,10 +9259,13 @@ version = "0.1.0"
dependencies = [
"anyhow",
"crossbeam-channel",
"crossbeam-utils",
"once_cell",
"parking_lot",
"postcard",
"rustc-hash 1.1.0",
"serde",
"thread_local",
"tokio",
"tracing",
"tracing-subscriber",
......
......@@ -13,12 +13,15 @@ bench = false
[dependencies]
anyhow = { workspace = true }
crossbeam-utils = { version = "0.8.20" }
crossbeam-channel = { workspace = true }
once_cell = { workspace = true }
parking_lot = { workspace = true }
postcard = { workspace = true, features = ["alloc", "use-std"] }
rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["macros", "signal", "sync", "rt"] }
thread_local = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
turbo-tasks-malloc = { workspace = true }
use postcard::ser_flavors::Flavor;
pub struct BufFlavor {
pub buf: Vec<u8>,
use crate::trace_writer::WriteGuard;
pub struct WriteGuardFlavor<'l> {
pub guard: WriteGuard<'l>,
}
impl Flavor for BufFlavor {
type Output = Vec<u8>;
impl Flavor for WriteGuardFlavor<'_> {
type Output = ();
fn try_push(&mut self, data: u8) -> postcard::Result<()> {
self.buf.push(data);
self.guard.push(data);
Ok(())
}
fn finalize(self) -> postcard::Result<Self::Output> {
Ok(self.buf)
Ok(())
}
fn try_extend(&mut self, data: &[u8]) -> postcard::Result<()> {
self.buf.extend_from_slice(data);
self.guard.extend(data);
Ok(())
}
}
use std::{borrow::Cow, fmt::Write, marker::PhantomData, thread, time::Instant};
use std::{
borrow::Cow, fmt::Write, marker::PhantomData, sync::atomic::AtomicU64, thread, time::Instant,
};
use tracing::{
field::{display, Visit},
......@@ -8,16 +10,35 @@ use tracing_subscriber::{registry::LookupSpan, Layer};
use turbo_tasks_malloc::TurboMalloc;
use crate::{
flavor::BufFlavor,
flavor::WriteGuardFlavor,
trace_writer::TraceWriter,
tracing::{TraceRow, TraceValue},
};
pub struct RawTraceLayerOptions {}
struct RawTraceLayerExtension {
id: u64,
}
fn get_id<S: Subscriber + for<'a> LookupSpan<'a>>(
ctx: tracing_subscriber::layer::Context<'_, S>,
id: &span::Id,
) -> u64 {
ctx.span(id)
.unwrap()
.extensions()
.get::<RawTraceLayerExtension>()
.unwrap()
.id
}
/// A tracing layer that writes raw trace data to a writer. The data format is
/// defined by [FullTraceRow].
pub struct RawTraceLayer<S: Subscriber + for<'a> LookupSpan<'a>> {
trace_writer: TraceWriter,
start: Instant,
next_id: AtomicU64,
_phantom: PhantomData<fn(S)>,
}
......@@ -26,16 +47,15 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> RawTraceLayer<S> {
Self {
trace_writer,
start: Instant::now(),
next_id: AtomicU64::new(1),
_phantom: PhantomData,
}
}
fn write(&self, data: TraceRow<'_>) {
let start = TurboMalloc::allocation_counters();
// Buffer is recycled
let buf = self.trace_writer.try_get_buffer().unwrap_or_default();
let buf = postcard::serialize_with_flavor(&data, BufFlavor { buf }).unwrap();
self.trace_writer.write(buf);
let guard = self.trace_writer.start_write();
postcard::serialize_with_flavor(&data, WriteGuardFlavor { guard }).unwrap();
TurboMalloc::reset_allocation_counters(start);
}
......@@ -62,13 +82,20 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> {
let ts = self.start.elapsed().as_micros() as u64;
let mut values = ValuesVisitor::new();
attrs.values().record(&mut values);
let external_id = self
.next_id
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
ctx.span(id)
.unwrap()
.extensions_mut()
.insert(RawTraceLayerExtension { id: external_id });
self.write(TraceRow::Start {
ts,
id: id.into_u64(),
id: external_id,
parent: if attrs.is_contextual() {
ctx.current_span().id().map(|p| p.into_u64())
ctx.current_span().id().map(|p| get_id(ctx, p))
} else {
attrs.parent().map(|p| p.into_u64())
attrs.parent().map(|p| get_id(ctx, p))
},
name: attrs.metadata().name().into(),
target: attrs.metadata().target().into(),
......@@ -76,32 +103,32 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> {
});
}
fn on_close(&self, id: span::Id, _ctx: tracing_subscriber::layer::Context<'_, S>) {
fn on_close(&self, id: span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
let ts = self.start.elapsed().as_micros() as u64;
self.write(TraceRow::End {
ts,
id: id.into_u64(),
id: get_id(ctx, &id),
});
}
fn on_enter(&self, id: &span::Id, _ctx: tracing_subscriber::layer::Context<'_, S>) {
fn on_enter(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
let ts = self.start.elapsed().as_micros() as u64;
let thread_id = thread::current().id().as_u64().into();
self.report_allocations(ts, thread_id);
self.write(TraceRow::Enter {
ts,
id: id.into_u64(),
id: get_id(ctx, id),
thread_id,
});
}
fn on_exit(&self, id: &span::Id, _ctx: tracing_subscriber::layer::Context<'_, S>) {
fn on_exit(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
let ts = self.start.elapsed().as_micros() as u64;
let thread_id = thread::current().id().as_u64().into();
self.report_allocations(ts, thread_id);
self.write(TraceRow::Exit {
ts,
id: id.into_u64(),
id: get_id(ctx, id),
thread_id,
});
}
......@@ -113,9 +140,9 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> {
self.write(TraceRow::Event {
ts,
parent: if event.is_contextual() {
ctx.current_span().id().map(|p| p.into_u64())
ctx.current_span().id().map(|p| get_id(ctx, p))
} else {
event.parent().map(|p| p.into_u64())
event.parent().map(|p| get_id(ctx, p))
},
values: values.values,
});
......@@ -125,12 +152,12 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> {
&self,
id: &span::Id,
record: &span::Record<'_>,
_ctx: tracing_subscriber::layer::Context<'_, S>,
ctx: tracing_subscriber::layer::Context<'_, S>,
) {
let mut values = ValuesVisitor::new();
record.record(&mut values);
self.write(TraceRow::Record {
id: id.into_u64(),
id: get_id(ctx, id),
values: values.values,
});
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment