Unverified Commit a10d278e authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add no-memory flag to NEXT_TURBOPACK_TRACING (#99777)

## Summary

Adds a `no-memory` flag to `NEXT_TURBOPACK_TRACING`, next to
`gz`/`gz-best` (e.g. `NEXT_TURBOPACK_TRACING=1,no-memory`).

With the flag, `RawTraceLayer` writes no allocation counters for span
enters/exits and no `MemorySample` rows. Without memory data, a trace of
`bench/heavy-npm-deps` is about a third of the size. Use it when only
timing data matters.

- `RawTraceLayerOptions { memory: bool }` (default `true`), with
`RawTraceLayer::with_options`. `RawTraceLayer::new` keeps the old
behavior.
- The flag is documented in `contributing/turbopack/tracing.md`.

Part of a stack that reduces the trace file size (#99765 adds the
`turbo-trace-size` CLI used for the measurements):

1. #99777: **This PR**: `no-memory` flag
2. #99778: Merge allocation counters into Enter/Exit rows
3. #99779: Delta-encode allocation counters, skip unchanged ones
4. #99780: Omit Exit+Enter pairs with (almost) no gap
5. #99781: Bump the trace header to `TRACEv1`

## Measurements

`next build` of `bench/heavy-npm-deps`, release native binding built
from source (`image-extended,tracing/release_max_level_trace`):

| flags | trace size |
| --- | --- |
| `1` | 68.2 MiB |
| `1,no-memory` | 23.3 MiB |

## Verification

- New unit tests in `raw_trace.rs` decode a captured trace (with and
without memory tracking)
- Captured both traces above with a locally built binding

<!-- NEXT_JS_LLM -->


<!-- fleet 4a52842b-f1a2-43cb-9779-3655caada6a0 -->
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 0b9fffb1
...@@ -15,6 +15,12 @@ It supports [the following special preset values][presets]: ...@@ -15,6 +15,12 @@ It supports [the following special preset values][presets]:
Alternatively, any directives syntax supported by [`tracing_subscriber::filter::EnvFilter`][directives] can be used. Alternatively, any directives syntax supported by [`tracing_subscriber::filter::EnvFilter`][directives] can be used.
Additionally, these flags can be added to the comma-separated list (e.g. `NEXT_TURBOPACK_TRACING=1,no-memory,gz`):
- **`gz`:** Compress the trace file with gzip (fast compression level).
- **`gz-best`:** Compress the trace file with gzip (best compression level).
- **`no-memory`:** Don't track memory. Skips the per-thread allocation counters and the process memory samples, which make up a large part of the trace file size. Allocation and memory value modes in the viewer will be empty.
> [!WARNING] > [!WARNING]
> A normal Next.js canary/stable release only includes the info level tracing. This is the tracing level intended for user-facing tracing. > A normal Next.js canary/stable release only includes the info level tracing. This is the tracing level intended for user-facing tracing.
> >
......
...@@ -78,7 +78,7 @@ use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, Issue, Resource ...@@ -78,7 +78,7 @@ use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, Issue, Resource
use turbopack_trace_utils::{ use turbopack_trace_utils::{
exit::{ExitHandler, ExitReceiver}, exit::{ExitHandler, ExitReceiver},
filter_layer::FilterLayer, filter_layer::FilterLayer,
raw_trace::RawTraceLayer, raw_trace::{RawTraceLayer, RawTraceLayerOptions},
trace_writer::TraceWriter, trace_writer::TraceWriter,
}; };
use url::Url; use url::Url;
...@@ -470,6 +470,7 @@ pub fn project_new<'env>( ...@@ -470,6 +470,7 @@ pub fn project_new<'env>(
GzipBest, GzipBest,
} }
let mut compress = Compression::None; let mut compress = Compression::None;
let mut raw_trace_options = RawTraceLayerOptions::default();
if let Some(mut trace) = trace { if let Some(mut trace) = trace {
let trace_path_override = std::env::var_os("NEXT_TURBOPACK_TRACING_PATH") let trace_path_override = std::env::var_os("NEXT_TURBOPACK_TRACING_PATH")
.filter(|v| !v.is_empty()) .filter(|v| !v.is_empty())
...@@ -515,6 +516,10 @@ pub fn project_new<'env>( ...@@ -515,6 +516,10 @@ pub fn project_new<'env>(
compress = Compression::GzipBest; compress = Compression::GzipBest;
return None; return None;
} }
"no-memory" => {
raw_trace_options.memory = false;
return None;
}
_ => Cow::Borrowed(item), _ => Cow::Borrowed(item),
}) })
}) })
...@@ -558,7 +563,8 @@ pub fn project_new<'env>( ...@@ -558,7 +563,8 @@ pub fn project_new<'env>(
TraceWriter::new(trace_writer) TraceWriter::new(trace_writer)
} }
}; };
let subscriber = subscriber.with(RawTraceLayer::new(trace_writer)); let subscriber =
subscriber.with(RawTraceLayer::with_options(trace_writer, raw_trace_options));
exit.on_exit(async move { exit.on_exit(async move {
tokio::task::spawn_blocking(move || drop(trace_writer_guard)) tokio::task::spawn_blocking(move || drop(trace_writer_guard))
......
...@@ -32,7 +32,21 @@ thread_local! { ...@@ -32,7 +32,21 @@ thread_local! {
static THREAD_LOCAL_LAST_MEMORY_SAMPLE: Cell<u64> = const { Cell::new(0) }; static THREAD_LOCAL_LAST_MEMORY_SAMPLE: Cell<u64> = const { Cell::new(0) };
} }
pub struct RawTraceLayerOptions {} /// Options for [`RawTraceLayer`].
#[derive(Clone, Debug)]
pub struct RawTraceLayerOptions {
/// Track memory: write the per-thread allocation counters on every span enter and exit
/// (`AllocationCounters` rows) and periodic process memory samples (`MemorySample` rows).
/// These make up a large part of the trace size, so disabling them significantly reduces
/// trace file size, at the cost of losing all memory information. Defaults to `true`.
pub memory: bool,
}
impl Default for RawTraceLayerOptions {
fn default() -> Self {
Self { memory: true }
}
}
struct RawTraceLayerExtension { struct RawTraceLayerExtension {
id: u64, id: u64,
...@@ -56,15 +70,23 @@ pub struct RawTraceLayer<S: Subscriber + for<'a> LookupSpan<'a>> { ...@@ -56,15 +70,23 @@ pub struct RawTraceLayer<S: Subscriber + for<'a> LookupSpan<'a>> {
trace_writer: TraceWriter, trace_writer: TraceWriter,
start: Instant, start: Instant,
next_id: AtomicU64, next_id: AtomicU64,
memory: bool,
_phantom: PhantomData<fn(S)>, _phantom: PhantomData<fn(S)>,
} }
impl<S: Subscriber + for<'a> LookupSpan<'a>> RawTraceLayer<S> { impl<S: Subscriber + for<'a> LookupSpan<'a>> RawTraceLayer<S> {
/// Creates a layer with the default [`RawTraceLayerOptions`].
pub fn new(trace_writer: TraceWriter) -> Self { pub fn new(trace_writer: TraceWriter) -> Self {
Self::with_options(trace_writer, RawTraceLayerOptions::default())
}
pub fn with_options(trace_writer: TraceWriter, options: RawTraceLayerOptions) -> Self {
let RawTraceLayerOptions { memory } = options;
Self { Self {
trace_writer, trace_writer,
start: Instant::now(), start: Instant::now(),
next_id: AtomicU64::new(1), next_id: AtomicU64::new(1),
memory,
_phantom: PhantomData, _phantom: PhantomData,
} }
} }
...@@ -173,8 +195,10 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> { ...@@ -173,8 +195,10 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> {
fn on_enter(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) { fn on_enter(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
let ts = self.start.elapsed().as_micros() as u64; let ts = self.start.elapsed().as_micros() as u64;
let thread_id = thread::current().id().as_u64().into(); let thread_id = thread::current().id().as_u64().into();
self.maybe_report_memory_sample(ts); if self.memory {
self.report_allocations(ts, thread_id); self.maybe_report_memory_sample(ts);
self.report_allocations(ts, thread_id);
}
self.write(TraceRow::Enter { self.write(TraceRow::Enter {
ts, ts,
id: get_id(ctx, id), id: get_id(ctx, id),
...@@ -185,7 +209,9 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> { ...@@ -185,7 +209,9 @@ impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for RawTraceLayer<S> {
fn on_exit(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) { fn on_exit(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
let ts = self.start.elapsed().as_micros() as u64; let ts = self.start.elapsed().as_micros() as u64;
let thread_id = thread::current().id().as_u64().into(); let thread_id = thread::current().id().as_u64().into();
self.report_allocations(ts, thread_id); if self.memory {
self.report_allocations(ts, thread_id);
}
self.write(TraceRow::Exit { self.write(TraceRow::Exit {
ts, ts,
id: get_id(ctx, id), id: get_id(ctx, id),
...@@ -284,3 +310,100 @@ impl Visit for ValuesVisitor { ...@@ -284,3 +310,100 @@ impl Visit for ValuesVisitor {
self.record_debug(field, &display(value)) self.record_debug(field, &display(value))
} }
} }
#[cfg(test)]
pub(crate) mod tests {
use std::{
io::{self, Write},
sync::{Arc, Mutex},
};
use tracing_subscriber::{Registry, layer::SubscriberExt};
use crate::{
raw_trace::{RawTraceLayer, RawTraceLayerOptions},
trace_writer::TraceWriter,
tracing::TraceRow,
};
#[derive(Clone, Default)]
struct SharedBuffer(Arc<Mutex<Vec<u8>>>);
impl Write for SharedBuffer {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
/// Runs `f` with a [`RawTraceLayer`] installed as the thread's default subscriber and
/// returns the complete trace file content.
pub(crate) fn capture(options: RawTraceLayerOptions, f: impl FnOnce()) -> Vec<u8> {
let buffer = SharedBuffer::default();
let (trace_writer, guard) = TraceWriter::new(buffer.clone());
let subscriber =
Registry::default().with(RawTraceLayer::with_options(trace_writer, options));
tracing::subscriber::with_default(subscriber, f);
// Flushes all buffers and waits for the writer thread to finish.
drop(guard);
Arc::try_unwrap(buffer.0).unwrap().into_inner().unwrap()
}
/// Decodes the rows of a trace file, skipping the header.
pub(crate) fn decode(data: &[u8]) -> Vec<TraceRow<'_>> {
let header = b"TRACEv0";
assert!(data.starts_with(header), "missing trace header");
let mut remaining = &data[header.len()..];
let mut rows = Vec::new();
while !remaining.is_empty() {
let (row, rest) = postcard::take_from_bytes(remaining).unwrap();
rows.push(row);
remaining = rest;
}
rows
}
fn enter_exit_span() {
let span = tracing::info_span!("test span");
let _guard = span.enter();
}
fn count(rows: &[TraceRow<'_>], predicate: impl Fn(&TraceRow<'_>) -> bool) -> usize {
rows.iter().filter(|row| predicate(row)).count()
}
#[test]
fn writes_allocation_counters_by_default() {
let data = capture(RawTraceLayerOptions::default(), enter_exit_span);
let rows = decode(&data);
assert_eq!(count(&rows, |r| matches!(r, TraceRow::Enter { .. })), 1);
assert_eq!(count(&rows, |r| matches!(r, TraceRow::Exit { .. })), 1);
assert_eq!(
count(&rows, |r| matches!(r, TraceRow::AllocationCounters { .. })),
2
);
}
#[test]
fn memory_tracking_can_be_disabled() {
let data = capture(RawTraceLayerOptions { memory: false }, enter_exit_span);
let rows = decode(&data);
assert_eq!(count(&rows, |r| matches!(r, TraceRow::Start { .. })), 1);
assert_eq!(count(&rows, |r| matches!(r, TraceRow::Enter { .. })), 1);
assert_eq!(count(&rows, |r| matches!(r, TraceRow::Exit { .. })), 1);
assert_eq!(count(&rows, |r| matches!(r, TraceRow::End { .. })), 1);
assert_eq!(
count(&rows, |r| matches!(
r,
TraceRow::AllocationCounters { .. }
| TraceRow::Allocation { .. }
| TraceRow::MemorySample { .. }
)),
0
);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment