Unverified Commit 2998a2e8 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: fix trace crates after concurrent trace format changes (#100656)

## Summary

Fixes the canary build. `turbopack-trace-utils` currently fails to
compile (`rust check`, `rustdoc`, cargo unit/bench tests, wasi and every
native build are red since #99809).

#99809 (typed, delta-encoded trace timestamps) landed after two PRs it
was never built together with:

- **#99527** added `memory_footprint` to `TraceRow::MemorySample`. The
new `TraceRow::try_map_timestamp` from #99809 rebuilds each variant
field by field and doesn't copy it (`E0027`/`E0063` in `tracing.rs`). It
now copies the field, and a new test checks that a `MemorySample` keeps
all its fields through timestamp encoding, serialization and decoding.
- **#100205** added trace-server test helpers for the untyped format.
Their rows now have the wrong timestamp type, and a second `encode`
writes rows without a `TimestampBase` (`E0428` plus 18 `E0308` in
`reader/turbopack.rs` tests). The helpers now build `TraceRow<'_, u64>`
rows and use the shared encoder. The partial-read test encodes one
continuous trace and splits it before the last row, instead of encoding
the last row on its own (that would now give it its own base). No
assertions changed.

The only production change is copying `memory_footprint`; the serialized
format is unchanged.

## Verification

- `cargo test` for `turbopack-trace-utils`, `turbopack-trace-server` and
`turbopack-trace-size`, including all #100205 re-entry tests
- `cargo clippy --all-targets -- -D warnings` for those crates; rustdoc
with `-D warnings`

<!-- NEXT_JS_LLM -->


<!-- fleet 4a52842b-f1a2-43cb-9779-3655caada6a0 -->
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent eb5bafb7
......@@ -544,19 +544,27 @@ mod tests {
/// Serializes the rows like the trace writer does: with delta encoded timestamps and a
/// [`TraceRow::TimestampBase`] row before the first one.
fn encode(rows: &[TraceRow<'_, u64>]) -> Vec<u8> {
encode_rows(rows).concat()
}
/// Like [`encode`], but returns the bytes of each row separately. A
/// [`TraceRow::TimestampBase`] is part of the bytes of the row it precedes.
fn encode_rows(rows: &[TraceRow<'_, u64>]) -> Vec<Vec<u8>> {
let mut encoder = TimestampEncoder::default();
let mut data = Vec::new();
for row in rows {
// A copy of the row, through its serialized form
let bytes = postcard::to_stdvec(row).unwrap();
let row: TraceRow<'_, u64> = postcard::from_bytes(&bytes).unwrap();
let (base, row) = encoder.encode_row(row);
if let Some(base) = base {
data.extend(postcard::to_stdvec(&base).unwrap());
}
data.extend(postcard::to_stdvec(&row).unwrap());
}
data
rows.iter()
.map(|row| {
// A copy of the row, through its serialized form
let bytes = postcard::to_stdvec(row).unwrap();
let row: TraceRow<'_, u64> = postcard::from_bytes(&bytes).unwrap();
let (base, row) = encoder.encode_row(row);
let mut data = Vec::new();
if let Some(base) = base {
data.extend(postcard::to_stdvec(&base).unwrap());
}
data.extend(postcard::to_stdvec(&row).unwrap());
data
})
.collect()
}
/// Reads the rows (without header) and returns the self allocations, allocation counts and
......@@ -585,7 +593,7 @@ mod tests {
.collect()
}
fn enter(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static> {
fn enter(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static, u64> {
TraceRow::Enter {
ts,
id,
......@@ -594,7 +602,7 @@ mod tests {
}
}
fn exit(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static> {
fn exit(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static, u64> {
TraceRow::Exit {
ts,
id,
......@@ -603,13 +611,7 @@ mod tests {
}
}
fn encode(rows: &[TraceRow<'_>]) -> Vec<u8> {
rows.iter()
.flat_map(|row| postcard::to_stdvec(row).unwrap())
.collect()
}
fn ingest(format: &mut TurbopackFormat, rows: &[TraceRow<'_>]) {
fn ingest(format: &mut TurbopackFormat, rows: &[TraceRow<'_, u64>]) {
let data = encode(rows);
assert_eq!(
format
......@@ -824,13 +826,16 @@ mod tests {
fn live_trace_with_partial_exit_preserves_the_resumed_frame() {
let mut format = TurbopackFormat::new(Arc::new(StoreContainer::new()));
let mut reuse = TurbopackFormat::create_reused();
let prefix = encode(&[
// One trace, split before its last row
let rows = encode_rows(&[
start(1, None, "span"),
enter(10, 1, 1),
enter(20, 1, 1),
exit(30, 1, 1),
exit(40, 1, 1),
]);
let final_row = encode(&[exit(40, 1, 1)]);
let prefix = rows[..4].concat();
let final_row = rows[4].clone();
let mut data = prefix.clone();
data.extend_from_slice(&final_row[..final_row.len() - 1]);
let consumed = format.read(&data, &mut reuse).unwrap();
......
......@@ -229,11 +229,13 @@ impl<'a, T> TraceRow<'a, T> {
ts,
memory,
memory_pressure,
memory_footprint,
active_worker_threads,
} => TraceRow::MemorySample {
ts: f(ts)?,
memory,
memory_pressure,
memory_footprint,
active_worker_threads,
},
TraceRow::TimestampBase { ts } => TraceRow::TimestampBase { ts },
......@@ -556,6 +558,46 @@ mod tests {
assert!(encoder.encode_row(end(1)).0.is_some());
}
#[test]
fn memory_sample_keeps_all_fields() {
let mut encoder = TimestampEncoder::default();
let (base, row) = encoder.encode_row(TraceRow::<u64>::MemorySample {
ts: 1234,
memory: 1 << 30,
memory_pressure: 42,
memory_footprint: 3 << 30,
active_worker_threads: 7,
});
// Through the serialized form, like a reader sees it
let bytes: Vec<Vec<u8>> = base
.into_iter()
.chain([row])
.map(|row| postcard::to_stdvec(&row).unwrap())
.collect();
let mut decoder = TimestampDecoder::default();
let mut decoded = None;
for bytes in &bytes {
let row: TraceRow<'_> = postcard::from_bytes(bytes).unwrap();
decoded = Some(decoder.decode(row).unwrap());
}
match decoded.unwrap() {
TraceRow::MemorySample {
ts,
memory,
memory_pressure,
memory_footprint,
active_worker_threads,
} => {
assert_eq!(ts, 1234);
assert_eq!(memory, 1 << 30);
assert_eq!(memory_pressure, 42);
assert_eq!(memory_footprint, 3 << 30);
assert_eq!(active_worker_threads, 7);
}
row => panic!("expected a MemorySample, got {row:?}"),
}
}
#[test]
fn decoder_keeps_timestamp_bases() {
let mut decoder = TimestampDecoder::default();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment