Unverified Commit 422c890f authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Turbopack: fix trace crates after concurrent trace format changes

#99809 (typed, delta-encoded trace timestamps) landed after two PRs that it was never built together with:

- #99527 added `memory_footprint` to `TraceRow::MemorySample`, but the new `TraceRow::try_map_timestamp` doesn't copy it, so `turbopack-trace-utils` doesn't compile.
- #100205 added trace-server test helpers for the untyped format. Their rows now have the wrong timestamp type, and a second `encode` writes no `TimestampBase`.

Copy `memory_footprint` in `try_map_timestamp` and cover it with a round-trip test. Port the #100205 test helpers to `TraceRow<'_, u64>` and the shared encoder; their tests are unchanged apart from encoding one continuous trace in the partial-read test.
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent c0f54e5d
...@@ -544,19 +544,27 @@ mod tests { ...@@ -544,19 +544,27 @@ mod tests {
/// Serializes the rows like the trace writer does: with delta encoded timestamps and a /// Serializes the rows like the trace writer does: with delta encoded timestamps and a
/// [`TraceRow::TimestampBase`] row before the first one. /// [`TraceRow::TimestampBase`] row before the first one.
fn encode(rows: &[TraceRow<'_, u64>]) -> Vec<u8> { fn encode(rows: &[TraceRow<'_, u64>]) -> Vec<u8> {
encode_rows(rows).concat()
}
/// Like [`encode`], but returns the bytes of each row separately. A
/// [`TraceRow::TimestampBase`] is part of the bytes of the row it precedes.
fn encode_rows(rows: &[TraceRow<'_, u64>]) -> Vec<Vec<u8>> {
let mut encoder = TimestampEncoder::default(); let mut encoder = TimestampEncoder::default();
let mut data = Vec::new(); rows.iter()
for row in rows { .map(|row| {
// A copy of the row, through its serialized form // A copy of the row, through its serialized form
let bytes = postcard::to_stdvec(row).unwrap(); let bytes = postcard::to_stdvec(row).unwrap();
let row: TraceRow<'_, u64> = postcard::from_bytes(&bytes).unwrap(); let row: TraceRow<'_, u64> = postcard::from_bytes(&bytes).unwrap();
let (base, row) = encoder.encode_row(row); let (base, row) = encoder.encode_row(row);
if let Some(base) = base { let mut data = Vec::new();
data.extend(postcard::to_stdvec(&base).unwrap()); if let Some(base) = base {
} data.extend(postcard::to_stdvec(&base).unwrap());
data.extend(postcard::to_stdvec(&row).unwrap()); }
} data.extend(postcard::to_stdvec(&row).unwrap());
data data
})
.collect()
} }
/// Reads the rows (without header) and returns the self allocations, allocation counts and /// Reads the rows (without header) and returns the self allocations, allocation counts and
...@@ -585,7 +593,7 @@ mod tests { ...@@ -585,7 +593,7 @@ mod tests {
.collect() .collect()
} }
fn enter(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static> { fn enter(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static, u64> {
TraceRow::Enter { TraceRow::Enter {
ts, ts,
id, id,
...@@ -594,7 +602,7 @@ mod tests { ...@@ -594,7 +602,7 @@ mod tests {
} }
} }
fn exit(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static> { fn exit(ts: u64, id: u64, thread_id: u64) -> TraceRow<'static, u64> {
TraceRow::Exit { TraceRow::Exit {
ts, ts,
id, id,
...@@ -603,13 +611,7 @@ mod tests { ...@@ -603,13 +611,7 @@ mod tests {
} }
} }
fn encode(rows: &[TraceRow<'_>]) -> Vec<u8> { fn ingest(format: &mut TurbopackFormat, rows: &[TraceRow<'_, u64>]) {
rows.iter()
.flat_map(|row| postcard::to_stdvec(row).unwrap())
.collect()
}
fn ingest(format: &mut TurbopackFormat, rows: &[TraceRow<'_>]) {
let data = encode(rows); let data = encode(rows);
assert_eq!( assert_eq!(
format format
...@@ -824,13 +826,16 @@ mod tests { ...@@ -824,13 +826,16 @@ mod tests {
fn live_trace_with_partial_exit_preserves_the_resumed_frame() { fn live_trace_with_partial_exit_preserves_the_resumed_frame() {
let mut format = TurbopackFormat::new(Arc::new(StoreContainer::new())); let mut format = TurbopackFormat::new(Arc::new(StoreContainer::new()));
let mut reuse = TurbopackFormat::create_reused(); let mut reuse = TurbopackFormat::create_reused();
let prefix = encode(&[ // One trace, split before its last row
let rows = encode_rows(&[
start(1, None, "span"), start(1, None, "span"),
enter(10, 1, 1), enter(10, 1, 1),
enter(20, 1, 1), enter(20, 1, 1),
exit(30, 1, 1), exit(30, 1, 1),
exit(40, 1, 1),
]); ]);
let final_row = encode(&[exit(40, 1, 1)]); let prefix = rows[..4].concat();
let final_row = rows[4].clone();
let mut data = prefix.clone(); let mut data = prefix.clone();
data.extend_from_slice(&final_row[..final_row.len() - 1]); data.extend_from_slice(&final_row[..final_row.len() - 1]);
let consumed = format.read(&data, &mut reuse).unwrap(); let consumed = format.read(&data, &mut reuse).unwrap();
......
...@@ -229,11 +229,13 @@ impl<'a, T> TraceRow<'a, T> { ...@@ -229,11 +229,13 @@ impl<'a, T> TraceRow<'a, T> {
ts, ts,
memory, memory,
memory_pressure, memory_pressure,
memory_footprint,
active_worker_threads, active_worker_threads,
} => TraceRow::MemorySample { } => TraceRow::MemorySample {
ts: f(ts)?, ts: f(ts)?,
memory, memory,
memory_pressure, memory_pressure,
memory_footprint,
active_worker_threads, active_worker_threads,
}, },
TraceRow::TimestampBase { ts } => TraceRow::TimestampBase { ts }, TraceRow::TimestampBase { ts } => TraceRow::TimestampBase { ts },
...@@ -556,6 +558,46 @@ mod tests { ...@@ -556,6 +558,46 @@ mod tests {
assert!(encoder.encode_row(end(1)).0.is_some()); assert!(encoder.encode_row(end(1)).0.is_some());
} }
#[test]
fn memory_sample_keeps_all_fields() {
let mut encoder = TimestampEncoder::default();
let (base, row) = encoder.encode_row(TraceRow::<u64>::MemorySample {
ts: 1234,
memory: 1 << 30,
memory_pressure: 42,
memory_footprint: 3 << 30,
active_worker_threads: 7,
});
// Through the serialized form, like a reader sees it
let bytes: Vec<Vec<u8>> = base
.into_iter()
.chain([row])
.map(|row| postcard::to_stdvec(&row).unwrap())
.collect();
let mut decoder = TimestampDecoder::default();
let mut decoded = None;
for bytes in &bytes {
let row: TraceRow<'_> = postcard::from_bytes(bytes).unwrap();
decoded = Some(decoder.decode(row).unwrap());
}
match decoded.unwrap() {
TraceRow::MemorySample {
ts,
memory,
memory_pressure,
memory_footprint,
active_worker_threads,
} => {
assert_eq!(ts, 1234);
assert_eq!(memory, 1 << 30);
assert_eq!(memory_pressure, 42);
assert_eq!(memory_footprint, 3 << 30);
assert_eq!(active_worker_threads, 7);
}
row => panic!("expected a MemorySample, got {row:?}"),
}
}
#[test] #[test]
fn decoder_keeps_timestamp_bases() { fn decoder_keeps_timestamp_bases() {
let mut decoder = TimestampDecoder::default(); let mut decoder = TimestampDecoder::default();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment