Unverified Commit f2b544db authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub
parent 48f1b3a5
...@@ -4,6 +4,16 @@ This crate exposes the transport-independent Turbopack trace viewer engine throu ...@@ -4,6 +4,16 @@ This crate exposes the transport-independent Turbopack trace viewer engine throu
The exported `TurbopackTraceServer` class accepts a complete raw or gzip-compressed trace as a `Uint8Array`. Its `handleMessage()` method accepts the existing trace viewer client JSON protocol and returns the corresponding server messages in protocol order. The exported `TurbopackTraceServer` class accepts a complete raw or gzip-compressed trace as a `Uint8Array`. Its `handleMessage()` method accepts the existing trace viewer client JSON protocol and returns the corresponding server messages in protocol order.
Loading is synchronous, so pass an optional progress callback to the constructor before parsing starts. It runs at most once every 250 milliseconds and always receives a final update with `done: true`:
```js
const server = new TurbopackTraceServer(trace, (progress) => {
worker.postMessage({ type: 'trace-load-progress', progress })
})
```
The callback receives `bytesRead`, `totalBytes`, `uncompressedBytesRead`, `percentage`, `elapsedMs`, `bytesPerSecond`, `etaMs`, `stats`, and `done`. Because construction is synchronous, use the callback from a Web Worker if progress needs to update the browser UI while the trace is loading.
Zstd-compressed traces and live file tailing remain native-only. Zstd-compressed traces and live file tailing remain native-only.
## Build ## Build
......
...@@ -11,6 +11,21 @@ if (!tracePath) { ...@@ -11,6 +11,21 @@ if (!tracePath) {
} }
const trace = readFileSync(tracePath) const trace = readFileSync(tracePath)
new TurbopackTraceServer(trace) new TurbopackTraceServer(trace, (progress) => {
const line = `${progress.percentage.toFixed(0)}% read (${(
progress.bytesRead /
1024 /
1024
).toFixed(0)}/${(progress.totalBytes / 1024 / 1024).toFixed(0)} MB, ${(
progress.bytesPerSecond /
1024 /
1024
).toFixed(0)} MB/s) - ${progress.stats}`
if (process.stderr.isTTY) {
process.stderr.write(`\r\x1b[2K${line}${progress.done ? '\n' : ''}`)
} else {
console.error(line)
}
})
console.log(`Loaded trace file: ${tracePath}`) console.log(`Loaded trace file: ${tracePath}`)
use napi::{Result, bindgen_prelude::Uint8Array}; use std::{
ops::ControlFlow,
time::{Duration, Instant},
};
use napi::{
Result,
bindgen_prelude::{Function, Uint8Array},
};
use napi_derive::napi; use napi_derive::napi;
use turbopack_trace_server::{protocol::ProtocolSession, read_trace_bytes}; use turbopack_trace_server::{
protocol::ProtocolSession, read_trace_bytes, read_trace_bytes_with_progress,
};
const PROGRESS_INTERVAL: Duration = Duration::from_millis(250);
#[napi(object)]
pub struct TraceLoadProgress {
pub bytes_read: f64,
pub total_bytes: f64,
pub uncompressed_bytes_read: f64,
pub percentage: f64,
pub elapsed_ms: f64,
pub bytes_per_second: f64,
pub eta_ms: Option<f64>,
pub stats: String,
pub done: bool,
}
/// In-memory implementation of the turbo trace viewer protocol. /// In-memory implementation of the turbo trace viewer protocol.
#[napi] #[napi]
...@@ -10,10 +35,66 @@ pub struct TurbopackTraceServer { ...@@ -10,10 +35,66 @@ pub struct TurbopackTraceServer {
#[napi] #[napi]
impl TurbopackTraceServer { impl TurbopackTraceServer {
/// Loads a complete raw or gzip-compressed trace. /// Loads a complete raw or gzip-compressed trace, optionally reporting progress.
#[napi(constructor)] #[napi(constructor)]
pub fn new(trace: Uint8Array) -> Result<Self> { pub fn new(
let store = read_trace_bytes(trace.as_ref()).map_err(|error| { trace: Uint8Array,
on_progress: Option<Function<'_, TraceLoadProgress, ()>>,
) -> Result<Self> {
let started = Instant::now();
let mut last_reported = Duration::ZERO;
let mut callback_error = None;
let store = if let Some(on_progress) = on_progress {
read_trace_bytes_with_progress(trace.as_ref(), |progress| {
let elapsed = started.elapsed();
if !progress.done && elapsed.saturating_sub(last_reported) < PROGRESS_INTERVAL {
return ControlFlow::Continue(());
}
last_reported = elapsed;
let bytes_read = progress.bytes_read as f64;
let total_bytes = progress.total_bytes as f64;
let elapsed_ms = elapsed.as_secs_f64() * 1000.0;
let bytes_per_second = if elapsed_ms > 0.0 {
bytes_read * 1000.0 / elapsed_ms
} else {
0.0
};
let eta_ms = if bytes_read > 0.0 && bytes_read < total_bytes {
Some(elapsed_ms * (total_bytes - bytes_read) / bytes_read)
} else {
None
};
if let Err(error) = on_progress.call(TraceLoadProgress {
bytes_read,
total_bytes,
uncompressed_bytes_read: progress.uncompressed_bytes_read as f64,
percentage: if total_bytes > 0.0 {
bytes_read * 100.0 / total_bytes
} else {
100.0
},
elapsed_ms,
bytes_per_second,
eta_ms,
stats: progress.stats(),
done: progress.done,
}) {
callback_error = Some(error);
ControlFlow::Break(())
} else {
ControlFlow::Continue(())
}
})
} else {
read_trace_bytes(trace.as_ref())
};
if let Some(error) = callback_error {
return Err(error);
}
let store = store.map_err(|error| {
napi::Error::from_reason(format!("failed to load Turbopack trace: {error:#}")) napi::Error::from_reason(format!("failed to load Turbopack trace: {error:#}"))
})?; })?;
Ok(Self { Ok(Self {
......
...@@ -6,6 +6,7 @@ use std::path::PathBuf; ...@@ -6,6 +6,7 @@ use std::path::PathBuf;
use std::{ use std::{
hash::BuildHasherDefault, hash::BuildHasherDefault,
io::Read, io::Read,
ops::ControlFlow,
sync::Arc, sync::Arc,
thread, thread,
time::{Duration, Instant}, time::{Duration, Instant},
...@@ -69,6 +70,40 @@ pub fn start_turbopack_trace_server(path: PathBuf, port: Option<u16>) -> Arc<Sto ...@@ -69,6 +70,40 @@ pub fn start_turbopack_trace_server(path: PathBuf, port: Option<u16>) -> Arc<Sto
/// Zstd is intentionally unsupported here because the browser WASM build does /// Zstd is intentionally unsupported here because the browser WASM build does
/// not include the native zstd decoder used by the file reader. /// not include the native zstd decoder used by the file reader.
pub fn read_trace_bytes(bytes: &[u8]) -> anyhow::Result<Arc<StoreContainer>> { pub fn read_trace_bytes(bytes: &[u8]) -> anyhow::Result<Arc<StoreContainer>> {
read_trace_bytes_impl(bytes, None)
}
/// Progress made while parsing a complete trace held in memory.
pub struct TraceReadProgress<'a> {
pub bytes_read: usize,
pub total_bytes: usize,
pub uncompressed_bytes_read: usize,
pub done: bool,
parser: &'a TraceParser,
}
impl TraceReadProgress<'_> {
pub fn stats(&self) -> String {
self.parser.stats()
}
}
type TraceProgressCallback<'a> = dyn FnMut(TraceReadProgress<'_>) -> ControlFlow<()> + 'a;
/// Parses a complete trace and reports progress after each parsing batch.
///
/// Returning `ControlFlow::Break(())` from the callback aborts parsing.
pub fn read_trace_bytes_with_progress(
bytes: &[u8],
mut progress: impl FnMut(TraceReadProgress<'_>) -> ControlFlow<()>,
) -> anyhow::Result<Arc<StoreContainer>> {
read_trace_bytes_impl(bytes, Some(&mut progress))
}
fn read_trace_bytes_impl(
bytes: &[u8],
mut progress: Option<&mut TraceProgressCallback<'_>>,
) -> anyhow::Result<Arc<StoreContainer>> {
const PARSE_CHUNK_SIZE: usize = 2 * 1024 * 1024; const PARSE_CHUNK_SIZE: usize = 2 * 1024 * 1024;
const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b]; const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b];
const ZSTD_MAGIC: &[u8] = &[0x28, 0xb5, 0x2f, 0xfd]; const ZSTD_MAGIC: &[u8] = &[0x28, 0xb5, 0x2f, 0xfd];
...@@ -79,6 +114,7 @@ pub fn read_trace_bytes(bytes: &[u8]) -> anyhow::Result<Arc<StoreContainer>> { ...@@ -79,6 +114,7 @@ pub fn read_trace_bytes(bytes: &[u8]) -> anyhow::Result<Arc<StoreContainer>> {
let store = Arc::new(StoreContainer::new()); let store = Arc::new(StoreContainer::new());
let mut parser = TraceParser::new(store.clone()); let mut parser = TraceParser::new(store.clone());
let mut uncompressed_bytes_read = 0;
if bytes.starts_with(GZIP_MAGIC) { if bytes.starts_with(GZIP_MAGIC) {
let mut decoder = flate2::read::GzDecoder::new(bytes); let mut decoder = flate2::read::GzDecoder::new(bytes);
...@@ -89,17 +125,69 @@ pub fn read_trace_bytes(bytes: &[u8]) -> anyhow::Result<Arc<StoreContainer>> { ...@@ -89,17 +125,69 @@ pub fn read_trace_bytes(bytes: &[u8]) -> anyhow::Result<Arc<StoreContainer>> {
break; break;
} }
parser.push(&chunk[..bytes_read])?; parser.push(&chunk[..bytes_read])?;
uncompressed_bytes_read += bytes_read;
let bytes_read = bytes.len() - decoder.get_ref().len();
report_trace_read_progress(
&mut progress,
&parser,
bytes_read,
bytes.len(),
uncompressed_bytes_read,
false,
)?;
} }
} else { } else {
let mut bytes_read = 0;
for chunk in bytes.chunks(PARSE_CHUNK_SIZE) { for chunk in bytes.chunks(PARSE_CHUNK_SIZE) {
parser.push(chunk)?; parser.push(chunk)?;
bytes_read += chunk.len();
uncompressed_bytes_read = bytes_read;
report_trace_read_progress(
&mut progress,
&parser,
bytes_read,
bytes.len(),
uncompressed_bytes_read,
false,
)?;
} }
} }
parser.finish()?; parser.finish()?;
report_trace_read_progress(
&mut progress,
&parser,
bytes.len(),
bytes.len(),
uncompressed_bytes_read,
true,
)?;
Ok(store) Ok(store)
} }
fn report_trace_read_progress(
progress: &mut Option<&mut TraceProgressCallback<'_>>,
parser: &TraceParser,
bytes_read: usize,
total_bytes: usize,
uncompressed_bytes_read: usize,
done: bool,
) -> anyhow::Result<()> {
if let Some(progress) = progress
&& progress(TraceReadProgress {
bytes_read,
total_bytes,
uncompressed_bytes_read,
done,
parser,
})
.is_break()
{
anyhow::bail!("trace loading was aborted by the progress callback");
}
Ok(())
}
const PAGE_SIZE: usize = 20; const PAGE_SIZE: usize = 20;
/// How spans should be sorted. /// How spans should be sorted.
......
...@@ -169,7 +169,6 @@ impl HeaptrackFormat { ...@@ -169,7 +169,6 @@ impl HeaptrackFormat {
} }
impl TraceFormat for HeaptrackFormat { impl TraceFormat for HeaptrackFormat {
#[cfg(not(target_arch = "wasm32"))]
fn stats(&self) -> String { fn stats(&self) -> String {
format!( format!(
"{} spans, {} strings, {} ips, {} traces, {} allocations, {:.2} GB allocated, {:.2} \ "{} spans, {} strings, {} ips, {} traces, {} allocations, {:.2} GB allocated, {:.2} \
......
...@@ -55,7 +55,6 @@ impl IdMapping { ...@@ -55,7 +55,6 @@ impl IdMapping {
*entry = Some(span); *entry = Some(span);
} }
#[cfg(not(target_arch = "wasm32"))]
fn len(&self) -> usize { fn len(&self) -> usize {
self.len self.len
} }
...@@ -505,7 +504,6 @@ impl TraceFormat for TurbopackFormat { ...@@ -505,7 +504,6 @@ impl TraceFormat for TurbopackFormat {
Vec::with_capacity(4_096) Vec::with_capacity(4_096)
} }
#[cfg(not(target_arch = "wasm32"))]
fn stats(&self) -> String { fn stats(&self) -> String {
use std::fmt::Write; use std::fmt::Write;
......
...@@ -22,7 +22,6 @@ pub(crate) trait TraceFormat { ...@@ -22,7 +22,6 @@ pub(crate) trait TraceFormat {
fn read(&mut self, buffer: &[u8], reuse: &mut Self::Reused) -> Result<usize>; fn read(&mut self, buffer: &[u8], reuse: &mut Self::Reused) -> Result<usize>;
#[cfg(not(target_arch = "wasm32"))]
fn stats(&self) -> String { fn stats(&self) -> String {
String::new() String::new()
} }
...@@ -35,7 +34,6 @@ struct ErasedTraceFormat(Box<dyn ObjectSafeTraceFormat>); ...@@ -35,7 +34,6 @@ struct ErasedTraceFormat(Box<dyn ObjectSafeTraceFormat>);
trait ObjectSafeTraceFormat { trait ObjectSafeTraceFormat {
fn create_reused(&self) -> ErasedReused; fn create_reused(&self) -> ErasedReused;
fn read(&mut self, buffer: &[u8], reuse: &mut ErasedReused) -> Result<usize>; fn read(&mut self, buffer: &[u8], reuse: &mut ErasedReused) -> Result<usize>;
#[cfg(not(target_arch = "wasm32"))]
fn stats(&self) -> String; fn stats(&self) -> String;
} }
...@@ -52,7 +50,6 @@ where ...@@ -52,7 +50,6 @@ where
TraceFormat::read(self, buffer, reuse) TraceFormat::read(self, buffer, reuse)
} }
#[cfg(not(target_arch = "wasm32"))]
fn stats(&self) -> String { fn stats(&self) -> String {
TraceFormat::stats(self) TraceFormat::stats(self)
} }
...@@ -67,7 +64,6 @@ impl ObjectSafeTraceFormat for ErasedTraceFormat { ...@@ -67,7 +64,6 @@ impl ObjectSafeTraceFormat for ErasedTraceFormat {
self.0.read(buffer, reuse) self.0.read(buffer, reuse)
} }
#[cfg(not(target_arch = "wasm32"))]
fn stats(&self) -> String { fn stats(&self) -> String {
self.0.stats() self.0.stats()
} }
...@@ -127,7 +123,7 @@ impl TraceParser { ...@@ -127,7 +123,7 @@ impl TraceParser {
/// Completes a finite trace input and rejects an incomplete final record. /// Completes a finite trace input and rejects an incomplete final record.
#[allow(dead_code, reason = "unused by the native binary's streaming reader")] #[allow(dead_code, reason = "unused by the native binary's streaming reader")]
pub fn finish(mut self) -> Result<()> { pub fn finish(&mut self) -> Result<()> {
self.push(&[])?; self.push(&[])?;
if self.format.is_none() { if self.format.is_none() {
bail!("trace is too short to determine its format"); bail!("trace is too short to determine its format");
...@@ -142,8 +138,7 @@ impl TraceParser { ...@@ -142,8 +138,7 @@ impl TraceParser {
Ok(()) Ok(())
} }
#[cfg(not(target_arch = "wasm32"))] pub fn stats(&self) -> String {
pub(crate) fn stats(&self) -> String {
self.format self.format
.as_ref() .as_ref()
.map(|(format, _)| format.stats()) .map(|(format, _)| format.stats())
......
use std::{borrow::Cow, io::Write, sync::Arc}; use std::{borrow::Cow, io::Write, ops::ControlFlow, sync::Arc};
use flate2::{Compression, write::GzEncoder}; use flate2::{Compression, write::GzEncoder};
use turbopack_trace_server::{ use turbopack_trace_server::{
QueryOptions, SortMode, TraceParser, protocol::ProtocolSession, query_spans, read_trace_bytes, QueryOptions, SortMode, TraceParser, protocol::ProtocolSession, query_spans, read_trace_bytes,
store_container::StoreContainer, read_trace_bytes_with_progress, store_container::StoreContainer,
}; };
use turbopack_trace_utils::tracing::TraceRow; use turbopack_trace_utils::tracing::TraceRow;
...@@ -80,6 +80,52 @@ fn parses_gzip_trace() { ...@@ -80,6 +80,52 @@ fn parses_gzip_trace() {
assert_eq!(root_names(&store), ["test compile"]); assert_eq!(root_names(&store), ["test compile"]);
} }
#[test]
fn reports_raw_and_gzip_load_progress() {
let raw = raw_trace();
let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
encoder.write_all(&raw).unwrap();
let gzip = encoder.finish().unwrap();
for (input, is_compressed) in [(&raw, false), (&gzip, true)] {
let mut reports = Vec::new();
read_trace_bytes_with_progress(input, |progress| {
reports.push((
progress.bytes_read,
progress.total_bytes,
progress.uncompressed_bytes_read,
progress.done,
progress.stats(),
));
ControlFlow::Continue(())
})
.unwrap();
let final_report = reports.last().unwrap();
assert_eq!(final_report.0, input.len());
assert_eq!(final_report.1, input.len());
assert_eq!(final_report.2, raw.len());
assert!(final_report.3);
assert!(final_report.4.contains("1 spans"));
assert!(reports[..reports.len() - 1].iter().all(|report| !report.3));
if is_compressed {
assert_ne!(final_report.0, final_report.2);
} else {
assert_eq!(final_report.0, final_report.2);
}
}
}
#[test]
fn progress_callback_can_abort_loading() {
let error = read_trace_bytes_with_progress(&raw_trace(), |_| ControlFlow::Break(()))
.err()
.unwrap()
.to_string();
assert!(error.contains("aborted by the progress callback"));
}
#[test] #[test]
fn rejects_unsupported_or_incomplete_input() { fn rejects_unsupported_or_incomplete_input() {
let error = read_trace_bytes(&[0x28, 0xb5, 0x2f, 0xfd]) let error = read_trace_bytes(&[0x28, 0xb5, 0x2f, 0xfd])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment