Unverified Commit 9b96a51b authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: forward module graph, entrypoint write, and emit spans to the JS trace (#98408)

<img width="1273" height="207" alt="Bildschirmfoto 2026-09-09 um 16 32
57"
src="https://github.com/user-attachments/assets/849e55d4-bf3f-4ef0-bf0d-bd2879f89e25"
/>



## Summary

Adds three Turbopack build-phase spans to the JS-side trace
(`.next/trace`, `.next/trace-build`, and trace uploads), forwarded from
Rust via `turbo_tasks.send_compilation_event` like the existing
`turbopack-persistence` span:

- `turbopack-module-graph` — whole-app module graph computation
(`whole_app_module_graph_operation`)
- `turbopack-write-entrypoints` — computing all output assets for the
entrypoints write phase (`all_entrypoints_write_to_disk_operation`)
- `turbopack-emit` — writing all output assets to disk
(`emit_all_output_assets_once_with_issues_operation`)

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
parent 719d7cea
use std::{
iter,
path::{Path, PathBuf},
time::Duration,
sync::Arc,
time::{Duration, Instant, SystemTime},
};
use anyhow::{Context, Result, bail};
......@@ -45,7 +46,8 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, Completions, FxIndexMap, InvalidationReason, NonLocalValue, OperationValue,
OperationVc, ReadRef, ResolvedVc, State, TransientInstance, TryFlatJoinIterExt, TryJoinIterExt,
Vc, debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs,
Vc, debug::ValueDebugFormat, fxindexmap, message_queue::TraceEvent, trace::TraceRawVcs,
turbo_tasks,
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{
......@@ -2893,9 +2895,11 @@ async fn scale_down_node_pool(project: ResolvedVc<Project>) -> Result<()> {
async fn whole_app_module_graph_operation(
project: ResolvedVc<Project>,
) -> Result<Vc<BaseAndFullModuleGraph>> {
let start = Instant::now();
let wall_start = SystemTime::now();
let span = tracing::info_span!("whole app module graph", modules = Empty, edges = Empty);
let span_clone = span.clone();
async move {
let result = async move {
let next_mode = project.next_mode();
let should_trace = *project.should_write_nft_manifests().await?;
let should_read_binding_usage = next_mode.await?.is_production();
......@@ -2980,7 +2984,14 @@ async fn whole_app_module_graph_operation(
.cell())
}
.instrument(span_clone)
.await
.await;
turbo_tasks().send_compilation_event(Arc::new(TraceEvent::new_with_duration(
"turbopack-module-graph",
wall_start,
start.elapsed(),
serde_json::json!([]),
)));
result
}
#[turbo_tasks::value(shared)]
......
......@@ -5,7 +5,7 @@ use std::{
path::{Path, PathBuf},
sync::{Arc, LazyLock, Mutex},
thread,
time::Duration,
time::{Duration, SystemTime},
};
use anyhow::{Context, Result, anyhow, bail};
......@@ -55,10 +55,10 @@ use turbo_tasks::{
Effects, FxIndexSet, GcRoot, OperationValue, OperationVc, PrettyPrintError, ReadRef,
ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasksApi, TurboTasksCallApi, UpdateInfo,
Vc, mark_top_level_task,
message_queue::{CompilationEvent, Severity},
message_queue::{CompilationEvent, Severity, TraceEvent},
read_strongly_consistent_and_apply_effects, take_effects,
trace::TraceRawVcs,
unmark_top_level_task_may_leak_eventually_consistent_state,
turbo_tasks, unmark_top_level_task_may_leak_eventually_consistent_state,
};
use turbo_tasks_backend::{BackingStorageOptions, db_invalidation::invalidation_reasons};
#[cfg(windows)]
......@@ -1620,11 +1620,22 @@ pub async fn all_entrypoints_write_to_disk_operation(
app_dir_only: bool,
write_phase: EntrypointsWritePhase,
) -> Result<Vc<Entrypoints>> {
let start = Instant::now();
let wall_start = SystemTime::now();
// Compute all outputs for this phase but do not emit to disk yet.
let output_assets_operation = output_assets_operation(project, app_dir_only, write_phase);
let _ = output_assets_operation.connect().await?;
let result = output_assets_operation
.connect()
.await
.map(|_| project.entrypoints());
Ok(project.entrypoints())
turbo_tasks().send_compilation_event(Arc::new(TraceEvent::new_with_duration(
"turbopack-write-entrypoints",
wall_start,
start.elapsed(),
serde_json::json!([]),
)));
result
}
#[turbo_tasks::function(operation)]
......@@ -1681,16 +1692,29 @@ async fn emit_all_output_assets_once_with_issues_operation(
app_dir_only: bool,
has_deferred_entrypoints: bool,
) -> Result<Vc<OperationResult>> {
let entrypoints_operation = EntrypointsOperation::new(emit_all_output_assets_once_operation(
container,
app_dir_only,
has_deferred_entrypoints,
));
let filter = container.project().issue_filter().await?;
let (_, issues, effects) =
strongly_consistent_catch_collectables(entrypoints_operation, &filter).await?;
Ok(OperationResult { issues, effects }.cell())
let start = Instant::now();
let wall_start = SystemTime::now();
let result = async {
let entrypoints_operation =
EntrypointsOperation::new(emit_all_output_assets_once_operation(
container,
app_dir_only,
has_deferred_entrypoints,
));
let filter = container.project().issue_filter().await?;
let (_, issues, effects) =
strongly_consistent_catch_collectables(entrypoints_operation, &filter).await?;
Ok(OperationResult { issues, effects }.cell())
}
.await;
turbo_tasks().send_compilation_event(Arc::new(TraceEvent::new_with_duration(
"turbopack-emit",
wall_start,
start.elapsed(),
serde_json::json!([]),
)));
result
}
#[turbo_tasks::function(operation)]
......
......@@ -17,6 +17,9 @@ const allowlistedEvents = new Set([
'telemetry-flush',
'turbopack-persistence',
'turbopack-compaction',
'turbopack-module-graph',
'turbopack-write-entrypoints',
'turbopack-emit',
])
export default createJsonReporter({
......
......@@ -11,6 +11,9 @@ const COMMON_ALLOWED_EVENTS = [
'memory-usage',
'turbopack-persistence',
'turbopack-compaction',
'turbopack-module-graph',
'turbopack-write-entrypoints',
'turbopack-emit',
]
// Predefined set of the event names to be included in the trace.
......
import { nextTestSetup, isNextDev, isNextStart } from 'e2e-utils'
import { join } from 'path'
import { existsSync } from 'fs'
import { existsSync, rmSync } from 'fs'
import { parseTraceFile } from '../../../lib/parse-trace-file'
// The trace assertions require access to local build output.
......@@ -56,6 +56,13 @@ describe('trace-build-file', () => {
})
it('should only contain allowlisted events', async () => {
// Remove the persistent cache so this build re-executes (and re-emits
// the trace spans of) all turbo-tasks operations instead of serving
// them from the cache.
rmSync(join(next.testDir, '.next', 'cache', 'turbopack'), {
recursive: true,
force: true,
})
await next.build()
const traceBuildPath = join(next.testDir, '.next/trace-build')
......@@ -90,7 +97,10 @@ describe('trace-build-file', () => {
"static-check",
"static-generation",
"telemetry-flush",
"turbopack-emit",
"turbopack-module-graph",
"turbopack-persistence",
"turbopack-write-entrypoints",
]
`)
} else {
......
......@@ -1599,29 +1599,21 @@ impl TurboTasksBackend {
)));
}
let wall_start_ms = wall_start
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
// as_millis_f64 is not stable yet
.as_secs_f64()
* 1000.0;
let wall_end_ms: f64 = wall_start_ms + elapsed.as_secs_f64() * 1000.0;
let (persist_start_ms, persist_end_ms) = if let Some(gc_elapsed) = gc_elapsed {
let persist_begin_ms = wall_start_ms + gc_elapsed.as_secs_f64() * 1000.0;
turbo_tasks.send_compilation_event(Arc::new(TraceEvent::new(
let (persist_wall_start, persist_wall_duration) = if let Some(gc_elapsed) = gc_elapsed {
turbo_tasks.send_compilation_event(Arc::new(TraceEvent::new_with_duration(
"turbopack-gc",
wall_start_ms,
persist_begin_ms,
wall_start,
gc_elapsed,
serde_json::json!([]),
)));
(persist_begin_ms, wall_end_ms)
(wall_start + gc_elapsed, elapsed.saturating_sub(gc_elapsed))
} else {
(wall_start_ms, wall_end_ms)
(wall_start, elapsed)
};
turbo_tasks.send_compilation_event(Arc::new(TraceEvent::new(
turbo_tasks.send_compilation_event(Arc::new(TraceEvent::new_with_duration(
"turbopack-persistence",
persist_start_ms,
persist_end_ms,
persist_wall_start,
persist_wall_duration,
serde_json::json!([
["reason", reason.as_str()],
[
......
......@@ -183,16 +183,10 @@ fn do_compact(
turbo_tasks()
.send_compilation_event(Arc::new(TimingEvent::new(message.to_string(), elapsed)));
}
let wall_start_ms = wall_start
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64()
* 1000.0;
let wall_end_ms = wall_start_ms + elapsed.as_secs_f64() * 1000.0;
turbo_tasks().send_compilation_event(Arc::new(TraceEvent::new(
turbo_tasks().send_compilation_event(Arc::new(TraceEvent::new_with_duration(
"turbopack-compaction",
wall_start_ms,
wall_end_ms,
wall_start,
elapsed,
serde_json::json!([
["bytes_written", stats.bytes_written],
["bytes_deleted", stats.bytes_deleted],
......
use std::{any::Any, collections::VecDeque, fmt::Display, sync::Arc, time::Duration};
use std::{
any::Any,
collections::VecDeque,
fmt::Display,
sync::Arc,
time::{Duration, SystemTime},
};
use dashmap::DashMap;
use parking_lot::Mutex;
......@@ -340,6 +346,31 @@ impl TraceEvent {
attributes,
}
}
/// Creates a `TraceEvent` that started at `wall_start` (wall clock) and ran for `duration`.
///
/// Prefer passing a duration measured with a monotonic clock (e.g.
/// [`std::time::Instant::elapsed`]) over computing it from two [`SystemTime`]s, so that
/// wall-clock adjustments don't skew (or negate) the span's duration.
pub fn new_with_duration(
name: &'static str,
wall_start: SystemTime,
duration: Duration,
attributes: serde_json::Value,
) -> Self {
let start_time_ms = wall_start
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
// as_millis_f64 is not stable yet
.as_secs_f64()
* 1000.0;
Self::new(
name,
start_time_ms,
start_time_ms + duration.as_secs_f64() * 1000.0,
attributes,
)
}
}
impl CompilationEvent for TraceEvent {
......@@ -399,6 +430,20 @@ mod tests {
assert!(rx2.recv().await.is_none());
}
#[test]
fn test_trace_event_new_with_duration() {
let start = SystemTime::UNIX_EPOCH + Duration::from_secs(10);
let event = TraceEvent::new_with_duration(
"test",
start,
Duration::from_millis(1500),
serde_json::json!([]),
);
assert_eq!(event.name, "test");
assert_eq!(event.start_time_ms, 10_000.0);
assert_eq!(event.end_time_ms, 11_500.0);
}
#[test]
fn test_timing_event_string_formatting() {
let tests = vec![
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment