Unverified Commit df85ad1e authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

add tracing for turbopack (#49920)

### What?

adds `NEXT_TURBOPACK_TRACING` env var to enable tracing. Writes into `.next/trace.log`

There are 4 presets:

* `NEXT_TURBOPACK_TRACING=overview` gives a overview of requests and modules processed.
* `NEXT_TURBOPACK_TRACING=next` above plus details for next.js
* `NEXT_TURBOPACK_TRACING=turbopack` above plus details for turbopack
* `NEXT_TURBOPACK_TRACING=turbo-tasks` above plus details for turbo-tasks

Published release builds will only allow `overview` to work, since all detailed instrumentation is statically disabled.

see https://github.com/vercel/turbo/pull/4966 for more details

### Why?

get more insight into build times

### Turbopack changes

* https://github.com/vercel/turbo/pull/4995 
* https://github.com/vercel/turbo/pull/5049 
* https://github.com/vercel/turbo/pull/5053 
* https://github.com/vercel/turbo/pull/4966 
parent 3b6f286d
This diff is collapsed.
...@@ -42,11 +42,11 @@ swc_core = { version = "0.76.18" } ...@@ -42,11 +42,11 @@ swc_core = { version = "0.76.18" }
testing = { version = "0.33.11" } testing = { version = "0.33.11" }
# Turbo crates # Turbo crates
turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230518.4" } turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230522.2" }
# [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros.. # [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros..
turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230518.4" } turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230522.2" }
# [TODO]: need to refactor embed_directory! macro usage in next-core # [TODO]: need to refactor embed_directory! macro usage in next-core
turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230518.4" } turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230522.2" }
# General Deps # General Deps
...@@ -83,6 +83,7 @@ concurrent-queue = "2.1.0" ...@@ -83,6 +83,7 @@ concurrent-queue = "2.1.0"
console = "0.15.5" console = "0.15.5"
console-subscriber = "0.1.8" console-subscriber = "0.1.8"
criterion = "0.4.0" criterion = "0.4.0"
crossbeam-channel = "0.5.8"
dashmap = "5.4.0" dashmap = "5.4.0"
dialoguer = "0.10.3" dialoguer = "0.10.3"
dunce = "1.0.3" dunce = "1.0.3"
...@@ -102,6 +103,7 @@ owo-colors = "3.5.0" ...@@ -102,6 +103,7 @@ owo-colors = "3.5.0"
parking_lot = "0.12.1" parking_lot = "0.12.1"
pathdiff = "0.2.1" pathdiff = "0.2.1"
pin-project-lite = "0.2.9" pin-project-lite = "0.2.9"
postcard = "1.0.4"
predicates = "2.1.5" predicates = "2.1.5"
pretty_assertions = "1.3.0" pretty_assertions = "1.3.0"
proc-macro2 = "1.0.51" proc-macro2 = "1.0.51"
...@@ -123,6 +125,7 @@ tiny-gradient = "0.1.0" ...@@ -123,6 +125,7 @@ tiny-gradient = "0.1.0"
tokio = "1.25.0" tokio = "1.25.0"
tokio-util = { version = "0.7.7", features = ["io"] } tokio-util = { version = "0.7.7", features = ["io"] }
tracing = "0.1.37" tracing = "0.1.37"
tracing-subscriber = "0.3.16"
url = "2.2.2" url = "2.2.2"
urlencoding = "2.1.2" urlencoding = "2.1.2"
webbrowser = "0.8.7" webbrowser = "0.8.7"
......
...@@ -10,8 +10,8 @@ ...@@ -10,8 +10,8 @@
"check": "tsc --noEmit" "check": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@vercel/turbopack-dev": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230518.4", "@vercel/turbopack-dev": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230522.2",
"@vercel/turbopack-node": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230518.4", "@vercel/turbopack-node": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230522.2",
"anser": "^2.1.1", "anser": "^2.1.1",
"css.escape": "^1.5.1", "css.escape": "^1.5.1",
"next": "*", "next": "*",
......
...@@ -56,6 +56,7 @@ futures = { workspace = true } ...@@ -56,6 +56,7 @@ futures = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
mime = { workspace = true } mime = { workspace = true }
next-core = { workspace = true } next-core = { workspace = true }
once_cell = { workspace = true }
owo-colors = { workspace = true } owo-colors = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
tokio = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["full"] }
...@@ -72,6 +73,8 @@ turbopack-binding = { workspace = true, features = [ ...@@ -72,6 +73,8 @@ turbopack-binding = { workspace = true, features = [
"__turbopack_dev_server", "__turbopack_dev_server",
]} ]}
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
webbrowser = { workspace = true } webbrowser = { workspace = true }
[dev-dependencies] [dev-dependencies]
......
...@@ -10,7 +10,7 @@ use std::{ ...@@ -10,7 +10,7 @@ use std::{
future::{join, Future}, future::{join, Future},
io::{stdout, Write}, io::{stdout, Write},
net::{IpAddr, SocketAddr}, net::{IpAddr, SocketAddr},
path::{PathBuf, MAIN_SEPARATOR}, path::{Path, PathBuf, MAIN_SEPARATOR},
sync::Arc, sync::Arc,
time::{Duration, Instant}, time::{Duration, Instant},
}; };
...@@ -25,7 +25,9 @@ use next_core::{ ...@@ -25,7 +25,9 @@ use next_core::{
next_image::NextImageContentSourceVc, pages_structure::find_pages_structure, next_image::NextImageContentSourceVc, pages_structure::find_pages_structure,
router_source::NextRouterContentSourceVc, source_map::NextSourceMapTraceContentSourceVc, router_source::NextRouterContentSourceVc, source_map::NextSourceMapTraceContentSourceVc,
}; };
use once_cell::sync::Lazy;
use owo_colors::OwoColorize; use owo_colors::OwoColorize;
use tracing_subscriber::{prelude::*, EnvFilter, Registry};
use turbo_tasks::{ use turbo_tasks::{
util::{FormatBytes, FormatDuration}, util::{FormatBytes, FormatDuration},
StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi, UpdateInfo, Value, StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi, UpdateInfo, Value,
...@@ -38,7 +40,15 @@ use turbopack_binding::{ ...@@ -38,7 +40,15 @@ use turbopack_binding::{
tasks_memory::MemoryBackend, tasks_memory::MemoryBackend,
}, },
turbopack::{ turbopack::{
cli_utils::issue::{ConsoleUiVc, LogOptions}, cli_utils::{
exit::ExitGuard,
issue::{ConsoleUiVc, LogOptions},
raw_trace::RawTraceLayer,
trace_writer::TraceWriter,
tracing_presets::{
TRACING_OVERVIEW_TARGETS, TRACING_TURBOPACK_TARGETS, TRACING_TURBO_TASKS_TARGETS,
},
},
core::{ core::{
environment::{ServerAddr, ServerAddrVc}, environment::{ServerAddr, ServerAddrVc},
issue::{IssueReporterVc, IssueSeverity}, issue::{IssueReporterVc, IssueSeverity},
...@@ -424,12 +434,81 @@ pub fn register() { ...@@ -424,12 +434,81 @@ pub fn register() {
include!(concat!(env!("OUT_DIR"), "/register.rs")); include!(concat!(env!("OUT_DIR"), "/register.rs"));
} }
static TRACING_NEXT_TARGETS: Lazy<Vec<&str>> = Lazy::new(|| {
[
&TRACING_OVERVIEW_TARGETS[..],
&[
"next_dev=trace",
"next_core=trace",
"next_font=trace",
"turbopack_node=trace",
],
]
.concat()
});
static TRACING_NEXT_TURBOPACK_TARGETS: Lazy<Vec<&str>> =
Lazy::new(|| [&TRACING_NEXT_TARGETS[..], &TRACING_TURBOPACK_TARGETS[..]].concat());
static TRACING_NEXT_TURBO_TASKS_TARGETS: Lazy<Vec<&str>> = Lazy::new(|| {
[
&TRACING_TURBOPACK_TARGETS[..],
&TRACING_TURBO_TASKS_TARGETS[..],
]
.concat()
});
/// Start a devserver with the given options. /// Start a devserver with the given options.
pub async fn start_server(options: &DevServerOptions) -> Result<()> { pub async fn start_server(options: &DevServerOptions) -> Result<()> {
let start = Instant::now(); let start = Instant::now();
#[cfg(feature = "tokio_console")] #[cfg(feature = "tokio_console")]
console_subscriber::init(); console_subscriber::init();
let trace = std::env::var("NEXT_TURBOPACK_TRACING").ok();
let _guard = if let Some(mut trace) = trace {
// Trace presets
match trace.as_str() {
"overview" => {
trace = TRACING_OVERVIEW_TARGETS.join(",");
}
"next" => {
trace = TRACING_NEXT_TARGETS.join(",");
}
"turbopack" => {
trace = TRACING_NEXT_TURBOPACK_TARGETS.join(",");
}
"turbo-tasks" => {
trace = TRACING_NEXT_TURBO_TASKS_TARGETS.join(",");
}
_ => {}
}
let subscriber = Registry::default();
let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap());
let internal_dir = options
.dir
.as_deref()
.unwrap_or_else(|| Path::new("."))
.join(".next");
std::fs::create_dir_all(&internal_dir)
.context("Unable to create .next directory")
.unwrap();
let trace_file = internal_dir.join("trace.log");
let trace_writer = std::fs::File::create(trace_file).unwrap();
let (trace_writer, guard) = TraceWriter::new(trace_writer);
let subscriber = subscriber.with(RawTraceLayer::new(trace_writer));
let guard = ExitGuard::new(guard).unwrap();
subscriber.init();
Some(guard)
} else {
None
};
register(); register();
let dir = options let dir = options
......
...@@ -1012,8 +1012,8 @@ importers: ...@@ -1012,8 +1012,8 @@ importers:
'@types/react': 18.2.5 '@types/react': 18.2.5
'@types/react-dom': 18.2.3 '@types/react-dom': 18.2.3
'@vercel/ncc': ^0.36.0 '@vercel/ncc': ^0.36.0
'@vercel/turbopack-dev': https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230518.4 '@vercel/turbopack-dev': https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230522.2
'@vercel/turbopack-node': https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230518.4 '@vercel/turbopack-node': https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230522.2
anser: ^2.1.1 anser: ^2.1.1
css.escape: ^1.5.1 css.escape: ^1.5.1
find-up: ^6.3.0 find-up: ^6.3.0
...@@ -1025,8 +1025,8 @@ importers: ...@@ -1025,8 +1025,8 @@ importers:
stacktrace-parser: ^0.1.10 stacktrace-parser: ^0.1.10
strip-ansi: ^7.0.1 strip-ansi: ^7.0.1
dependencies: dependencies:
'@vercel/turbopack-dev': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230518.4_react-refresh@0.12.0' '@vercel/turbopack-dev': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230522.2_react-refresh@0.12.0'
'@vercel/turbopack-node': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230518.4' '@vercel/turbopack-node': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230522.2'
anser: 2.1.1 anser: 2.1.1
css.escape: 1.5.1 css.escape: 1.5.1
next: link:../../../../next next: link:../../../../next
...@@ -5997,7 +5997,7 @@ packages: ...@@ -5997,7 +5997,7 @@ packages:
dependencies: dependencies:
'@mdx-js/mdx': 2.2.1 '@mdx-js/mdx': 2.2.1
source-map: 0.7.3 source-map: 0.7.3
webpack: 5.74.0 webpack: 5.74.0_@swc+core@1.3.55
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
...@@ -6669,7 +6669,6 @@ packages: ...@@ -6669,7 +6669,6 @@ packages:
cpu: [arm64] cpu: [arm64]
os: [darwin] os: [darwin]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-darwin-x64/1.3.55: /@swc/core-darwin-x64/1.3.55:
...@@ -6678,7 +6677,6 @@ packages: ...@@ -6678,7 +6677,6 @@ packages:
cpu: [x64] cpu: [x64]
os: [darwin] os: [darwin]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-linux-arm-gnueabihf/1.3.55: /@swc/core-linux-arm-gnueabihf/1.3.55:
...@@ -6687,7 +6685,6 @@ packages: ...@@ -6687,7 +6685,6 @@ packages:
cpu: [arm] cpu: [arm]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-linux-arm64-gnu/1.3.55: /@swc/core-linux-arm64-gnu/1.3.55:
...@@ -6696,7 +6693,6 @@ packages: ...@@ -6696,7 +6693,6 @@ packages:
cpu: [arm64] cpu: [arm64]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-linux-arm64-musl/1.3.55: /@swc/core-linux-arm64-musl/1.3.55:
...@@ -6705,7 +6701,6 @@ packages: ...@@ -6705,7 +6701,6 @@ packages:
cpu: [arm64] cpu: [arm64]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-linux-x64-gnu/1.3.55: /@swc/core-linux-x64-gnu/1.3.55:
...@@ -6714,7 +6709,6 @@ packages: ...@@ -6714,7 +6709,6 @@ packages:
cpu: [x64] cpu: [x64]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-linux-x64-musl/1.3.55: /@swc/core-linux-x64-musl/1.3.55:
...@@ -6723,7 +6717,6 @@ packages: ...@@ -6723,7 +6717,6 @@ packages:
cpu: [x64] cpu: [x64]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-win32-arm64-msvc/1.3.55: /@swc/core-win32-arm64-msvc/1.3.55:
...@@ -6732,7 +6725,6 @@ packages: ...@@ -6732,7 +6725,6 @@ packages:
cpu: [arm64] cpu: [arm64]
os: [win32] os: [win32]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-win32-ia32-msvc/1.3.55: /@swc/core-win32-ia32-msvc/1.3.55:
...@@ -6741,7 +6733,6 @@ packages: ...@@ -6741,7 +6733,6 @@ packages:
cpu: [ia32] cpu: [ia32]
os: [win32] os: [win32]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core-win32-x64-msvc/1.3.55: /@swc/core-win32-x64-msvc/1.3.55:
...@@ -6750,7 +6741,6 @@ packages: ...@@ -6750,7 +6741,6 @@ packages:
cpu: [x64] cpu: [x64]
os: [win32] os: [win32]
requiresBuild: true requiresBuild: true
dev: true
optional: true optional: true
/@swc/core/1.3.55_@swc+helpers@0.5.1: /@swc/core/1.3.55_@swc+helpers@0.5.1:
...@@ -6775,7 +6765,6 @@ packages: ...@@ -6775,7 +6765,6 @@ packages:
'@swc/core-win32-arm64-msvc': 1.3.55 '@swc/core-win32-arm64-msvc': 1.3.55
'@swc/core-win32-ia32-msvc': 1.3.55 '@swc/core-win32-ia32-msvc': 1.3.55
'@swc/core-win32-x64-msvc': 1.3.55 '@swc/core-win32-x64-msvc': 1.3.55
dev: true
/@swc/helpers/0.4.14: /@swc/helpers/0.4.14:
resolution: {integrity: sha512-4C7nX/dvpzB7za4Ql9K81xK3HPxCpHMgwTZVyf+9JQ6VUbn9jjZVN7/Nkdz/Ugzs2CSjqnL/UPXroiVBVHUWUw==} resolution: {integrity: sha512-4C7nX/dvpzB7za4Ql9K81xK3HPxCpHMgwTZVyf+9JQ6VUbn9jjZVN7/Nkdz/Ugzs2CSjqnL/UPXroiVBVHUWUw==}
...@@ -23669,7 +23658,6 @@ packages: ...@@ -23669,7 +23658,6 @@ packages:
source-map: 0.6.1 source-map: 0.6.1
terser: 5.14.1 terser: 5.14.1
webpack: 5.74.0_@swc+core@1.3.55 webpack: 5.74.0_@swc+core@1.3.55
dev: true
/terser-webpack-plugin/5.2.4_webpack@5.74.0: /terser-webpack-plugin/5.2.4_webpack@5.74.0:
resolution: {integrity: sha512-E2CkNMN+1cho04YpdANyRrn8CyN4yMy+WdFKZIySFZrGXZxJwJP6PMNGGc/Mcr6qygQHUUqRxnAPmi0M9f00XA==} resolution: {integrity: sha512-E2CkNMN+1cho04YpdANyRrn8CyN4yMy+WdFKZIySFZrGXZxJwJP6PMNGGc/Mcr6qygQHUUqRxnAPmi0M9f00XA==}
...@@ -25079,7 +25067,6 @@ packages: ...@@ -25079,7 +25067,6 @@ packages:
- '@swc/core' - '@swc/core'
- esbuild - esbuild
- uglify-js - uglify-js
dev: true
/websocket-driver/0.7.3: /websocket-driver/0.7.3:
resolution: {integrity: sha512-bpxWlvbbB459Mlipc5GBzzZwhoZgGEZLuqPaR0INBGnPAY1vdBX6hPnoFXiw+3yWxDuHyQjO2oXTMyS8A5haFg==} resolution: {integrity: sha512-bpxWlvbbB459Mlipc5GBzzZwhoZgGEZLuqPaR0INBGnPAY1vdBX6hPnoFXiw+3yWxDuHyQjO2oXTMyS8A5haFg==}
...@@ -25490,9 +25477,9 @@ packages: ...@@ -25490,9 +25477,9 @@ packages:
/zwitch/2.0.4: /zwitch/2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
'@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230518.4_react-refresh@0.12.0': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230522.2_react-refresh@0.12.0':
resolution: {tarball: https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230518.4} resolution: {tarball: https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230522.2}
id: '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230518.4' id: '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230522.2'
name: '@vercel/turbopack-dev' name: '@vercel/turbopack-dev'
version: 0.0.0 version: 0.0.0
dependencies: dependencies:
...@@ -25502,8 +25489,8 @@ packages: ...@@ -25502,8 +25489,8 @@ packages:
- webpack - webpack
dev: false dev: false
'@gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230518.4': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230522.2':
resolution: {tarball: https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230518.4} resolution: {tarball: https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230522.2}
name: '@vercel/turbopack-node' name: '@vercel/turbopack-node'
version: 0.0.0 version: 0.0.0
dependencies: dependencies:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment