Unverified Commit aae734b3 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: next build --analyze (#85197)

This implements a portion of the first iteration of a bundle analyzer for Turbopack in Next.js by writing data files to disk. A separate visualization and explorer tool will follow.

This data is written when builds are run with `next build --experimental-analyze`. An upcoming PR will implement a dedicated `next analyze` command that will write the data files without writing a complete build.

The data is derived from Turbopack’s module and chunk graphs, and precise module sizes are reversed from the source map data of each chunk. Importantly, this requires that source maps are generated within Turbopack and the project is recomputed with them -- however, no source maps are written as part of the build artifacts if the user did not configure their Next.js config that way.

The binary format written in these files is subject to change, so the feature is experimental for the time being. Until it stabilizes, no guarantees will be provided between it and the visualizer tool.

Test Plan: 

In a test app:
- do not configure `productionBrowserSourceMaps`
- run `next build` and confirm **no files are written** to `.next/diagnostics/analyze`
- run `next build --analyze` and confirm files **are** written to `.next/diagnostics/analyze`
- Open the data with the yet-to-be-published visualization tool and confirm data is visible
parent 014a2549
...@@ -4253,6 +4253,7 @@ name = "next-api" ...@@ -4253,6 +4253,7 @@ name = "next-api"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"byteorder",
"either", "either",
"futures", "futures",
"indexmap 2.9.0", "indexmap 2.9.0",
...@@ -4274,6 +4275,7 @@ dependencies = [ ...@@ -4274,6 +4275,7 @@ dependencies = [
"turbo-tasks-malloc", "turbo-tasks-malloc",
"turbo-unix-path", "turbo-unix-path",
"turbopack", "turbopack",
"turbopack-analyze",
"turbopack-browser", "turbopack-browser",
"turbopack-core", "turbopack-core",
"turbopack-ecmascript", "turbopack-ecmascript",
...@@ -9469,6 +9471,23 @@ dependencies = [ ...@@ -9469,6 +9471,23 @@ dependencies = [
"turbopack-wasm", "turbopack-wasm",
] ]
[[package]]
name = "turbopack-analyze"
version = "0.1.0"
dependencies = [
"anyhow",
"rustc-hash 2.1.1",
"serde",
"serde_json",
"tokio",
"turbo-rcstr",
"turbo-tasks",
"turbo-tasks-backend",
"turbo-tasks-fs",
"turbo-tasks-testing",
"turbopack-core",
]
[[package]] [[package]]
name = "turbopack-bench" name = "turbopack-bench"
version = "0.1.0" version = "0.1.0"
......
...@@ -317,6 +317,7 @@ turbopack-cli-utils = { path = "turbopack/crates/turbopack-cli-utils" } ...@@ -317,6 +317,7 @@ turbopack-cli-utils = { path = "turbopack/crates/turbopack-cli-utils" }
turbopack-core = { path = "turbopack/crates/turbopack-core" } turbopack-core = { path = "turbopack/crates/turbopack-core" }
turbopack-create-test-app = { path = "turbopack/crates/turbopack-create-test-app" } turbopack-create-test-app = { path = "turbopack/crates/turbopack-create-test-app" }
turbopack-css = { path = "turbopack/crates/turbopack-css" } turbopack-css = { path = "turbopack/crates/turbopack-css" }
turbopack-analyze = { path = "turbopack/crates/turbopack-analyze" }
turbopack-browser = { path = "turbopack/crates/turbopack-browser" } turbopack-browser = { path = "turbopack/crates/turbopack-browser" }
turbopack-dev-server = { path = "turbopack/crates/turbopack-dev-server" } turbopack-dev-server = { path = "turbopack/crates/turbopack-dev-server" }
turbopack-ecmascript = { path = "turbopack/crates/turbopack-ecmascript" } turbopack-ecmascript = { path = "turbopack/crates/turbopack-ecmascript" }
...@@ -377,6 +378,7 @@ async-compression = { version = "0.3.13", default-features = false, features = [ ...@@ -377,6 +378,7 @@ async-compression = { version = "0.3.13", default-features = false, features = [
] } ] }
async-trait = "0.1.64" async-trait = "0.1.64"
bitfield = "0.18.0" bitfield = "0.18.0"
byteorder = "1.5.0"
bytes = "1.1.0" bytes = "1.1.0"
bytes-str = "0.2.7" bytes-str = "0.2.7"
chrono = "0.4.23" chrono = "0.4.23"
......
use std::{iter::once, sync::Arc};
use anyhow::Result;
use next_api::{
analyze::{AnalyzeDataOutputAsset, ModulesDataOutputAsset},
project::ProjectContainer,
};
use turbo_tasks::{Effects, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{diagnostics::PlainDiagnostic, issue::PlainIssue, output::OutputAssets};
use crate::next_api::utils::strongly_consistent_catch_collectables;
#[turbo_tasks::value(serialization = "none")]
pub struct WriteAnalyzeResult {
pub issues: Arc<Vec<ReadRef<PlainIssue>>>,
pub diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
pub effects: Arc<Effects>,
}
#[turbo_tasks::function(operation)]
pub async fn write_analyze_data_with_issues_operation(
project: ResolvedVc<ProjectContainer>,
app_dir_only: bool,
) -> Result<Vc<WriteAnalyzeResult>> {
let analyze_data_op = write_analyze_data_with_issues_operation_inner(project, app_dir_only);
let (_analyze_data, issues, diagnostics, effects) =
strongly_consistent_catch_collectables(analyze_data_op).await?;
Ok(WriteAnalyzeResult {
issues,
diagnostics,
effects,
}
.cell())
}
#[turbo_tasks::function(operation)]
async fn write_analyze_data_with_issues_operation_inner(
project: ResolvedVc<ProjectContainer>,
app_dir_only: bool,
) -> Result<()> {
let analyze_data_op = get_analyze_data_operation(project, app_dir_only);
project
.project()
.emit_all_output_assets(analyze_data_op)
.as_side_effect()
.await?;
Ok(())
}
#[turbo_tasks::function(operation)]
async fn get_analyze_data_operation(
container: ResolvedVc<ProjectContainer>,
app_dir_only: bool,
) -> Result<Vc<OutputAssets>> {
let project = container.project();
let project =
project.with_next_config(project.next_config().with_production_browser_source_maps());
let analyze_output_root = project
.node_root()
.owned()
.await?
.join("diagnostics/analyze")?;
let whole_app_module_graphs = project.whole_app_module_graphs();
let analyze_output_root = &analyze_output_root;
let analyze_data = project
.get_all_endpoint_groups(app_dir_only)
.await?
.iter()
.map(|(key, endpoint_group)| async move {
let output_assets = endpoint_group.output_assets();
let analyze_data = AnalyzeDataOutputAsset::new(
analyze_output_root
.join(&key.to_string())?
.join("analyze.data")?,
output_assets,
)
.to_resolved()
.await?;
Ok(ResolvedVc::upcast(analyze_data))
})
.try_join()
.await?;
whole_app_module_graphs.as_side_effect().await?;
let modules_data = ResolvedVc::upcast(
ModulesDataOutputAsset::new(
analyze_output_root.join("modules.data")?,
Vc::cell(vec![whole_app_module_graphs.await?.full]),
)
.to_resolved()
.await?,
);
Ok(Vc::cell(
analyze_data
.iter()
.cloned()
.chain(once(modules_data))
.collect(),
))
}
pub mod analyze;
pub mod endpoint; pub mod endpoint;
pub mod project; pub mod project;
pub mod turbopack_ctx; pub mod turbopack_ctx;
......
...@@ -64,6 +64,7 @@ use url::Url; ...@@ -64,6 +64,7 @@ use url::Url;
use crate::{ use crate::{
next_api::{ next_api::{
analyze::{WriteAnalyzeResult, write_analyze_data_with_issues_operation},
endpoint::ExternalEndpoint, endpoint::ExternalEndpoint,
turbopack_ctx::{ turbopack_ctx::{
NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject, NextTurboTasks, NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject, NextTurboTasks,
...@@ -915,6 +916,13 @@ fn project_container_entrypoints_operation( ...@@ -915,6 +916,13 @@ fn project_container_entrypoints_operation(
container.entrypoints() container.entrypoints()
} }
#[turbo_tasks::value(serialization = "none")]
struct OperationResult {
issues: Arc<Vec<ReadRef<PlainIssue>>>,
diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
effects: Arc<Effects>,
}
#[turbo_tasks::value(serialization = "none")] #[turbo_tasks::value(serialization = "none")]
struct AllWrittenEntrypointsWithIssues { struct AllWrittenEntrypointsWithIssues {
entrypoints: Option<ReadRef<EntrypointsOperation>>, entrypoints: Option<ReadRef<EntrypointsOperation>>,
...@@ -995,9 +1003,10 @@ pub async fn all_entrypoints_write_to_disk_operation( ...@@ -995,9 +1003,10 @@ pub async fn all_entrypoints_write_to_disk_operation(
project: ResolvedVc<ProjectContainer>, project: ResolvedVc<ProjectContainer>,
app_dir_only: bool, app_dir_only: bool,
) -> Result<Vc<Entrypoints>> { ) -> Result<Vc<Entrypoints>> {
let output_assets_operation = output_assets_operation(project, app_dir_only);
project project
.project() .project()
.emit_all_output_assets(output_assets_operation(project, app_dir_only)) .emit_all_output_assets(output_assets_operation)
.as_side_effect() .as_side_effect()
.await?; .await?;
...@@ -1037,6 +1046,49 @@ async fn output_assets_operation( ...@@ -1037,6 +1046,49 @@ async fn output_assets_operation(
} }
#[tracing::instrument(level = "info", name = "get entrypoints", skip_all)] #[tracing::instrument(level = "info", name = "get entrypoints", skip_all)]
#[napi]
pub async fn project_entrypoints(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
) -> napi::Result<TurbopackResult<Option<NapiEntrypoints>>> {
let container = project.container;
let (entrypoints, issues, diags) = project
.turbopack_ctx
.turbo_tasks()
.run_once(async move {
let entrypoints_with_issues_op = get_entrypoints_with_issues_operation(container);
// Read and compile the files
let EntrypointsWithIssues {
entrypoints,
issues,
diagnostics,
effects: _,
} = &*entrypoints_with_issues_op
.read_strongly_consistent()
.await?;
Ok((entrypoints.clone(), issues.clone(), diagnostics.clone()))
})
.await
.map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
let result = match entrypoints {
Some(entrypoints) => Some(NapiEntrypoints::from_entrypoints_op(
&entrypoints,
&project.turbopack_ctx,
)?),
None => None,
};
Ok(TurbopackResult {
result,
issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
diagnostics: diags.iter().map(|d| NapiDiagnostic::from(d)).collect(),
})
}
#[tracing::instrument(level = "info", name = "subscribe to entrypoints", skip_all)]
#[napi(ts_return_type = "{ __napiType: \"RootTask\" }")] #[napi(ts_return_type = "{ __napiType: \"RootTask\" }")]
pub fn project_entrypoints_subscribe( pub fn project_entrypoints_subscribe(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>, #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
...@@ -1718,3 +1770,37 @@ pub fn project_get_source_map_sync( ...@@ -1718,3 +1770,37 @@ pub fn project_get_source_map_sync(
tokio::runtime::Handle::current().block_on(project_get_source_map(project, file_path)) tokio::runtime::Handle::current().block_on(project_get_source_map(project, file_path))
}) })
} }
#[napi]
pub async fn project_write_analyze_data(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
app_dir_only: bool,
) -> napi::Result<TurbopackResult<()>> {
let container = project.container;
let (issues, diagnostics) = project
.turbopack_ctx
.turbo_tasks()
.run_once(async move {
let analyze_data_op = write_analyze_data_with_issues_operation(container, app_dir_only);
let WriteAnalyzeResult {
issues,
diagnostics,
effects,
} = &*analyze_data_op.read_strongly_consistent().await?;
// Write the files to disk
effects.apply().await?;
Ok((issues.clone(), diagnostics.clone()))
})
.await
.map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
Ok(TurbopackResult {
result: (),
issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
diagnostics: diagnostics
.iter()
.map(|d| NapiDiagnostic::from(d))
.collect(),
})
}
...@@ -14,6 +14,7 @@ workspace = true ...@@ -14,6 +14,7 @@ workspace = true
[dependencies] [dependencies]
anyhow = { workspace = true, features = ["backtrace"] } anyhow = { workspace = true, features = ["backtrace"] }
byteorder = { workspace = true }
either = { workspace = true } either = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
...@@ -31,6 +32,7 @@ turbo-tasks-env = { workspace = true } ...@@ -31,6 +32,7 @@ turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-unix-path = { workspace = true } turbo-unix-path = { workspace = true }
turbopack = { workspace = true } turbopack = { workspace = true }
turbopack-analyze = { workspace = true }
turbopack-browser = { workspace = true } turbopack-browser = { workspace = true }
turbopack-core = { workspace = true } turbopack-core = { workspace = true }
turbopack-ecmascript = { workspace = true } turbopack-ecmascript = { workspace = true }
......
This diff is collapsed.
...@@ -81,8 +81,10 @@ use crate::{ ...@@ -81,8 +81,10 @@ use crate::{
all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root,
get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings,
}, },
project::{ModuleGraphs, Project}, project::{BaseAndFullModuleGraph, Project},
route::{AppPageRoute, Endpoint, EndpointOutput, EndpointOutputPaths, Route, Routes}, route::{
AppPageRoute, Endpoint, EndpointOutput, EndpointOutputPaths, ModuleGraphs, Route, Routes,
},
server_actions::{build_server_actions_loader, create_server_actions_manifest}, server_actions::{build_server_actions_loader, create_server_actions_manifest},
webpack_stats::generate_webpack_stats, webpack_stats::generate_webpack_stats,
}; };
...@@ -850,7 +852,7 @@ impl AppProject { ...@@ -850,7 +852,7 @@ impl AppProject {
rsc_entry: ResolvedVc<Box<dyn Module>>, rsc_entry: ResolvedVc<Box<dyn Module>>,
client_shared_entries: Vc<EvaluatableAssets>, client_shared_entries: Vc<EvaluatableAssets>,
has_layout_segments: bool, has_layout_segments: bool,
) -> Result<Vc<ModuleGraphs>> { ) -> Result<Vc<BaseAndFullModuleGraph>> {
if *self.project.per_page_module_graph().await? { if *self.project.per_page_module_graph().await? {
let should_trace = self.project.next_mode().await?.is_production(); let should_trace = self.project.next_mode().await?.is_production();
let client_shared_entries = client_shared_entries let client_shared_entries = client_shared_entries
...@@ -946,7 +948,7 @@ impl AppProject { ...@@ -946,7 +948,7 @@ impl AppProject {
graphs.push(additional_module_graph); graphs.push(additional_module_graph);
let full = ModuleGraph::from_graphs(graphs); let full = ModuleGraph::from_graphs(graphs);
Ok(ModuleGraphs { Ok(BaseAndFullModuleGraph {
base: base.to_resolved().await?, base: base.to_resolved().await?,
full: full.to_resolved().await?, full: full.to_resolved().await?,
} }
...@@ -2102,6 +2104,22 @@ impl Endpoint for AppEndpoint { ...@@ -2102,6 +2104,22 @@ impl Endpoint for AppEndpoint {
server_actions_loader, server_actions_loader,
])])) ])]))
} }
#[turbo_tasks::function]
async fn module_graphs(self: Vc<Self>) -> Result<Vc<ModuleGraphs>> {
let this = self.await?;
let app_entry = self.app_endpoint_entry().await?;
let module_graphs = this
.app_project
.app_module_graphs(
self,
*app_entry.rsc_entry,
this.app_project.client_runtime_entries(),
matches!(this.ty, AppEndpointType::Page { .. }),
)
.await?;
Ok(Vc::cell(vec![module_graphs.full]))
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
......
...@@ -2,7 +2,7 @@ use anyhow::{Result, bail}; ...@@ -2,7 +2,7 @@ use anyhow::{Result, bail};
use turbo_tasks::{Completion, Vc}; use turbo_tasks::{Completion, Vc};
use turbopack_core::module_graph::GraphEntries; use turbopack_core::module_graph::GraphEntries;
use crate::route::{Endpoint, EndpointOutput}; use crate::route::{Endpoint, EndpointOutput, ModuleGraphs};
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct EmptyEndpoint; pub struct EmptyEndpoint;
...@@ -36,4 +36,9 @@ impl Endpoint for EmptyEndpoint { ...@@ -36,4 +36,9 @@ impl Endpoint for EmptyEndpoint {
fn entries(self: Vc<Self>) -> Vc<GraphEntries> { fn entries(self: Vc<Self>) -> Vc<GraphEntries> {
GraphEntries::empty() GraphEntries::empty()
} }
#[turbo_tasks::function]
fn module_graphs(self: Vc<Self>) -> Vc<ModuleGraphs> {
Vc::cell(vec![])
}
} }
...@@ -32,7 +32,7 @@ use crate::{ ...@@ -32,7 +32,7 @@ use crate::{
all_server_paths, get_js_paths_from_root, get_wasm_paths_from_root, wasm_paths_to_bindings, all_server_paths, get_js_paths_from_root, get_wasm_paths_from_root, wasm_paths_to_bindings,
}, },
project::Project, project::Project,
route::{Endpoint, EndpointOutput, EndpointOutputPaths}, route::{Endpoint, EndpointOutput, EndpointOutputPaths, ModuleGraphs},
}; };
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -69,7 +69,7 @@ impl InstrumentationEndpoint { ...@@ -69,7 +69,7 @@ impl InstrumentationEndpoint {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn core_modules(&self) -> Result<Vc<InstrumentationCoreModules>> { async fn entry_module(&self) -> Result<Vc<Box<dyn Module>>> {
let userland_module = self let userland_module = self
.asset_context .asset_context
.process( .process(
...@@ -80,6 +80,10 @@ impl InstrumentationEndpoint { ...@@ -80,6 +80,10 @@ impl InstrumentationEndpoint {
.to_resolved() .to_resolved()
.await?; .await?;
if !self.is_edge {
return Ok(*userland_module);
}
let edge_entry_module = wrap_edge_entry( let edge_entry_module = wrap_edge_entry(
*self.asset_context, *self.asset_context,
self.project.project_path().owned().await?, self.project.project_path().owned().await?,
...@@ -89,17 +93,13 @@ impl InstrumentationEndpoint { ...@@ -89,17 +93,13 @@ impl InstrumentationEndpoint {
.to_resolved() .to_resolved()
.await?; .await?;
Ok(InstrumentationCoreModules { Ok(*edge_entry_module)
userland_module,
edge_entry_module,
}
.cell())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn edge_chunk_group(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> { async fn edge_chunk_group(self: Vc<Self>) -> Result<Vc<OutputAssetsWithReferenced>> {
let this = self.await?; let this = self.await?;
let module = self.core_modules().await?.edge_entry_module; let module = self.entry_module().to_resolved().await?;
let module_graph = this.project.module_graph(*module); let module_graph = this.project.module_graph(*module);
...@@ -118,7 +118,7 @@ impl InstrumentationEndpoint { ...@@ -118,7 +118,7 @@ impl InstrumentationEndpoint {
let chunking_context = this.project.server_chunking_context(false); let chunking_context = this.project.server_chunking_context(false);
let userland_module = self.core_modules().await?.userland_module; let userland_module = self.entry_module().to_resolved().await?;
let module_graph = this.project.module_graph(*userland_module); let module_graph = this.project.module_graph(*userland_module);
let Some(module) = ResolvedVc::try_downcast(userland_module) else { let Some(module) = ResolvedVc::try_downcast(userland_module) else {
...@@ -200,12 +200,6 @@ impl InstrumentationEndpoint { ...@@ -200,12 +200,6 @@ impl InstrumentationEndpoint {
} }
} }
#[turbo_tasks::value]
struct InstrumentationCoreModules {
pub userland_module: ResolvedVc<Box<dyn Module>>,
pub edge_entry_module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Endpoint for InstrumentationEndpoint { impl Endpoint for InstrumentationEndpoint {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -249,13 +243,15 @@ impl Endpoint for InstrumentationEndpoint { ...@@ -249,13 +243,15 @@ impl Endpoint for InstrumentationEndpoint {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> { async fn entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
let core_modules = self.core_modules().await?; let entry_module = self.entry_module().to_resolved().await?;
Ok(Vc::cell(vec![ChunkGroupEntry::Entry( Ok(Vc::cell(vec![ChunkGroupEntry::Entry(vec![entry_module])]))
if self.await?.is_edge { }
vec![core_modules.edge_entry_module]
} else { #[turbo_tasks::function]
vec![core_modules.userland_module] async fn module_graphs(self: Vc<Self>) -> Result<Vc<ModuleGraphs>> {
}, let this = self.await?;
)])) let module = self.entry_module();
let module_graph = this.project.module_graph(module).to_resolved().await?;
Ok(Vc::cell(vec![module_graph]))
} }
} }
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
#![feature(impl_trait_in_assoc_type)] #![feature(impl_trait_in_assoc_type)]
pub mod analyze;
mod app; mod app;
mod client_references; mod client_references;
mod dynamic_imports; mod dynamic_imports;
......
...@@ -38,7 +38,7 @@ use crate::{ ...@@ -38,7 +38,7 @@ use crate::{
get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings,
}, },
project::Project, project::Project,
route::{Endpoint, EndpointOutput, EndpointOutputPaths}, route::{Endpoint, EndpointOutput, EndpointOutputPaths, ModuleGraphs},
}; };
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -386,4 +386,15 @@ impl Endpoint for MiddlewareEndpoint { ...@@ -386,4 +386,15 @@ impl Endpoint for MiddlewareEndpoint {
self.entry_module().to_resolved().await?, self.entry_module().to_resolved().await?,
])])) ])]))
} }
#[turbo_tasks::function]
async fn module_graphs(self: Vc<Self>) -> Result<Vc<ModuleGraphs>> {
let this = self.await?;
let module_graph = this
.project
.module_graph(self.entry_module())
.to_resolved()
.await?;
Ok(Vc::cell(vec![module_graph]))
}
} }
...@@ -78,7 +78,7 @@ use crate::{ ...@@ -78,7 +78,7 @@ use crate::{
get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings,
}, },
project::Project, project::Project,
route::{Endpoint, EndpointOutput, EndpointOutputPaths, Route, Routes}, route::{Endpoint, EndpointOutput, EndpointOutputPaths, ModuleGraphs, Route, Routes},
webpack_stats::generate_webpack_stats, webpack_stats::generate_webpack_stats,
}; };
...@@ -1728,6 +1728,13 @@ impl Endpoint for PageEndpoint { ...@@ -1728,6 +1728,13 @@ impl Endpoint for PageEndpoint {
Ok(Vc::cell(modules)) Ok(Vc::cell(modules))
} }
#[turbo_tasks::function]
async fn module_graphs(self: Vc<Self>) -> Result<Vc<ModuleGraphs>> {
let client_module_graph = self.client_module_graph().to_resolved().await?;
let ssr_module_graph = self.ssr_module_graph().to_resolved().await?;
Ok(Vc::cell(vec![client_module_graph, ssr_module_graph]))
}
} }
#[turbo_tasks::value] #[turbo_tasks::value]
......
...@@ -84,7 +84,7 @@ use crate::{ ...@@ -84,7 +84,7 @@ use crate::{
instrumentation::InstrumentationEndpoint, instrumentation::InstrumentationEndpoint,
middleware::MiddlewareEndpoint, middleware::MiddlewareEndpoint,
pages::PagesProject, pages::PagesProject,
route::{AppPageRoute, Endpoint, Endpoints, Route}, route::{Endpoint, EndpointGroup, EndpointGroupKey, EndpointGroups, Endpoints, Route},
versioned_content_map::VersionedContentMap, versioned_content_map::VersionedContentMap,
}; };
...@@ -534,6 +534,7 @@ impl ProjectContainer { ...@@ -534,6 +534,7 @@ impl ProjectContainer {
} }
} }
#[derive(Clone)]
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct Project { pub struct Project {
/// An absolute root path (Windows or Unix path) from which all files must be nested under. /// An absolute root path (Windows or Unix path) from which all files must be nested under.
...@@ -793,7 +794,7 @@ impl Project { ...@@ -793,7 +794,7 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub(super) fn next_config(&self) -> Vc<NextConfig> { pub fn next_config(&self) -> Vc<NextConfig> {
*self.next_config *self.next_config
} }
...@@ -866,67 +867,77 @@ impl Project { ...@@ -866,67 +867,77 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn get_all_endpoints(self: Vc<Self>, app_dir_only: bool) -> Result<Vc<Endpoints>> { pub async fn get_all_endpoint_groups(
let mut endpoints = Vec::new(); self: Vc<Self>,
app_dir_only: bool,
) -> Result<Vc<EndpointGroups>> {
let mut endpoint_groups = Vec::new();
let entrypoints = self.entrypoints().await?; let entrypoints = self.entrypoints().await?;
let mut is_pages_entries_added = false; let mut add_pages_entries = false;
if let Some(middleware) = &entrypoints.middleware { if let Some(middleware) = &entrypoints.middleware {
endpoints.push(middleware.endpoint); endpoint_groups.push((
EndpointGroupKey::Middleware,
EndpointGroup::from(middleware.endpoint),
));
} }
if let Some(instrumentation) = &entrypoints.instrumentation { if let Some(instrumentation) = &entrypoints.instrumentation {
endpoints.push(instrumentation.node_js); endpoint_groups.push((
endpoints.push(instrumentation.edge); EndpointGroupKey::Instrumentation,
EndpointGroup::from(instrumentation.node_js),
));
endpoint_groups.push((
EndpointGroupKey::InstrumentationEdge,
EndpointGroup::from(instrumentation.edge),
));
} }
for (_, route) in entrypoints.routes.iter() { for (key, route) in entrypoints.routes.iter() {
match route { match route {
Route::Page { Route::Page {
html_endpoint, html_endpoint,
data_endpoint, data_endpoint,
} => { } => {
if !app_dir_only { if !app_dir_only {
endpoints.push(*html_endpoint); endpoint_groups.push((
if !is_pages_entries_added { EndpointGroupKey::Route(key.clone()),
endpoints.push(entrypoints.pages_error_endpoint); EndpointGroup {
endpoints.push(entrypoints.pages_app_endpoint); primary: vec![*html_endpoint],
endpoints.push(entrypoints.pages_document_endpoint); // This only exists in development mode for HMR
is_pages_entries_added = true; additional: data_endpoint.iter().copied().collect(),
} },
// This only exists in development mode for HMR ));
if let Some(data_endpoint) = data_endpoint { add_pages_entries = true;
endpoints.push(*data_endpoint);
}
} }
} }
Route::PageApi { endpoint } => { Route::PageApi { endpoint } => {
if !app_dir_only { if !app_dir_only {
endpoints.push(*endpoint); endpoint_groups.push((
if !is_pages_entries_added { EndpointGroupKey::Route(key.clone()),
endpoints.push(entrypoints.pages_error_endpoint); EndpointGroup::from(*endpoint),
endpoints.push(entrypoints.pages_app_endpoint); ));
endpoints.push(entrypoints.pages_document_endpoint); add_pages_entries = true;
is_pages_entries_added = true;
}
} }
} }
Route::AppPage(page_routes) => { Route::AppPage(page_routes) => {
for AppPageRoute { endpoint_groups.push((
original_name: _, EndpointGroupKey::Route(key.clone()),
html_endpoint, EndpointGroup {
rsc_endpoint: _, primary: page_routes.iter().map(|r| r.html_endpoint).collect(),
} in page_routes additional: Vec::new(),
{ },
endpoints.push(*html_endpoint); ));
}
} }
Route::AppRoute { Route::AppRoute {
original_name: _, original_name: _,
endpoint, endpoint,
} => { } => {
endpoints.push(*endpoint); endpoint_groups.push((
EndpointGroupKey::Route(key.clone()),
EndpointGroup::from(*endpoint),
));
} }
Route::Conflict => { Route::Conflict => {
tracing::info!("WARN: conflict"); tracing::info!("WARN: conflict");
...@@ -934,6 +945,36 @@ impl Project { ...@@ -934,6 +945,36 @@ impl Project {
} }
} }
if add_pages_entries {
endpoint_groups.push((
EndpointGroupKey::PagesError,
EndpointGroup::from(entrypoints.pages_error_endpoint),
));
endpoint_groups.push((
EndpointGroupKey::PagesApp,
EndpointGroup::from(entrypoints.pages_app_endpoint),
));
endpoint_groups.push((
EndpointGroupKey::PagesDocument,
EndpointGroup::from(entrypoints.pages_document_endpoint),
));
}
Ok(Vc::cell(endpoint_groups))
}
#[turbo_tasks::function]
pub async fn get_all_endpoints(self: Vc<Self>, app_dir_only: bool) -> Result<Vc<Endpoints>> {
let mut endpoints = Vec::new();
for (_key, group) in self.get_all_endpoint_groups(app_dir_only).await?.iter() {
for &endpoint in group.primary.iter() {
endpoints.push(endpoint);
}
for &endpoint in group.additional.iter() {
endpoints.push(endpoint);
}
}
Ok(Vc::cell(endpoints)) Ok(Vc::cell(endpoints))
} }
...@@ -999,7 +1040,9 @@ impl Project { ...@@ -999,7 +1040,9 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn whole_app_module_graphs(self: ResolvedVc<Self>) -> Result<Vc<ModuleGraphs>> { pub async fn whole_app_module_graphs(
self: ResolvedVc<Self>,
) -> Result<Vc<BaseAndFullModuleGraph>> {
async move { async move {
let module_graphs_op = whole_app_module_graph_operation(self); let module_graphs_op = whole_app_module_graph_operation(self);
let module_graphs_vc = if self.next_mode().await?.is_production() { let module_graphs_vc = if self.next_mode().await?.is_production() {
...@@ -1843,6 +1886,15 @@ impl Project { ...@@ -1843,6 +1886,15 @@ impl Project {
Ok(Vc::cell(None)) Ok(Vc::cell(None))
} }
} }
#[turbo_tasks::function]
pub async fn with_next_config(&self, next_config: Vc<NextConfig>) -> Result<Vc<Self>> {
Ok(Self {
next_config: next_config.to_resolved().await?,
..(*self).clone()
}
.cell())
}
} }
// This is a performance optimization. This function is a root aggregation function that // This is a performance optimization. This function is a root aggregation function that
...@@ -1850,7 +1902,7 @@ impl Project { ...@@ -1850,7 +1902,7 @@ impl Project {
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn whole_app_module_graph_operation( async fn whole_app_module_graph_operation(
project: ResolvedVc<Project>, project: ResolvedVc<Project>,
) -> Result<Vc<ModuleGraphs>> { ) -> Result<Vc<BaseAndFullModuleGraph>> {
mark_root(); mark_root();
let should_trace = project.next_mode().await?.is_production(); let should_trace = project.next_mode().await?.is_production();
...@@ -1868,7 +1920,7 @@ async fn whole_app_module_graph_operation( ...@@ -1868,7 +1920,7 @@ async fn whole_app_module_graph_operation(
); );
let full = ModuleGraph::from_graphs(vec![base_single_module_graph, additional_module_graph]); let full = ModuleGraph::from_graphs(vec![base_single_module_graph, additional_module_graph]);
Ok(ModuleGraphs { Ok(BaseAndFullModuleGraph {
base: base.to_resolved().await?, base: base.to_resolved().await?,
full: full.to_resolved().await?, full: full.to_resolved().await?,
} }
...@@ -1876,7 +1928,7 @@ async fn whole_app_module_graph_operation( ...@@ -1876,7 +1928,7 @@ async fn whole_app_module_graph_operation(
} }
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct ModuleGraphs { pub struct BaseAndFullModuleGraph {
pub base: ResolvedVc<ModuleGraph>, pub base: ResolvedVc<ModuleGraph>,
pub full: ResolvedVc<ModuleGraph>, pub full: ResolvedVc<ModuleGraph>,
} }
......
use std::fmt::Display;
use anyhow::Result; use anyhow::Result;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
Completion, FxIndexMap, NonLocalValue, OperationVc, ResolvedVc, Vc, debug::ValueDebugFormat, Completion, FxIndexMap, FxIndexSet, NonLocalValue, OperationVc, ResolvedVc, TryFlatJoinIterExt,
trace::TraceRawVcs, TryJoinIterExt, Vc, debug::ValueDebugFormat, trace::TraceRawVcs,
}; };
use turbopack_core::{ use turbopack_core::{
module_graph::{GraphEntries, ModuleGraph}, module_graph::{GraphEntries, ModuleGraph},
...@@ -47,6 +49,9 @@ pub enum Route { ...@@ -47,6 +49,9 @@ pub enum Route {
Conflict, Conflict,
} }
#[turbo_tasks::value(transparent)]
pub struct ModuleGraphs(Vec<ResolvedVc<ModuleGraph>>);
#[turbo_tasks::value_trait] #[turbo_tasks::value_trait]
pub trait Endpoint { pub trait Endpoint {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -65,8 +70,108 @@ pub trait Endpoint { ...@@ -65,8 +70,108 @@ pub trait Endpoint {
fn additional_entries(self: Vc<Self>, _graph: Vc<ModuleGraph>) -> Vc<GraphEntries> { fn additional_entries(self: Vc<Self>, _graph: Vc<ModuleGraph>) -> Vc<GraphEntries> {
GraphEntries::empty() GraphEntries::empty()
} }
#[turbo_tasks::function]
fn module_graphs(self: Vc<Self>) -> Vc<ModuleGraphs>;
} }
#[derive(
TraceRawVcs,
Serialize,
Deserialize,
PartialEq,
Eq,
ValueDebugFormat,
Clone,
Debug,
NonLocalValue,
)]
pub enum EndpointGroupKey {
Instrumentation,
InstrumentationEdge,
Middleware,
PagesError,
PagesApp,
PagesDocument,
Route(RcStr),
}
impl Display for EndpointGroupKey {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
EndpointGroupKey::Instrumentation => write!(f, "instrumentation"),
EndpointGroupKey::InstrumentationEdge => write!(f, "instrumentation-edge"),
EndpointGroupKey::Middleware => write!(f, "middleware"),
EndpointGroupKey::PagesError => write!(f, "_error"),
EndpointGroupKey::PagesApp => write!(f, "_app"),
EndpointGroupKey::PagesDocument => write!(f, "_document"),
EndpointGroupKey::Route(route) => write!(f, "/{}", route),
}
}
}
#[derive(
TraceRawVcs,
Serialize,
Deserialize,
PartialEq,
Eq,
ValueDebugFormat,
Clone,
Debug,
NonLocalValue,
)]
pub struct EndpointGroup {
pub primary: Vec<ResolvedVc<Box<dyn Endpoint>>>,
pub additional: Vec<ResolvedVc<Box<dyn Endpoint>>>,
}
impl EndpointGroup {
pub fn from(endpoint: ResolvedVc<Box<dyn Endpoint>>) -> Self {
Self {
primary: vec![endpoint],
additional: vec![],
}
}
pub fn output_assets(&self) -> Vc<OutputAssets> {
output_of_endpoints(self.primary.iter().map(|endpoint| **endpoint).collect())
}
pub fn module_graphs(&self) -> Vc<ModuleGraphs> {
module_graphs_of_endpoints(self.primary.iter().map(|endpoint| **endpoint).collect())
}
}
#[turbo_tasks::function]
async fn output_of_endpoints(endpoints: Vec<Vc<Box<dyn Endpoint>>>) -> Result<Vc<OutputAssets>> {
let assets = endpoints
.iter()
.map(async |endpoint| Ok(*endpoint.output().await?.output_assets))
.try_join()
.await?;
Ok(OutputAssets::concat(assets))
}
#[turbo_tasks::function]
async fn module_graphs_of_endpoints(
endpoints: Vec<Vc<Box<dyn Endpoint>>>,
) -> Result<Vc<ModuleGraphs>> {
let module_graphs = endpoints
.iter()
.map(async |endpoint| Ok(endpoint.module_graphs().await?.into_iter()))
.try_flat_join()
.await?
.into_iter()
.copied()
.collect::<FxIndexSet<_>>()
.into_iter()
.collect::<Vec<_>>();
Ok(Vc::cell(module_graphs))
}
#[turbo_tasks::value(transparent)]
pub struct EndpointGroups(Vec<(EndpointGroupKey, EndpointGroup)>);
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct Endpoints(Vec<ResolvedVc<Box<dyn Endpoint>>>); pub struct Endpoints(Vec<ResolvedVc<Box<dyn Endpoint>>>);
......
...@@ -158,6 +158,18 @@ pub struct NextConfig { ...@@ -158,6 +158,18 @@ pub struct NextConfig {
webpack: Option<serde_json::Value>, webpack: Option<serde_json::Value>,
} }
#[turbo_tasks::value_impl]
impl NextConfig {
#[turbo_tasks::function]
pub fn with_production_browser_source_maps(&self) -> Vc<Self> {
Self {
production_browser_source_maps: true,
..self.clone()
}
.cell()
}
}
#[derive( #[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue, OperationValue, Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue, OperationValue,
)] )]
......
...@@ -146,6 +146,10 @@ program ...@@ -146,6 +146,10 @@ program
'If no directory is provided, the current directory will be used.' 'If no directory is provided, the current directory will be used.'
)}` )}`
) )
.option(
'--experimental-analyze',
'Analyze bundle output. Only compatible with Turbopack.'
)
.option('-d, --debug', 'Enables a more verbose build output.') .option('-d, --debug', 'Enables a more verbose build output.')
.option( .option(
'--debug-prerender', '--debug-prerender',
......
...@@ -95,4 +95,5 @@ export const NextBuildContext: Partial<{ ...@@ -95,4 +95,5 @@ export const NextBuildContext: Partial<{
allowedRevalidateHeaderKeys?: string[] allowedRevalidateHeaderKeys?: string[]
isCompileMode?: boolean isCompileMode?: boolean
debugPrerender: boolean debugPrerender: boolean
analyze: boolean
}> = {} }> = {}
...@@ -904,6 +904,7 @@ async function getBuildId( ...@@ -904,6 +904,7 @@ async function getBuildId(
export default async function build( export default async function build(
dir: string, dir: string,
experimentalAnalyze = false,
reactProductionProfiling = false, reactProductionProfiling = false,
debugOutput = false, debugOutput = false,
debugPrerender = false, debugPrerender = false,
...@@ -918,6 +919,7 @@ export default async function build( ...@@ -918,6 +919,7 @@ export default async function build(
const isCompileMode = experimentalBuildMode === 'compile' const isCompileMode = experimentalBuildMode === 'compile'
const isGenerateMode = experimentalBuildMode === 'generate' const isGenerateMode = experimentalBuildMode === 'generate'
NextBuildContext.isCompileMode = isCompileMode NextBuildContext.isCompileMode = isCompileMode
NextBuildContext.analyze = experimentalAnalyze
const buildStartTime = Date.now() const buildStartTime = Date.now()
let loadedConfig: NextConfigComplete | undefined let loadedConfig: NextConfigComplete | undefined
......
...@@ -297,6 +297,9 @@ export declare function projectWriteAllEntrypointsToDisk( ...@@ -297,6 +297,9 @@ export declare function projectWriteAllEntrypointsToDisk(
project: { __napiType: 'Project' }, project: { __napiType: 'Project' },
appDirOnly: boolean appDirOnly: boolean
): Promise<TurbopackResult> ): Promise<TurbopackResult>
export declare function projectEntrypoints(project: {
__napiType: 'Project'
}): Promise<TurbopackResult>
export declare function projectEntrypointsSubscribe( export declare function projectEntrypointsSubscribe(
project: { __napiType: 'Project' }, project: { __napiType: 'Project' },
func: (...args: any[]) => any func: (...args: any[]) => any
...@@ -373,6 +376,10 @@ export declare function projectGetSourceMapSync( ...@@ -373,6 +376,10 @@ export declare function projectGetSourceMapSync(
project: { __napiType: 'Project' }, project: { __napiType: 'Project' },
filePath: RcStr filePath: RcStr
): string | null ): string | null
export declare function projectWriteAnalyzeData(
project: { __napiType: 'Project' },
appDirOnly: boolean
): Promise<TurbopackResult>
/** /**
* A version of [`NapiNextTurbopackCallbacks`] that can accepted as an argument to a napi function. * A version of [`NapiNextTurbopackCallbacks`] that can accepted as an argument to a napi function.
* *
......
...@@ -655,6 +655,16 @@ function bindingToApi( ...@@ -655,6 +655,16 @@ function bindingToApi(
) )
} }
async writeAnalyzeData(
appDirOnly: boolean
): Promise<TurbopackResult<void>> {
const napiResult = (await binding.projectWriteAnalyzeData(
this._nativeProject,
appDirOnly
)) as TurbopackResult<void>
return napiResult
}
async writeAllEntrypointsToDisk( async writeAllEntrypointsToDisk(
appDirOnly: boolean appDirOnly: boolean
): Promise<TurbopackResult<Partial<RawEntrypoints>>> { ): Promise<TurbopackResult<Partial<RawEntrypoints>>> {
......
...@@ -231,6 +231,8 @@ export interface UpdateInfo { ...@@ -231,6 +231,8 @@ export interface UpdateInfo {
export interface Project { export interface Project {
update(options: Partial<ProjectOptions>): Promise<void> update(options: Partial<ProjectOptions>): Promise<void>
writeAnalyzeData(appDirOnly: boolean): Promise<TurbopackResult<void>>
writeAllEntrypointsToDisk( writeAllEntrypointsToDisk(
appDirOnly: boolean appDirOnly: boolean
): Promise<TurbopackResult<Partial<RawEntrypoints>>> ): Promise<TurbopackResult<Partial<RawEntrypoints>>>
......
...@@ -193,6 +193,10 @@ export async function turbopackBuild(): Promise<{ ...@@ -193,6 +193,10 @@ export async function turbopackBuild(): Promise<{
entrypoints: currentEntrypoints, entrypoints: currentEntrypoints,
}) })
if (NextBuildContext.analyze) {
await project.writeAnalyzeData(appDirOnly)
}
const shutdownPromise = project.shutdown() const shutdownPromise = project.shutdown()
const time = process.hrtime(startTime) const time = process.hrtime(startTime)
......
...@@ -10,13 +10,14 @@ import isError from '../lib/is-error' ...@@ -10,13 +10,14 @@ import isError from '../lib/is-error'
import { getProjectDir } from '../lib/get-project-dir' import { getProjectDir } from '../lib/get-project-dir'
import { enableMemoryDebuggingMode } from '../lib/memory/startup' import { enableMemoryDebuggingMode } from '../lib/memory/startup'
import { disableMemoryDebuggingMode } from '../lib/memory/shutdown' import { disableMemoryDebuggingMode } from '../lib/memory/shutdown'
import { parseBundlerArgs } from '../lib/bundler' import { Bundler, parseBundlerArgs } from '../lib/bundler'
import { import {
resolveBuildPaths, resolveBuildPaths,
parseBuildPathsInput, parseBuildPathsInput,
} from '../lib/resolve-build-paths' } from '../lib/resolve-build-paths'
export type NextBuildOptions = { export type NextBuildOptions = {
experimentalAnalyze?: boolean
debug?: boolean debug?: boolean
debugPrerender?: boolean debugPrerender?: boolean
profile?: boolean profile?: boolean
...@@ -38,6 +39,7 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => { ...@@ -38,6 +39,7 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
process.on('SIGINT', () => process.exit(130)) process.on('SIGINT', () => process.exit(130))
const { const {
experimentalAnalyze,
debug, debug,
debugPrerender, debugPrerender,
experimentalDebugMemoryUsage, experimentalDebugMemoryUsage,
...@@ -54,6 +56,14 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => { ...@@ -54,6 +56,14 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
traceUploadUrl = experimentalUploadTrace traceUploadUrl = experimentalUploadTrace
} }
const bundler = parseBundlerArgs(options)
if (experimentalAnalyze && bundler !== Bundler.Turbopack) {
printAndExit(
'--experimental-analyze is only compatible with the Turbopack bundler.'
)
}
if (!mangling) { if (!mangling) {
warn( warn(
`Mangling is disabled. ${italic('Note: This may affect performance and should only be used for debugging purposes.')}` `Mangling is disabled. ${italic('Note: This may affect performance and should only be used for debugging purposes.')}`
...@@ -85,8 +95,6 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => { ...@@ -85,8 +95,6 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
printAndExit(`> No such directory exists as the project root: ${dir}`) printAndExit(`> No such directory exists as the project root: ${dir}`)
} }
const bundler = parseBundlerArgs(options)
// Resolve selective build paths // Resolve selective build paths
let resolvedAppPaths: string[] | undefined let resolvedAppPaths: string[] | undefined
let resolvedPagePaths: string[] | undefined let resolvedPagePaths: string[] | undefined
...@@ -110,6 +118,7 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => { ...@@ -110,6 +118,7 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
return build( return build(
dir, dir,
experimentalAnalyze,
profile, profile,
debug || Boolean(process.env.NEXT_DEBUG_BUILD), debug || Boolean(process.env.NEXT_DEBUG_BUILD),
debugPrerender, debugPrerender,
......
...@@ -123,7 +123,11 @@ module.exports = function (task) { ...@@ -123,7 +123,11 @@ module.exports = function (task) {
const distFilePath = path.dirname( const distFilePath = path.dirname(
// we must strip src from filePath as it isn't carried into // we must strip src from filePath as it isn't carried into
// the dist file path // the dist file path
path.join(__dirname, 'dist', filePath.replace(/^src[/\\]/, '')) path.join(
__dirname,
esm ? 'dist/esm' : 'dist',
filePath.replace(/^src[/\\]/, '')
)
) )
const options = { const options = {
......
/// <reference types="next" /> /// <reference types="next" />
/// <reference types="next/image-types/global" /> /// <reference types="next/image-types/global" />
import "./.next/dev/types/routes.d.ts"; import './.next/dev/types/routes.d.ts'
// NOTE: This file should not be edited // NOTE: This file should not be edited
// see https://nextjs.org/docs/pages/api-reference/config/typescript for more information. // see https://nextjs.org/docs/pages/api-reference/config/typescript for more information.
...@@ -15,7 +15,7 @@ print_stats = ["stats"] ...@@ -15,7 +15,7 @@ print_stats = ["stats"]
anyhow = { workspace = true } anyhow = { workspace = true }
either = { workspace = true } either = { workspace = true }
pot = "3.0.0" pot = "3.0.0"
byteorder = "1.5.0" byteorder = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
lzzzz = "1.1.0" lzzzz = "1.1.0"
memmap2 = "0.9.5" memmap2 = "0.9.5"
......
...@@ -31,7 +31,7 @@ anyhow = { workspace = true } ...@@ -31,7 +31,7 @@ anyhow = { workspace = true }
arc-swap = { version = "1.7.1" } arc-swap = { version = "1.7.1" }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
bitfield = { workspace = true } bitfield = { workspace = true }
byteorder = "1.5.0" byteorder = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]} dashmap = { workspace = true, features = ["raw-api"]}
either = { workspace = true } either = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] } hashbrown = { workspace = true, features = ["raw"] }
......
...@@ -2289,6 +2289,17 @@ pub struct FileLine { ...@@ -2289,6 +2289,17 @@ pub struct FileLine {
pub bytes_offset: u32, pub bytes_offset: u32,
} }
impl FileLine {
pub fn len(&self) -> usize {
self.content.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[turbo_tasks::value(shared, serialization = "none")] #[turbo_tasks::value(shared, serialization = "none")]
pub enum FileLinesContent { pub enum FileLinesContent {
Lines(#[turbo_tasks(trace_ignore)] Vec<FileLine>), Lines(#[turbo_tasks(trace_ignore)] Vec<FileLine>),
......
...@@ -164,6 +164,14 @@ impl RopeBuilder { ...@@ -164,6 +164,14 @@ impl RopeBuilder {
self.uncommitted.push_bytes(bytes); self.uncommitted.push_bytes(bytes);
} }
/// Reserve additional capacity for owned bytes in the Rope.
///
/// This is useful to call before multiple `push_bytes` calls to avoid
/// multiple allocations.
pub fn reserve_bytes(&mut self, additional: usize) {
self.uncommitted.reserve_bytes(additional);
}
/// Push static lifetime bytes into the Rope. /// Push static lifetime bytes into the Rope.
/// ///
/// This is more efficient than pushing owned bytes, because the internal /// This is more efficient than pushing owned bytes, because the internal
...@@ -305,6 +313,24 @@ impl Uncommitted { ...@@ -305,6 +313,24 @@ impl Uncommitted {
} }
} }
/// Reserves additional capacity for owned bytes, converting the current
/// representation to an Owned if it's not already.
fn reserve_bytes(&mut self, additional: usize) {
match self {
Self::None => {
*self = Self::Owned(Vec::with_capacity(additional));
}
Self::Static(s) => {
let mut v = Vec::with_capacity(s.len() + additional);
v.extend_from_slice(s);
*self = Self::Owned(v);
}
Self::Owned(v) => {
v.reserve(additional);
}
}
}
/// Pushes static lifetime bytes, but only if the current representation is /// Pushes static lifetime bytes, but only if the current representation is
/// None. Else, it coverts to an Owned. /// None. Else, it coverts to an Owned.
fn push_static_bytes(&mut self, bytes: &'static [u8]) { fn push_static_bytes(&mut self, bytes: &'static [u8]) {
......
[package]
name = "turbopack-analyze"
version = "0.1.0"
edition = "2024"
license = "MIT"
autobenches = false
[lib]
bench = false
[dependencies]
anyhow = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive"] }
turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbopack-core = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
tokio = { workspace = true, features = ["full"] }
turbo-tasks-testing = { workspace = true }
turbo-tasks-backend = { workspace = true }
[lints]
workspace = true
#![feature(arbitrary_self_types_pointers)]
pub mod split_chunk;
use std::mem::replace;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, NonLocalValue, ValueToString, Vc, trace::TraceRawVcs};
use turbo_tasks_fs::{FileContent, FileLine, FileLinesContent, rope::Rope};
use turbopack_core::{
asset::{Asset, AssetContent},
output::OutputAsset,
source_map::{GenerateSourceMap, OriginalToken, SourceMap, SyntheticToken, Token},
};
#[derive(Clone, Debug, Deserialize, Eq, NonLocalValue, PartialEq, Serialize, TraceRawVcs)]
pub struct ChunkPartRange {
pub line: u32,
pub start_column: u32,
pub end_column: u32,
}
#[derive(Clone, Debug, Deserialize, Eq, NonLocalValue, PartialEq, Serialize, TraceRawVcs)]
pub struct ChunkPart {
pub source: RcStr,
pub real_size: u32,
pub unaccounted_size: u32,
pub ranges: Vec<ChunkPartRange>,
}
#[turbo_tasks::value(transparent)]
#[derive(Debug)]
pub struct ChunkParts(Vec<ChunkPart>);
#[turbo_tasks::function]
pub async fn split_output_asset_into_parts(
asset: Vc<Box<dyn OutputAsset>>,
) -> Result<Vc<ChunkParts>> {
let content = asset.content().await?;
let AssetContent::File(file_content) = &*content else {
return Ok(Vc::cell(vec![]));
};
let FileContent::Content(content) = &*file_content.await? else {
return Ok(Vc::cell(vec![]));
};
let content = content.content();
let Some(generate_source_map) =
Vc::try_resolve_sidecast::<Box<dyn GenerateSourceMap>>(asset).await?
else {
return self_mapped(asset, content).await;
};
let Some(source_map) = &*generate_source_map.generate_source_map().await? else {
return self_mapped(asset, content).await;
};
let Some(source_map) = SourceMap::new_from_rope(source_map)? else {
return unaccounted(asset, content).await;
};
let lines = file_content.lines().await?;
let FileLinesContent::Lines(lines) = &*lines else {
return unaccounted(asset, content).await;
};
fn end_of_mapping_column(
start_line: u32,
end_line: u32,
end_column: u32,
lines: &[FileLine],
) -> u32 {
let start_line = start_line.min(lines.len() as u32 - 1);
let line_end = lines[start_line as usize].len() as u32;
if start_line == end_line {
end_column.min(line_end)
} else {
line_end
}
}
fn len_between(
start_line: u32,
start_column: u32,
end_line: u32,
end_column: u32,
lines: &[FileLine],
) -> u32 {
let start_line = start_line.min(lines.len() as u32 - 1);
let end_line = end_line.min(lines.len() as u32 - 1);
if start_line == end_line {
return end_column - start_column;
}
let mut len = lines[start_line as usize].len() as u32 - start_column + 1;
for line in &lines[start_line as usize + 1..end_line as usize] {
len += line.len() as u32 + 1;
}
len += end_column;
len
}
let mut chunk_parts = FxIndexMap::default();
fn add_chunk_part_range(
source: RcStr,
chunk_part_range: ChunkPartRange,
size: u32,
chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
) {
let entry = chunk_parts
.entry(source)
.or_insert_with_key(|source| ChunkPart {
source: source.clone(),
real_size: 0,
unaccounted_size: 0,
ranges: vec![],
});
entry.real_size += size;
entry.ranges.push(chunk_part_range);
}
fn add_unaccounted_chunk_part(
source: RcStr,
unaccounted: u32,
chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
) {
let entry = chunk_parts
.entry(source)
.or_insert_with_key(|source| ChunkPart {
source: source.clone(),
real_size: 0,
unaccounted_size: 0,
ranges: vec![],
});
entry.unaccounted_size += unaccounted;
}
enum State {
StartOfFile,
InMapping {
source: RcStr,
current_generated_line: u32,
current_generated_column: u32,
},
AfterMapping {
source: RcStr,
current_generated_line: u32,
current_generated_column: u32,
},
}
let mut state: State = State::StartOfFile;
fn end_current_token(
lines: &[FileLine],
chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
state: &mut State,
token: &Token,
) {
if let State::InMapping {
ref source,
current_generated_line,
current_generated_column,
} = *state
{
let (Token::Original(OriginalToken {
generated_line,
generated_column,
..
})
| Token::Synthetic(SyntheticToken {
generated_line,
generated_column,
..
})) = *token;
let mapping_end_column = end_of_mapping_column(
current_generated_line,
generated_line,
generated_column,
lines,
);
// TODO: Handle this better
let len = mapping_end_column.saturating_sub(current_generated_column);
add_chunk_part_range(
source.clone(),
ChunkPartRange {
line: current_generated_line,
start_column: current_generated_column,
end_column: mapping_end_column,
},
len,
chunk_parts,
);
*state = State::AfterMapping {
source: source.clone(),
current_generated_line,
current_generated_column: mapping_end_column,
};
}
}
fn start_new_mapping(
lines: &[FileLine],
chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
state: &mut State,
original_file: RcStr,
generated_line: u32,
generated_column: u32,
) {
match replace(
state,
State::InMapping {
source: original_file.clone(),
current_generated_line: generated_line,
current_generated_column: generated_column,
},
) {
State::InMapping { .. } => {
unreachable!();
}
State::AfterMapping {
source,
current_generated_line,
current_generated_column,
} => {
let len = len_between(
current_generated_line,
current_generated_column,
generated_line,
generated_column,
lines,
);
let half = len / 2;
add_unaccounted_chunk_part(source, half, chunk_parts);
add_unaccounted_chunk_part(original_file.clone(), len - half, chunk_parts);
}
State::StartOfFile => {
let len = len_between(0, 0, generated_line, generated_column, lines);
add_unaccounted_chunk_part(original_file.clone(), len, chunk_parts);
}
}
}
for token in source_map.tokens() {
// First end the previous mapping if we were in one
end_current_token(lines, &mut chunk_parts, &mut state, &token);
if let Token::Original(OriginalToken {
original_file,
generated_line,
generated_column,
..
}) = token
{
// Start a new mapping and put the unaccounted part in between
// somewhere
start_new_mapping(
lines,
&mut chunk_parts,
&mut state,
original_file,
generated_line,
generated_column,
);
}
}
let last_line = lines.len() as u32 - 1;
let last_column = lines[last_line as usize].len() as u32;
end_current_token(
lines,
&mut chunk_parts,
&mut state,
&Token::Synthetic(SyntheticToken {
generated_line: last_line,
generated_column: last_column,
guessed_original_file: None,
}),
);
match state {
State::InMapping { .. } => {
unreachable!();
}
State::AfterMapping {
source,
current_generated_line,
current_generated_column,
} => {
let len = len_between(
current_generated_line,
current_generated_column,
last_line,
last_column,
lines,
);
add_unaccounted_chunk_part(source, len, &mut chunk_parts);
}
State::StartOfFile => {
return unaccounted(asset, content).await;
}
}
Ok(Vc::cell(chunk_parts.into_values().collect()))
}
async fn self_mapped(asset: Vc<Box<dyn OutputAsset>>, content: &Rope) -> Result<Vc<ChunkParts>> {
let len = content.len().try_into().unwrap_or(u32::MAX);
Ok(Vc::cell(vec![ChunkPart {
source: asset.path().to_string().owned().await?,
real_size: len,
unaccounted_size: 0,
ranges: vec![],
}]))
}
async fn unaccounted(asset: Vc<Box<dyn OutputAsset>>, content: &Rope) -> Result<Vc<ChunkParts>> {
let len = content.len().try_into().unwrap_or(u32::MAX);
Ok(Vc::cell(vec![ChunkPart {
source: asset.path().to_string().owned().await?,
real_size: 0,
unaccounted_size: len,
ranges: vec![],
}]))
}
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
#![cfg(test)]
use anyhow::Result;
use serde_json::json;
use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{File, FileSystem, FileSystemPath, VirtualFileSystem, rope::Rope};
use turbo_tasks_testing::{Registration, register, run_once};
use turbopack_analyze::split_chunk::{ChunkPart, ChunkPartRange, split_output_asset_into_parts};
use turbopack_core::{
asset::{Asset, AssetContent},
code_builder::{Code, CodeBuilder},
output::OutputAsset,
source_map::{GenerateSourceMap, OptionStringifiedSourceMap},
};
static REGISTRATION: Registration = register!(turbo_tasks_fetch::register);
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn split_chunk() {
run_once(&REGISTRATION, || async {
let mut code = CodeBuilder::new(true, false);
code += "Hello world!\n";
code += "This is a test file.\n";
code.push_source(
&Rope::from("Hello world!\n123"),
Some(Rope::from(serde_json::to_string_pretty(&json! ({
"version": 3,
"mappings": "AAAA;AACA",
"sources": ["source1.js"],
"names": [],
"sourcesContent": ["console.log('Hello world!');"]
}))?)),
);
code += "This is the middle of the file.\n";
code.push_source(
&Rope::from("This is the middle of the file.\n"),
Some(Rope::from(serde_json::to_string_pretty(&json! ({
"version": 3,
"mappings": "AAAA",
"sources": ["source2.js"],
"names": [],
"sourcesContent": ["console.log('Middle of file');"]
}))?)),
);
code += "This is the end of the file.\n";
let code = code.build();
let asset = Vc::upcast(
TestAsset {
code: code.resolved_cell(),
}
.cell(),
);
let parts = split_output_asset_into_parts(asset).await.unwrap();
assert_eq!(
&*parts,
&vec![
ChunkPart {
source: rcstr!("source1.js"),
real_size: 15,
unaccounted_size: 51,
ranges: vec![
ChunkPartRange {
line: 2,
start_column: 0,
end_column: 12,
},
ChunkPartRange {
line: 3,
start_column: 0,
end_column: 3,
},
],
},
ChunkPart {
source: rcstr!("source2.js"),
real_size: 31,
unaccounted_size: 46,
ranges: vec![ChunkPartRange {
line: 4,
start_column: 0,
end_column: 31,
}],
},
]
);
println!("{:#?}", parts);
anyhow::Ok(())
})
.await
.unwrap()
}
#[turbo_tasks::value]
struct TestAsset {
code: ResolvedVc<Code>,
}
#[turbo_tasks::value_impl]
impl OutputAsset for TestAsset {
#[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> {
Ok(VirtualFileSystem::new()
.root()
.await?
.join("test.js")?
.cell())
}
}
#[turbo_tasks::value_impl]
impl Asset for TestAsset {
#[turbo_tasks::function]
async fn content(&self) -> Result<Vc<AssetContent>> {
Ok(AssetContent::file(
File::from(self.code.await?.source_code().clone()).into(),
))
}
}
#[turbo_tasks::value_impl]
impl GenerateSourceMap for TestAsset {
#[turbo_tasks::function]
pub fn generate_source_map(&self) -> Vc<OptionStringifiedSourceMap> {
self.code.generate_source_map()
}
}
|name, initial| {
let path = std::path::PathBuf::from(
format!("{}/.cache/{}", env!("CARGO_TARGET_TMPDIR"), name));
if initial {
let _ = std::fs::remove_dir_all(&path);
}
std::fs::create_dir_all(&path).unwrap();
turbo_tasks::TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new(
turbo_tasks_backend::BackendOptions {
num_workers: Some(2),
small_preallocation: true,
..Default::default()
},
turbo_tasks_backend::default_backing_storage(
path.as_path(),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
dirty: false,
},
false,
true,
).unwrap().0
)
)
}
...@@ -52,6 +52,15 @@ impl OutputAssets { ...@@ -52,6 +52,15 @@ impl OutputAssets {
assets.extend(other.await?.iter().copied()); assets.extend(other.await?.iter().copied());
Ok(Vc::cell(assets.into_iter().collect())) Ok(Vc::cell(assets.into_iter().collect()))
} }
#[turbo_tasks::function]
pub async fn concat(other: Vec<Vc<Self>>) -> Result<Vc<Self>> {
let mut assets: FxIndexSet<_> = FxIndexSet::default();
for other in other {
assets.extend(other.await?.iter().copied());
}
Ok(Vc::cell(assets.into_iter().collect()))
}
} }
impl OutputAssets { impl OutputAssets {
......
...@@ -2,6 +2,7 @@ use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc}; ...@@ -2,6 +2,7 @@ use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc};
use anyhow::Result; use anyhow::Result;
use bytes_str::BytesStr; use bytes_str::BytesStr;
use either::Either;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use ref_cast::RefCast; use ref_cast::RefCast;
use regex::Regex; use regex::Regex;
...@@ -93,9 +94,6 @@ impl OptionSourceMap { ...@@ -93,9 +94,6 @@ impl OptionSourceMap {
ResolvedVc::cell(None) ResolvedVc::cell(None)
} }
} }
#[turbo_tasks::value(transparent)]
#[derive(Clone, Debug)]
pub struct Tokens(Vec<Token>);
/// A token represents a mapping in a source map. /// A token represents a mapping in a source map.
/// ///
...@@ -153,6 +151,32 @@ impl Token { ...@@ -153,6 +151,32 @@ impl Token {
Self::Synthetic(t) => t.generated_column, Self::Synthetic(t) => t.generated_column,
} }
} }
pub fn with_offset(&self, line_offset: u32, column_offset: u32) -> Self {
match self {
Self::Original(t) => Self::Original(OriginalToken {
generated_line: t.generated_line + line_offset,
generated_column: if t.generated_line == 0 {
t.generated_column + column_offset
} else {
t.generated_column
},
original_file: t.original_file.clone(),
original_line: t.original_line,
original_column: t.original_column,
name: t.name.clone(),
}),
Self::Synthetic(t) => Self::Synthetic(SyntheticToken {
generated_line: t.generated_line + line_offset,
generated_column: if t.generated_line == 0 {
t.generated_column + column_offset
} else {
t.generated_column
},
guessed_original_file: t.guessed_original_file.clone(),
}),
}
}
} }
impl From<swc_sourcemap::Token<'_>> for Token { impl From<swc_sourcemap::Token<'_>> for Token {
...@@ -554,6 +578,62 @@ impl SourceMap { ...@@ -554,6 +578,62 @@ impl SourceMap {
} }
} }
impl SourceMap {
pub fn tokens(&self) -> impl Iterator<Item = Token> + '_ {
let map = &self.map;
fn regular_map_to_tokens(
map: &RegularMap,
offset_line: u32,
offset_column: u32,
) -> impl Iterator<Item = Token> + '_ {
map.tokens()
.map(move |t| Token::from(t).with_offset(offset_line, offset_column))
}
fn index_map_to_tokens(
map: &SourceMapIndex,
offset_line: u32,
offset_column: u32,
) -> impl Iterator<Item = Token> + '_ {
map.sections().flat_map(move |section| {
let (line, col) = section.get_offset();
let offset_line = offset_line + line;
let offset_column = if line == 0 { offset_column + col } else { col };
if let Some(source_map) = section.get_sourcemap() {
Either::Left(Box::new(decoded_map_to_tokens(
source_map,
offset_line,
offset_column,
)) as Box<dyn Iterator<Item = Token>>)
} else {
Either::Right(std::iter::empty())
}
})
}
fn decoded_map_to_tokens(
map: &DecodedMap,
offset_line: u32,
offset_column: u32,
) -> impl Iterator<Item = Token> + '_ {
match map {
DecodedMap::Regular(map) => {
Either::Left(regular_map_to_tokens(map, offset_line, offset_column))
}
DecodedMap::Index(map) => {
Either::Right(index_map_to_tokens(map, offset_line, offset_column))
}
DecodedMap::Hermes(_) => {
todo!("hermes source maps are not implemented");
}
}
}
decoded_map_to_tokens(&map.0, 0, 0)
}
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl GenerateSourceMap for SourceMap { impl GenerateSourceMap for SourceMap {
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -50,6 +50,6 @@ impl OutputAsset for TracedAsset { ...@@ -50,6 +50,6 @@ impl OutputAsset for TracedAsset {
impl Asset for TracedAsset { impl Asset for TracedAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
fn content(&self) -> Vc<AssetContent> { fn content(&self) -> Vc<AssetContent> {
panic!("TracedAsset::content() should never be called"); self.module.content()
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment