Unverified Commit 84e75d11 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: uncell AstPath (#75695)

- Remove the last remaining `AstPath` cells
- Also box a rare `CodeGen` enum variant (brings down enum size from 128 bytes to 48 bytes)
- `code_gens.shrink_to_fit()`

Might help memory slightly, but it's bordering on measurement noise:
```
testing against 0293c96cf32

canary b6f8743e
13,75 GB 
TURBO_ENGINE_READ_ONLY=1 NEXT_TURBOPACK_TRACING= TURBOPACK=1 TURBOPACK_BUILD=  507.86s user 77.80s system 847% cpu 1:09.11 total
TURBO_ENGINE_READ_ONLY=1 NEXT_TURBOPACK_TRACING= TURBOPACK=1 TURBOPACK_BUILD=  503.13s user 77.86s system 855% cpu 1:07.90 total


uncell AstPath 3ec676fea5
13,7 GB 
TURBO_ENGINE_READ_ONLY=1 NEXT_TURBOPACK_TRACING= TURBOPACK=1 TURBOPACK_BUILD=  505.04s user 83.92s system 853% cpu 1:09.00 total
TURBO_ENGINE_READ_ONLY=1 NEXT_TURBOPACK_TRACING= TURBOPACK=1 TURBOPACK_BUILD=  509.03s user 84.26s system 829% cpu 1:11.53 total
```
parent df0573a4
......@@ -109,7 +109,8 @@ pub trait CodeGenerateable {
#[derive(PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, ValueDebugFormat, NonLocalValue)]
pub enum CodeGen {
AmdDefineWithDependenciesCodeGen(AmdDefineWithDependenciesCodeGen),
// AMD occurs very rarely and makes the enum more than 2x bigger
AmdDefineWithDependenciesCodeGen(Box<AmdDefineWithDependenciesCodeGen>),
CjsRequireCacheAccess(CjsRequireCacheAccess),
ConstantConditionCodeGen(ConstantConditionCodeGen),
ConstantValueCodeGen(ConstantValueCodeGen),
......
......@@ -215,7 +215,7 @@ impl AmdDefineWithDependenciesCodeGen {
impl From<AmdDefineWithDependenciesCodeGen> for CodeGen {
fn from(val: AmdDefineWithDependenciesCodeGen) -> Self {
CodeGen::AmdDefineWithDependenciesCodeGen(val)
CodeGen::AmdDefineWithDependenciesCodeGen(Box::new(val))
}
}
......
......@@ -86,7 +86,7 @@ impl ChunkableModuleReference for CjsAssetReference {}
pub struct CjsRequireAssetReference {
pub origin: ResolvedVc<Box<dyn ResolveOrigin>>,
pub request: ResolvedVc<Request>,
pub path: ResolvedVc<AstPath>,
pub path: AstPath,
pub issue_source: IssueSource,
pub in_try: bool,
}
......@@ -97,7 +97,7 @@ impl CjsRequireAssetReference {
pub fn new(
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
request: ResolvedVc<Request>,
path: ResolvedVc<AstPath>,
path: AstPath,
issue_source: IssueSource,
in_try: bool,
) -> Vc<Self> {
......@@ -161,8 +161,7 @@ impl CodeGenerateable for CjsRequireAssetReference {
.await?;
let mut visitors = Vec::new();
let path = &self.path.await?;
visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
visitors.push(create_visitor!(self.path, visit_mut_expr(expr: &mut Expr) {
let old_expr = expr.take();
let message = if let Expr::Call(CallExpr { args, ..}) = old_expr {
match args.into_iter().next() {
......@@ -195,7 +194,7 @@ impl CodeGenerateable for CjsRequireAssetReference {
pub struct CjsRequireResolveAssetReference {
pub origin: ResolvedVc<Box<dyn ResolveOrigin>>,
pub request: ResolvedVc<Request>,
pub path: ResolvedVc<AstPath>,
pub path: AstPath,
pub issue_source: IssueSource,
pub in_try: bool,
}
......@@ -206,7 +205,7 @@ impl CjsRequireResolveAssetReference {
pub fn new(
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
request: ResolvedVc<Request>,
path: ResolvedVc<AstPath>,
path: AstPath,
issue_source: IssueSource,
in_try: bool,
) -> Vc<Self> {
......@@ -270,9 +269,8 @@ impl CodeGenerateable for CjsRequireResolveAssetReference {
.await?;
let mut visitors = Vec::new();
let path = &self.path.await?;
// Inline the result of the `require.resolve` call as a literal.
visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
visitors.push(create_visitor!(self.path, visit_mut_expr(expr: &mut Expr) {
if let Expr::Call(call_expr) = expr {
let args = std::mem::take(&mut call_expr.args);
*expr = match args.into_iter().next() {
......
......@@ -34,7 +34,7 @@ use crate::{
pub struct EsmAsyncAssetReference {
pub origin: ResolvedVc<Box<dyn ResolveOrigin>>,
pub request: ResolvedVc<Request>,
pub path: ResolvedVc<AstPath>,
pub path: AstPath,
pub annotations: ImportAnnotations,
pub issue_source: IssueSource,
pub in_try: bool,
......@@ -57,7 +57,7 @@ impl EsmAsyncAssetReference {
pub fn new(
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
request: ResolvedVc<Request>,
path: ResolvedVc<AstPath>,
path: AstPath,
issue_source: IssueSource,
annotations: Value<ImportAnnotations>,
in_try: bool,
......@@ -140,10 +140,9 @@ impl CodeGenerateable for EsmAsyncAssetReference {
)
.await?;
let path = &self.path.await?;
let import_externals = self.import_externals;
let visitor = create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
let visitor = create_visitor!(self.path, visit_mut_expr(expr: &mut Expr) {
let old_expr = expr.take();
let message = if let Expr::Call(CallExpr { args, ..}) = old_expr {
match args.into_iter().next() {
......
......@@ -24,13 +24,13 @@ use crate::{
#[derive(Hash, Debug)]
pub struct EsmModuleIdAssetReference {
inner: ResolvedVc<EsmAssetReference>,
ast_path: ResolvedVc<AstPath>,
ast_path: AstPath,
}
#[turbo_tasks::value_impl]
impl EsmModuleIdAssetReference {
#[turbo_tasks::function]
pub fn new(inner: ResolvedVc<EsmAssetReference>, ast_path: ResolvedVc<AstPath>) -> Vc<Self> {
pub fn new(inner: ResolvedVc<EsmAssetReference>, ast_path: AstPath) -> Vc<Self> {
Self::cell(EsmModuleIdAssetReference { inner, ast_path })
}
}
......@@ -78,7 +78,7 @@ impl CodeGenerateable for EsmModuleIdAssetReference {
.await?;
let id = module_id_to_lit(&id);
visitors.push(
create_visitor!(self.ast_path.await?, visit_mut_expr(expr: &mut Expr) {
create_visitor!(self.ast_path, visit_mut_expr(expr: &mut Expr) {
*expr = id.clone()
}),
);
......@@ -87,7 +87,7 @@ impl CodeGenerateable for EsmModuleIdAssetReference {
// This can happen if the referenced asset is an external, or doesn't resolve
// to anything.
visitors.push(
create_visitor!(self.ast_path.await?, visit_mut_expr(expr: &mut Expr) {
create_visitor!(self.ast_path, visit_mut_expr(expr: &mut Expr) {
*expr = quote!("null" as Expr);
}),
);
......
......@@ -69,7 +69,7 @@ pub struct UrlAssetReference {
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
request: ResolvedVc<Request>,
rendering: ResolvedVc<Rendering>,
ast_path: ResolvedVc<AstPath>,
ast_path: AstPath,
issue_source: IssueSource,
in_try: bool,
url_rewrite_behavior: UrlRewriteBehavior,
......@@ -82,7 +82,7 @@ impl UrlAssetReference {
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
request: ResolvedVc<Request>,
rendering: ResolvedVc<Rendering>,
ast_path: ResolvedVc<AstPath>,
ast_path: AstPath,
issue_source: IssueSource,
in_try: bool,
url_rewrite_behavior: UrlRewriteBehavior,
......@@ -168,7 +168,6 @@ impl CodeGenerateable for UrlAssetReference {
match this.url_rewrite_behavior {
UrlRewriteBehavior::Relative => {
let referenced_asset = self.get_referenced_asset().await?;
let ast_path = this.ast_path.await?;
// if the referenced url is in the module graph of turbopack, replace it into
// the chunk item will be emitted into output path to point the
......@@ -185,7 +184,7 @@ impl CodeGenerateable for UrlAssetReference {
.id()
.await?;
visitors.push(create_visitor!(ast_path, visit_mut_expr(new_expr: &mut Expr) {
visitors.push(create_visitor!(this.ast_path, visit_mut_expr(new_expr: &mut Expr) {
let should_rewrite_to_relative = if let Expr::New(NewExpr { args: Some(args), .. }) = new_expr {
matches!(args.first(), Some(ExprOrSpread { .. }))
} else {
......@@ -204,7 +203,7 @@ impl CodeGenerateable for UrlAssetReference {
}
ReferencedAsset::External(request, ExternalType::Url) => {
let request = request.to_string();
visitors.push(create_visitor!(ast_path, visit_mut_expr(new_expr: &mut Expr) {
visitors.push(create_visitor!(this.ast_path, visit_mut_expr(new_expr: &mut Expr) {
let should_rewrite_to_relative = if let Expr::New(NewExpr { args: Some(args), .. }) = new_expr {
matches!(args.first(), Some(ExprOrSpread { .. }))
} else {
......@@ -232,7 +231,6 @@ impl CodeGenerateable for UrlAssetReference {
}
UrlRewriteBehavior::Full => {
let referenced_asset = self.get_referenced_asset().await?;
let ast_path = this.ast_path.await?;
// For rendering environments (CSR), we rewrite the `import.meta.url` to
// be a location.origin because it allows us to access files from the root of
......@@ -276,7 +274,7 @@ impl CodeGenerateable for UrlAssetReference {
)
};
visitors.push(create_visitor!(ast_path, visit_mut_expr(new_expr: &mut Expr) {
visitors.push(create_visitor!(this.ast_path, visit_mut_expr(new_expr: &mut Expr) {
if let Expr::New(NewExpr { args: Some(args), .. }) = new_expr {
if let Some(ExprOrSpread { box expr, spread: None }) = args.get_mut(0) {
*expr = url_segment_resolver.clone();
......@@ -296,7 +294,7 @@ impl CodeGenerateable for UrlAssetReference {
}
ReferencedAsset::External(request, ExternalType::Url) => {
let request = request.to_string();
visitors.push(create_visitor!(ast_path, visit_mut_expr(new_expr: &mut Expr) {
visitors.push(create_visitor!(this.ast_path, visit_mut_expr(new_expr: &mut Expr) {
if let Expr::New(NewExpr { args: Some(args), .. }) = new_expr {
if let Some(ExprOrSpread { box expr, spread: None }) = args.get_mut(0) {
*expr = request.as_str().into()
......
......@@ -29,6 +29,7 @@ use once_cell::sync::Lazy;
use parking_lot::Mutex;
use regex::Regex;
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use sourcemap::decode_data_url;
use swc_core::{
atoms::JsWord,
......@@ -50,7 +51,8 @@ use swc_core::{
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexMap, FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, Upcast, Value, ValueToString, Vc,
trace::TraceRawVcs, FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput,
TryJoinIterExt, Upcast, Value, ValueToString, Vc,
};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
......@@ -262,7 +264,7 @@ impl AnalyzeEcmascriptModuleResultBuilder {
/// Builds the final analysis result. Resolves internal Vcs.
pub async fn build(
self,
mut self,
track_reexport_references: bool,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let references = self.references.into_iter().collect();
......@@ -287,6 +289,8 @@ impl AnalyzeEcmascriptModuleResultBuilder {
} else {
OptionSourceMap::none().to_resolved().await?
};
self.code_gens.shrink_to_fit();
Ok(AnalyzeEcmascriptModuleResult::cell(
AnalyzeEcmascriptModuleResult {
references: ResolvedVc::cell(references),
......@@ -1260,7 +1264,7 @@ pub(crate) async fn analyse_ecmascript_module_internal(
if let Some(&r) = import_references.get(esm_reference_index) {
if let Some("__turbopack_module_id__") = export.as_deref() {
analysis.add_reference(
EsmModuleIdAssetReference::new(*r, Vc::cell(ast_path))
EsmModuleIdAssetReference::new(*r, ast_path.into())
.to_resolved()
.await?,
)
......@@ -1473,7 +1477,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
*origin,
Request::parse(Value::new(pat)),
compile_time_info.environment().rendering(),
Vc::cell(ast_path.to_vec()),
ast_path.to_vec().into(),
issue_source(source, span),
in_try,
url_rewrite_behavior.unwrap_or(UrlRewriteBehavior::Relative),
......@@ -1508,7 +1512,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
WorkerAssetReference::new(
*origin,
Request::parse(Value::new(pat)),
Vc::cell(ast_path.to_vec()),
ast_path.to_vec().into(),
issue_source(source, span),
in_try,
)
......@@ -1607,7 +1611,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
EsmAsyncAssetReference::new(
*origin,
Request::parse(Value::new(pat)),
Vc::cell(ast_path.to_vec()),
ast_path.to_vec().into(),
issue_source(source, span),
Value::new(import_annotations),
in_try,
......@@ -1649,7 +1653,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
CjsRequireAssetReference::new(
*origin,
Request::parse(Value::new(pat)),
Vc::cell(ast_path.to_vec()),
ast_path.to_vec().into(),
issue_source(source, span),
in_try,
)
......@@ -1703,7 +1707,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
CjsRequireResolveAssetReference::new(
*origin,
Request::parse(Value::new(pat)),
Vc::cell(ast_path.to_vec()),
ast_path.to_vec().into(),
issue_source(source, span),
in_try,
)
......@@ -1749,7 +1753,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
options.dir,
options.include_subdirs,
Vc::cell(options.filter),
Vc::cell(ast_path.to_vec()),
ast_path.to_vec().into(),
Some(issue_source(source, span)),
in_try,
)
......@@ -3352,10 +3356,16 @@ async fn resolve_as_webpack_runtime(
}
}
#[turbo_tasks::value(transparent)]
#[derive(Hash, Debug, Clone)]
#[derive(Hash, Debug, Clone, Eq, Serialize, Deserialize, PartialEq, TraceRawVcs)]
pub struct AstPath(#[turbo_tasks(trace_ignore)] Vec<AstParentKind>);
impl TaskInput for AstPath {
fn is_transient(&self) -> bool {
false
}
}
unsafe impl NonLocalValue for AstPath {}
impl Deref for AstPath {
type Target = [AstParentKind];
......
......@@ -212,7 +212,7 @@ pub struct RequireContextAssetReference {
pub dir: RcStr,
pub include_subdirs: bool,
pub path: ResolvedVc<AstPath>,
pub path: AstPath,
pub issue_source: Option<IssueSource>,
pub in_try: bool,
}
......@@ -226,7 +226,7 @@ impl RequireContextAssetReference {
dir: RcStr,
include_subdirs: bool,
filter: Vc<Regex>,
path: ResolvedVc<AstPath>,
path: AstPath,
issue_source: Option<IssueSource>,
in_try: bool,
) -> Result<Vc<Self>> {
......@@ -302,8 +302,7 @@ impl CodeGenerateable for RequireContextAssetReference {
let mut visitors = Vec::new();
let path = &self.path.await?;
visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
visitors.push(create_visitor!(self.path, visit_mut_expr(expr: &mut Expr) {
if let Expr::Call(_) = expr {
*expr = quote!(
"$turbopack_module_context($turbopack_require($id))" as Expr,
......
......@@ -29,7 +29,7 @@ use crate::{
pub struct WorkerAssetReference {
pub origin: ResolvedVc<Box<dyn ResolveOrigin>>,
pub request: ResolvedVc<Request>,
pub path: ResolvedVc<AstPath>,
pub path: AstPath,
pub issue_source: IssueSource,
pub in_try: bool,
}
......@@ -40,7 +40,7 @@ impl WorkerAssetReference {
pub fn new(
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
request: ResolvedVc<Request>,
path: ResolvedVc<AstPath>,
path: AstPath,
issue_source: IssueSource,
in_try: bool,
) -> Vc<Self> {
......@@ -131,9 +131,7 @@ impl CodeGenerateable for WorkerAssetReference {
.chunk_item_id_from_ident(loader.ident())
.await?;
let path = &self.path.await?;
let visitor = create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
let visitor = create_visitor!(self.path, visit_mut_expr(expr: &mut Expr) {
let message = if let Expr::New(NewExpr { args, ..}) = expr {
if let Some(args) = args {
match args.first_mut() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment