Unverified Commit a2fe4aec authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: optimize client manifest generation (#75248)

**View diff without whitespace changes!**

This brings the number of `AssetIdent::path` calls down from 5M to 100k

```
### canary (de984c2695b):
~16,60GB
582.76s user 109.72s system 815% cpu 1:24.95 total
583.30s user 108.45s system 805% cpu 1:25.86 total

### plus mischnic/client-manifest-optimization:
still ~16,60GB
547.70s user 105.24s system 811% cpu 1:20.48 total
550.16s user 108.14s system 782% cpu 1:24.10 total
```
parent 70c0a66c
...@@ -4476,6 +4476,7 @@ dependencies = [ ...@@ -4476,6 +4476,7 @@ dependencies = [
"async-trait", "async-trait",
"auto-hash-map", "auto-hash-map",
"base64 0.21.4", "base64 0.21.4",
"either",
"futures", "futures",
"indexmap 2.5.0", "indexmap 2.5.0",
"indoc", "indoc",
......
...@@ -15,6 +15,7 @@ workspace = true ...@@ -15,6 +15,7 @@ workspace = true
anyhow = { workspace = true } anyhow = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
base64 = "0.21.0" base64 = "0.21.0"
either = { workspace = true }
lazy-regex = "3.0.1" lazy-regex = "3.0.1"
next-custom-transforms = { workspace = true } next-custom-transforms = { workspace = true }
once_cell = { workspace = true } once_cell = { workspace = true }
......
use std::collections::HashMap;
use anyhow::Result; use anyhow::Result;
use either::Either;
use indoc::formatdoc; use indoc::formatdoc;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TaskInput, TryJoinIterExt, Value, ValueToString, Vc}; use turbo_tasks::{
FxIndexSet, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, Value, ValueToString, Vc,
};
use turbo_tasks_fs::{File, FileSystemPath}; use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -63,6 +69,11 @@ impl ClientReferenceManifest { ...@@ -63,6 +69,11 @@ impl ClientReferenceManifest {
runtime, runtime,
mode, mode,
} = options; } = options;
let span = tracing::info_span!(
"ClientReferenceManifest build output",
entry_name = display(&entry_name)
);
async move {
let mut entry_manifest: ClientReferenceManifest = Default::default(); let mut entry_manifest: ClientReferenceManifest = Default::default();
let mut references = FxIndexSet::default(); let mut references = FxIndexSet::default();
entry_manifest.module_loading.prefix = next_config entry_manifest.module_loading.prefix = next_config
...@@ -77,10 +88,56 @@ impl ClientReferenceManifest { ...@@ -77,10 +88,56 @@ impl ClientReferenceManifest {
.cross_origin .cross_origin
.as_ref() .as_ref()
.map(|p| p.to_owned()); .map(|p| p.to_owned());
let client_references_chunks = client_references_chunks.await?; let ClientReferencesChunks {
let client_relative_path = client_relative_path.await?; client_component_client_chunks,
let node_root_ref = node_root.await?; layout_segment_client_chunks,
let rsc_app_entry_chunks = rsc_app_entry_chunks.await?; client_component_ssr_chunks,
} = &*client_references_chunks.await?;
let client_relative_path = &*client_relative_path.await?;
let node_root_ref = &*node_root.await?;
let rsc_app_entry_chunks = &*rsc_app_entry_chunks.await?;
async fn cached_chunk_paths(
cache: &mut HashMap<ResolvedVc<Box<dyn OutputAsset>>, ReadRef<FileSystemPath>>,
chunks: impl Iterator<Item = ResolvedVc<Box<dyn OutputAsset>>>,
) -> Result<
impl Iterator<Item = (ResolvedVc<Box<dyn OutputAsset>>, ReadRef<FileSystemPath>)>,
> {
let results = chunks
.into_iter()
.map(|chunk| (chunk, cache.get(&chunk).cloned()))
.map(async |(chunk, path)| {
Ok(if let Some(path) = path {
(chunk, Either::Left(path))
} else {
(chunk, Either::Right(chunk.ident().path().await?))
})
})
.try_join()
.await?;
for (chunk, path) in &results {
if let Either::Right(path) = path {
cache.insert(*chunk, path.clone());
}
}
Ok(results.into_iter().map(|(chunk, path)| match path {
Either::Left(path) => (chunk, path),
Either::Right(path) => (chunk, path),
}))
}
let mut client_chunk_path_cache: HashMap<
ResolvedVc<Box<dyn OutputAsset>>,
ReadRef<FileSystemPath>,
> = HashMap::new();
let mut ssr_chunk_path_cache: HashMap<
ResolvedVc<Box<dyn OutputAsset>>,
ReadRef<FileSystemPath>,
> = HashMap::new();
let mut rsc_chunk_path_cache: HashMap<
ResolvedVc<Box<dyn OutputAsset>>,
ReadRef<FileSystemPath>,
> = HashMap::new();
for app_client_reference in client_references.await?.client_references.iter() { for app_client_reference in client_references.await?.client_references.iter() {
let app_client_reference_ty = app_client_reference.ty(); let app_client_reference_ty = app_client_reference.ty();
...@@ -92,7 +149,6 @@ impl ClientReferenceManifest { ...@@ -92,7 +149,6 @@ impl ClientReferenceManifest {
let client_reference_module_ref = client_reference_module.await?; let client_reference_module_ref = client_reference_module.await?;
let server_path = client_reference_module_ref.server_ident.to_string().await?; let server_path = client_reference_module_ref.server_ident.to_string().await?;
let client_module = client_reference_module_ref.client_module; let client_module = client_reference_module_ref.client_module;
let client_chunk_item = client_module let client_chunk_item = client_module
.as_chunk_item(module_graph, Vc::upcast(client_chunking_context)) .as_chunk_item(module_graph, Vc::upcast(client_chunking_context))
...@@ -103,22 +159,22 @@ impl ClientReferenceManifest { ...@@ -103,22 +159,22 @@ impl ClientReferenceManifest {
let (client_chunks_paths, client_is_async) = let (client_chunks_paths, client_is_async) =
if let Some((client_chunks, client_availability_info)) = if let Some((client_chunks, client_availability_info)) =
client_references_chunks client_component_client_chunks.get(&app_client_reference_ty)
.client_component_client_chunks
.get(&app_client_reference_ty)
{ {
let client_chunks = client_chunks.await?; let client_chunks = client_chunks.await?;
references.extend(client_chunks.iter()); references.extend(client_chunks.iter());
let client_chunks_paths = client_chunks let client_chunks_paths = cached_chunk_paths(
.iter() &mut client_chunk_path_cache,
.map(|chunk| chunk.ident().path()) client_chunks.iter().copied(),
.try_join() )
.await?; .await?;
let chunk_paths = client_chunks_paths let chunk_paths = client_chunks_paths
.iter() .filter_map(|(_, chunk_path)| {
.filter_map(|chunk_path| client_relative_path.get_path_to(chunk_path)) client_relative_path
.get_path_to(&chunk_path)
.map(ToString::to_string) .map(ToString::to_string)
})
// It's possible that a chunk also emits CSS files, that will // It's possible that a chunk also emits CSS files, that will
// be handled separatedly. // be handled separatedly.
.filter(|path| path.ends_with(".js")) .filter(|path| path.ends_with(".js"))
...@@ -157,28 +213,29 @@ impl ClientReferenceManifest { ...@@ -157,28 +213,29 @@ impl ClientReferenceManifest {
// loading chunks. // loading chunks.
(Vec::new(), false) (Vec::new(), false)
} else if let Some((ssr_chunks, ssr_availability_info)) = } else if let Some((ssr_chunks, ssr_availability_info)) =
client_references_chunks client_component_ssr_chunks.get(&app_client_reference_ty)
.client_component_ssr_chunks
.get(&app_client_reference_ty)
{ {
let ssr_chunks = ssr_chunks.await?; let ssr_chunks = ssr_chunks.await?;
references.extend(ssr_chunks.iter()); references.extend(ssr_chunks.iter());
let ssr_chunks_paths = ssr_chunks let ssr_chunks_paths = cached_chunk_paths(
.iter() &mut ssr_chunk_path_cache,
.map(|chunk| chunk.ident().path()) ssr_chunks.iter().copied(),
.try_join() )
.await?; .await?;
let chunk_paths = ssr_chunks_paths let chunk_paths = ssr_chunks_paths
.iter() .filter_map(|(_, chunk_path)| {
.filter_map(|chunk_path| node_root_ref.get_path_to(chunk_path)) node_root_ref
.get_path_to(&chunk_path)
.map(ToString::to_string) .map(ToString::to_string)
})
.map(RcStr::from) .map(RcStr::from)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let is_async = let is_async = is_item_async(
is_item_async(ssr_availability_info, ResolvedVc::upcast(ssr_module)) ssr_availability_info,
ResolvedVc::upcast(ssr_module),
)
.await?; .await?;
(chunk_paths, is_async) (chunk_paths, is_async)
...@@ -193,16 +250,18 @@ impl ClientReferenceManifest { ...@@ -193,16 +250,18 @@ impl ClientReferenceManifest {
// loading chunks. // loading chunks.
(Vec::new(), false) (Vec::new(), false)
} else { } else {
let rsc_chunks_paths = rsc_app_entry_chunks let rsc_chunks_paths = cached_chunk_paths(
.iter() &mut rsc_chunk_path_cache,
.map(|chunk| chunk.ident().path()) rsc_app_entry_chunks.iter().copied(),
.try_join() )
.await?; .await?;
let chunk_paths = rsc_chunks_paths let chunk_paths = rsc_chunks_paths
.iter() .filter_map(|(_, chunk_path)| {
.filter_map(|chunk_path| node_root_ref.get_path_to(chunk_path)) node_root_ref
.get_path_to(&chunk_path)
.map(ToString::to_string) .map(ToString::to_string)
})
.map(RcStr::from) .map(RcStr::from)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
...@@ -221,8 +280,9 @@ impl ClientReferenceManifest { ...@@ -221,8 +280,9 @@ impl ClientReferenceManifest {
name: "*".into(), name: "*".into(),
id: (&*client_module_id).into(), id: (&*client_module_id).into(),
chunks: client_chunks_paths, chunks: client_chunks_paths,
// This should of course be client_is_async, but SSR can become async // This should of course be client_is_async, but SSR can become
// due to ESM externals, and the ssr_manifest_node is currently ignored // async due to ESM externals, and
// the ssr_manifest_node is currently ignored
// by React. // by React.
r#async: client_is_async || ssr_is_async, r#async: client_is_async || ssr_is_async,
}, },
...@@ -274,15 +334,12 @@ impl ClientReferenceManifest { ...@@ -274,15 +334,12 @@ impl ClientReferenceManifest {
} }
// per layout segment chunks need to be emitted into the manifest too // per layout segment chunks need to be emitted into the manifest too
for (server_component, client_chunks) in for (server_component, client_chunks) in layout_segment_client_chunks.iter() {
client_references_chunks.layout_segment_client_chunks.iter()
{
let server_component_name = server_component let server_component_name = server_component
.server_path() .server_path()
.with_extension("".into()) .with_extension("".into())
.to_string() .to_string()
.await?; .await?;
let mut entry_css_files_with_chunk = Vec::new(); let mut entry_css_files_with_chunk = Vec::new();
let entry_js_files = entry_manifest let entry_js_files = entry_manifest
.entry_js_files .entry_js_files
...@@ -290,10 +347,8 @@ impl ClientReferenceManifest { ...@@ -290,10 +347,8 @@ impl ClientReferenceManifest {
.or_default(); .or_default();
let client_chunks = &client_chunks.await?; let client_chunks = &client_chunks.await?;
let client_chunks_with_path = client_chunks let client_chunks_with_path =
.iter() cached_chunk_paths(&mut client_chunk_path_cache, client_chunks.iter().copied())
.map(|chunk| async move { Ok((chunk, chunk.ident().path().await?)) })
.try_join()
.await?; .await?;
for (chunk, chunk_path) in client_chunks_with_path { for (chunk, chunk_path) in client_chunks_with_path {
...@@ -311,7 +366,7 @@ impl ClientReferenceManifest { ...@@ -311,7 +366,7 @@ impl ClientReferenceManifest {
&& mode.await?.is_production(); && mode.await?.is_production();
let entry_css_files_vec = entry_css_files_with_chunk let entry_css_files_vec = entry_css_files_with_chunk
.into_iter() .into_iter()
.map(|(path, chunk)| async { .map(async |(path, chunk)| {
let content = if inlined { let content = if inlined {
if let Some(content_file) = if let Some(content_file) =
chunk.content().file_content().await?.as_content() chunk.content().file_content().await?.as_content()
...@@ -366,6 +421,9 @@ impl ClientReferenceManifest { ...@@ -366,6 +421,9 @@ impl ClientReferenceManifest {
Vc::cell(references.into_iter().collect()), Vc::cell(references.into_iter().collect()),
))) )))
} }
.instrument(span)
.await
}
} }
impl From<&TurbopackModuleId> for ModuleId { impl From<&TurbopackModuleId> for ModuleId {
...@@ -394,9 +452,7 @@ async fn is_item_async( ...@@ -394,9 +452,7 @@ async fn is_item_async(
return Ok(false); return Ok(false);
}; };
let Some(info) = &*available_modules.get(*module).await? else { let available_modules = available_modules.snapshot().await?;
return Ok(false);
};
Ok(info.is_async) Ok(available_modules.get(module).is_some_and(|i| i.is_async))
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment