Unverified Commit a2fe4aec authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: optimize client manifest generation (#75248)

**View diff without whitespace changes!**

This brings the number of `AssetIdent::path` calls down from 5M to 100k

```
### canary (de984c2695b):
~16,60GB
582.76s user 109.72s system 815% cpu 1:24.95 total
583.30s user 108.45s system 805% cpu 1:25.86 total

### plus mischnic/client-manifest-optimization:
still ~16,60GB
547.70s user 105.24s system 811% cpu 1:20.48 total
550.16s user 108.14s system 782% cpu 1:24.10 total
```
parent 70c0a66c
...@@ -4476,6 +4476,7 @@ dependencies = [ ...@@ -4476,6 +4476,7 @@ dependencies = [
"async-trait", "async-trait",
"auto-hash-map", "auto-hash-map",
"base64 0.21.4", "base64 0.21.4",
"either",
"futures", "futures",
"indexmap 2.5.0", "indexmap 2.5.0",
"indoc", "indoc",
......
...@@ -15,6 +15,7 @@ workspace = true ...@@ -15,6 +15,7 @@ workspace = true
anyhow = { workspace = true } anyhow = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
base64 = "0.21.0" base64 = "0.21.0"
either = { workspace = true }
lazy-regex = "3.0.1" lazy-regex = "3.0.1"
next-custom-transforms = { workspace = true } next-custom-transforms = { workspace = true }
once_cell = { workspace = true } once_cell = { workspace = true }
......
use std::collections::HashMap;
use anyhow::Result; use anyhow::Result;
use either::Either;
use indoc::formatdoc; use indoc::formatdoc;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TaskInput, TryJoinIterExt, Value, ValueToString, Vc}; use turbo_tasks::{
FxIndexSet, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, Value, ValueToString, Vc,
};
use turbo_tasks_fs::{File, FileSystemPath}; use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -63,308 +69,360 @@ impl ClientReferenceManifest { ...@@ -63,308 +69,360 @@ impl ClientReferenceManifest {
runtime, runtime,
mode, mode,
} = options; } = options;
let mut entry_manifest: ClientReferenceManifest = Default::default(); let span = tracing::info_span!(
let mut references = FxIndexSet::default(); "ClientReferenceManifest build output",
entry_manifest.module_loading.prefix = next_config entry_name = display(&entry_name)
.computed_asset_prefix() );
.await? async move {
.as_ref() let mut entry_manifest: ClientReferenceManifest = Default::default();
.map(|p| p.clone()) let mut references = FxIndexSet::default();
.unwrap_or_default(); entry_manifest.module_loading.prefix = next_config
.computed_asset_prefix()
entry_manifest.module_loading.cross_origin = next_config .await?
.await? .as_ref()
.cross_origin .map(|p| p.clone())
.as_ref() .unwrap_or_default();
.map(|p| p.to_owned());
let client_references_chunks = client_references_chunks.await?; entry_manifest.module_loading.cross_origin = next_config
let client_relative_path = client_relative_path.await?; .await?
let node_root_ref = node_root.await?; .cross_origin
let rsc_app_entry_chunks = rsc_app_entry_chunks.await?; .as_ref()
.map(|p| p.to_owned());
for app_client_reference in client_references.await?.client_references.iter() { let ClientReferencesChunks {
let app_client_reference_ty = app_client_reference.ty(); client_component_client_chunks,
layout_segment_client_chunks,
// An client component need to be emitted into the client reference manifest client_component_ssr_chunks,
if let ClientReferenceType::EcmascriptClientReference(client_reference_module) = } = &*client_references_chunks.await?;
app_client_reference_ty let client_relative_path = &*client_relative_path.await?;
{ let node_root_ref = &*node_root.await?;
let client_reference_module_ref = client_reference_module.await?; let rsc_app_entry_chunks = &*rsc_app_entry_chunks.await?;
let server_path = client_reference_module_ref.server_ident.to_string().await?; async fn cached_chunk_paths(
cache: &mut HashMap<ResolvedVc<Box<dyn OutputAsset>>, ReadRef<FileSystemPath>>,
let client_module = client_reference_module_ref.client_module; chunks: impl Iterator<Item = ResolvedVc<Box<dyn OutputAsset>>>,
let client_chunk_item = client_module ) -> Result<
.as_chunk_item(module_graph, Vc::upcast(client_chunking_context)) impl Iterator<Item = (ResolvedVc<Box<dyn OutputAsset>>, ReadRef<FileSystemPath>)>,
.to_resolved() > {
let results = chunks
.into_iter()
.map(|chunk| (chunk, cache.get(&chunk).cloned()))
.map(async |(chunk, path)| {
Ok(if let Some(path) = path {
(chunk, Either::Left(path))
} else {
(chunk, Either::Right(chunk.ident().path().await?))
})
})
.try_join()
.await?; .await?;
let client_module_id = client_chunk_item.id().await?; for (chunk, path) in &results {
if let Either::Right(path) = path {
let (client_chunks_paths, client_is_async) = cache.insert(*chunk, path.clone());
if let Some((client_chunks, client_availability_info)) = }
client_references_chunks }
.client_component_client_chunks Ok(results.into_iter().map(|(chunk, path)| match path {
.get(&app_client_reference_ty) Either::Left(path) => (chunk, path),
{ Either::Right(path) => (chunk, path),
let client_chunks = client_chunks.await?; }))
references.extend(client_chunks.iter()); }
let client_chunks_paths = client_chunks let mut client_chunk_path_cache: HashMap<
.iter() ResolvedVc<Box<dyn OutputAsset>>,
.map(|chunk| chunk.ident().path()) ReadRef<FileSystemPath>,
.try_join() > = HashMap::new();
let mut ssr_chunk_path_cache: HashMap<
ResolvedVc<Box<dyn OutputAsset>>,
ReadRef<FileSystemPath>,
> = HashMap::new();
let mut rsc_chunk_path_cache: HashMap<
ResolvedVc<Box<dyn OutputAsset>>,
ReadRef<FileSystemPath>,
> = HashMap::new();
for app_client_reference in client_references.await?.client_references.iter() {
let app_client_reference_ty = app_client_reference.ty();
// An client component need to be emitted into the client reference manifest
if let ClientReferenceType::EcmascriptClientReference(client_reference_module) =
app_client_reference_ty
{
let client_reference_module_ref = client_reference_module.await?;
let server_path = client_reference_module_ref.server_ident.to_string().await?;
let client_module = client_reference_module_ref.client_module;
let client_chunk_item = client_module
.as_chunk_item(module_graph, Vc::upcast(client_chunking_context))
.to_resolved()
.await?;
let client_module_id = client_chunk_item.id().await?;
let (client_chunks_paths, client_is_async) =
if let Some((client_chunks, client_availability_info)) =
client_component_client_chunks.get(&app_client_reference_ty)
{
let client_chunks = client_chunks.await?;
references.extend(client_chunks.iter());
let client_chunks_paths = cached_chunk_paths(
&mut client_chunk_path_cache,
client_chunks.iter().copied(),
)
.await?; .await?;
let chunk_paths = client_chunks_paths let chunk_paths = client_chunks_paths
.iter() .filter_map(|(_, chunk_path)| {
.filter_map(|chunk_path| client_relative_path.get_path_to(chunk_path)) client_relative_path
.map(ToString::to_string) .get_path_to(&chunk_path)
// It's possible that a chunk also emits CSS files, that will .map(ToString::to_string)
// be handled separatedly. })
.filter(|path| path.ends_with(".js")) // It's possible that a chunk also emits CSS files, that will
.map(RcStr::from) // be handled separatedly.
.collect::<Vec<_>>(); .filter(|path| path.ends_with(".js"))
.map(RcStr::from)
let is_async = is_item_async( .collect::<Vec<_>>();
client_availability_info,
ResolvedVc::upcast(client_module), let is_async = is_item_async(
) client_availability_info,
.await?; ResolvedVc::upcast(client_module),
)
.await?;
(chunk_paths, is_async) (chunk_paths, is_async)
} else { } else {
(Vec::new(), false) (Vec::new(), false)
}; };
if let Some(ssr_chunking_context) = ssr_chunking_context {
let ssr_module = client_reference_module_ref.ssr_module;
let ssr_chunk_item = ssr_module
.as_chunk_item(module_graph, Vc::upcast(ssr_chunking_context))
.to_resolved()
.await?;
let ssr_module_id = ssr_chunk_item.id().await?;
if let Some(ssr_chunking_context) = ssr_chunking_context { let rsc_chunk_item = client_reference_module
let ssr_module = client_reference_module_ref.ssr_module; .as_chunk_item(module_graph, Vc::upcast(ssr_chunking_context))
let ssr_chunk_item = ssr_module .to_resolved()
.as_chunk_item(module_graph, Vc::upcast(ssr_chunking_context)) .await?;
.to_resolved() let rsc_module_id = rsc_chunk_item.id().await?;
.await?;
let ssr_module_id = ssr_chunk_item.id().await?; let (ssr_chunks_paths, ssr_is_async) = if runtime == NextRuntime::Edge {
// the chunks get added to the middleware-manifest.json instead
// of this file because the
// edge runtime doesn't support dynamically
// loading chunks.
(Vec::new(), false)
} else if let Some((ssr_chunks, ssr_availability_info)) =
client_component_ssr_chunks.get(&app_client_reference_ty)
{
let ssr_chunks = ssr_chunks.await?;
references.extend(ssr_chunks.iter());
let rsc_chunk_item = client_reference_module let ssr_chunks_paths = cached_chunk_paths(
.as_chunk_item(module_graph, Vc::upcast(ssr_chunking_context)) &mut ssr_chunk_path_cache,
.to_resolved() ssr_chunks.iter().copied(),
.await?; )
let rsc_module_id = rsc_chunk_item.id().await?; .await?;
let chunk_paths = ssr_chunks_paths
let (ssr_chunks_paths, ssr_is_async) = if runtime == NextRuntime::Edge { .filter_map(|(_, chunk_path)| {
// the chunks get added to the middleware-manifest.json instead node_root_ref
// of this file because the .get_path_to(&chunk_path)
// edge runtime doesn't support dynamically .map(ToString::to_string)
// loading chunks. })
(Vec::new(), false) .map(RcStr::from)
} else if let Some((ssr_chunks, ssr_availability_info)) = .collect::<Vec<_>>();
client_references_chunks
.client_component_ssr_chunks let is_async = is_item_async(
.get(&app_client_reference_ty) ssr_availability_info,
{ ResolvedVc::upcast(ssr_module),
let ssr_chunks = ssr_chunks.await?; )
references.extend(ssr_chunks.iter());
let ssr_chunks_paths = ssr_chunks
.iter()
.map(|chunk| chunk.ident().path())
.try_join()
.await?; .await?;
let chunk_paths = ssr_chunks_paths (chunk_paths, is_async)
.iter() } else {
.filter_map(|chunk_path| node_root_ref.get_path_to(chunk_path)) (Vec::new(), false)
.map(ToString::to_string) };
.map(RcStr::from)
.collect::<Vec<_>>(); let (rsc_chunks_paths, rsc_is_async) = if runtime == NextRuntime::Edge {
// the chunks get added to the middleware-manifest.json instead
let is_async = // of this file because the
is_item_async(ssr_availability_info, ResolvedVc::upcast(ssr_module)) // edge runtime doesn't support dynamically
.await?; // loading chunks.
(Vec::new(), false)
(chunk_paths, is_async) } else {
} else { let rsc_chunks_paths = cached_chunk_paths(
(Vec::new(), false) &mut rsc_chunk_path_cache,
}; rsc_app_entry_chunks.iter().copied(),
)
let (rsc_chunks_paths, rsc_is_async) = if runtime == NextRuntime::Edge {
// the chunks get added to the middleware-manifest.json instead
// of this file because the
// edge runtime doesn't support dynamically
// loading chunks.
(Vec::new(), false)
} else {
let rsc_chunks_paths = rsc_app_entry_chunks
.iter()
.map(|chunk| chunk.ident().path())
.try_join()
.await?; .await?;
let chunk_paths = rsc_chunks_paths let chunk_paths = rsc_chunks_paths
.iter() .filter_map(|(_, chunk_path)| {
.filter_map(|chunk_path| node_root_ref.get_path_to(chunk_path)) node_root_ref
.map(ToString::to_string) .get_path_to(&chunk_path)
.map(RcStr::from) .map(ToString::to_string)
.collect::<Vec<_>>(); })
.map(RcStr::from)
let is_async = is_item_async( .collect::<Vec<_>>();
&rsc_app_entry_chunks_availability,
ResolvedVc::upcast(client_reference_module), let is_async = is_item_async(
) &rsc_app_entry_chunks_availability,
.await?; ResolvedVc::upcast(client_reference_module),
)
.await?;
(chunk_paths, is_async) (chunk_paths, is_async)
}; };
entry_manifest.client_modules.module_exports.insert( entry_manifest.client_modules.module_exports.insert(
get_client_reference_module_key(&server_path, "*"), get_client_reference_module_key(&server_path, "*"),
ManifestNodeEntry { ManifestNodeEntry {
name: "*".into(), name: "*".into(),
id: (&*client_module_id).into(), id: (&*client_module_id).into(),
chunks: client_chunks_paths, chunks: client_chunks_paths,
// This should of course be client_is_async, but SSR can become async // This should of course be client_is_async, but SSR can become
// due to ESM externals, and the ssr_manifest_node is currently ignored // async due to ESM externals, and
// by React. // the ssr_manifest_node is currently ignored
r#async: client_is_async || ssr_is_async, // by React.
}, r#async: client_is_async || ssr_is_async,
); },
);
let mut ssr_manifest_node = ManifestNode::default();
ssr_manifest_node.module_exports.insert( let mut ssr_manifest_node = ManifestNode::default();
"*".into(), ssr_manifest_node.module_exports.insert(
ManifestNodeEntry { "*".into(),
name: "*".into(), ManifestNodeEntry {
id: (&*ssr_module_id).into(), name: "*".into(),
chunks: ssr_chunks_paths, id: (&*ssr_module_id).into(),
// See above chunks: ssr_chunks_paths,
r#async: client_is_async || ssr_is_async, // See above
}, r#async: client_is_async || ssr_is_async,
); },
);
let mut rsc_manifest_node = ManifestNode::default();
rsc_manifest_node.module_exports.insert( let mut rsc_manifest_node = ManifestNode::default();
"*".into(), rsc_manifest_node.module_exports.insert(
ManifestNodeEntry { "*".into(),
name: "*".into(), ManifestNodeEntry {
id: (&*rsc_module_id).into(), name: "*".into(),
chunks: rsc_chunks_paths, id: (&*rsc_module_id).into(),
r#async: rsc_is_async, chunks: rsc_chunks_paths,
}, r#async: rsc_is_async,
); },
);
match runtime {
NextRuntime::NodeJs => { match runtime {
entry_manifest NextRuntime::NodeJs => {
.ssr_module_mapping entry_manifest
.insert((&*client_module_id).into(), ssr_manifest_node); .ssr_module_mapping
entry_manifest .insert((&*client_module_id).into(), ssr_manifest_node);
.rsc_module_mapping entry_manifest
.insert((&*client_module_id).into(), rsc_manifest_node); .rsc_module_mapping
} .insert((&*client_module_id).into(), rsc_manifest_node);
NextRuntime::Edge => { }
entry_manifest NextRuntime::Edge => {
.edge_ssr_module_mapping entry_manifest
.insert((&*client_module_id).into(), ssr_manifest_node); .edge_ssr_module_mapping
entry_manifest .insert((&*client_module_id).into(), ssr_manifest_node);
.edge_rsc_module_mapping entry_manifest
.insert((&*client_module_id).into(), rsc_manifest_node); .edge_rsc_module_mapping
.insert((&*client_module_id).into(), rsc_manifest_node);
}
} }
} }
} }
} }
}
// per layout segment chunks need to be emitted into the manifest too // per layout segment chunks need to be emitted into the manifest too
for (server_component, client_chunks) in for (server_component, client_chunks) in layout_segment_client_chunks.iter() {
client_references_chunks.layout_segment_client_chunks.iter() let server_component_name = server_component
{ .server_path()
let server_component_name = server_component .with_extension("".into())
.server_path() .to_string()
.with_extension("".into()) .await?;
.to_string() let mut entry_css_files_with_chunk = Vec::new();
.await?; let entry_js_files = entry_manifest
.entry_js_files
let mut entry_css_files_with_chunk = Vec::new(); .entry(server_component_name.clone_value())
let entry_js_files = entry_manifest .or_default();
.entry_js_files
.entry(server_component_name.clone_value()) let client_chunks = &client_chunks.await?;
.or_default(); let client_chunks_with_path =
cached_chunk_paths(&mut client_chunk_path_cache, client_chunks.iter().copied())
let client_chunks = &client_chunks.await?; .await?;
let client_chunks_with_path = client_chunks
.iter() for (chunk, chunk_path) in client_chunks_with_path {
.map(|chunk| async move { Ok((chunk, chunk.ident().path().await?)) }) if let Some(path) = client_relative_path.get_path_to(&chunk_path) {
.try_join() let path = path.into();
.await?; if chunk_path.extension_ref() == Some("css") {
entry_css_files_with_chunk.push((path, chunk));
for (chunk, chunk_path) in client_chunks_with_path { } else {
if let Some(path) = client_relative_path.get_path_to(&chunk_path) { entry_js_files.insert(path);
let path = path.into(); }
if chunk_path.extension_ref() == Some("css") {
entry_css_files_with_chunk.push((path, chunk));
} else {
entry_js_files.insert(path);
} }
} }
}
let inlined = next_config.await?.experimental.inline_css.unwrap_or(false) let inlined = next_config.await?.experimental.inline_css.unwrap_or(false)
&& mode.await?.is_production(); && mode.await?.is_production();
let entry_css_files_vec = entry_css_files_with_chunk let entry_css_files_vec = entry_css_files_with_chunk
.into_iter() .into_iter()
.map(|(path, chunk)| async { .map(async |(path, chunk)| {
let content = if inlined { let content = if inlined {
if let Some(content_file) = if let Some(content_file) =
chunk.content().file_content().await?.as_content() chunk.content().file_content().await?.as_content()
{ {
Some(content_file.content().to_str()?.into()) Some(content_file.content().to_str()?.into())
} else {
Some("".into())
}
} else { } else {
Some("".into()) None
} };
} else { Ok(CssResource {
None path,
}; inlined,
Ok(CssResource { content,
path, })
inlined,
content,
}) })
}) .try_join()
.try_join() .await?;
.await?;
let entry_css_files = entry_manifest
.entry_css_files
.entry(server_component_name.clone_value())
.or_default();
entry_css_files.extend(entry_css_files_vec);
}
let client_reference_manifest_json = serde_json::to_string(&entry_manifest).unwrap(); let entry_css_files = entry_manifest
.entry_css_files
// We put normalized path for the each entry key and the manifest output path, .entry(server_component_name.clone_value())
// to conform next.js's load client reference manifest behavior: .or_default();
// https://github.com/vercel/next.js/blob/2f9d718695e4c90be13c3bf0f3647643533071bf/packages/next/src/server/load-components.ts#L162-L164 entry_css_files.extend(entry_css_files_vec);
// note this only applies to the manifests, assets are placed to the original }
// path still (same as webpack does)
let normalized_manifest_entry = entry_name.replace("%5F", "_"); let client_reference_manifest_json = serde_json::to_string(&entry_manifest).unwrap();
Ok(Vc::upcast(VirtualOutputAsset::new_with_references(
node_root.join( // We put normalized path for the each entry key and the manifest output path,
format!("server/app{normalized_manifest_entry}_client-reference-manifest.js",) // to conform next.js's load client reference manifest behavior:
.into(), // https://github.com/vercel/next.js/blob/2f9d718695e4c90be13c3bf0f3647643533071bf/packages/next/src/server/load-components.ts#L162-L164
), // note this only applies to the manifests, assets are placed to the original
AssetContent::file( // path still (same as webpack does)
File::from(formatdoc! { let normalized_manifest_entry = entry_name.replace("%5F", "_");
r#" Ok(Vc::upcast(VirtualOutputAsset::new_with_references(
node_root.join(
format!("server/app{normalized_manifest_entry}_client-reference-manifest.js",)
.into(),
),
AssetContent::file(
File::from(formatdoc! {
r#"
globalThis.__RSC_MANIFEST = globalThis.__RSC_MANIFEST || {{}}; globalThis.__RSC_MANIFEST = globalThis.__RSC_MANIFEST || {{}};
globalThis.__RSC_MANIFEST[{entry_name}] = {manifest} globalThis.__RSC_MANIFEST[{entry_name}] = {manifest}
"#, "#,
entry_name = StringifyJs(&normalized_manifest_entry), entry_name = StringifyJs(&normalized_manifest_entry),
manifest = &client_reference_manifest_json manifest = &client_reference_manifest_json
}) })
.into(), .into(),
), ),
Vc::cell(references.into_iter().collect()), Vc::cell(references.into_iter().collect()),
))) )))
}
.instrument(span)
.await
} }
} }
...@@ -394,9 +452,7 @@ async fn is_item_async( ...@@ -394,9 +452,7 @@ async fn is_item_async(
return Ok(false); return Ok(false);
}; };
let Some(info) = &*available_modules.get(*module).await? else { let available_modules = available_modules.snapshot().await?;
return Ok(false);
};
Ok(info.is_async) Ok(available_modules.get(module).is_some_and(|i| i.is_async))
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment