Unverified Commit b7e32858 authored by Sam Poder's avatar Sam Poder Committed by GitHub

[turbopack] Generate component chunks for each merged group to increase cache hits (#95261)

This is a WIP PR to attempt to achieve this:

<img width="653" height="291" alt="Screenshot 2026-06-28 at 11 53 17 pm"
src="https://github.com/user-attachments/assets/68a78003-b76c-40e2-bbea-ca581f468eef"
/>

This does involve emitting extra chunks, though in theory, they are
already being emitted.

I'm reusing the `module_chunks` field on `OutputChunkRuntimeInfo`:

```rust
/// List of paths of chunks containing individual modules that are part of
/// this chunk. This is useful for selectively loading modules from a chunk
/// without loading the whole chunk.
pub module_chunks: Option<ResolvedVc<OutputAssets>>,
```

This does cause larger build sizes:

| App | Client chunk files (baseline → PR) | Difference | Client chunk
bytes (baseline → PR) | Difference |
| --- | ---: | ---: | ---: | ---: |
| vercel-site | 3651 → 5245 | +1594 (43.7%) | 344.5 MiB → 418.4 MiB |
+73.9 MiB (21.5%) |
| vercel-marketing | 2242 → 2634 | +392 (17.5%) | 163.5 MiB → 180.7 MiB
| +17.2 MiB (10.5%) |
| **Total** | **5893 → 7879** | **+1986 (33.7%)** | **508.0 MiB → 599.1
MiB** | **+91.1 MiB (17.9%)** |
parent 83e99f0a
......@@ -1637,6 +1637,7 @@ impl Project {
chunking_first_page_load_priority: chunking_heuristics.first_page_load_priority,
chunking_priority_boost_percent: chunking_heuristics.priority_boost_percent,
chunking_request_cost: chunking_heuristics.request_cost,
generate_component_chunks: self.next_config().turbopack_generate_component_chunks(),
}))
}
......
......@@ -493,6 +493,7 @@ pub struct ClientChunkingContextOptions {
pub chunking_first_page_load_priority: Option<u32>,
pub chunking_priority_boost_percent: Option<u32>,
pub chunking_request_cost: Option<u64>,
pub generate_component_chunks: Vc<bool>,
}
/// Next.js' chunk-load retry policy for the Turbopack browser runtime.
......@@ -535,6 +536,7 @@ pub async fn get_client_chunking_context(
chunking_first_page_load_priority,
chunking_priority_boost_percent,
chunking_request_cost,
generate_component_chunks,
} = options;
let next_mode = mode.await?;
......@@ -604,6 +606,10 @@ pub async fn get_client_chunking_context(
first_page_load_priority: chunking_first_page_load_priority,
priority_boost_percent: chunking_priority_boost_percent,
request_cost: chunking_request_cost,
// Generate component chunks alongside the merged chunk so that the browser
// runtime can fetch an already-cached one instead of the whole merged chunk.
generate_component_chunks: *generate_component_chunks.await?,
min_component_chunk_size: 20_000,
..Default::default()
},
)
......
......@@ -1371,6 +1371,7 @@ pub struct ExperimentalConfig {
turbopack_input_source_maps: Option<bool>,
turbopack_tree_shaking: Option<bool>,
turbopack_scope_hoisting: Option<bool>,
turbopack_generate_component_chunks: Option<bool>,
/// Custom URL prefix for Web Worker URLs (the entrypoint and the module
/// chunks loaded inside the worker) produced by
/// `new Worker(new URL(..., import.meta.url))`. Mirrors webpack's
......@@ -2517,6 +2518,15 @@ impl NextConfig {
}))
}
#[turbo_tasks::function]
pub fn turbopack_generate_component_chunks(&self) -> Vc<bool> {
Vc::cell(
self.experimental
.turbopack_generate_component_chunks
.unwrap_or(false),
)
}
#[turbo_tasks::function]
pub async fn turbo_nested_async_chunking(
&self,
......
......@@ -2,7 +2,7 @@ use anyhow::Result;
use either::Either;
use indoc::formatdoc;
use itertools::Itertools;
use rustc_hash::FxHashMap;
use rustc_hash::{FxHashMap, FxHashSet};
use serde::Serialize;
use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr};
......@@ -13,7 +13,10 @@ use turbo_tasks::{
use turbo_tasks_fs::{File, FileContent, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{ChunkingContext, CrossOrigin, ModuleChunkItemIdExt, ModuleId as TurbopackModuleId},
chunk::{
ChunkingContext, CrossOrigin, ModuleChunkItemIdExt, ModuleId as TurbopackModuleId,
OutputChunk,
},
module_graph::async_module_info::AsyncModulesInfo,
output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
};
......@@ -86,11 +89,25 @@ pub struct ManifestNodeEntry {
/// Export name.
pub name: RcStr,
/// Chunks for the module. JS and CSS.
pub chunks: Vec<RcStr>,
pub chunks: Vec<ClientChunk>,
// TODO(WEB-434)
pub r#async: bool,
}
/// One entry in a `ManifestNodeEntry.chunks` array, as consumed by React's Flight client via
/// `__turbopack_load_by_url__`.
///
/// Most chunks are a plain URL string. A *merged* chunk (one that bundles several component
/// chunks) is instead emitted as a `[url, componentChunkPaths, componentChunkSizes]` array. This
/// us to dynamically choose to load the whole chunk or individual components of it, as neeeded.
/// The sizes (bytes of the emitted files) feed the runtime's split-vs-whole cost heuristic.
#[derive(Serialize, Debug, Clone)]
#[serde(untagged)]
pub enum ClientChunk {
Path(RcStr),
Merged(RcStr, Vec<RcStr>, Vec<u64>),
}
#[turbo_tasks::value(shared)]
pub struct ClientReferenceManifest {
pub node_root: FileSystemPath,
......@@ -255,6 +272,8 @@ async fn build_manifest(
let mut ssr_chunk_path_cache: FxHashMap<ResolvedVc<Box<dyn OutputAsset>>, FileSystemPath> =
FxHashMap::default();
let mut client_reference_chunk_paths: FxHashSet<RcStr> = FxHashSet::default();
for (client_reference_module, client_reference_module_ref) in client_references_ecmascript {
let app_client_reference_ty =
ClientReferenceType::EcmascriptClientReference(client_reference_module);
......@@ -277,25 +296,43 @@ async fn build_manifest(
cached_chunk_paths(&mut client_chunk_path_cache, client_chunks.iter().copied())
.await?;
let chunk_paths = client_chunks_paths
.filter_map(|(_, chunk_path)| {
let js_chunks = client_chunks_paths
.filter_map(|(chunk, chunk_path)| {
client_relative_path
.get_path_to(&chunk_path)
.map(ToString::to_string)
.map(|path| (chunk, path.to_string()))
})
// It's possible that a chunk also emits CSS files, that will
// be handled separately.
.filter(|path| path.ends_with(".js"))
.map(|path| {
format!(
.filter(|(_, path)| path.ends_with(".js"))
.collect::<Vec<_>>();
for (_, path) in &js_chunks {
client_reference_chunk_paths.insert(RcStr::from(path.as_str()));
}
let chunk_paths = js_chunks
.into_iter()
.map(async |(chunk, path)| {
let url = RcStr::from(format!(
"{}{}{}",
prefix_path,
path.split('/').map(encode_uri_component).format("/"),
suffix_path
)
));
// If this is a merged chunk, emit its component chunk paths alongside the
// URL so the browser runtime can split it during navigation.
let components =
client_chunk_components(chunk, &client_relative_path).await?;
Ok(if components.is_empty() {
ClientChunk::Path(url)
} else {
let (paths, sizes) = components.into_iter().unzip();
ClientChunk::Merged(url, paths, sizes)
})
})
.map(RcStr::from)
.collect::<Vec<_>>();
.try_join()
.await?;
let is_async = async_modules.contains(&ResolvedVc::upcast(client_module));
......@@ -371,7 +408,10 @@ async fn build_manifest(
ManifestNodeEntry {
name: rcstr!("*"),
id: (&ssr_chunk_item_id).into(),
chunks: ssr_chunks_paths,
chunks: ssr_chunks_paths
.into_iter()
.map(ClientChunk::Path)
.collect(),
// See above
r#async: client_is_async || ssr_is_async,
},
......@@ -436,6 +476,11 @@ async fn build_manifest(
.await?;
// Inlining breaks HMR so it is always disabled in dev.
let inlined_css = *next_config.inline_css().await? && mode.is_production();
// Component chunks are also exposed via `clientModules[].chunks`, so when the feature
// is on we drop them from `entryJSFiles` to avoid double-listing. When it's off the
// manifest must match the non-component-chunk output exactly.
let generate_component_chunks =
*next_config.turbopack_generate_component_chunks().await?;
for (chunk, chunk_path) in client_chunks_with_path {
if let Some(path) = client_relative_path.get_path_to(&chunk_path) {
......@@ -461,7 +506,10 @@ async fn build_manifest(
inlined: inlined_css,
content,
});
} else {
} else if !mode.is_production()
|| !generate_component_chunks
|| !client_reference_chunk_paths.contains(&path)
{
entry_js_files.insert(path);
}
}
......@@ -522,6 +570,35 @@ impl From<&TurbopackModuleId> for ModuleId {
}
}
async fn client_chunk_components(
chunk: ResolvedVc<Box<dyn OutputAsset>>,
client_relative_path: &FileSystemPath,
) -> Result<Vec<(RcStr, u64)>> {
let Some(output_chunk) = ResolvedVc::try_sidecast::<Box<dyn OutputChunk>>(chunk) else {
return Ok(Vec::new());
};
let Some(component_chunks) = output_chunk.runtime_info().await?.module_chunks else {
return Ok(Vec::new());
};
let component_assets = component_chunks.await?;
let mut components = Vec::with_capacity(component_assets.len());
for component in component_assets.iter() {
let component_path = component.path().await?;
if let Some(rel) = client_relative_path.get_path_to(&component_path)
&& rel.ends_with(".js")
{
let size = component
.content()
.file_content()
.await?
.as_content()
.map_or(0, |file| file.content().len() as u64);
components.push((RcStr::from(rel), size));
}
}
Ok(components)
}
/// See next.js/packages/next/src/lib/client-reference.ts
pub fn get_client_reference_module_key(server_path: &str, export_name: &str) -> RcStr {
if export_name == "*" {
......
......@@ -386,6 +386,7 @@ export const experimentalSchema = {
turbopackRemoveUnusedImports: z.boolean().optional(),
turbopackRemoveUnusedExports: z.boolean().optional(),
turbopackScopeHoisting: z.boolean().optional(),
turbopackGenerateComponentChunks: z.boolean().optional(),
turbopackChunkingHeuristics: z
.object({
firstPageLoadPriority: z.number().min(0).max(1).optional(),
......
......@@ -776,6 +776,13 @@ export interface ExperimentalConfig {
*/
turbopackScopeHoisting?: boolean
/**
* (`next --turbopack` only) Emit each merged production chunk's constituent component chunks
* alongside it, so the browser runtime can load only the ones it doesn't already have.
* Defaults to `false`. Only applies in build mode.
*/
turbopackGenerateComponentChunks?: boolean
/**
* (`next --turbopack` only) Traffic-related hints for the production chunker. These change the
* assumptions Turbopack makes when making chunk merging decisions.
......
......@@ -577,6 +577,16 @@ impl BrowserChunkingContext {
pub fn chunk_load_retry(&self) -> Vc<ChunkLoadRetry> {
self.chunk_load_retry.cell()
}
/// Whether the ECMAScript chunking config emits component chunks alongside merged chunks.
#[turbo_tasks::function]
pub async fn generate_component_chunks(&self) -> Result<Vc<bool>> {
let ecmascript_ty: ResolvedVc<Box<dyn ChunkType>> =
ResolvedVc::upcast(Vc::<EcmascriptChunkType>::default().to_resolved().await?);
Ok(Vc::cell(self.chunking_configs.iter().any(
|(ty, config)| *ty == ecmascript_ty && config.generate_component_chunks,
)))
}
}
#[turbo_tasks::value_impl]
......@@ -1049,6 +1059,7 @@ impl ChunkingContext for BrowserChunkingContext {
batch_groups: Vec::new(),
}
.cell(),
Vec::new(),
)
.to_resolved()
.await?
......
use anyhow::Result;
use anyhow::{Context, Result};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexSet, ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{FileContent, FileSystemPath};
......@@ -51,6 +51,21 @@ impl EcmascriptBrowserChunk {
}
impl EcmascriptBrowserChunk {
async fn component_chunk_assets(&self) -> Result<Vec<ResolvedVc<Box<dyn OutputAsset>>>> {
let component_chunks = self.chunk.component_chunks().await?;
let mut assets = Vec::with_capacity(component_chunks.len());
for &component in component_chunks.iter() {
let component_chunk = ResolvedVc::try_downcast_type::<EcmascriptChunk>(component)
.context("merged chunk component_chunks must be ecmascript chunks")?;
assets.push(ResolvedVc::upcast(
EcmascriptBrowserChunk::new(*self.chunking_context, *component_chunk)
.to_resolved()
.await?,
));
}
Ok(assets)
}
async fn ident_for_path(&self) -> Result<Vc<AssetIdent>> {
Ok(self
.chunk
......@@ -66,8 +81,15 @@ impl EcmascriptBrowserChunk {
impl OutputChunk for EcmascriptBrowserChunk {
#[turbo_tasks::function]
async fn runtime_info(&self) -> Result<Vc<OutputChunkRuntimeInfo>> {
let component_assets = self.component_chunk_assets().await?;
let module_chunks = if component_assets.is_empty() {
None
} else {
Some(ResolvedVc::cell(component_assets))
};
Ok(OutputChunkRuntimeInfo {
included_ids: Some(self.chunk.entry_ids().to_resolved().await?),
module_chunks,
..Default::default()
}
.cell())
......@@ -113,9 +135,26 @@ impl OutputAssetsReference for EcmascriptBrowserChunk {
assets.push(ResolvedVc::upcast(self.source_map().to_resolved().await?));
}
// Constituent component chunks of a merged chunk are emitted as referenced assets
// so the runtime can fetch an individual component when it's already cached, without
// them being eagerly loaded as primary chunks.
let component_assets = this.component_chunk_assets().await?;
let referenced_assets = if component_assets.is_empty() {
chunk_references.referenced_assets
} else {
let mut referenced: Vec<_> = chunk_references
.referenced_assets
.await?
.iter()
.copied()
.collect();
referenced.extend(component_assets);
ResolvedVc::cell(referenced)
};
Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(assets),
referenced_assets: chunk_references.referenced_assets,
referenced_assets,
references: chunk_references.references,
}
.cell())
......
......@@ -195,6 +195,7 @@ impl EcmascriptBrowserEvaluateChunk {
this.chunking_context.chunk_load_retry(),
has_async_modules,
this.chunking_context.chunk_loading(),
*this.chunking_context.generate_component_chunks().await?,
);
code.push_code(&*runtime_code.await?);
}
......
......@@ -24,7 +24,14 @@ async fn handle_split_group<'l>(
Ok(match (chunk_size(chunk_items), remaining) {
(ChunkSize::Large, _) => false,
(ChunkSize::Perfect, _) | (ChunkSize::Small, None) => {
make_chunk(take(chunk_items), Vec::new(), key, split_context).await?;
make_chunk(
take(chunk_items),
Vec::new(),
Vec::new(),
key,
split_context,
)
.await?;
true
}
(ChunkSize::Small, Some(remaining)) => {
......@@ -109,6 +116,7 @@ pub async fn app_vendors_split(
make_chunk(
chunk_group_specific_chunk_items,
Vec::new(),
Vec::new(),
&mut name,
split_context,
)
......@@ -216,7 +224,7 @@ async fn folder_split<'l>(
continue;
} else {
let mut key = format!("{name}-{folder_name}");
make_chunk(list, Vec::new(), &mut key, split_context).await?;
make_chunk(list, Vec::new(), Vec::new(), &mut key, split_context).await?;
return Ok(());
}
} else {
......@@ -236,7 +244,7 @@ async fn folder_split<'l>(
))
.await?;
} else {
make_chunk(list, Vec::new(), &mut key, split_context).await?;
make_chunk(list, Vec::new(), Vec::new(), &mut key, split_context).await?;
}
}
}
......@@ -246,7 +254,7 @@ async fn folder_split<'l>(
};
let mut key = format!("{}-{}", name, &asset_ident[..location]);
if !handle_split_group(&mut remaining, &mut key, split_context, None).await? {
make_chunk(remaining, Vec::new(), &mut key, split_context).await?;
make_chunk(remaining, Vec::new(), Vec::new(), &mut key, split_context).await?;
}
}
Ok(())
......
......@@ -361,6 +361,7 @@ pub async fn make_chunks(
make_chunk(
chunk_items,
Vec::new(),
Vec::new(),
&mut format!("{key_prefix}{ty_name}"),
&mut split_context,
)
......@@ -401,30 +402,59 @@ struct SplitContext<'a> {
chunks: &'a mut Vec<Vc<Box<dyn Chunk>>>,
}
fn build_chunk(
chunk_items: Vec<&'_ ChunkItemOrBatchWithInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
component_chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
split_context: &SplitContext<'_>,
) -> Vc<Box<dyn Chunk>> {
split_context.ty.chunk(
*split_context.chunking_context,
chunk_items
.into_iter()
.map(|item| match item {
ChunkItemOrBatchWithInfo::ChunkItem { chunk_item, .. } => {
ChunkItemOrBatchWithAsyncModuleInfo::ChunkItem(*chunk_item)
}
&ChunkItemOrBatchWithInfo::Batch { batch, .. } => {
ChunkItemOrBatchWithAsyncModuleInfo::Batch(batch)
}
})
.collect(),
ResolvedVc::deref_vec(batch_groups),
ResolvedVc::deref_vec(component_chunks),
)
}
/// The chunk items and batch groups that make up one component chunk of a merged chunk.
type ComponentChunkItems<'l> = (
Vec<&'l ChunkItemOrBatchWithInfo>,
Vec<ResolvedVc<ChunkItemBatchGroup>>,
);
/// Creates a chunk with the given `chunk_items. `key` should be unique.
#[tracing::instrument(level = Level::TRACE, skip_all, fields(key = display(key)))]
async fn make_chunk(
chunk_items: Vec<&'_ ChunkItemOrBatchWithInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
components: Vec<ComponentChunkItems<'_>>,
key: &mut String,
split_context: &mut SplitContext<'_>,
) -> Result<()> {
split_context.chunks.push(
split_context.ty.chunk(
*split_context.chunking_context,
chunk_items
.into_iter()
.map(|item| match item {
ChunkItemOrBatchWithInfo::ChunkItem { chunk_item, .. } => {
ChunkItemOrBatchWithAsyncModuleInfo::ChunkItem(*chunk_item)
}
&ChunkItemOrBatchWithInfo::Batch { batch, .. } => {
ChunkItemOrBatchWithAsyncModuleInfo::Batch(batch)
}
})
.collect(),
ResolvedVc::deref_vec(batch_groups),
),
);
let mut component_chunks = Vec::with_capacity(components.len());
for (component_items, component_batch_groups) in components {
component_chunks.push(
build_chunk(
component_items,
component_batch_groups,
Vec::new(),
split_context,
)
.to_resolved()
.await?,
);
}
let chunk = build_chunk(chunk_items, batch_groups, component_chunks, split_context);
split_context.chunks.push(chunk);
Ok(())
}
......@@ -11,7 +11,7 @@ use crate::{
chunk::{
ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo, ChunkItemWithAsyncModuleInfo,
ChunkingConfig,
chunking::{ChunkItemOrBatchWithInfo, SplitContext, make_chunk},
chunking::{ChunkItemOrBatchWithInfo, ComponentChunkItems, SplitContext, make_chunk},
},
module_graph::{
ModuleGraph,
......@@ -131,6 +131,8 @@ pub async fn make_production_chunks(
first_page_load_priority,
priority_boost_percent,
request_cost,
generate_component_chunks,
min_component_chunk_size,
..
} = chunking_config;
......@@ -140,6 +142,7 @@ pub async fn make_production_chunks(
make_chunk(
group.chunk_items,
group.batch_group.into_iter().collect(),
Vec::new(),
&mut String::new(),
&mut split_context,
)
......@@ -162,6 +165,11 @@ pub async fn make_production_chunks(
.sum::<usize>();
ChunkCandidate {
size,
components: vec![ChunkComponent {
size,
chunk_items: chunk_items.clone(),
batch_groups: batch_group.into_iter().collect(),
}],
chunk_items,
batch_groups: batch_group.into_iter().collect(),
chunk_groups: key.map(Cow::Borrowed),
......@@ -199,6 +207,7 @@ pub async fn make_production_chunks(
chunk_items,
batch_groups,
chunk_groups,
components,
} = heap.pop().unwrap();
chunks_to_merge_size += size;
chunks_to_merge.push(MergeCandidate {
......@@ -206,6 +215,7 @@ pub async fn make_production_chunks(
chunk_items,
batch_groups,
chunk_groups,
components,
});
continue;
}
......@@ -532,7 +542,9 @@ pub async fn make_production_chunks(
chunk_items,
mut batch_groups,
chunk_groups,
components: other_components,
} = other;
candidate.components.extend(other_components);
candidate.size += size;
candidate.chunk_items.extend(chunk_items);
if batch_groups.len() + candidate.batch_groups.len() > 16 {
......@@ -569,6 +581,7 @@ pub async fn make_production_chunks(
chunk_items: unused.chunk_items,
batch_groups: unused.batch_groups,
chunk_groups: unused.chunk_groups,
components: unused.components,
});
} else {
chunks_to_merge.push(unused);
......@@ -589,6 +602,7 @@ pub async fn make_production_chunks(
chunk_items,
batch_groups,
chunk_groups,
components,
} in chunks_to_merge.into_iter()
{
if size > merge_threshold {
......@@ -597,6 +611,7 @@ pub async fn make_production_chunks(
chunk_items,
batch_groups,
chunk_groups,
components,
});
} else {
remained_size += size;
......@@ -613,6 +628,8 @@ pub async fn make_production_chunks(
chunk_items: remained_chunk_items,
batch_groups: remained_batch_groups.into_iter().collect(),
chunk_groups: None,
// The remained chunk holds unsharable left-overs; no split benefit.
components: Vec::new(),
});
}
}
......@@ -624,13 +641,22 @@ pub async fn make_production_chunks(
chunk_items,
batch_groups,
size,
components,
..
} in heap.into_iter()
{
total_size += size;
// Merged chunks also emit their constituent components as referenced "module
// chunks" for cache-aware loading; a plain chunk passes no
// components.
let components = generate_component_chunks
.then(|| split_into_component_chunks(components, min_component_chunk_size))
.flatten()
.unwrap_or_default();
make_chunk(
chunk_items,
batch_groups.into_vec(),
components,
&mut String::new(),
&mut split_context,
)
......@@ -645,11 +671,22 @@ pub async fn make_production_chunks(
.await
}
/// One original (pre-merge) atomic chunk group. A [`ChunkCandidate`]/[`MergeCandidate`] tracks the
/// components it was merged from so the emitted merged chunk can also expose them as individual
/// "module chunks" for cache-aware loading at runtime.
struct ChunkComponent<'l> {
size: usize,
chunk_items: Vec<&'l ChunkItemOrBatchWithInfo>,
batch_groups: SmallVec<[ResolvedVc<ChunkItemBatchGroup>; 1]>,
}
struct ChunkCandidate<'l> {
size: usize,
chunk_items: Vec<&'l ChunkItemOrBatchWithInfo>,
batch_groups: SmallVec<[ResolvedVc<ChunkItemBatchGroup>; 1]>,
chunk_groups: Option<Cow<'l, RoaringBitmapWrapper>>,
/// Original groups this candidate covers; one per chunk, > 1 once merged.
components: Vec<ChunkComponent<'l>>,
}
impl Ord for ChunkCandidate<'_> {
......@@ -677,6 +714,8 @@ struct MergeCandidate<'l> {
chunk_items: Vec<&'l ChunkItemOrBatchWithInfo>,
batch_groups: SmallVec<[ResolvedVc<ChunkItemBatchGroup>; 1]>,
chunk_groups: Option<Cow<'l, RoaringBitmapWrapper>>,
/// Original groups this candidate covers; one per chunk, > 1 once merged.
components: Vec<ChunkComponent<'l>>,
}
impl MergeCandidate<'_> {
......@@ -709,6 +748,45 @@ impl PartialEq for MergeCandidate<'_> {
}
}
fn split_into_component_chunks<'l>(
components: Vec<ChunkComponent<'l>>,
min_component_chunk_size: usize,
) -> Option<Vec<ComponentChunkItems<'l>>> {
// A single original part can't benefit from splitting.
if components.len() <= 1 {
return None;
}
let mut component_chunks: Vec<ComponentChunkItems<'l>> = Vec::new();
let mut remainder_items: Vec<&'l ChunkItemOrBatchWithInfo> = Vec::new();
let mut remainder_batch_groups = FxIndexSet::default();
for part in components {
if part.size >= min_component_chunk_size {
component_chunks.push((part.chunk_items, part.batch_groups.into_vec()));
} else {
// we create a "remainder" chunk with smaller component chunks so that
// an entire chunk can be loaded by loading all of its component chunks
remainder_items.extend(part.chunk_items);
remainder_batch_groups.extend(part.batch_groups);
}
}
// TODO (@sampoder): handle the case where there are many the component chunks
// that are only slightly smaller than the min_component_chunk_size. this may
// mean that there is no benefit to component chunking.
if !remainder_items.is_empty() {
component_chunks.push((
remainder_items,
remainder_batch_groups.into_iter().collect(),
));
}
// A split is only worthwhile if it yields more than one component chunk.
if component_chunks.len() <= 1 {
None
} else {
Some(component_chunks)
}
}
fn overlap(
chunk_groups: &Option<Cow<'_, RoaringBitmapWrapper>>,
chunk_groups2: &Option<Cow<'_, RoaringBitmapWrapper>>,
......
......@@ -53,6 +53,7 @@ pub async fn make_style_production_chunks(
make_chunk(
vec![&ChunkItemOrBatchWithInfo::Batch { batch, size: 0 }],
vec![],
vec![],
&mut String::new(),
&mut split_context,
)
......@@ -67,6 +68,7 @@ pub async fn make_style_production_chunks(
asset_ident: rcstr!(""),
}],
vec![],
vec![],
&mut String::new(),
&mut split_context,
)
......
......@@ -255,6 +255,16 @@ pub struct ChunkingConfig {
/// This makes sure that code in big chunks is not duplicated in multiple chunks.
pub max_merge_chunk_size: usize,
/// When enabled, a merged chunk also emits its constituent component chunks (referenced,
/// loaded on demand) so the runtime can fetch an individual component chunk instead of the
/// whole merged chunk when it is already cached.
pub generate_component_chunks: bool,
/// Minimum size for a component chunk to be emitted on its own when
/// `generate_component_chunks` is enabled. Component chunks smaller than this are folded
/// into a single remainder component chunk.
pub min_component_chunk_size: usize,
/// Selects the algorithm used to compute
/// [`crate::module_graph::style_groups::StyleGroups`]. Only consulted for the CSS chunk
/// type.
......
......@@ -538,6 +538,7 @@ pub trait ChunkType: ValueToString {
chunking_context: Vc<Box<dyn ChunkingContext>>,
chunk_items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
component_chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
) -> Vc<Box<dyn Chunk>>;
#[turbo_tasks::function]
......
......@@ -532,6 +532,7 @@ impl ChunkType for CssChunkType {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
chunk_items_or_batches: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
_batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
_component_chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
) -> Result<Vc<Box<dyn Chunk>>> {
let mut chunk_items = Vec::new();
// TODO operate with batches
......
......@@ -88,6 +88,9 @@ function registerChunk(registration: ChunkRegistration) {
const chunk = getChunkFromRegistration(registration[0]) as
| ChunkScript
| ChunkPath
if (SUPPORT_COMPONENT_CHUNKS) {
markChunkComponentsAvailable(chunk)
}
let runtimeParams: RuntimeParams | undefined
// When bootstrapping we are passed a single runtimeParams object so we can distinguish purely based on length
if (registration.length === 2) {
......
......@@ -545,6 +545,9 @@ function registerChunk(registration: ChunkRegistration) {
const chunk = getChunkFromRegistration(registration[0]) as
| ChunkPath
| ChunkScript
if (SUPPORT_COMPONENT_CHUNKS) {
markChunkComponentsAvailable(chunk)
}
let runtimeParams: RuntimeParams | undefined
// When bootstrapping we are passed a single runtimeParams object so we can distinguish purely based on length
if (registration.length === 2) {
......
......@@ -28,6 +28,7 @@ pub async fn get_browser_runtime_code(
chunk_load_retry: Vc<ChunkLoadRetry>,
has_async_modules: bool,
chunk_loading: Vc<ChunkLoading>,
support_component_chunks: bool,
) -> Result<Vc<Code>> {
let asset_context = *asset_context;
let environment = asset_context.compile_time_info().environment();
......@@ -114,11 +115,13 @@ pub async fn get_browser_runtime_code(
var CHUNK_BASE_PATH = {};
var RELATIVE_ROOT_PATH = {};
var RUNTIME_PUBLIC_PATH = {};
const SUPPORT_COMPONENT_CHUNKS = {};
"#,
StringifyJs(&chunk_loading_global),
StringifyJs(chunk_base_path),
StringifyJs(relative_root_path.as_str()),
StringifyJs(chunk_base_path),
support_component_chunks,
)?;
match &*asset_suffix {
......
......@@ -26,6 +26,7 @@ impl ChunkType for EcmascriptChunkType {
chunking_context: Vc<Box<dyn ChunkingContext>>,
chunk_items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
component_chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
) -> Result<Vc<Box<dyn Chunk>>> {
let content = EcmascriptChunkContent {
chunk_items: chunk_items
......@@ -43,7 +44,11 @@ impl ChunkType for EcmascriptChunkType {
.await?,
}
.cell();
Ok(Vc::upcast(EcmascriptChunk::new(chunking_context, content)))
Ok(Vc::upcast(EcmascriptChunk::new(
chunking_context,
content,
ResolvedVc::deref_vec(component_chunks),
)))
}
#[turbo_tasks::function]
......
......@@ -14,7 +14,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbo_tasks_fs::FileSystem;
use turbopack_core::{
chunk::{Chunk, ChunkItem, ChunkItems, ChunkingContext, ModuleIds},
chunk::{Chunk, ChunkItem, ChunkItems, ChunkingContext, Chunks, ModuleIds},
ident::AssetIdent,
introspect::{
Introspectable, IntrospectableChildren, module::IntrospectableModule,
......@@ -48,6 +48,7 @@ pub use self::{
pub struct EcmascriptChunk {
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
pub content: ResolvedVc<EcmascriptChunkContent>,
pub component_chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
}
#[turbo_tasks::value_impl]
......@@ -56,14 +57,21 @@ impl EcmascriptChunk {
pub fn new(
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
content: ResolvedVc<EcmascriptChunkContent>,
component_chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
) -> Vc<Self> {
EcmascriptChunk {
chunking_context,
content,
component_chunks,
}
.cell()
}
#[turbo_tasks::function]
pub fn component_chunks(&self) -> Vc<Chunks> {
Vc::cell(self.component_chunks.clone())
}
#[turbo_tasks::function]
pub fn entry_ids(self: Vc<Self>) -> Vc<ModuleIds> {
// TODO return something useful
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment