Unverified Commit 26365e05 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

refactor(turbopack): chunk several chunk groups in one chunk_group call (#99686)

### What?

`ChunkingContext::chunk_group` takes a list of chunk groups and chunks
them together, as one chunk group, in a single chunking task. It
produces one `ChunkGroupResult` (one `make_chunks` run) for the whole
list.

`evaluated_chunk_group`, `entry_chunk_group` and the
`ChunkingContextExt` helpers still take a single group and wrap it in a
one-element list. All existing callers pass a one-element list, so
**this PR changes no output** (zero snapshot changes).

### Why?

Review feedback on #98734
([1](https://github.com/vercel/next.js/pull/98734#discussion_r4182977091),
[2](https://github.com/vercel/next.js/pull/98734#discussion_r4182978846)):
a layout segment's client references can derive several isolated merged
chunk groups, and #98734 chunked each one with a separate `chunk_group`
call. That costs an extra chunking task per group and threads
availability by hand. This PR adds the list form; #98734 then calls
`chunk_group` once per segment and side with all of the segment's
derived groups.

### How?

`make_chunk_group` / `chunk_group_content` /
`chunk_group_content_operation` take the list as a
`SmallVec<[ChunkGroup; 1]>`: almost every call passes a single group, so
the common case doesn't heap-allocate. The list crosses turbo-task
boundaries, so this PR also adds `impl TaskInput for SmallVec<A>` in
turbo-tasks, mirroring the `Vec` impl. Semantics:

- **Traversal:** the entries of each group, in list order, with one
shared traversal state. A module already reached from an earlier group
isn't collected again, which matches what the previous
one-call-per-group sequence produced (CSS order unchanged).
- **Availability:** every listed group is added to the resulting
availability info.
- **Entry groups:** if several `Entry` groups are listed, the active
page entries and the resulting entry group are the union of their
modules, in list order.
- **Edge cases:** an empty list is an error; groups listed more than
once (same key) are chunked once.

Unit tests cover the edge cases, the `Entry` union, and `SmallVec` as a
task input (inline and spilled).

This is PR 2 of an eight-PR stack (on top of #99100, already merged):

1. #98733 — canonical chunk-group identity
2. **This PR** — chunk several chunk groups in one `chunk_group` call
3. #99749 — register the chunk groups that are chunked
4. #98734 — use graph-derived merged groups
5. #99750 — chunking takes a `ChunkGroupKey`
6. #98823 — Pages Router dev client graph chaining
7. #98685 — chunk-group availability bitmaps
8. #99050 — keyed chunk-group index lookups

<!-- NEXT_JS_LLM -->


<!-- fleet 4f67c6f7-bbe2-454b-9a17-822a60ef8f7e -->
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 62bf969d
...@@ -4155,6 +4155,7 @@ dependencies = [ ...@@ -4155,6 +4155,7 @@ dependencies = [
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"serde_json", "serde_json",
"smallvec",
"swc_core", "swc_core",
"tempfile", "tempfile",
"tokio", "tokio",
...@@ -9523,6 +9524,7 @@ dependencies = [ ...@@ -9523,6 +9524,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_qs", "serde_qs",
"smallvec",
"tracing", "tracing",
"turbo-bincode", "turbo-bincode",
"turbo-rcstr", "turbo-rcstr",
...@@ -9973,6 +9975,7 @@ dependencies = [ ...@@ -9973,6 +9975,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_qs", "serde_qs",
"smallvec",
"tracing", "tracing",
"turbo-bincode", "turbo-bincode",
"turbo-rcstr", "turbo-rcstr",
......
...@@ -32,6 +32,7 @@ roaring = { workspace = true } ...@@ -32,6 +32,7 @@ roaring = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
smallvec = { workspace = true }
swc_core = { workspace = true } swc_core = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
......
...@@ -36,6 +36,7 @@ use next_core::{ ...@@ -36,6 +36,7 @@ use next_core::{
segment_config::{NextSegmentConfig, ParseSegmentMode}, segment_config::{NextSegmentConfig, ParseSegmentMode},
util::{NextRuntime, app_function_name, module_styles_rule_condition, styles_rule_condition}, util::{NextRuntime, app_function_name, module_styles_rule_condition, styles_rule_condition},
}; };
use smallvec::smallvec;
use tracing::{Instrument, field::Empty}; use tracing::{Instrument, field::Empty};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
...@@ -1932,7 +1933,9 @@ impl AppEndpoint { ...@@ -1932,7 +1933,9 @@ impl AppEndpoint {
NextRuntime::Edge => { NextRuntime::Edge => {
let chunk_group1 = chunking_context.chunk_group( let chunk_group1 = chunking_context.chunk_group(
server_action_manifest_loader.ident(), server_action_manifest_loader.ident(),
ChunkGroup::Shared(ResolvedVc::upcast(server_action_manifest_loader)), smallvec![ChunkGroup::Shared(ResolvedVc::upcast(
server_action_manifest_loader,
))],
module_graph, module_graph,
AvailabilityInfo::root(), AvailabilityInfo::root(),
); );
...@@ -1992,11 +1995,11 @@ impl AppEndpoint { ...@@ -1992,11 +1995,11 @@ impl AppEndpoint {
) )
.with_modifier(rcstr!("server-utils")) .with_modifier(rcstr!("server-utils"))
.into_vc(), .into_vc(),
ChunkGroup::SharedMerged { smallvec![ChunkGroup::SharedMerged {
merge_tag: NEXT_SERVER_UTILITY_MERGE_TAG.clone(), merge_tag: NEXT_SERVER_UTILITY_MERGE_TAG.clone(),
entries: server_utils, entries: server_utils,
parent: parent_chunk_group, parent: parent_chunk_group,
}, }],
module_graph, module_graph,
AvailabilityInfo::root(), AvailabilityInfo::root(),
) )
...@@ -2028,7 +2031,7 @@ impl AppEndpoint { ...@@ -2028,7 +2031,7 @@ impl AppEndpoint {
async { async {
let chunk_group = chunking_context.chunk_group( let chunk_group = chunking_context.chunk_group(
server_component.ident(), server_component.ident(),
ChunkGroup::Shared(ResolvedVc::upcast(server_component)), smallvec![ChunkGroup::Shared(ResolvedVc::upcast(server_component))],
module_graph, module_graph,
current_chunk_group.await?.availability_info, current_chunk_group.await?.availability_info,
); );
...@@ -2047,7 +2050,9 @@ impl AppEndpoint { ...@@ -2047,7 +2050,9 @@ impl AppEndpoint {
{ {
let chunk_group = chunking_context.chunk_group( let chunk_group = chunking_context.chunk_group(
server_action_manifest_loader.ident(), server_action_manifest_loader.ident(),
ChunkGroup::Shared(ResolvedVc::upcast(server_action_manifest_loader)), smallvec![ChunkGroup::Shared(ResolvedVc::upcast(
server_action_manifest_loader,
))],
module_graph, module_graph,
current_chunk_group.await?.availability_info, current_chunk_group.await?.availability_info,
); );
......
...@@ -29,6 +29,7 @@ use next_core::{ ...@@ -29,6 +29,7 @@ use next_core::{
segment_config::ParseSegmentMode, segment_config::ParseSegmentMode,
util::{NextRuntime, get_asset_prefix_from_pathname, pages_function_name}, util::{NextRuntime, get_asset_prefix_from_pathname, pages_function_name},
}; };
use smallvec::smallvec;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Completion, FxIndexMap, ResolvedVc, ValueToString, Vc, fxindexmap, fxindexset}; use turbo_tasks::{Completion, FxIndexMap, ResolvedVc, ValueToString, Vc, fxindexmap, fxindexset};
...@@ -1027,7 +1028,7 @@ impl PageEndpoint { ...@@ -1027,7 +1028,7 @@ impl PageEndpoint {
async { async {
let chunk_group = chunking_context.chunk_group( let chunk_group = chunking_context.chunk_group(
layout.ident(), layout.ident(),
ChunkGroup::Shared(layout), smallvec![ChunkGroup::Shared(layout)],
ssr_module_graph, ssr_module_graph,
current_chunk_group.await?.availability_info, current_chunk_group.await?.availability_info,
); );
......
use anyhow::Result; use anyhow::Result;
use smallvec::smallvec;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::{ use turbo_tasks::{
...@@ -229,11 +230,11 @@ pub async fn get_app_client_references_chunks( ...@@ -229,11 +230,11 @@ pub async fn get_app_client_references_chunks(
.clone() .clone()
.with_modifier(rcstr!("ssr modules")) .with_modifier(rcstr!("ssr modules"))
.into_vc(), .into_vc(),
ChunkGroup::IsolatedMerged { smallvec![ChunkGroup::IsolatedMerged {
parent: parent_chunk_group, parent: parent_chunk_group,
merge_tag: ecmascript_client_reference_merge_tag_ssr(), merge_tag: ecmascript_client_reference_merge_tag_ssr(),
entries: ssr_modules, entries: ssr_modules,
}, }],
module_graph, module_graph,
availability_info, availability_info,
), ),
...@@ -268,11 +269,11 @@ pub async fn get_app_client_references_chunks( ...@@ -268,11 +269,11 @@ pub async fn get_app_client_references_chunks(
Some(client_chunking_context.chunk_group( Some(client_chunking_context.chunk_group(
base_ident.with_modifier(rcstr!("client modules")).into_vc(), base_ident.with_modifier(rcstr!("client modules")).into_vc(),
ChunkGroup::IsolatedMerged { smallvec![ChunkGroup::IsolatedMerged {
parent: parent_chunk_group, parent: parent_chunk_group,
merge_tag: ecmascript_client_reference_merge_tag(), merge_tag: ecmascript_client_reference_merge_tag(),
entries: client_modules, entries: client_modules,
}, }],
module_graph, module_graph,
availability_info, availability_info,
)) ))
......
...@@ -18,6 +18,7 @@ use bincode::{ ...@@ -18,6 +18,7 @@ use bincode::{
error::{DecodeError, EncodeError}, error::{DecodeError, EncodeError},
}; };
use either::Either; use either::Either;
use smallvec::{Array, SmallVec};
use turbo_frozenmap::{FrozenMap, FrozenSet}; use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks_hash::HashAlgorithm; use turbo_tasks_hash::HashAlgorithm;
...@@ -195,6 +196,29 @@ where ...@@ -195,6 +196,29 @@ where
} }
} }
impl<A> TaskInput for SmallVec<A>
where
A: Array + Send + Sync,
A::Item: TaskInput,
SmallVec<A>: Encode + Decode<()>,
{
fn is_resolved(&self) -> bool {
self.iter().all(TaskInput::is_resolved)
}
fn is_transient(&self) -> bool {
self.iter().any(TaskInput::is_transient)
}
async fn resolve_input(&self) -> Result<Self> {
let mut resolved = SmallVec::with_capacity(self.len());
for value in self {
resolved.push(value.resolve_input().await?);
}
Ok(resolved)
}
}
impl<T> TaskInput for Box<T> impl<T> TaskInput for Box<T>
where where
T: TaskInput, T: TaskInput,
...@@ -584,6 +608,23 @@ mod tests { ...@@ -584,6 +608,23 @@ mod tests {
Ok(()) Ok(())
} }
#[tokio::test]
async fn test_small_vec() -> Result<()> {
let inline: SmallVec<[RcStr; 1]> = smallvec::smallvec![rcstr!("a")];
assert_task_input(inline.clone());
assert!(inline.is_resolved());
assert!(!inline.is_transient());
let resolved = inline.resolve_input().await?;
assert_eq!(resolved, inline);
assert!(!resolved.spilled());
let spilled: SmallVec<[u32; 1]> = smallvec::smallvec![1, 2, 3];
assert!(spilled.is_resolved());
assert!(!spilled.is_transient());
assert_eq!(spilled.resolve_input().await?, spilled);
Ok(())
}
#[test] #[test]
fn test_one_unnamed_field() -> Result<()> { fn test_one_unnamed_field() -> Result<()> {
#[turbo_tasks::task_input] #[turbo_tasks::task_input]
......
...@@ -27,6 +27,7 @@ indoc = { workspace = true } ...@@ -27,6 +27,7 @@ indoc = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
serde_qs = { workspace = true } serde_qs = { workspace = true }
smallvec = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
......
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use async_trait::async_trait; use async_trait::async_trait;
use smallvec::{SmallVec, smallvec};
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
...@@ -877,7 +878,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -877,7 +878,7 @@ impl ChunkingContext for BrowserChunkingContext {
async fn chunk_group( async fn chunk_group(
self: ResolvedVc<Self>, self: ResolvedVc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
chunk_group: ChunkGroup, chunk_groups: SmallVec<[ChunkGroup; 1]>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
availability_info: AvailabilityInfo, availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> { ) -> Result<Vc<ChunkGroupResult>> {
...@@ -889,7 +890,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -889,7 +890,7 @@ impl ChunkingContext for BrowserChunkingContext {
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
chunk_group, chunk_groups,
module_graph, module_graph,
ResolvedVc::upcast(self), ResolvedVc::upcast(self),
input_availability_info, input_availability_info,
...@@ -940,7 +941,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -940,7 +941,7 @@ impl ChunkingContext for BrowserChunkingContext {
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
chunk_group.clone(), smallvec![chunk_group.clone()],
module_graph, module_graph,
ResolvedVc::upcast(self), ResolvedVc::upcast(self),
input_availability_info, input_availability_info,
...@@ -1130,7 +1131,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -1130,7 +1131,7 @@ impl ChunkingContext for BrowserChunkingContext {
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
chunk_group.clone(), smallvec![chunk_group.clone()],
module_graph, module_graph,
ResolvedVc::upcast(self), ResolvedVc::upcast(self),
availability_info, availability_info,
......
...@@ -3,6 +3,7 @@ use std::sync::atomic::AtomicBool; ...@@ -3,6 +3,7 @@ use std::sync::atomic::AtomicBool;
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use smallvec::SmallVec;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::{ use turbo_tasks::{
...@@ -40,9 +41,11 @@ pub struct MakeChunkGroupResult { ...@@ -40,9 +41,11 @@ pub struct MakeChunkGroupResult {
pub availability_info: AvailabilityInfo, pub availability_info: AvailabilityInfo,
} }
/// Creates a chunk group from a set of entries. /// Creates one chunk group from `chunk_groups`, chunking all of them together.
///
/// See [`chunk_group_content`] for how several groups are combined.
pub async fn make_chunk_group( pub async fn make_chunk_group(
chunk_group: ChunkGroup, chunk_groups: SmallVec<[ChunkGroup; 1]>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
availability_info: AvailabilityInfo, availability_info: AvailabilityInfo,
...@@ -59,7 +62,7 @@ pub async fn make_chunk_group( ...@@ -59,7 +62,7 @@ pub async fn make_chunk_group(
availability_info: new_availability_info, availability_info: new_availability_info,
} = chunk_group_content( } = chunk_group_content(
module_graph, module_graph,
chunk_group, chunk_groups,
ChunkGroupContentOptions { ChunkGroupContentOptions {
availability_info, availability_info,
can_split_async, can_split_async,
...@@ -210,22 +213,25 @@ pub struct ChunkGroupContentOptions { ...@@ -210,22 +213,25 @@ pub struct ChunkGroupContentOptions {
pub batching_config: ResolvedVc<BatchingConfig>, pub batching_config: ResolvedVc<BatchingConfig>,
} }
/// Computes the content of a chunk group. /// Computes the content of one chunk group made of all `chunk_groups`.
///
/// The groups are chunked together, in a single task: the traversal starts from the entries of each
/// group in list order and shares its state, so a module reached from an earlier group is not
/// collected again for a later one. The resulting availability includes all of them. The modules
/// of all [`ChunkGroup::Entry`] groups together form the entry group. Groups listed more than once
/// are chunked once; an empty list is an error.
pub async fn chunk_group_content( pub async fn chunk_group_content(
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
chunk_group: ChunkGroup, chunk_groups: SmallVec<[ChunkGroup; 1]>,
options: ChunkGroupContentOptions, options: ChunkGroupContentOptions,
) -> Result<ChunkGroupContent> { ) -> Result<ChunkGroupContent> {
let availability_info = options.availability_info; let availability_info = options.availability_info;
let chunk_groups = dedup_chunk_groups(chunk_groups)?;
let entry_group: Option<ResolvedVc<Modules>> = let entry_group: Option<ResolvedVc<Modules>> =
if let ChunkGroup::Entry(entry_modules) = &chunk_group { entry_group_modules(&chunk_groups).map(ResolvedVc::cell);
Some(ResolvedVc::cell(entry_modules.clone()))
} else {
None
};
let chunk_group_content = chunk_group_content_operation(module_graph, chunk_group, options); let chunk_group_content = chunk_group_content_operation(module_graph, chunk_groups, options);
let available_modules = available_modules_operation(chunk_group_content); let available_modules = available_modules_operation(chunk_group_content);
let availability_info = availability_info.with_modules(available_modules).await?; let availability_info = availability_info.with_modules(available_modules).await?;
...@@ -242,6 +248,35 @@ pub async fn chunk_group_content( ...@@ -242,6 +248,35 @@ pub async fn chunk_group_content(
}) })
} }
/// Rejects an empty list and drops groups that are listed more than once (by key), keeping list
/// order.
fn dedup_chunk_groups(
chunk_groups: SmallVec<[ChunkGroup; 1]>,
) -> Result<SmallVec<[ChunkGroup; 1]>> {
if chunk_groups.is_empty() {
bail!("Cannot chunk an empty list of chunk groups");
}
let mut keys = FxHashSet::default();
Ok(chunk_groups
.into_iter()
.filter(|chunk_group| keys.insert(chunk_group.key()))
.collect())
}
/// The modules of all [`ChunkGroup::Entry`] groups in `chunk_groups`, in list order, or `None` if
/// there are none.
fn entry_group_modules(chunk_groups: &[ChunkGroup]) -> Option<Vec<ResolvedVc<Box<dyn Module>>>> {
let mut modules = FxIndexSet::default();
let mut has_entry_group = false;
for chunk_group in chunk_groups {
if let ChunkGroup::Entry(entries) = chunk_group {
has_entry_group = true;
modules.extend(entries.iter().copied());
}
}
has_entry_group.then(|| modules.into_iter().collect())
}
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn available_modules_operation( async fn available_modules_operation(
chunk_group_content: OperationVc<ChunkGroupContentInner>, chunk_group_content: OperationVc<ChunkGroupContentInner>,
...@@ -252,7 +287,7 @@ async fn available_modules_operation( ...@@ -252,7 +287,7 @@ async fn available_modules_operation(
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn chunk_group_content_operation( async fn chunk_group_content_operation(
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
chunk_group: ChunkGroup, chunk_groups: SmallVec<[ChunkGroup; 1]>,
ChunkGroupContentOptions { ChunkGroupContentOptions {
availability_info, availability_info,
can_split_async, can_split_async,
...@@ -283,15 +318,20 @@ async fn chunk_group_content_operation( ...@@ -283,15 +318,20 @@ async fn chunk_group_content_operation(
None => None, None => None,
}; };
let entries = chunk_group // The entries of all groups, in list order. An entry shared by several groups is traversed
.entries() // once.
let entries = chunk_groups
.iter()
.flat_map(|chunk_group| chunk_group.entries())
.collect::<FxIndexSet<_>>()
.into_iter()
.map(|entry| module_batches_graph.get_entry_index(entry)) .map(|entry| module_batches_graph.get_entry_index(entry))
.try_join() .try_join()
.await?; .await?;
let active_page_entries: Option<FxHashSet<ResolvedVc<Box<dyn Module>>>> = let active_page_entries: Option<FxHashSet<ResolvedVc<Box<dyn Module>>>> =
if let ChunkGroup::Entry(entry_modules) = &chunk_group { if let Some(entry_modules) = entry_group_modules(&chunk_groups) {
Some(entry_modules.iter().copied().collect()) Some(entry_modules.into_iter().collect())
} else if let Some(entry_group) = availability_info.entry_group() { } else if let Some(entry_group) = availability_info.entry_group() {
Some(entry_group.await?.iter().copied().collect()) Some(entry_group.await?.iter().copied().collect())
} else { } else {
...@@ -603,3 +643,82 @@ async fn map_module_batch_group( ...@@ -603,3 +643,82 @@ async fn map_module_batch_group(
Ok(group) Ok(group)
} }
} }
#[cfg(test)]
mod tests {
use anyhow::Result;
use smallvec::smallvec;
use turbo_rcstr::rcstr;
use turbo_tasks::{Completion, ResolvedVc, Vc};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{File, FileContent, FileSystem, VirtualFileSystem};
use super::{dedup_chunk_groups, entry_group_modules};
use crate::{
asset::AssetContent, module::Module, module_graph::chunk_group_info::ChunkGroup,
raw_module::RawModule, source::Source, virtual_source::VirtualSource,
};
async fn test_module(name: &str) -> Result<ResolvedVc<Box<dyn Module>>> {
let root = VirtualFileSystem::new_with_name(rcstr!("chunk-group-test"))
.root()
.await?;
let source = Vc::upcast::<Box<dyn Source>>(VirtualSource::new(
root.join(name)?,
AssetContent::file(FileContent::Content(File::from("")).cell()),
));
Vc::upcast::<Box<dyn Module>>(RawModule::new(source))
.to_resolved()
.await
}
#[turbo_tasks::function(operation, root)]
async fn combines_chunk_group_lists_operation() -> Result<Vc<Completion>> {
let error = dedup_chunk_groups(smallvec![]).unwrap_err().to_string();
assert_eq!(error, "Cannot chunk an empty list of chunk groups");
let a = test_module("a.js").await?;
let b = test_module("b.js").await?;
let c = test_module("c.js").await?;
let shared_a = ChunkGroup::Shared(a);
let shared_b = ChunkGroup::Shared(b);
let groups = dedup_chunk_groups(smallvec![
shared_a.clone(),
shared_b.clone(),
shared_a.clone()
])?;
assert_eq!(
groups.iter().map(ChunkGroup::key).collect::<Vec<_>>(),
vec![shared_a.key(), shared_b.key()]
);
assert_eq!(entry_group_modules(&[ChunkGroup::Shared(a)]), None);
assert_eq!(
entry_group_modules(&[
ChunkGroup::Entry(vec![a, b]),
ChunkGroup::Shared(c),
ChunkGroup::Entry(vec![b, c]),
]),
Some(vec![a, b, c])
);
Ok(Completion::new())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn combines_chunk_group_lists() {
let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
BackendOptions::default(),
noop_backing_storage(),
));
tt.run_once(async {
combines_chunk_group_lists_operation()
.read_strongly_consistent()
.await?;
Ok(())
})
.await
.unwrap();
}
}
...@@ -2,6 +2,7 @@ use anyhow::Result; ...@@ -2,6 +2,7 @@ use anyhow::Result;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use smallvec::{SmallVec, smallvec};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Upcast, Vc, turbobail}; use turbo_tasks::{ResolvedVc, Upcast, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
...@@ -473,17 +474,21 @@ pub trait ChunkingContext { ...@@ -473,17 +474,21 @@ pub trait ChunkingContext {
chunk_item: ResolvedVc<Box<dyn ChunkItem>>, chunk_item: ResolvedVc<Box<dyn ChunkItem>>,
) -> Vc<Box<dyn OutputAsset>>; ) -> Vc<Box<dyn OutputAsset>>;
/// Chunks `chunk_groups` together as one chunk group, in a single task.
///
/// The modules of all groups are collected once each and chunked together, and the resulting
/// availability includes every group. `chunk_groups` must not be empty.
#[turbo_tasks::function] #[turbo_tasks::function]
fn chunk_group( fn chunk_group(
self: Vc<Self>, self: Vc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
chunk_group: ChunkGroup, chunk_groups: SmallVec<[ChunkGroup; 1]>,
module_graph: Vc<ModuleGraph>, module_graph: Vc<ModuleGraph>,
availability_info: AvailabilityInfo, availability_info: AvailabilityInfo,
) -> Vc<ChunkGroupResult>; ) -> Vc<ChunkGroupResult>;
/// Like [`Self::chunk_group`], but additionally produces an evaluate chunk /// Like [`Self::chunk_group`] for a single `chunk_group`, but additionally produces an
/// (and, in dev, a chunk-list register chunk) that bootstraps and runs /// evaluate chunk (and, in dev, a chunk-list register chunk) that bootstraps and runs
/// `chunk_group`'s entries. /// `chunk_group`'s entries.
/// ///
/// `extra_chunks` are not part of this chunk group's module graph, but they /// `extra_chunks` are not part of this chunk group's module graph, but they
...@@ -648,7 +653,12 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte ...@@ -648,7 +653,12 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
chunk_group: ChunkGroup, chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>, module_graph: Vc<ModuleGraph>,
) -> Vc<ChunkGroupResult> { ) -> Vc<ChunkGroupResult> {
self.chunk_group(ident, chunk_group, module_graph, AvailabilityInfo::root()) self.chunk_group(
ident,
smallvec![chunk_group],
module_graph,
AvailabilityInfo::root(),
)
} }
fn root_chunk_group_assets( fn root_chunk_group_assets(
...@@ -866,6 +876,11 @@ fn chunk_group_assets( ...@@ -866,6 +876,11 @@ fn chunk_group_assets(
availability_info: AvailabilityInfo, availability_info: AvailabilityInfo,
) -> Vc<OutputAssetsWithReferenced> { ) -> Vc<OutputAssetsWithReferenced> {
chunking_context chunking_context
.chunk_group(ident, chunk_group, module_graph, availability_info) .chunk_group(
ident,
smallvec![chunk_group],
module_graph,
availability_info,
)
.output_assets_with_referenced() .output_assets_with_referenced()
} }
...@@ -26,6 +26,7 @@ indoc = { workspace = true } ...@@ -26,6 +26,7 @@ indoc = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
serde_qs = { workspace = true } serde_qs = { workspace = true }
smallvec = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
......
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use smallvec::{SmallVec, smallvec};
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
...@@ -565,7 +566,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -565,7 +566,7 @@ impl ChunkingContext for NodeJsChunkingContext {
async fn chunk_group( async fn chunk_group(
self: ResolvedVc<Self>, self: ResolvedVc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
chunk_group: ChunkGroup, chunk_groups: SmallVec<[ChunkGroup; 1]>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
availability_info: AvailabilityInfo, availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> { ) -> Result<Vc<ChunkGroupResult>> {
...@@ -576,7 +577,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -576,7 +577,7 @@ impl ChunkingContext for NodeJsChunkingContext {
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
chunk_group, chunk_groups,
module_graph, module_graph,
ResolvedVc::upcast(self), ResolvedVc::upcast(self),
availability_info, availability_info,
...@@ -625,7 +626,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -625,7 +626,7 @@ impl ChunkingContext for NodeJsChunkingContext {
references, references,
availability_info, availability_info,
} = make_chunk_group( } = make_chunk_group(
chunk_group.clone(), smallvec![chunk_group.clone()],
module_graph, module_graph,
ResolvedVc::upcast(self), ResolvedVc::upcast(self),
availability_info, availability_info,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment