Unverified Commit 00705147 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: make available_modules an OperationVc and simplify merged module bookkeeping (#94055)

### What?

Reworks how `chunk_group_content` exposes `available_modules` for
downstream chunking, and consolidates the bookkeeping for merged modules
in `ChunkGroup`.

- Splits `ChunkGroupContent` into a `turbo_tasks::value`
`ChunkGroupContentInner` (carrying `chunkable_items`, `batch_groups`,
`async_modules`, `traced_modules`, and `available_modules`) plus a thin
wrapper that pairs it with the resulting `AvailabilityInfo`.
`chunk_group_content_operation` now runs entirely inside `turbo_tasks`,
allowing `AvailableModules` to store its set as an
`OperationVc<AvailableModulesSet>` and `.connect()` it on read.
- Replaces the separate `included` set and
`should_create_chunk_item_for` method on `MergedModuleInfo` with an
`Option<ChunkableModule>` value type on the `replacements` map. Call
sites now decide all three cases (replace / skip / keep) in one match
via `should_replace_module`.

### Why?

- `AvailableModules::snapshot` did not correctly derive its invalidation
priority, which caused double invalidations: the snapshot was treated as
a low-priority dependency even though it gated chunking work, so when
its inputs changed turbo-tasks would invalidate it after dependent
computations had already started, triggering them a second time.
Exposing `available_modules` as an `OperationVc` makes
`AvailableModules` inherit the correct invalidation priority from the
operation that produces it, which removes the double-invalidation.
- The previous `MergedModuleInfo` API had two parallel data structures
(`replacements` and `included`) and a helper that combined them. Folding
`included` into the value of `replacements` removes the duplication,
makes the three states (replace, skip-because-merged, keep) explicit,
and shrinks the call-site logic in `chunk_group.rs`.

### How?

**`available_modules` as an `OperationVc`:**

- New `ChunkGroupContentInner` is a `turbo_tasks::value` holding the
chunkable items, batch groups, async/traced modules, and an
`OperationVc<AvailableModulesSet>` for `available_modules`.
- `ChunkGroupContent` becomes a plain struct wrapping
`ResolvedVc<ChunkGroupContentInner>` together with the
`AvailabilityInfo` computed for the group.
- `chunk_group_content_operation` is an `OperationVc`-producing function
so its inner `available_modules` can be returned as an
`OperationVc<AvailableModulesSet>`. `AvailableModules` now stores that
operation Vc and `.connect()`s it whenever the set is read, so consumers
reach the set through the operation and inherit its priority.
- Browser and Node.js chunking contexts are updated to thread the new
shape through `make_chunk_group` and friends.

**`MergedModuleInfo` simplification:**

- `replacements` changes from `HashMap<ChunkableModule,
ChunkableModule>` to `HashMap<ChunkableModule,
Option<ChunkableModule>>`: `Some(m)` means "replace with `m`", `None`
means "skip — already included in another merged module", absent means
"keep as-is".
- The old `included` set and `should_create_chunk_item_for` helper are
removed; `should_replace_module` returns the three-way outcome and
consumers match on it once.
- Call sites in `chunk_group.rs` are restructured around the single
match so the logic is no longer split across two lookups.

### Notes

- No user-facing behavior change is intended — this is an internal
refactor in `turbopack-core` (`chunk_group_content`,
`available_modules`, `chunk_group`) and the browser/nodejs chunking
contexts.
- Existing chunking tests in `turbopack-core` exercise the new shape.

<!-- NEXT_JS_LLM_PR -->
parent 01c9cbfe
......@@ -763,13 +763,12 @@ impl ChunkingContext for BrowserChunkingContext {
self: ResolvedVc<Self>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> {
let span = tracing::info_span!("chunking", name = display(ident.to_string().await?));
async move {
let this = self.await?;
let entries = chunk_group.entries();
let input_availability_info = availability_info;
let MakeChunkGroupResult {
chunks,
......@@ -777,7 +776,7 @@ impl ChunkingContext for BrowserChunkingContext {
references,
availability_info,
} = make_chunk_group(
entries,
chunk_group,
module_graph,
ResolvedVc::upcast(self),
input_availability_info,
......@@ -834,7 +833,7 @@ impl ChunkingContext for BrowserChunkingContext {
self: ResolvedVc<Self>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
input_availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> {
let span = tracing::info_span!(
......@@ -844,14 +843,13 @@ impl ChunkingContext for BrowserChunkingContext {
);
async move {
let this = self.await?;
let entries = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
chunk_group.clone(),
module_graph,
ResolvedVc::upcast(self),
input_availability_info,
......@@ -903,7 +901,7 @@ impl ChunkingContext for BrowserChunkingContext {
}
assets.push(
self.generate_evaluate_chunk(ident, other_assets, entries, module_graph)
self.generate_evaluate_chunk(ident, other_assets, entries, *module_graph)
.to_resolved()
.await?,
);
......
......@@ -2,7 +2,7 @@ use anyhow::Result;
use bincode::{Decode, Encode};
use bitfield::bitfield;
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, Vc, trace::TraceRawVcs};
use turbo_tasks::{NonLocalValue, OperationVc, ResolvedVc, TaskInput, trace::TraceRawVcs};
use crate::chunk::available_modules::{AvailableModules, AvailableModulesSet};
......@@ -34,7 +34,7 @@ impl AvailabilityInfo {
self.available_modules
}
pub async fn with_modules(self, modules: Vc<AvailableModulesSet>) -> Result<Self> {
pub async fn with_modules(self, modules: OperationVc<AvailableModulesSet>) -> Result<Self> {
Ok(if let Some(available_modules) = self.available_modules {
Self {
flags: self.flags,
......
use anyhow::Result;
use bincode::{Decode, Encode};
use turbo_tasks::{
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, ValueToString, Vc,
trace::TraceRawVcs, turbofmt,
FxIndexSet, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt,
ValueToString, Vc, trace::TraceRawVcs, turbofmt,
};
use turbo_tasks_hash::Xxh3Hash64Hasher;
......@@ -61,13 +61,13 @@ pub struct AvailableModulesSet(
#[turbo_tasks::value]
pub struct AvailableModules {
parent: Option<ResolvedVc<AvailableModules>>,
modules: ResolvedVc<AvailableModulesSet>,
modules: OperationVc<AvailableModulesSet>,
}
#[turbo_tasks::value_impl]
impl AvailableModules {
#[turbo_tasks::function]
pub fn new(modules: ResolvedVc<AvailableModulesSet>) -> Vc<Self> {
pub fn new(modules: OperationVc<AvailableModulesSet>) -> Vc<Self> {
AvailableModules {
parent: None,
modules,
......@@ -78,7 +78,7 @@ impl AvailableModules {
#[turbo_tasks::function]
pub fn with_modules(
self: ResolvedVc<Self>,
modules: ResolvedVc<AvailableModulesSet>,
modules: OperationVc<AvailableModulesSet>,
) -> Result<Vc<Self>> {
Ok(AvailableModules {
parent: Some(self),
......@@ -97,6 +97,7 @@ impl AvailableModules {
}
let item_idents = self
.modules
.connect()
.await?
.iter()
.map(async |&module| module.ident_strings().await)
......@@ -118,7 +119,7 @@ impl AvailableModules {
#[turbo_tasks::function]
pub async fn get(&self, item: AvailableModuleItem) -> Result<Vc<bool>> {
if self.modules.await?.contains(&item) {
if self.modules.connect().await?.contains(&item) {
return Ok(Vc::cell(true));
};
if let Some(parent) = self.parent {
......@@ -129,7 +130,7 @@ impl AvailableModules {
#[turbo_tasks::function]
pub async fn snapshot(&self) -> Result<Vc<AvailableModulesSnapshot>> {
let modules = self.modules.await?;
let modules = self.modules.connect().await?;
let parent = if let Some(parent) = self.parent {
Some(parent.snapshot().await?)
} else {
......
......@@ -16,7 +16,7 @@ use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexSet, NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs,
};
use turbo_tasks_hash::DeterministicHash;
......@@ -36,7 +36,7 @@ pub use crate::chunk::{
};
use crate::{
asset::Asset,
chunk::availability_info::AvailabilityInfo,
chunk::{availability_info::AvailabilityInfo, available_modules::AvailableModulesSet},
ident::AssetIdent,
module::Module,
module_graph::{
......@@ -427,11 +427,19 @@ impl ChunkingType {
}
}
pub struct ChunkGroupContent {
#[turbo_tasks::value(cell = "new")]
pub struct ChunkGroupContentInner {
pub chunkable_items: Vec<ChunkableModuleOrBatch>,
pub batch_groups: Vec<ResolvedVc<ModuleBatchGroup>>,
#[bincode(with = "turbo_bincode::indexset")]
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
#[bincode(with = "turbo_bincode::indexset")]
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
pub available_modules: ResolvedVc<AvailableModulesSet>,
}
pub struct ChunkGroupContent {
pub inner: ReadRef<ChunkGroupContentInner>,
pub availability_info: AvailabilityInfo,
}
......
......@@ -161,7 +161,9 @@ impl ChunkGroupEntry {
}
}
#[derive(Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, Encode, Decode)]
#[derive(
Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, Encode, Decode, NonLocalValue,
)]
pub enum ChunkGroup {
/// The entry chunk group of the compilation, e.g. src/index.js for a SPA, or app/foo/page.js
/// for Next.js.
......
......@@ -20,7 +20,7 @@ use crate::{
#[turbo_tasks::value(transparent, cell = "keyed")]
#[allow(clippy::type_complexity)]
pub struct MergedModulesReplacements(
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn ChunkableModule>>>,
FxHashMap<ResolvedVc<Box<dyn Module>>, Option<ResolvedVc<Box<dyn ChunkableModule>>>>,
);
#[turbo_tasks::value(transparent, cell = "keyed")]
......@@ -29,27 +29,30 @@ pub struct MergedModulesOriginalModules(
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn Module>>>,
);
#[turbo_tasks::value(transparent, cell = "keyed")]
#[allow(clippy::type_complexity)]
pub struct MergedModulesIncluded(FxHashSet<ResolvedVc<Box<dyn Module>>>);
#[turbo_tasks::value]
pub struct MergedModuleInfo {
/// A map of modules to the merged module containing the module plus additional modules.
/// A map of modules describing how they participate in module merging:
/// - Not present: the module is not affected by merging and a regular chunk item should be
/// created for it.
/// - Present with `Some(replacement)`: the module should be replaced with the given merged
/// module when creating a chunk item.
/// - Present with `None`: the module is already included in some other merged module returned
/// by a `Some(replacement)` entry and no chunk item should be created for it.
pub replacements: ResolvedVc<MergedModulesReplacements>,
/// A map of replacement modules to their corresponding chunk group info (which is the same as
/// the chunk group info of the original module it replaced).
pub replacements_to_original: ResolvedVc<MergedModulesOriginalModules>,
/// A map of modules that are already contained as values in replacements.
pub included: ResolvedVc<MergedModulesIncluded>,
}
impl MergedModuleInfo {
/// Whether the given module should be replaced with a merged module.
/// Returns the merging decision for the given module:
/// - `None`: the module is not affected by merging, keep it as-is.
/// - `Some(None)`: the module is already included in another merged module, skip it.
/// - `Some(Some(replacement))`: the module should be replaced with `replacement`.
pub async fn should_replace_module(
&self,
module: ResolvedVc<Box<dyn Module>>,
) -> Result<Option<ResolvedVc<Box<dyn ChunkableModule>>>> {
) -> Result<Option<Option<ResolvedVc<Box<dyn ChunkableModule>>>>> {
Ok(self.replacements.get(&module).await?.as_deref().copied())
}
......@@ -66,15 +69,6 @@ impl MergedModuleInfo {
.as_deref()
.copied())
}
// Whether the given module should be skipped during chunking, as it is already included in a
// module returned by some `should_replace_module` call.
pub async fn should_create_chunk_item_for(
&self,
module: ResolvedVc<Box<dyn Module>>,
) -> Result<bool> {
Ok(!self.included.contains_key(&module).await?)
}
}
/// Determine which modules can be merged together:
......@@ -774,28 +768,32 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
#[allow(clippy::type_complexity)]
let mut replacements: FxHashMap<
ResolvedVc<Box<dyn Module>>,
ResolvedVc<Box<dyn ChunkableModule>>,
Option<ResolvedVc<Box<dyn ChunkableModule>>>,
> = Default::default();
#[allow(clippy::type_complexity)]
let mut replacements_to_original: FxHashMap<
ResolvedVc<Box<dyn Module>>,
ResolvedVc<Box<dyn Module>>,
> = Default::default();
let mut included: FxHashSet<ResolvedVc<Box<dyn Module>>> = FxHashSet::default();
let mut merged_groups = 0;
let mut included_modules = 0;
for (original, replacement, replacement_included) in result.into_iter().flatten() {
replacements.insert(original, replacement);
replacements.insert(original, Some(replacement));
replacements_to_original.insert(ResolvedVc::upcast(replacement), original);
included.extend(replacement_included);
merged_groups += 1;
included_modules += replacement_included.len();
for included in replacement_included {
replacements.insert(included, None);
}
}
span.record("merged_groups", replacements.len());
span.record("included_modules", included.len());
span.record("merged_groups", merged_groups);
span.record("included_modules", included_modules);
Ok(MergedModuleInfo {
replacements: ResolvedVc::cell(replacements),
replacements_to_original: ResolvedVc::cell(replacements_to_original),
included: ResolvedVc::cell(included),
}
.cell())
}
......
......@@ -528,19 +528,18 @@ impl ChunkingContext for NodeJsChunkingContext {
self: ResolvedVc<Self>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> {
let span = tracing::info_span!("chunking", name = display(ident.to_string().await?));
async move {
let modules = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
modules,
chunk_group,
module_graph,
ResolvedVc::upcast(self),
availability_info,
......@@ -572,7 +571,7 @@ impl ChunkingContext for NodeJsChunkingContext {
self: ResolvedVc<Self>,
path: FileSystemPath,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
......@@ -583,14 +582,13 @@ impl ChunkingContext for NodeJsChunkingContext {
chunking_type = "entry",
);
async move {
let entries = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
mut referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
chunk_group.clone(),
module_graph,
ResolvedVc::upcast(self),
availability_info,
......@@ -630,7 +628,7 @@ impl ChunkingContext for NodeJsChunkingContext {
*module,
Vc::cell(referenced_output_assets),
Vc::cell(references),
module_graph,
*module_graph,
*self,
)
.to_resolved()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment