Commit a27b2581 authored by Tobias Koppers's avatar Tobias Koppers

Refactor AvailableModules to a flat AvailableModulesSet

Replace the linked-list `AvailableModules` (with `parent` reference) and
its `AvailableModulesSnapshot` with a direct flat `AvailableModulesSet`.
`AvailabilityInfo` now holds `ResolvedVc<AvailableModulesSet>` directly.
`with_modules` eagerly merges sets instead of building a chain, and
`hash` and `contains` operate on the flat set. The `snapshot` method
and turbo-task round-trips through the linked-list are eliminated.
Co-Authored-By: 's avatarClaude <noreply@anthropic.com>
parent f8de120d
...@@ -4,7 +4,7 @@ use bitfield::bitfield; ...@@ -4,7 +4,7 @@ use bitfield::bitfield;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, Vc, trace::TraceRawVcs}; use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, Vc, trace::TraceRawVcs};
use crate::chunk::available_modules::{AvailableModules, AvailableModulesSet}; use crate::chunk::available_modules::AvailableModulesSet;
bitfield! { bitfield! {
#[derive(Clone, Copy, Default, TaskInput, TraceRawVcs, NonLocalValue, PartialEq, Eq, Hash, Encode, Decode)] #[derive(Clone, Copy, Default, TaskInput, TraceRawVcs, NonLocalValue, PartialEq, Eq, Hash, Encode, Decode)]
...@@ -19,7 +19,7 @@ bitfield! { ...@@ -19,7 +19,7 @@ bitfield! {
pub struct AvailabilityInfo { pub struct AvailabilityInfo {
flags: AvailabilityFlags, flags: AvailabilityFlags,
/// There are modules already available. /// There are modules already available.
available_modules: Option<ResolvedVc<AvailableModules>>, available_modules: Option<ResolvedVc<AvailableModulesSet>>,
} }
impl AvailabilityInfo { impl AvailabilityInfo {
...@@ -30,7 +30,7 @@ impl AvailabilityInfo { ...@@ -30,7 +30,7 @@ impl AvailabilityInfo {
} }
} }
pub fn available_modules(&self) -> Option<ResolvedVc<AvailableModules>> { pub fn available_modules(&self) -> Option<ResolvedVc<AvailableModulesSet>> {
self.available_modules self.available_modules
} }
...@@ -48,7 +48,7 @@ impl AvailabilityInfo { ...@@ -48,7 +48,7 @@ impl AvailabilityInfo {
} else { } else {
Self { Self {
flags: self.flags, flags: self.flags,
available_modules: Some(AvailableModules::new(modules).to_resolved().await?), available_modules: Some(modules.to_resolved().await?),
} }
}) })
} }
......
use anyhow::Result; use anyhow::Result;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use turbo_tasks::{ use turbo_tasks::{
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, ValueToString, Vc, FxIndexSet, NonLocalValue, ResolvedVc, TaskInput, TryJoinIterExt, ValueToString, Vc,
trace::TraceRawVcs, turbofmt, trace::TraceRawVcs, turbofmt,
}; };
use turbo_tasks_hash::Xxh3Hash64Hasher; use turbo_tasks_hash::Xxh3Hash64Hasher;
...@@ -49,55 +49,44 @@ impl From<ChunkableModuleOrBatch> for AvailableModuleItem { ...@@ -49,55 +49,44 @@ impl From<ChunkableModuleOrBatch> for AvailableModuleItem {
} }
} }
/// A flat set of modules/items that are already available in the current chunk
/// group context and therefore do not need to be included again.
///
/// This is a `#[turbo_tasks::value(transparent)]` wrapping
/// `FxIndexSet<AvailableModuleItem>`. Awaiting a `Vc<AvailableModulesSet>`
/// yields a `ReadRef` whose `Deref` target is the inner `FxIndexSet`, and
/// `Vc::cell` takes an `FxIndexSet<AvailableModuleItem>` directly.
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct AvailableModulesSet( pub struct AvailableModulesSet(
#[bincode(with = "turbo_bincode::indexset")] FxIndexSet<AvailableModuleItem>, #[bincode(with = "turbo_bincode::indexset")] FxIndexSet<AvailableModuleItem>,
); );
/// Allows to gather information about which assets are already available.
/// Adding more roots will form a linked list like structure to allow caching
/// `include` queries.
#[turbo_tasks::value]
pub struct AvailableModules {
parent: Option<ResolvedVc<AvailableModules>>,
modules: ResolvedVc<AvailableModulesSet>,
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl AvailableModules { impl AvailableModulesSet {
#[turbo_tasks::function] /// Returns a new set that is the union of `self` and `extra`.
pub fn new(modules: ResolvedVc<AvailableModulesSet>) -> Vc<Self> {
AvailableModules {
parent: None,
modules,
}
.cell()
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn with_modules( pub async fn with_modules(
self: ResolvedVc<Self>, self: ResolvedVc<Self>,
modules: ResolvedVc<AvailableModulesSet>, extra: Vc<AvailableModulesSet>,
) -> Result<Vc<Self>> { ) -> Result<Vc<Self>> {
Ok(AvailableModules { // For transparent Vc, .await? yields a ReadRef whose Deref target is the
parent: Some(self), // inner FxIndexSet<AvailableModuleItem>. Explicitly dereferencing with `*`
modules, // gives the inner set.
} let base = self.await?;
.cell()) let extra = extra.await?;
let mut merged = (*base).clone();
merged.extend(extra.iter().copied());
Ok(Vc::cell(merged))
} }
/// Returns a stable hash of the set contents, suitable for use in asset
/// identifiers.
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn hash(&self) -> Result<Vc<u64>> { pub async fn hash(self: Vc<Self>) -> Result<Vc<u64>> {
let set = self.await?;
let mut hasher = Xxh3Hash64Hasher::new(); let mut hasher = Xxh3Hash64Hasher::new();
if let Some(parent) = self.parent { let item_idents = set
hasher.write_value(parent.hash().await?);
} else {
hasher.write_value(0u64);
}
let item_idents = self
.modules
.await?
.iter() .iter()
.map(async |&module| module.ident_strings().await) .map(async |&module| module.ident_strings().await)
.try_join() .try_join()
...@@ -115,40 +104,12 @@ impl AvailableModules { ...@@ -115,40 +104,12 @@ impl AvailableModules {
} }
Ok(Vc::cell(hasher.finish())) Ok(Vc::cell(hasher.finish()))
} }
#[turbo_tasks::function]
pub async fn get(&self, item: AvailableModuleItem) -> Result<Vc<bool>> {
if self.modules.await?.contains(&item) {
return Ok(Vc::cell(true));
};
if let Some(parent) = self.parent {
return Ok(parent.get(item));
}
Ok(Vc::cell(false))
}
#[turbo_tasks::function]
pub async fn snapshot(&self) -> Result<Vc<AvailableModulesSnapshot>> {
let modules = self.modules.await?;
let parent = if let Some(parent) = self.parent {
Some(parent.snapshot().await?)
} else {
None
};
Ok(AvailableModulesSnapshot { parent, modules }.cell())
}
}
#[turbo_tasks::value(serialization = "none")]
#[derive(Debug, Clone)]
pub struct AvailableModulesSnapshot {
parent: Option<ReadRef<AvailableModulesSnapshot>>,
modules: ReadRef<AvailableModulesSet>,
} }
impl AvailableModulesSnapshot { impl AvailableModulesSet {
pub fn get(&self, item: AvailableModuleItem) -> bool { /// Returns true if this set contains the given item.
self.modules.contains(&item) || self.parent.as_ref().is_some_and(|parent| parent.get(item)) pub fn contains(&self, item: &AvailableModuleItem) -> bool {
// Self is #[repr(transparent)] over FxIndexSet; field .0 is accessible here.
self.0.contains(item)
} }
} }
...@@ -239,7 +239,7 @@ pub async fn chunk_group_content( ...@@ -239,7 +239,7 @@ pub async fn chunk_group_content(
}; };
let available_modules = match availability_info.available_modules() { let available_modules = match availability_info.available_modules() {
Some(available_modules) => Some(available_modules.snapshot().await?), Some(available_modules) => Some(available_modules.await?),
None => None, None => None,
}; };
...@@ -278,9 +278,9 @@ pub async fn chunk_group_content( ...@@ -278,9 +278,9 @@ pub async fn chunk_group_content(
return Ok(GraphTraversalAction::Exclude); return Ok(GraphTraversalAction::Exclude);
}; };
let is_available = available_modules let is_available = available_modules.as_ref().is_some_and(|available_modules| {
.as_ref() available_modules.contains(&AvailableModuleItem::from(chunkable_node))
.is_some_and(|available_modules| available_modules.get(chunkable_node.into())); });
let Some((_, edge)) = parent_info else { let Some((_, edge)) = parent_info else {
// An entry from the entries list // An entry from the entries list
...@@ -318,7 +318,7 @@ pub async fn chunk_group_content( ...@@ -318,7 +318,7 @@ pub async fn chunk_group_content(
let is_async_loader_available = let is_async_loader_available =
available_modules.as_ref().is_some_and(|available_modules| { available_modules.as_ref().is_some_and(|available_modules| {
available_modules available_modules
.get(AvailableModuleItem::AsyncLoader(chunkable_module)) .contains(&AvailableModuleItem::AsyncLoader(chunkable_module))
}); });
if !is_async_loader_available { if !is_async_loader_available {
state.async_modules.insert(chunkable_module); state.async_modules.insert(chunkable_module);
......
...@@ -7,6 +7,7 @@ use turbopack_core::{ ...@@ -7,6 +7,7 @@ use turbopack_core::{
chunk::{ chunk::{
AsyncModuleInfo, ChunkData, ChunkableModule, ChunkingContext, ChunkingContextExt, AsyncModuleInfo, ChunkData, ChunkableModule, ChunkingContext, ChunkingContextExt,
ChunksData, ModuleChunkItemIdExt, availability_info::AvailabilityInfo, ChunksData, ModuleChunkItemIdExt, availability_info::AvailabilityInfo,
available_modules::AvailableModuleItem,
}, },
ident::AssetIdent, ident::AssetIdent,
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
...@@ -68,7 +69,9 @@ impl AsyncLoaderModule { ...@@ -68,7 +69,9 @@ impl AsyncLoaderModule {
let module_or_batch = batches.get_entry(inner_module).await?; let module_or_batch = batches.get_entry(inner_module).await?;
if let Some(chunkable_module_or_batch) = if let Some(chunkable_module_or_batch) =
ChunkableModuleOrBatch::from_module_or_batch(module_or_batch) ChunkableModuleOrBatch::from_module_or_batch(module_or_batch)
&& *chunk_items.get(chunkable_module_or_batch.into()).await? && chunk_items
.await?
.contains(&AvailableModuleItem::from(chunkable_module_or_batch))
{ {
return Ok(OutputAssetsWithReferenced { return Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![]), assets: ResolvedVc::cell(vec![]),
......
...@@ -5,7 +5,7 @@ use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc}; ...@@ -5,7 +5,7 @@ use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{ use turbopack_core::{
chunk::{ chunk::{
AsyncModuleInfo, ChunkData, ChunkableModule, ChunkingContext, ChunkingContextExt, AsyncModuleInfo, ChunkData, ChunkableModule, ChunkingContext, ChunkingContextExt,
ChunksData, availability_info::AvailabilityInfo, ChunksData, availability_info::AvailabilityInfo, available_modules::AvailableModuleItem,
}, },
ident::AssetIdent, ident::AssetIdent,
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
...@@ -85,7 +85,9 @@ impl ManifestAsyncModule { ...@@ -85,7 +85,9 @@ impl ManifestAsyncModule {
let module_or_batch = batches.get_entry(inner_module).await?; let module_or_batch = batches.get_entry(inner_module).await?;
if let Some(chunkable_module_or_batch) = if let Some(chunkable_module_or_batch) =
ChunkableModuleOrBatch::from_module_or_batch(module_or_batch) ChunkableModuleOrBatch::from_module_or_batch(module_or_batch)
&& *chunk_items.get(chunkable_module_or_batch.into()).await? && chunk_items
.await?
.contains(&AvailableModuleItem::from(chunkable_module_or_batch))
{ {
return Ok(OutputAssetsWithReferenced { return Ok(OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![]), assets: ResolvedVc::cell(vec![]),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment