Unverified Commit bbf8aec9 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: selective read for module_chunk_groups in ChunkGroupInfo (#88559)

### What?

Use selective reads for module chunk groups from chunk group info

This is a preparation to make chunking be less dependent on module graph changes.
parent 8cfdcacf
...@@ -280,6 +280,7 @@ async fn batch_chunk_items_with_info_with_type( ...@@ -280,6 +280,7 @@ async fn batch_chunk_items_with_info_with_type(
} }
/// Creates chunks based on heuristics for the passed `chunk_items`. /// Creates chunks based on heuristics for the passed `chunk_items`.
#[tracing::instrument(level = Level::TRACE, skip_all)]
pub async fn make_chunks( pub async fn make_chunks(
module_graph: Vc<ModuleGraph>, module_graph: Vc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
......
...@@ -5,14 +5,18 @@ use rustc_hash::FxHasher; ...@@ -5,14 +5,18 @@ use rustc_hash::FxHasher;
use smallvec::SmallVec; use smallvec::SmallVec;
use tracing::{Instrument, field::Empty}; use tracing::{Instrument, field::Empty};
use turbo_prehash::BuildHasherExt; use turbo_prehash::BuildHasherExt;
use turbo_tasks::{FxIndexMap, FxIndexSet, ResolvedVc, TryJoinIterExt, Vc}; use turbo_tasks::{FxIndexMap, FxIndexSet, MappedReadRef, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use crate::{ use crate::{
chunk::{ chunk::{
ChunkItemBatchGroup, ChunkItemWithAsyncModuleInfo, ChunkingConfig, ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo, ChunkItemWithAsyncModuleInfo,
ChunkingConfig,
chunking::{ChunkItemOrBatchWithInfo, SplitContext, make_chunk}, chunking::{ChunkItemOrBatchWithInfo, SplitContext, make_chunk},
}, },
module_graph::{ModuleGraph, chunk_group_info::RoaringBitmapWrapper}, module_graph::{
ModuleGraph,
chunk_group_info::{ModuleToChunkGroups, RoaringBitmapWrapper},
},
}; };
pub async fn make_production_chunks( pub async fn make_production_chunks(
...@@ -32,7 +36,7 @@ pub async fn make_production_chunks( ...@@ -32,7 +36,7 @@ pub async fn make_production_chunks(
); );
let span = span_outer.clone(); let span = span_outer.clone();
async move { async move {
let chunk_group_info = module_graph.chunk_group_info().await?; let module_chunk_groups = module_graph.chunk_group_info().module_chunk_groups();
let merged_modules = module_graph.merged_modules().await?; let merged_modules = module_graph.merged_modules().await?;
#[derive(Default)] #[derive(Default)]
...@@ -43,27 +47,18 @@ pub async fn make_production_chunks( ...@@ -43,27 +47,18 @@ pub async fn make_production_chunks(
let mut grouped_chunk_items = FxIndexMap::<_, GroupedChunkItems<'_>>::default(); let mut grouped_chunk_items = FxIndexMap::<_, GroupedChunkItems<'_>>::default();
enum Prepared {
ChunkItem(MappedReadRef<ModuleToChunkGroups, RoaringBitmapWrapper>),
Batch(ReadRef<ChunkItemBatchWithAsyncModuleInfo>),
None,
}
// Helper Vec to keep ReadRefs on batches and allow references into them // Helper Vec to keep ReadRefs on batches and allow references into them
let batch_read_refs = chunk_items let prepared = chunk_items
.iter() .iter()
.copied() .copied()
.map(async |item| { .map(async |item| {
Ok( Ok(match item {
if let ChunkItemOrBatchWithInfo::Batch { batch, .. } = item {
Some(batch.await?)
} else {
None
},
)
})
.try_join()
.await?;
let batch_group_read_refs = batch_groups.iter().try_join().await?;
// Put chunk items into `grouped_chunk_items` based on their chunk groups
for (i, chunk_item) in chunk_items.into_iter().enumerate() {
let chunk_groups = match chunk_item {
&ChunkItemOrBatchWithInfo::ChunkItem { &ChunkItemOrBatchWithInfo::ChunkItem {
chunk_item: chunk_item:
ChunkItemWithAsyncModuleInfo { ChunkItemWithAsyncModuleInfo {
...@@ -71,28 +66,39 @@ pub async fn make_production_chunks( ...@@ -71,28 +66,39 @@ pub async fn make_production_chunks(
.. ..
}, },
.. ..
} => Some( } => Prepared::ChunkItem(
chunk_group_info if let Some(module_chunk_groups) =
.module_chunk_groups module_chunk_groups.get(&ResolvedVc::upcast(module)).await?
.get(&ResolvedVc::upcast(module)) {
.or_else(|| { module_chunk_groups
} else {
// Merged modules don't have a chunk group in chunk_group_info, so // Merged modules don't have a chunk group in chunk_group_info, so
// lookup using the original module. // lookup using the original module.
merged_modules let original_module = merged_modules
.get_original_module(ResolvedVc::upcast(module)) .get_original_module(ResolvedVc::upcast(module))
.and_then(|module| { .context("every module should have a chunk group")?;
chunk_group_info.module_chunk_groups.get(&module) module_chunk_groups
}) .get(&original_module)
}) .await?
.context("every module should have a chunk group")?, .context("every module should have a chunk group")?
},
), ),
&ChunkItemOrBatchWithInfo::ChunkItem { &ChunkItemOrBatchWithInfo::ChunkItem {
chunk_item: ChunkItemWithAsyncModuleInfo { module: None, .. }, chunk_item: ChunkItemWithAsyncModuleInfo { module: None, .. },
.. ..
} => None, } => Prepared::None,
ChunkItemOrBatchWithInfo::Batch { .. } => { ChunkItemOrBatchWithInfo::Batch { batch, .. } => Prepared::Batch(batch.await?),
batch_read_refs[i].as_ref().unwrap().chunk_groups.as_ref() })
} })
.try_join()
.await?;
// Put chunk items into `grouped_chunk_items` based on their chunk groups
for (chunk_item, prepared) in chunk_items.into_iter().zip(prepared.iter()) {
let chunk_groups = match prepared {
Prepared::None => None,
Prepared::ChunkItem(data) => Some(&**data),
Prepared::Batch(data) => data.chunk_groups.as_ref(),
}; };
let key = BuildHasherDefault::<FxHasher>::default().prehash(chunk_groups); let key = BuildHasherDefault::<FxHasher>::default().prehash(chunk_groups);
grouped_chunk_items grouped_chunk_items
...@@ -102,8 +108,12 @@ pub async fn make_production_chunks( ...@@ -102,8 +108,12 @@ pub async fn make_production_chunks(
.push(chunk_item); .push(chunk_item);
} }
for (i, batch_group) in batch_groups.into_iter().enumerate() { let batch_group_read_refs = batch_groups.iter().try_join().await?;
let data = &batch_group_read_refs[i].chunk_groups;
for (batch_group, batch_group_read_ref) in
batch_groups.into_iter().zip(batch_group_read_refs.iter())
{
let data = &batch_group_read_ref.chunk_groups;
let key = BuildHasherDefault::<FxHasher>::default().prehash(Some(data)); let key = BuildHasherDefault::<FxHasher>::default().prehash(Some(data));
grouped_chunk_items.entry(key).or_default().batch_group = Some(batch_group); grouped_chunk_items.entry(key).or_default().batch_group = Some(batch_group);
} }
......
...@@ -82,9 +82,12 @@ impl Hash for RoaringBitmapWrapper { ...@@ -82,9 +82,12 @@ impl Hash for RoaringBitmapWrapper {
} }
} }
#[turbo_tasks::value(transparent, cell = "keyed")]
pub struct ModuleToChunkGroups(FxHashMap<ResolvedVc<Box<dyn Module>>, RoaringBitmapWrapper>);
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct ChunkGroupInfo { pub struct ChunkGroupInfo {
pub module_chunk_groups: FxHashMap<ResolvedVc<Box<dyn Module>>, RoaringBitmapWrapper>, pub module_chunk_groups: ResolvedVc<ModuleToChunkGroups>,
#[turbo_tasks(trace_ignore)] #[turbo_tasks(trace_ignore)]
#[bincode(with = "turbo_bincode::indexset")] #[bincode(with = "turbo_bincode::indexset")]
pub chunk_groups: FxIndexSet<ChunkGroup>, pub chunk_groups: FxIndexSet<ChunkGroup>,
...@@ -95,6 +98,11 @@ pub struct ChunkGroupInfo { ...@@ -95,6 +98,11 @@ pub struct ChunkGroupInfo {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkGroupInfo { impl ChunkGroupInfo {
#[turbo_tasks::function]
pub fn module_chunk_groups(&self) -> Vc<ModuleToChunkGroups> {
*self.module_chunk_groups
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn get_index_of(&self, chunk_group: ChunkGroup) -> Result<Vc<usize>> { pub async fn get_index_of(&self, chunk_group: ChunkGroup) -> Result<Vc<usize>> {
if let Some(idx) = self.chunk_groups.get_index_of(&chunk_group) { if let Some(idx) = self.chunk_groups.get_index_of(&chunk_group) {
...@@ -736,7 +744,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro ...@@ -736,7 +744,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
} }
Ok(ChunkGroupInfo { Ok(ChunkGroupInfo {
module_chunk_groups, module_chunk_groups: ResolvedVc::cell(module_chunk_groups),
chunk_group_keys: chunk_groups_map.keys().cloned().collect(), chunk_group_keys: chunk_groups_map.keys().cloned().collect(),
chunk_groups: chunk_groups_map chunk_groups: chunk_groups_map
.into_iter() .into_iter()
......
...@@ -257,18 +257,17 @@ impl PreBatches { ...@@ -257,18 +257,17 @@ impl PreBatches {
fn ensure_pre_batch_for_module( fn ensure_pre_batch_for_module(
&mut self, &mut self,
module: ResolvedVc<Box<dyn Module>>, module: ResolvedVc<Box<dyn Module>>,
chunk_group_info: &ChunkGroupInfo, module_chunk_groups: &FxHashMap<ResolvedVc<Box<dyn Module>>, RoaringBitmapWrapper>,
queue: &mut VecDeque<(ResolvedVc<Box<dyn Module>>, PreBatchIndex)>, queue: &mut VecDeque<(ResolvedVc<Box<dyn Module>>, PreBatchIndex)>,
) -> Result<PreBatchIndex> { ) -> Result<PreBatchIndex> {
Ok(match self.entries.entry(module) { Ok(match self.entries.entry(module) {
Entry::Vacant(e) => { Entry::Vacant(e) => {
let index = self.batches.len(); let index = self.batches.len();
queue.push_back((module, index)); queue.push_back((module, index));
let chunk_groups = chunk_group_info let chunk_groups = module_chunk_groups
.module_chunk_groups
.get(&module) .get(&module)
.context("all modules need to have chunk group info")?; .context("all modules need to have chunk group info")?;
let batch = PreBatch::new(chunk_groups.clone()); let batch = PreBatch::new((*chunk_groups).clone());
self.batches.push(batch); self.batches.push(batch);
e.insert(index); e.insert(index);
index index
...@@ -280,7 +279,7 @@ impl PreBatches { ...@@ -280,7 +279,7 @@ impl PreBatches {
async fn get_pre_batch_items( async fn get_pre_batch_items(
&mut self, &mut self,
entry: ResolvedVc<Box<dyn Module>>, entry: ResolvedVc<Box<dyn Module>>,
chunk_group_info: &ChunkGroupInfo, module_chunk_groups: &FxHashMap<ResolvedVc<Box<dyn Module>>, RoaringBitmapWrapper>,
module_graph: &ModuleGraph, module_graph: &ModuleGraph,
queue: &mut VecDeque<(ResolvedVc<Box<dyn Module>>, PreBatchIndex)>, queue: &mut VecDeque<(ResolvedVc<Box<dyn Module>>, PreBatchIndex)>,
) -> Result<Vec<PreBatchItem>> { ) -> Result<Vec<PreBatchItem>> {
...@@ -312,7 +311,7 @@ impl PreBatches { ...@@ -312,7 +311,7 @@ impl PreBatches {
if parent_info.is_some() && state.this.boundary_modules.contains(&module) { if parent_info.is_some() && state.this.boundary_modules.contains(&module) {
let idx = state.this.ensure_pre_batch_for_module( let idx = state.this.ensure_pre_batch_for_module(
module, module,
chunk_group_info, module_chunk_groups,
queue, queue,
)?; )?;
state.items.push(PreBatchItem::ParallelReference(idx)); state.items.push(PreBatchItem::ParallelReference(idx));
...@@ -349,6 +348,7 @@ pub async fn compute_module_batches( ...@@ -349,6 +348,7 @@ pub async fn compute_module_batches(
let span = outer_span.clone(); let span = outer_span.clone();
async move { async move {
let chunk_group_info = module_graph.chunk_group_info().await?; let chunk_group_info = module_graph.chunk_group_info().await?;
let module_chunk_groups = chunk_group_info.module_chunk_groups.await?;
let module_graph = module_graph.await?; let module_graph = module_graph.await?;
let mut pre_batches = PreBatches::new(); let mut pre_batches = PreBatches::new();
...@@ -364,12 +364,10 @@ pub async fn compute_module_batches( ...@@ -364,12 +364,10 @@ pub async fn compute_module_batches(
return Ok(()); return Ok(());
}; };
if ty.chunking_type.is_parallel() { if ty.chunking_type.is_parallel() {
let parent_chunk_groups = chunk_group_info let parent_chunk_groups = module_chunk_groups
.module_chunk_groups
.get(&parent) .get(&parent)
.context("all modules need to have chunk group info")?; .context("all modules need to have chunk group info")?;
let chunk_groups = chunk_group_info let chunk_groups = module_chunk_groups
.module_chunk_groups
.get(&node) .get(&node)
.context("all modules need to have chunk group info")?; .context("all modules need to have chunk group info")?;
if parent_chunk_groups != chunk_groups { if parent_chunk_groups != chunk_groups {
...@@ -415,7 +413,7 @@ pub async fn compute_module_batches( ...@@ -415,7 +413,7 @@ pub async fn compute_module_batches(
// Start with the entries // Start with the entries
for chunk_group in &chunk_group_info.chunk_groups { for chunk_group in &chunk_group_info.chunk_groups {
for entry in chunk_group.entries() { for entry in chunk_group.entries() {
pre_batches.ensure_pre_batch_for_module(entry, &chunk_group_info, &mut queue)?; pre_batches.ensure_pre_batch_for_module(entry, &module_chunk_groups, &mut queue)?;
} }
if let Some(parent) = chunk_group.get_merged_parent() { if let Some(parent) = chunk_group.get_merged_parent() {
chunk_group_indices_with_merged_children.insert(parent); chunk_group_indices_with_merged_children.insert(parent);
...@@ -428,7 +426,7 @@ pub async fn compute_module_batches( ...@@ -428,7 +426,7 @@ pub async fn compute_module_batches(
let items = pre_batches let items = pre_batches
.get_pre_batch_items( .get_pre_batch_items(
chunkable_module, chunkable_module,
&chunk_group_info, &module_chunk_groups,
&module_graph, &module_graph,
&mut queue, &mut queue,
) )
...@@ -773,8 +771,7 @@ pub async fn compute_module_batches( ...@@ -773,8 +771,7 @@ pub async fn compute_module_batches(
batch_groups.entry(key).or_default().push(batch); batch_groups.entry(key).or_default().push(batch);
} }
for &module in &pre_batches.single_module_entries { for &module in &pre_batches.single_module_entries {
let chunk_groups = chunk_group_info let chunk_groups = module_chunk_groups
.module_chunk_groups
.get(&module) .get(&module)
.context("all modules need to have chunk group info")?; .context("all modules need to have chunk group info")?;
let key = BuildHasherDefault::<FxHasher>::default().prehash(chunk_groups); let key = BuildHasherDefault::<FxHasher>::default().prehash(chunk_groups);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment