Unverified Commit c38e57fd authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add caching to chunking (#88561)

### What?

Make make_chunks a cached function to allow it to be cached even when the module graph changes (assuming that chunk group's chunk items are not affected by the change)
parent a8b0f9db
...@@ -695,6 +695,8 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -695,6 +695,8 @@ impl ChunkingContext for BrowserChunkingContext {
) )
.await?; .await?;
let chunks = chunks.await?;
let mut assets = chunks let mut assets = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(**chunk).to_resolved())
...@@ -761,6 +763,8 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -761,6 +763,8 @@ impl ChunkingContext for BrowserChunkingContext {
) )
.await?; .await?;
let chunks = chunks.await?;
let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(**chunk).to_resolved())
......
...@@ -6,12 +6,12 @@ use turbo_rcstr::rcstr; ...@@ -6,12 +6,12 @@ use turbo_rcstr::rcstr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc}; use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
use super::{ use super::{
Chunk, ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext, ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext,
availability_info::AvailabilityInfo, chunking::make_chunks, availability_info::AvailabilityInfo, chunking::make_chunks,
}; };
use crate::{ use crate::{
chunk::{ chunk::{
ChunkableModule, ChunkingType, ChunkableModule, ChunkingType, Chunks,
available_modules::AvailableModuleItem, available_modules::AvailableModuleItem,
chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo}, chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
}, },
...@@ -35,7 +35,7 @@ use crate::{ ...@@ -35,7 +35,7 @@ use crate::{
}; };
pub struct MakeChunkGroupResult { pub struct MakeChunkGroupResult {
pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>, pub chunks: ResolvedVc<Chunks>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>, pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>, pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>,
pub availability_info: AvailabilityInfo, pub availability_info: AvailabilityInfo,
...@@ -154,11 +154,12 @@ pub async fn make_chunk_group( ...@@ -154,11 +154,12 @@ pub async fn make_chunk_group(
// Pass chunk items to chunking algorithm // Pass chunk items to chunking algorithm
let chunks = make_chunks( let chunks = make_chunks(
module_graph, module_graph,
chunking_context, *chunking_context,
chunk_items, Vc::cell(chunk_items),
chunk_item_batch_groups, Vc::cell(chunk_item_batch_groups),
rcstr!(""), rcstr!(""),
) )
.to_resolved()
.await?; .await?;
Ok(MakeChunkGroupResult { Ok(MakeChunkGroupResult {
......
...@@ -61,6 +61,9 @@ type ChunkItemOrBatchWithAsyncModuleInfoByChunkType = Either< ...@@ -61,6 +61,9 @@ type ChunkItemOrBatchWithAsyncModuleInfoByChunkType = Either<
ReadRef<ChunkItemBatchWithAsyncModuleInfoByChunkType>, ReadRef<ChunkItemBatchWithAsyncModuleInfoByChunkType>,
>; >;
#[turbo_tasks::value(transparent)]
pub struct ChunkItemOrBatchWithAsyncModuleInfos(Vec<ChunkItemOrBatchWithAsyncModuleInfo>);
impl ChunkItemOrBatchWithAsyncModuleInfo { impl ChunkItemOrBatchWithAsyncModuleInfo {
pub async fn from_chunkable_module_or_batch( pub async fn from_chunkable_module_or_batch(
chunkable_module_or_batch: ChunkableModuleOrBatch, chunkable_module_or_batch: ChunkableModuleOrBatch,
...@@ -222,6 +225,9 @@ type ChunkItemBatchGroupByChunkTypeT = SmallVec< ...@@ -222,6 +225,9 @@ type ChunkItemBatchGroupByChunkTypeT = SmallVec<
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct ChunkItemBatchGroupByChunkType(ChunkItemBatchGroupByChunkTypeT); pub struct ChunkItemBatchGroupByChunkType(ChunkItemBatchGroupByChunkTypeT);
#[turbo_tasks::value(transparent)]
pub struct ChunkItemBatchGroups(Vec<ResolvedVc<ChunkItemBatchGroup>>);
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct ChunkItemBatchGroup { pub struct ChunkItemBatchGroup {
pub items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>, pub items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
......
...@@ -13,10 +13,11 @@ use turbo_tasks::{ ...@@ -13,10 +13,11 @@ use turbo_tasks::{
use crate::{ use crate::{
chunk::{ chunk::{
Chunk, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkType, ChunkingContext, batch_info, Chunk, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkType, ChunkingContext, Chunks,
batch_info,
chunk_item_batch::{ chunk_item_batch::{
ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo, ChunkItemBatchGroup, ChunkItemBatchGroups, ChunkItemBatchWithAsyncModuleInfo,
ChunkItemOrBatchWithAsyncModuleInfo, ChunkItemOrBatchWithAsyncModuleInfo, ChunkItemOrBatchWithAsyncModuleInfos,
}, },
chunking::{ chunking::{
dev::{app_vendors_split, expand_batches}, dev::{app_vendors_split, expand_batches},
...@@ -280,15 +281,17 @@ async fn batch_chunk_items_with_info_with_type( ...@@ -280,15 +281,17 @@ async fn batch_chunk_items_with_info_with_type(
} }
/// Creates chunks based on heuristics for the passed `chunk_items`. /// Creates chunks based on heuristics for the passed `chunk_items`.
#[tracing::instrument(level = Level::TRACE, skip_all)] #[turbo_tasks::function]
pub async fn make_chunks( pub async fn make_chunks(
module_graph: Vc<ModuleGraph>, module_graph: Vc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
chunk_items_or_batches: Vec<ChunkItemOrBatchWithAsyncModuleInfo>, chunk_items_or_batches: ResolvedVc<ChunkItemOrBatchWithAsyncModuleInfos>,
batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>, batch_groups: ResolvedVc<ChunkItemBatchGroups>,
key_prefix: RcStr, key_prefix: RcStr,
) -> Result<Vec<ResolvedVc<Box<dyn Chunk>>>> { ) -> Result<Vc<Chunks>> {
let chunking_configs = &*chunking_context.chunking_configs().await?; let chunking_configs = &*chunking_context.chunking_configs().await?;
let chunk_items_or_batches = chunk_items_or_batches.await?;
let batch_groups = batch_groups.await?;
let span = tracing::trace_span!( let span = tracing::trace_span!(
"get chunk item info", "get chunk item info",
...@@ -384,7 +387,7 @@ pub async fn make_chunks( ...@@ -384,7 +387,7 @@ pub async fn make_chunks(
.try_join() .try_join()
.await?; .await?;
Ok(resolved_chunks) Ok(Vc::cell(resolved_chunks))
} }
struct SplitContext<'a> { struct SplitContext<'a> {
......
...@@ -472,6 +472,8 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -472,6 +472,8 @@ impl ChunkingContext for NodeJsChunkingContext {
) )
.await?; .await?;
let chunks = chunks.await?;
let assets = chunks let assets = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(**chunk).to_resolved())
...@@ -524,6 +526,8 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -524,6 +526,8 @@ impl ChunkingContext for NodeJsChunkingContext {
) )
.await?; .await?;
let chunks = chunks.await?;
let extra_chunks = extra_chunks.await?; let extra_chunks = extra_chunks.await?;
let mut other_chunks = chunks let mut other_chunks = chunks
.iter() .iter()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment