Unverified Commit 4ef926bc authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Turbopack chunking: use ChunkableModules in `chunk_content`, not ChunkItems (#74040)

This:

- Replaces all use of chunk items in the interface for `chunk_content`
with ChunkableModules
- Moves `is_self_async` from `ChunkItem` to `Module`
- Renames `AvailableChunkItems` to `AvailableModules` and all associated
structs and methods, using modules in place of chunk items

---------
Co-authored-by: 's avatarTobias Koppers <tobias.koppers@googlemail.com>
parent 40a4c3b7
......@@ -166,7 +166,7 @@ async fn build_manifest(
&key,
ActionManifestWorkerEntry {
module_id: ActionManifestModuleId::String(loader_id.as_str()),
is_async: *chunk_item.is_self_async().await?,
is_async: *chunk_item.module().is_self_async().await?,
},
);
entry.layer.insert(&key, *layer);
......
use anyhow::Result;
use anyhow::{Context, Result};
use indoc::formatdoc;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc};
......@@ -6,8 +6,8 @@ use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{
availability_info::AvailabilityInfo, ChunkItem, ChunkItemExt, ChunkableModule,
ChunkingContext, ModuleId as TurbopackModuleId,
availability_info::AvailabilityInfo, ChunkItemExt, ChunkableModule, ChunkingContext,
ModuleId as TurbopackModuleId,
},
output::{OutputAsset, OutputAssets},
virtual_output::VirtualOutputAsset,
......@@ -75,8 +75,8 @@ impl ClientReferenceManifest {
let server_path = ecmascript_client_reference.server_ident.to_string().await?;
let client_chunk_item = ecmascript_client_reference
.client_module
let client_module = ecmascript_client_reference.client_module;
let client_chunk_item = client_module
.as_chunk_item(Vc::upcast(client_chunking_context))
.to_resolved()
.await?;
......@@ -107,8 +107,11 @@ impl ClientReferenceManifest {
.map(RcStr::from)
.collect::<Vec<_>>();
let is_async =
is_item_async(client_availability_info, client_chunk_item).await?;
let is_async = is_item_async(
client_availability_info,
ResolvedVc::upcast(client_module),
)
.await?;
(chunk_paths, is_async)
} else {
......@@ -116,19 +119,20 @@ impl ClientReferenceManifest {
};
if let Some(ssr_chunking_context) = ssr_chunking_context {
let ssr_chunk_item = ecmascript_client_reference
.ssr_module
let ssr_module = ecmascript_client_reference.ssr_module;
let ssr_chunk_item = ssr_module
.as_chunk_item(Vc::upcast(ssr_chunking_context))
.to_resolved()
.await?;
let ssr_module_id = ssr_chunk_item.id().await?;
let rsc_chunk_item: ResolvedVc<Box<dyn ChunkItem>> =
ResolvedVc::try_downcast_type::<EcmascriptClientReferenceProxyModule>(
parent_module,
)
.await?
.unwrap()
let rsc_module = ResolvedVc::try_downcast_type::<
EcmascriptClientReferenceProxyModule,
>(parent_module)
.await?
.context("Expected EcmascriptClientReferenceProxyModule")?;
let rsc_chunk_item = rsc_module
.as_chunk_item(Vc::upcast(ssr_chunking_context))
.to_resolved()
.await?;
......@@ -161,7 +165,9 @@ impl ClientReferenceManifest {
.map(RcStr::from)
.collect::<Vec<_>>();
let is_async = is_item_async(ssr_availability_info, ssr_chunk_item).await?;
let is_async =
is_item_async(ssr_availability_info, ResolvedVc::upcast(ssr_module))
.await?;
(chunk_paths, is_async)
} else {
......@@ -188,9 +194,11 @@ impl ClientReferenceManifest {
.map(RcStr::from)
.collect::<Vec<_>>();
let is_async =
is_item_async(&rsc_app_entry_chunks_availability, rsc_chunk_item)
.await?;
let is_async = is_item_async(
&rsc_app_entry_chunks_availability,
ResolvedVc::upcast(rsc_module),
)
.await?;
(chunk_paths, is_async)
};
......@@ -368,13 +376,13 @@ pub fn get_client_reference_module_key(server_path: &str, export_name: &str) ->
async fn is_item_async(
availability_info: &AvailabilityInfo,
chunk_item: ResolvedVc<Box<dyn ChunkItem>>,
module: ResolvedVc<Box<dyn ChunkableModule>>,
) -> Result<bool> {
let Some(available_chunk_items) = availability_info.available_chunk_items() else {
let Some(available_modules) = availability_info.available_modules() else {
return Ok(false);
};
let Some(info) = available_chunk_items.await?.get(chunk_item).await? else {
let Some(info) = &*available_modules.get(*module).await? else {
return Ok(false);
};
......
......@@ -422,11 +422,9 @@ impl ChunkingContext for BrowserChunkingContext {
AvailabilityInfo::Untracked => {
ident = ident.with_modifier(Vc::cell("untracked".into()));
}
AvailabilityInfo::Complete {
available_chunk_items,
} => {
AvailabilityInfo::Complete { available_modules } => {
ident = ident.with_modifier(Vc::cell(
available_chunk_items.hash().await?.to_string().into(),
available_modules.hash().await?.to_string().into(),
));
}
}
......
......@@ -4,13 +4,14 @@ use anyhow::Result;
use tracing::{info_span, Instrument};
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ModuleId},
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkItemTy, ModuleId},
code_builder::{Code, CodeBuilder},
error::PrettyPrintError,
issue::{code_gen::CodeGenerationIssue, IssueExt, IssueSeverity, StyledString},
};
use turbopack_ecmascript::chunk::{
EcmascriptChunkContent, EcmascriptChunkItem, EcmascriptChunkItemExt,
EcmascriptChunkItemWithAsyncInfo,
};
/// A chunk item's content entry.
......@@ -28,7 +29,7 @@ pub struct EcmascriptDevChunkContentEntry {
impl EcmascriptDevChunkContentEntry {
pub async fn new(
chunk_item: Vc<Box<dyn EcmascriptChunkItem>>,
chunk_item: ResolvedVc<Box<dyn EcmascriptChunkItem>>,
async_module_info: Option<Vc<AsyncModuleInfo>>,
) -> Result<Self> {
let code = chunk_item.code(async_module_info).to_resolved().await?;
......@@ -52,15 +53,34 @@ impl EcmascriptDevChunkContentEntries {
) -> Result<Vc<EcmascriptDevChunkContentEntries>> {
let chunk_content = chunk_content.await?;
let entries: FxIndexMap<_, _> = chunk_content
let included_chunk_items = chunk_content
.chunk_items
.iter()
.map(|&(chunk_item, async_module_info)| async move {
.map(
async |EcmascriptChunkItemWithAsyncInfo {
ty,
chunk_item,
async_info,
}| {
if matches!(ty, ChunkItemTy::Included) {
Ok(Some((chunk_item, async_info)))
} else {
Ok(None)
}
},
)
.try_join()
.await?
.into_iter()
.flatten();
let entries: FxIndexMap<_, _> = included_chunk_items
.map(|(&chunk_item, &async_module_info)| async move {
async move {
Ok((
chunk_item.id().await?,
EcmascriptDevChunkContentEntry::new(
*chunk_item,
chunk_item,
async_module_info.map(|info| *info),
)
.await?,
......
#![feature(async_closure)]
#![feature(iter_intersperse)]
#![feature(int_roundings)]
#![feature(arbitrary_self_types)]
......
use anyhow::Result;
use turbo_tasks::{ResolvedVc, Vc};
use super::available_chunk_items::{AvailableChunkItemInfoMap, AvailableChunkItems};
use super::available_modules::{AvailableModuleInfoMap, AvailableModules};
#[turbo_tasks::value(serialization = "auto_for_input")]
#[derive(Hash, Clone, Copy, Debug)]
......@@ -12,36 +12,30 @@ pub enum AvailabilityInfo {
Root,
/// There are modules already available.
Complete {
available_chunk_items: ResolvedVc<AvailableChunkItems>,
available_modules: ResolvedVc<AvailableModules>,
},
}
impl AvailabilityInfo {
pub fn available_chunk_items(&self) -> Option<ResolvedVc<AvailableChunkItems>> {
pub fn available_modules(&self) -> Option<ResolvedVc<AvailableModules>> {
match self {
Self::Untracked => None,
Self::Root => None,
Self::Complete {
available_chunk_items,
..
} => Some(*available_chunk_items),
available_modules, ..
} => Some(*available_modules),
}
}
pub async fn with_chunk_items(
self,
chunk_items: Vc<AvailableChunkItemInfoMap>,
) -> Result<Self> {
pub async fn with_modules(self, modules: Vc<AvailableModuleInfoMap>) -> Result<Self> {
Ok(match self {
AvailabilityInfo::Untracked => AvailabilityInfo::Untracked,
AvailabilityInfo::Root => AvailabilityInfo::Complete {
available_chunk_items: AvailableChunkItems::new(chunk_items).to_resolved().await?,
available_modules: AvailableModules::new(modules).to_resolved().await?,
},
AvailabilityInfo::Complete {
available_chunk_items,
} => AvailabilityInfo::Complete {
available_chunk_items: available_chunk_items
.with_chunk_items(chunk_items)
AvailabilityInfo::Complete { available_modules } => AvailabilityInfo::Complete {
available_modules: available_modules
.with_modules(modules)
.to_resolved()
.await?,
},
......
use anyhow::Result;
use serde::{Deserialize, Serialize};
use turbo_tasks::{
debug::ValueDebugFormat, trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc,
TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
};
use turbo_tasks_hash::Xxh3Hash64Hasher;
use super::ChunkableModule;
use crate::module::Module;
#[derive(
PartialEq, Eq, TraceRawVcs, Copy, Clone, Serialize, Deserialize, ValueDebugFormat, NonLocalValue,
)]
pub struct AvailableModulesInfo {
pub is_async: bool,
}
#[turbo_tasks::value(transparent)]
pub struct OptionAvailableModulesInfo(Option<AvailableModulesInfo>);
#[turbo_tasks::value(transparent)]
pub struct AvailableModuleInfoMap(
FxIndexMap<ResolvedVc<Box<dyn ChunkableModule>>, AvailableModulesInfo>,
);
/// Allows to gather information about which assets are already available.
/// Adding more roots will form a linked list like structure to allow caching
/// `include` queries.
#[turbo_tasks::value]
pub struct AvailableModules {
parent: Option<ResolvedVc<AvailableModules>>,
modules: ResolvedVc<AvailableModuleInfoMap>,
}
#[turbo_tasks::value_impl]
impl AvailableModules {
#[turbo_tasks::function]
pub fn new(modules: ResolvedVc<AvailableModuleInfoMap>) -> Vc<Self> {
AvailableModules {
parent: None,
modules,
}
.cell()
}
#[turbo_tasks::function]
pub async fn with_modules(
self: ResolvedVc<Self>,
modules: ResolvedVc<AvailableModuleInfoMap>,
) -> Result<Vc<Self>> {
let modules = modules
.await?
.into_iter()
.map(|(&module, &info)| async move {
Ok(self.get(*module).await?.is_none().then_some((module, info)))
})
.try_flat_join()
.await?;
Ok(AvailableModules {
parent: Some(self),
modules: ResolvedVc::cell(modules.into_iter().collect()),
}
.cell())
}
#[turbo_tasks::function]
pub async fn hash(&self) -> Result<Vc<u64>> {
let mut hasher = Xxh3Hash64Hasher::new();
if let Some(parent) = self.parent {
hasher.write_value(parent.hash().await?);
} else {
hasher.write_value(0u64);
}
let item_idents = self
.modules
.await?
.iter()
.map(|(&module, _)| module.ident().to_string())
.try_join()
.await?;
for ident in item_idents {
hasher.write_value(ident);
}
Ok(Vc::cell(hasher.finish()))
}
#[turbo_tasks::function]
pub async fn get(
&self,
module: ResolvedVc<Box<dyn ChunkableModule>>,
) -> Result<Vc<OptionAvailableModulesInfo>> {
if let Some(&info) = self.modules.await?.get(&module) {
return Ok(Vc::cell(Some(info)));
};
if let Some(parent) = self.parent {
return Ok(parent.get(*module));
}
Ok(Vc::cell(None))
}
}
......@@ -2,15 +2,17 @@ use std::collections::HashSet;
use anyhow::Result;
use auto_hash_map::AutoSet;
use futures::future::Either;
use turbo_tasks::{FxIndexMap, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc};
use super::{
availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItemInfo,
chunk_content, chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem,
ChunkingContext,
availability_info::AvailabilityInfo, available_modules::AvailableModulesInfo, chunk_content,
chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem, ChunkItemTy,
ChunkItemWithAsyncModuleInfo, ChunkableModule, ChunkingContext,
};
use crate::{
module::Module, output::OutputAssets, rebase::RebasedAsset, reference::ModuleReference,
environment::ChunkLoading, module::Module, output::OutputAssets, rebase::RebasedAsset,
reference::ModuleReference,
};
pub struct MakeChunkGroupResult {
......@@ -24,40 +26,51 @@ pub async fn make_chunk_group(
chunk_group_entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
availability_info: AvailabilityInfo,
) -> Result<MakeChunkGroupResult> {
let can_split_async = !matches!(
*chunking_context.environment().chunk_loading().await?,
ChunkLoading::Edge
);
let should_trace = *chunking_context.is_tracing_enabled().await?;
let ChunkContentResult {
chunk_items,
external_output_assets,
external_module_references,
chunkable_modules,
async_modules,
traced_modules,
passthrough_modules,
forward_edges_inherit_async,
local_back_edges_inherit_async,
available_async_modules_back_edges_inherit_async,
} = chunk_content(chunking_context, chunk_group_entries, availability_info).await?;
} = chunk_content(
chunk_group_entries,
availability_info,
can_split_async,
should_trace,
)
.await?;
// Find all local chunk items that are self async
let self_async_children = chunk_items
let self_async_children = chunkable_modules
.iter()
.copied()
.map(|chunk_item| async move {
let is_self_async = *chunk_item.is_self_async().await?;
Ok(is_self_async.then_some(chunk_item))
.map(|m| async move {
let is_self_async = *m.is_self_async().await?;
Ok(is_self_async.then_some(m))
})
.try_flat_join()
.await?;
// Get all available async modules and concatenate with local async modules
let mut async_chunk_items = available_async_modules_back_edges_inherit_async
let mut all_async_modules = available_async_modules_back_edges_inherit_async
.keys()
.copied()
.chain(self_async_children.into_iter())
.map(|chunk_item| (chunk_item, AutoSet::<ResolvedVc<Box<dyn ChunkItem>>>::new()))
.map(|m| (m, AutoSet::<ResolvedVc<Box<dyn ChunkableModule>>>::new()))
.collect::<FxIndexMap<_, _>>();
// Propagate async inheritance
let mut i = 0;
loop {
let Some((&chunk_item, _)) = async_chunk_items.get_index(i) else {
let Some((&async_module, _)) = all_async_modules.get_index(i) else {
break;
};
// The first few entries are from
......@@ -68,26 +81,26 @@ pub async fn make_chunk_group(
} else {
&local_back_edges_inherit_async
};
if let Some(parents) = map.get(&chunk_item) {
if let Some(parents) = map.get(&async_module) {
for &parent in parents.iter() {
// Add item, it will be iterated by this loop too
async_chunk_items
all_async_modules
.entry(parent)
.or_default()
.insert(chunk_item);
.insert(async_module);
}
}
i += 1;
}
// Create map for chunk items with empty [Option<Vc<AsyncModuleInfo>>]
let mut chunk_items = chunk_items
let mut all_modules = chunkable_modules
.into_iter()
.map(|chunk_item| (chunk_item, None))
.map(|m| (m, None))
.collect::<FxIndexMap<_, Option<ResolvedVc<AsyncModuleInfo>>>>();
// Insert AsyncModuleInfo for every async module
for (async_item, referenced_async_modules) in async_chunk_items {
for (async_item, referenced_async_modules) in all_async_modules {
let referenced_async_modules =
if let Some(references) = forward_edges_inherit_async.get(&async_item) {
references
......@@ -99,7 +112,7 @@ pub async fn make_chunk_group(
} else {
Default::default()
};
chunk_items.insert(
all_modules.insert(
async_item,
Some(
AsyncModuleInfo::new(referenced_async_modules)
......@@ -111,12 +124,12 @@ pub async fn make_chunk_group(
// Compute new [AvailabilityInfo]
let availability_info = {
let map = chunk_items
let map = all_modules
.iter()
.map(|(&chunk_item, async_info)| async move {
.map(|(&module, async_info)| async move {
Ok((
chunk_item,
AvailableChunkItemInfo {
module,
AvailableModulesInfo {
is_async: async_info.is_some(),
},
))
......@@ -126,21 +139,29 @@ pub async fn make_chunk_group(
.into_iter()
.collect();
let map = Vc::cell(map);
availability_info.with_chunk_items(map).await?
availability_info.with_modules(map).await?
};
// Insert async chunk loaders for every referenced async module
let async_loaders = async_modules
.into_iter()
.map(|module| {
.map(async |module| {
chunking_context
.async_loader_chunk_item(*module, Value::new(availability_info))
.to_resolved()
.await
})
.try_join()
.await?;
let has_async_loaders = !async_loaders.is_empty();
let async_loader_chunk_items = async_loaders.iter().map(|&chunk_item| (chunk_item, None));
let async_loader_chunk_items =
async_loaders
.iter()
.map(|&chunk_item| ChunkItemWithAsyncModuleInfo {
ty: ChunkItemTy::Included,
chunk_item,
async_info: None,
});
// And also add output assets referenced by async chunk loaders
let async_loader_references = async_loaders
......@@ -155,36 +176,51 @@ pub async fn make_chunk_group(
.collect(),
);
let mut referenced_output_assets = (*external_output_assets.await?).clone();
referenced_output_assets.extend(
references_to_output_assets(&external_module_references)
.await?
.await?
.iter()
.copied(),
);
let rebased_modules = traced_modules
let traced_output_assets = traced_modules
.into_iter()
.map(|module| {
RebasedAsset::new(
*module,
module.ident().path().root(),
module.ident().path().root(),
)
.to_resolved()
.map(|module| async move {
Ok(ResolvedVc::upcast(
RebasedAsset::new(
*module,
module.ident().path().root(),
module.ident().path().root(),
)
.to_resolved()
.await?,
))
})
.try_join()
.await?;
referenced_output_assets.extend(rebased_modules.into_iter().map(ResolvedVc::upcast));
let chunk_items = all_modules
.iter()
.map(|(m, async_info)| {
Either::Left(async move {
Ok(ChunkItemWithAsyncModuleInfo {
ty: ChunkItemTy::Included,
chunk_item: m.as_chunk_item(*chunking_context).to_resolved().await?,
async_info: *async_info,
})
})
})
.chain(passthrough_modules.into_iter().map(|m| {
Either::Right(async move {
Ok(ChunkItemWithAsyncModuleInfo {
ty: ChunkItemTy::Passthrough,
chunk_item: m.as_chunk_item(*chunking_context).to_resolved().await?,
async_info: None,
})
})
}))
.try_join()
.await?;
// Pass chunk items to chunking algorithm
let mut chunks = make_chunks(
*chunking_context,
Vc::cell(chunk_items.into_iter().collect()),
Vc::cell(chunk_items),
"".into(),
Vc::cell(referenced_output_assets),
Vc::cell(traced_output_assets),
)
.await?
.clone_value();
......
......@@ -11,8 +11,8 @@ use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, Vc};
use super::{
AsyncModuleInfo, Chunk, ChunkItem, ChunkItemsWithAsyncModuleInfo, ChunkType, ChunkingContext,
Chunks,
AsyncModuleInfo, Chunk, ChunkItem, ChunkItemTy, ChunkItemWithAsyncModuleInfo,
ChunkItemsWithAsyncModuleInfo, ChunkType, ChunkingContext, Chunks,
};
use crate::output::OutputAssets;
......@@ -49,22 +49,26 @@ pub async fn make_chunks(
key_prefix: RcStr,
mut referenced_output_assets: Vc<OutputAssets>,
) -> Result<Vc<Chunks>> {
let chunk_items = chunk_items.await?;
let chunk_items = chunk_items
.await?
.iter()
.map(|&(chunk_item, async_info)| async move {
let chunk_item_info =
chunk_item_info(chunking_context, *chunk_item, async_info.map(|info| *info))
.await?;
Ok((chunk_item, async_info, chunk_item_info))
.map(|c| async move {
let chunk_item_info = chunk_item_info(
chunking_context,
*c.chunk_item,
c.async_info.map(|info| *info),
)
.await?;
Ok((c, chunk_item_info))
})
.try_join()
.await?;
let mut map = FxIndexMap::<_, Vec<_>>::default();
for (chunk_item, async_info, chunk_item_info) in chunk_items {
for (c, chunk_item_info) in chunk_items {
map.entry(chunk_item_info.ty)
.or_default()
.push((chunk_item, async_info, chunk_item_info));
.push((c, chunk_item_info));
}
let mut chunks = Vec::new();
......@@ -73,14 +77,24 @@ pub async fn make_chunks(
let chunk_items = chunk_items
.into_iter()
.map(|(chunk_item, async_info, chunk_item_info)| async move {
Ok((
chunk_item,
async_info,
chunk_item_info.size,
chunk_item_info.name.await?,
))
})
.map(
async |(
ChunkItemWithAsyncModuleInfo {
ty,
chunk_item,
async_info,
},
chunk_item_info,
)| {
Ok(ChunkItemWithInfo {
ty: *ty,
chunk_item: *chunk_item,
async_info: *async_info,
size: chunk_item_info.size,
asset_ident: chunk_item_info.name.await?,
})
},
)
.try_join()
.await?;
......@@ -119,12 +133,13 @@ pub async fn make_chunks(
Ok(Vc::cell(resolved_chunks))
}
type ChunkItemWithInfo = (
ResolvedVc<Box<dyn ChunkItem>>,
Option<ResolvedVc<AsyncModuleInfo>>,
usize,
ReadRef<RcStr>,
);
struct ChunkItemWithInfo {
ty: ChunkItemTy,
chunk_item: ResolvedVc<Box<dyn ChunkItem>>,
async_info: Option<ResolvedVc<AsyncModuleInfo>>,
size: usize,
asset_ident: ReadRef<RcStr>,
}
struct SplitContext<'a> {
ty: ResolvedVc<Box<dyn ChunkType>>,
......@@ -169,7 +184,18 @@ async fn make_chunk(
split_context.chunking_context,
chunk_items
.into_iter()
.map(|(chunk_item, async_info, ..)| (chunk_item, async_info))
.map(
|ChunkItemWithInfo {
ty,
chunk_item,
async_info,
..
}| ChunkItemWithAsyncModuleInfo {
ty,
chunk_item,
async_info,
},
)
.collect(),
replace(
split_context.referenced_output_assets,
......@@ -191,7 +217,7 @@ async fn app_vendors_split(
let mut app_chunk_items = Vec::new();
let mut vendors_chunk_items = Vec::new();
for item in chunk_items {
let (_, _, _, asset_ident) = &item;
let ChunkItemWithInfo { asset_ident, .. } = &item;
if is_app_code(asset_ident) {
app_chunk_items.push(item);
} else {
......@@ -239,7 +265,7 @@ async fn package_name_split(
) -> Result<()> {
let mut map = FxIndexMap::<_, Vec<ChunkItemWithInfo>>::default();
for item in chunk_items {
let (_, _, _, asset_ident) = &item;
let ChunkItemWithInfo { asset_ident, .. } = &item;
let package_name = package_name(asset_ident);
if let Some(list) = map.get_mut(package_name) {
list.push(item);
......@@ -273,7 +299,7 @@ async fn folder_split(
let mut map = FxIndexMap::<_, (_, Vec<ChunkItemWithInfo>)>::default();
loop {
for item in chunk_items {
let (_, _, _, asset_ident) = &item;
let ChunkItemWithInfo { asset_ident, .. } = &item;
let (folder_name, new_location) = folder_name(asset_ident, location);
if let Some((_, list)) = map.get_mut(folder_name) {
list.push(item);
......@@ -316,7 +342,7 @@ async fn folder_split(
}
}
if !remaining.is_empty() {
let (_, _, _, asset_ident) = &remaining[0];
let ChunkItemWithInfo { asset_ident, .. } = &remaining[0];
let mut key = format!("{}-{}", name, &asset_ident[..location]);
if !handle_split_group(&mut remaining, &mut key, split_context, None).await? {
make_chunk(remaining, &mut key, split_context).await?;
......@@ -365,7 +391,7 @@ enum ChunkSize {
/// large or perfect fit.
fn chunk_size(chunk_items: &[ChunkItemWithInfo]) -> ChunkSize {
let mut total_size = 0;
for (_, _, size, _) in chunk_items {
for ChunkItemWithInfo { size, .. } in chunk_items {
total_size += size;
}
if total_size >= LARGE_CHUNK {
......
......@@ -19,6 +19,11 @@ pub trait Module: Asset {
fn additional_layers_modules(self: Vc<Self>) -> Vc<Modules> {
Vc::cell(vec![])
}
/// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc.
fn is_self_async(self: Vc<Self>) -> Vc<bool> {
Vc::cell(false)
}
}
#[turbo_tasks::value(transparent)]
......
......@@ -3,7 +3,7 @@ pub mod source_map;
use std::fmt::Write;
use anyhow::{bail, Context, Result};
use anyhow::{bail, Result};
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Value, ValueDefault, ValueToString, Vc};
use turbo_tasks_fs::{rope::Rope, File, FileSystem};
......@@ -503,11 +503,14 @@ impl ChunkType for CssChunkType {
let content = CssChunkContent {
chunk_items: chunk_items
.iter()
.map(|(chunk_item, _async_info)| async move {
.map(async |ChunkItemWithAsyncModuleInfo { chunk_item, .. }| {
let Some(chunk_item) =
ResolvedVc::try_downcast::<Box<dyn CssChunkItem>>(*chunk_item).await?
else {
bail!("Chunk item is not an css chunk item but reporting chunk type css");
};
// CSS doesn't need to care about async_info, so we can discard it
ResolvedVc::try_downcast::<Box<dyn CssChunkItem>>(*chunk_item)
.await?
.context("Chunk item is not an css chunk item but reporting chunk type css")
Ok(chunk_item)
})
.try_join()
.await?,
......
#![feature(async_closure)]
#![feature(min_specialization)]
#![feature(box_patterns)]
#![feature(iter_intersperse)]
......
......@@ -32,19 +32,8 @@ impl AsyncLoaderChunkItem {
#[turbo_tasks::function]
pub(super) async fn chunks(&self) -> Result<Vc<OutputAssets>> {
let module = self.module.await?;
if let Some(chunk_items) = module.availability_info.available_chunk_items() {
if chunk_items
.await?
.get(
module
.inner
.as_chunk_item(*ResolvedVc::upcast(self.chunking_context))
.to_resolved()
.await?,
)
.await?
.is_some()
{
if let Some(chunk_items) = module.availability_info.available_modules() {
if chunk_items.get(*module.inner).await?.is_some() {
return Ok(Vc::cell(vec![]));
}
}
......@@ -168,7 +157,7 @@ impl ChunkItem for AsyncLoaderChunkItem {
async fn content_ident(&self) -> Result<Vc<AssetIdent>> {
let mut ident = self.module.ident();
if let Some(available_chunk_items) =
self.module.await?.availability_info.available_chunk_items()
self.module.await?.availability_info.available_modules()
{
ident = ident.with_modifier(Vc::cell(
available_chunk_items.hash().await?.to_string().into(),
......
......@@ -9,7 +9,10 @@ use turbopack_core::{
output::OutputAssets,
};
use super::{EcmascriptChunk, EcmascriptChunkContent, EcmascriptChunkItem};
use super::{
item::EcmascriptChunkItemWithAsyncInfo, EcmascriptChunk, EcmascriptChunkContent,
EcmascriptChunkItem,
};
#[turbo_tasks::value]
#[derive(Default)]
......@@ -45,18 +48,28 @@ impl ChunkType for EcmascriptChunkType {
let content = EcmascriptChunkContent {
chunk_items: chunk_items
.iter()
.map(|&(chunk_item, async_info)| async move {
let Some(chunk_item) =
ResolvedVc::try_downcast::<Box<dyn EcmascriptChunkItem>>(chunk_item)
.await?
else {
bail!(
"Chunk item is not an ecmascript chunk item but reporting chunk type \
ecmascript"
);
};
Ok((chunk_item, async_info))
})
.map(
async |ChunkItemWithAsyncModuleInfo {
ty,
chunk_item,
async_info,
}| {
let Some(chunk_item) =
ResolvedVc::try_downcast::<Box<dyn EcmascriptChunkItem>>(*chunk_item)
.await?
else {
bail!(
"Chunk item is not an ecmascript chunk item but reporting chunk \
type ecmascript"
);
};
Ok(EcmascriptChunkItemWithAsyncInfo {
ty: *ty,
chunk_item,
async_info: *async_info,
})
},
)
.try_join()
.await?,
referenced_output_assets: referenced_output_assets.await?.clone_value(),
......
use turbo_tasks::ResolvedVc;
use turbopack_core::{chunk::AsyncModuleInfo, output::OutputAsset};
use anyhow::Result;
use turbo_tasks::{ResolvedVc, Vc};
use turbopack_core::{
chunk::{ChunkItem, ChunkItemTy, ChunkItems},
output::OutputAsset,
};
use super::item::EcmascriptChunkItem;
type EcmascriptChunkItemWithAsyncInfo = (
ResolvedVc<Box<dyn EcmascriptChunkItem>>,
Option<ResolvedVc<AsyncModuleInfo>>,
);
use super::item::EcmascriptChunkItemWithAsyncInfo;
#[turbo_tasks::value(shared)]
pub struct EcmascriptChunkContent {
pub chunk_items: Vec<EcmascriptChunkItemWithAsyncInfo>,
pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkContent {
#[turbo_tasks::function]
pub async fn included_chunk_items(&self) -> Result<Vc<ChunkItems>> {
Ok(ChunkItems(
self.chunk_items
.iter()
.filter_map(|EcmascriptChunkItemWithAsyncInfo { ty, chunk_item, .. }| {
if matches!(ty, ChunkItemTy::Included) {
Some(chunk_item)
} else {
None
}
})
.map(|item| ResolvedVc::upcast::<Box<dyn ChunkItem>>(*item))
.collect(),
)
.cell())
}
}
......@@ -2,10 +2,12 @@ use std::io::Write;
use anyhow::{bail, Result};
use serde::{Deserialize, Serialize};
use turbo_tasks::{trace::TraceRawVcs, NonLocalValue, ResolvedVc, Upcast, ValueToString, Vc};
use turbo_tasks::{
trace::TraceRawVcs, NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
};
use turbo_tasks_fs::{rope::Rope, FileSystemPath};
use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkingContext},
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkItemTy, ChunkingContext},
code_builder::{fileify_source_map, Code, CodeBuilder},
error::PrettyPrintError,
issue::{code_gen::CodeGenerationIssue, IssueExt, IssueSeverity, StyledString},
......@@ -211,6 +213,15 @@ pub struct EcmascriptChunkItemOptions {
pub placeholder_for_future_extensions: (),
}
#[derive(
Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, TraceRawVcs, TaskInput, NonLocalValue,
)]
pub struct EcmascriptChunkItemWithAsyncInfo {
pub ty: ChunkItemTy,
pub chunk_item: ResolvedVc<Box<dyn EcmascriptChunkItem>>,
pub async_info: Option<ResolvedVc<AsyncModuleInfo>>,
}
#[turbo_tasks::value_trait]
pub trait EcmascriptChunkItem: ChunkItem {
fn content(self: Vc<Self>) -> Vc<EcmascriptChunkItemContent>;
......
......@@ -8,7 +8,7 @@ use std::fmt::Write;
use anyhow::{bail, Result};
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc};
use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, ValueToString, Vc};
use turbo_tasks_fs::FileSystem;
use turbopack_core::{
asset::{Asset, AssetContent},
......@@ -19,7 +19,7 @@ use turbopack_core::{
utils::{children_from_output_assets, content_to_details},
Introspectable, IntrospectableChildren,
},
output::OutputAssets,
output::{OutputAsset, OutputAssets},
server_fs::ServerFileSystem,
};
......@@ -29,7 +29,7 @@ pub use self::{
data::EcmascriptChunkData,
item::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemExt,
EcmascriptChunkItemOptions,
EcmascriptChunkItemOptions, EcmascriptChunkItemWithAsyncInfo,
},
placeable::{EcmascriptChunkPlaceable, EcmascriptExports},
};
......@@ -75,8 +75,8 @@ fn availability_root_key() -> Vc<RcStr> {
impl Chunk for EcmascriptChunk {
#[turbo_tasks::function]
async fn ident(&self) -> Result<Vc<AssetIdent>> {
let EcmascriptChunkContent { chunk_items, .. } = &*self.content.await?;
let mut common_path = if let Some((chunk_item, _)) = chunk_items.first() {
let chunk_items = &*self.content.included_chunk_items().await?;
let mut common_path = if let Some(chunk_item) = chunk_items.first() {
let path = chunk_item.asset_ident().path().to_resolved().await?;
Some((path, path.await?))
} else {
......@@ -84,7 +84,7 @@ impl Chunk for EcmascriptChunk {
};
// The included chunk items describe the chunk uniquely
for &(chunk_item, _) in chunk_items.iter() {
for &chunk_item in chunk_items.iter() {
if let Some((common_path_vc, common_path_ref)) = common_path.as_mut() {
let path = chunk_item.asset_ident().path().await?;
while !path.is_inside_or_equal_ref(common_path_ref) {
......@@ -102,7 +102,7 @@ impl Chunk for EcmascriptChunk {
let chunk_item_key = chunk_item_key().to_resolved().await?;
let assets = chunk_items
.iter()
.map(|&(chunk_item, _)| async move {
.map(|&chunk_item| async move {
Ok((
chunk_item_key,
chunk_item.content_ident().to_resolved().await?,
......@@ -136,19 +136,26 @@ impl Chunk for EcmascriptChunk {
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<OutputAssets>> {
let content = self.content.await?;
Ok(Vc::cell(content.referenced_output_assets.clone()))
let mut referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = content
.chunk_items
.iter()
.map(async |with_info| {
Ok(with_info
.chunk_item
.references()
.await?
.into_iter()
.copied())
})
.try_flat_join()
.await?;
referenced_output_assets.extend(content.referenced_output_assets.iter().copied());
Ok(Vc::cell(referenced_output_assets))
}
#[turbo_tasks::function]
async fn chunk_items(&self) -> Result<Vc<ChunkItems>> {
let EcmascriptChunkContent { chunk_items, .. } = &*self.content.await?;
Ok(ChunkItems(
chunk_items
.iter()
.map(|&(item, _)| ResolvedVc::upcast(item))
.collect(),
)
.cell())
async fn chunk_items(&self) -> Vc<ChunkItems> {
self.content.included_chunk_items()
}
}
......@@ -210,9 +217,8 @@ impl Introspectable for EcmascriptChunk {
let content = content_to_details(self.content());
let mut details = String::new();
let this = self.await?;
let chunk_content = this.content.await?;
details += "Chunk items:\n\n";
for (chunk_item, _) in chunk_content.chunk_items.iter() {
for chunk_item in this.content.included_chunk_items().await? {
writeln!(details, "- {}", chunk_item.asset_ident().to_string().await?)?;
}
details += "\nContent:\n\n";
......@@ -226,7 +232,7 @@ impl Introspectable for EcmascriptChunk {
.await?
.clone_value();
let chunk_item_module_key = chunk_item_module_key().to_resolved().await?;
for &(chunk_item, _) in self.await?.content.await?.chunk_items.iter() {
for chunk_item in self.await?.content.included_chunk_items().await? {
children.insert((
chunk_item_module_key,
IntrospectableModule::new(chunk_item.module())
......
// Needed for swc visit_ macros
#![allow(non_local_definitions)]
#![feature(async_closure)]
#![feature(box_patterns)]
#![feature(min_specialization)]
#![feature(iter_intersperse)]
......@@ -548,6 +549,15 @@ impl Module for EcmascriptModuleAsset {
let references = analyze.references.await?.iter().copied().collect();
Ok(Vc::cell(references))
}
#[turbo_tasks::function]
async fn is_self_async(self: Vc<Self>) -> Result<Vc<bool>> {
if let Some(async_module) = *self.get_async_module().await? {
Ok(async_module.is_self_async(*self.analyze().await?.references))
} else {
Ok(Vc::cell(false))
}
}
}
#[turbo_tasks::value_impl]
......@@ -643,15 +653,6 @@ impl ChunkItem for ModuleChunkItem {
fn module(&self) -> Vc<Box<dyn Module>> {
*ResolvedVc::upcast(self.module)
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
if let Some(async_module) = *self.module.get_async_module().await? {
Ok(async_module.is_self_async(*self.module.analyze().await?.references))
} else {
Ok(Vc::cell(false))
}
}
}
#[turbo_tasks::value_impl]
......@@ -735,11 +736,7 @@ impl EcmascriptModuleContent {
}
}
if let Some(async_module) = *async_module.await? {
code_gens.push(async_module.code_generation(
chunking_context,
async_module_info,
references,
));
code_gens.push(async_module.code_generation(async_module_info, references));
}
for c in code_generation.await?.iter() {
match c {
......
......@@ -64,18 +64,8 @@ impl ManifestAsyncModule {
#[turbo_tasks::function]
pub async fn manifest_chunks(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = self.await?;
if let Some(chunk_items) = this.availability_info.available_chunk_items() {
if chunk_items
.await?
.get(
this.inner
.as_chunk_item(*ResolvedVc::upcast(this.chunking_context))
.to_resolved()
.await?,
)
.await?
.is_some()
{
if let Some(chunk_items) = this.availability_info.available_modules() {
if chunk_items.get(*this.inner).await?.is_some() {
return Ok(Vc::cell(vec![]));
}
}
......@@ -92,7 +82,7 @@ impl ManifestAsyncModule {
#[turbo_tasks::function]
pub async fn content_ident(&self) -> Result<Vc<AssetIdent>> {
let mut ident = self.inner.ident();
if let Some(available_modules) = self.availability_info.available_chunk_items() {
if let Some(available_modules) = self.availability_info.available_modules() {
ident =
ident.with_modifier(Vc::cell(available_modules.hash().await?.to_string().into()));
}
......
......@@ -10,9 +10,7 @@ use turbo_tasks::{
TryJoinIterExt, Vc,
};
use turbopack_core::{
chunk::{
AsyncModuleInfo, ChunkableModule, ChunkableModuleReference, ChunkingContext, ChunkingType,
},
chunk::{AsyncModuleInfo, ChunkableModuleReference, ChunkingType},
reference::{ModuleReference, ModuleReferences},
resolve::ExternalType,
};
......@@ -102,7 +100,6 @@ impl AsyncModule {
#[turbo_tasks::function]
async fn get_async_idents(
&self,
chunking_context: Vc<Box<dyn ChunkingContext>>,
async_module_info: Vc<AsyncModuleInfo>,
references: Vc<ModuleReferences>,
) -> Result<Vc<AsyncModuleIdents>> {
......@@ -124,13 +121,9 @@ impl AsyncModule {
}
}
ReferencedAsset::Some(placeable) => {
let chunk_item = placeable
.as_chunk_item(Vc::upcast(chunking_context))
.to_resolved()
.await?;
if async_module_info
.referenced_async_modules
.contains(&chunk_item)
.contains(&ResolvedVc::upcast(*placeable))
{
referenced_asset.get_ident().await?
} else {
......@@ -194,14 +187,11 @@ impl AsyncModule {
#[turbo_tasks::function]
pub async fn code_generation(
self: Vc<Self>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
async_module_info: Option<Vc<AsyncModuleInfo>>,
references: Vc<ModuleReferences>,
) -> Result<Vc<CodeGeneration>> {
if let Some(async_module_info) = async_module_info {
let async_idents = self
.get_async_idents(chunking_context, async_module_info, references)
.await?;
let async_idents = self.get_async_idents(async_module_info, references).await?;
if !async_idents.is_empty() {
let idents = async_idents
......
......@@ -131,6 +131,13 @@ impl Module for CachedExternalModule {
async fn references(&self) -> Result<Vc<ModuleReferences>> {
Ok(Vc::cell(self.additional_references.clone()))
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
Ok(Vc::cell(
self.external_type == CachedExternalType::EcmaScriptViaImport,
))
}
}
#[turbo_tasks::value_impl]
......@@ -229,13 +236,6 @@ impl ChunkItem for CachedExternalModuleChunkItem {
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
Ok(Vc::cell(
self.module.await?.external_type == CachedExternalType::EcmaScriptViaImport,
))
}
}
#[turbo_tasks::value_impl]
......
......@@ -80,11 +80,11 @@ impl EcmascriptChunkItem for EcmascriptModuleFacadeChunkItem {
code_gens.push(code_gen.code_generation(*chunking_context));
}
}
code_gens.push(self.module.async_module().code_generation(
*chunking_context,
async_module_info,
references,
));
code_gens.push(
self.module
.async_module()
.code_generation(async_module_info, references),
);
code_gens.push(exports.code_generation(*chunking_context));
let code_gens = code_gens.into_iter().try_join().await?;
let code_gens = code_gens.iter().map(|cg| &**cg).collect::<Vec<_>>();
......@@ -150,18 +150,4 @@ impl ChunkItem for EcmascriptModuleFacadeChunkItem {
fn module(&self) -> Vc<Box<dyn Module>> {
*ResolvedVc::upcast(self.module)
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
let module = self.module;
let async_module = module.async_module();
let references = module.references();
let is_self_async = async_module
.resolve()
.await?
.is_self_async(references.resolve().await?)
.resolve()
.await?;
Ok(is_self_async)
}
}
......@@ -151,6 +151,19 @@ impl Module for EcmascriptModuleFacadeModule {
};
Ok(Vc::cell(references))
}
#[turbo_tasks::function]
async fn is_self_async(self: Vc<Self>) -> Result<Vc<bool>> {
let async_module = self.async_module();
let references = self.references();
let is_self_async = async_module
.resolve()
.await?
.is_self_async(references.resolve().await?)
.resolve()
.await?;
Ok(is_self_async)
}
}
#[turbo_tasks::value_impl]
......
......@@ -97,16 +97,4 @@ impl ChunkItem for EcmascriptModuleLocalsChunkItem {
fn module(&self) -> Vc<Box<dyn Module>> {
*ResolvedVc::upcast(self.module)
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
let module = self.module.await?;
let analyze = module.module.analyze().await?;
if let Some(async_module) = *analyze.async_module.await? {
let is_self_async = async_module.is_self_async(*analyze.local_references);
Ok(is_self_async)
} else {
Ok(Vc::cell(false))
}
}
}
......@@ -52,6 +52,17 @@ impl Module for EcmascriptModuleLocalsModule {
let result = self.module.analyze().await?;
Ok(*result.local_references)
}
#[turbo_tasks::function]
async fn is_self_async(&self) -> Result<Vc<bool>> {
let analyze = self.module.analyze().await?;
if let Some(async_module) = *analyze.async_module.await? {
let is_self_async = async_module.is_self_async(*analyze.local_references);
Ok(is_self_async)
} else {
Ok(Vc::cell(false))
}
}
}
#[turbo_tasks::value_impl]
......
......@@ -274,6 +274,11 @@ impl Module for EcmascriptModulePartAsset {
self.full_module.ident().with_part(*self.part)
}
#[turbo_tasks::function]
fn is_self_async(self: Vc<Self>) -> Vc<bool> {
self.is_async_module()
}
#[turbo_tasks::function]
async fn references(&self) -> Result<Vc<ModuleReferences>> {
let split_data = split_module(*self.full_module).await?;
......
......@@ -101,11 +101,6 @@ impl ChunkItem for EcmascriptModulePartChunkItem {
fn module(&self) -> Vc<Box<dyn Module>> {
*ResolvedVc::upcast(self.module)
}
#[turbo_tasks::function]
fn is_self_async(&self) -> Vc<bool> {
self.module.is_async_module()
}
}
#[turbo_tasks::value(shared)]
......
......@@ -13,7 +13,7 @@ use turbopack_core::{
version::{Version, VersionedContent},
};
use turbopack_ecmascript::{
chunk::{EcmascriptChunkContent, EcmascriptChunkItemExt},
chunk::{EcmascriptChunkContent, EcmascriptChunkItemExt, EcmascriptChunkItemWithAsyncInfo},
minify::minify,
utils::StringifyJs,
};
......@@ -52,12 +52,18 @@ pub(super) async fn chunk_items(
.await?
.chunk_items
.iter()
.map(|&(chunk_item, async_module_info)| async move {
Ok((
chunk_item.id().await?,
chunk_item.code(async_module_info.map(|info| *info)).await?,
))
})
.map(
async |&EcmascriptChunkItemWithAsyncInfo {
chunk_item,
async_info,
..
}| {
Ok((
chunk_item.id().await?,
chunk_item.code(async_info.map(|info| *info)).await?,
))
},
)
.try_join()
.await
}
......
#![feature(async_closure)]
#![feature(iter_intersperse)]
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
......
......@@ -136,6 +136,11 @@ impl Module for WebAssemblyModuleAsset {
fn references(self: Vc<Self>) -> Vc<ModuleReferences> {
self.loader().references()
}
#[turbo_tasks::function]
fn is_self_async(self: Vc<Self>) -> Vc<bool> {
Vc::cell(true)
}
}
#[turbo_tasks::value_impl]
......@@ -229,11 +234,6 @@ impl ChunkItem for ModuleChunkItem {
fn module(&self) -> Vc<Box<dyn Module>> {
Vc::upcast(*self.module)
}
#[turbo_tasks::function]
fn is_self_async(self: Vc<Self>) -> Vc<bool> {
Vc::cell(true)
}
}
#[turbo_tasks::value_impl]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment