Unverified Commit 6f936688 authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

Turbopack: thread tracing through to resolve results (#72314)

The bread and butter of NFT. The next two PRs split the work into two halves, one half plumbs as far as ResolveResultItem, and the next one fills in the remaining (and fixes some bugs).


Closes PACK-3379
parent b6b5ab88
......@@ -2,7 +2,7 @@ use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::resolve::{options::ImportMapping, ExternalType};
use turbopack_core::resolve::{options::ImportMapping, ExternalTraced, ExternalType};
use crate::next_import_map::get_next_package;
......@@ -29,6 +29,7 @@ pub async fn get_external_next_compiled_package_mapping(
Ok(ImportMapping::Alternatives(vec![ImportMapping::External(
Some(format!("next/dist/compiled/{}", &*package_name.await?).into()),
ExternalType::CommonJs,
ExternalTraced::Traced,
)
.resolved_cell()])
.cell())
......
......@@ -11,7 +11,7 @@ use turbopack_core::{
options::{ConditionValue, ImportMap, ImportMapping, ResolvedMap},
parse::Request,
pattern::Pattern,
resolve, AliasPattern, ExternalType, ResolveAliasMap, SubpathValue,
resolve, AliasPattern, ExternalTraced, ExternalType, ResolveAliasMap, SubpathValue,
},
source::Source,
};
......@@ -249,15 +249,20 @@ pub async fn get_next_build_import_map() -> Result<Vc<ImportMap>> {
next_js_fs().root().to_resolved().await?,
);
let external = ImportMapping::External(None, ExternalType::CommonJs).resolved_cell();
let external = ImportMapping::External(None, ExternalType::CommonJs, ExternalTraced::Traced)
.resolved_cell();
import_map.insert_exact_alias("next", external);
import_map.insert_wildcard_alias("next/", external);
import_map.insert_exact_alias("styled-jsx", external);
import_map.insert_exact_alias(
"styled-jsx/style",
ImportMapping::External(Some("styled-jsx/style.js".into()), ExternalType::CommonJs)
.resolved_cell(),
ImportMapping::External(
Some("styled-jsx/style.js".into()),
ExternalType::CommonJs,
ExternalTraced::Traced,
)
.resolved_cell(),
);
import_map.insert_wildcard_alias("styled-jsx/", external);
......@@ -322,7 +327,13 @@ pub async fn get_next_server_import_map(
let ty = ty.into_value();
let external = ImportMapping::External(None, ExternalType::CommonJs).resolved_cell();
let external = ImportMapping::External(
None,
ExternalType::CommonJs,
// TODO(arlyon): wiring up in a follow up PR
ExternalTraced::Untraced,
)
.resolved_cell();
import_map.insert_exact_alias("next/dist/server/require-hook", external);
match ty {
......@@ -337,8 +348,12 @@ pub async fn get_next_server_import_map(
import_map.insert_exact_alias("styled-jsx", external);
import_map.insert_exact_alias(
"styled-jsx/style",
ImportMapping::External(Some("styled-jsx/style.js".into()), ExternalType::CommonJs)
.resolved_cell(),
ImportMapping::External(
Some("styled-jsx/style.js".into()),
ExternalType::CommonJs,
ExternalTraced::Traced,
)
.resolved_cell(),
);
import_map.insert_wildcard_alias("styled-jsx/", external);
// TODO: we should not bundle next/dist/build/utils in the pages renderer at all
......@@ -563,12 +578,12 @@ async fn insert_next_server_special_aliases(
let external_cjs_if_node =
move |context_dir: ResolvedVc<FileSystemPath>, request: &str| match runtime {
NextRuntime::Edge => request_to_import_mapping(context_dir, request),
NextRuntime::NodeJs => external_request_to_cjs_import_mapping(request),
NextRuntime::NodeJs => external_request_to_cjs_import_mapping(context_dir, request),
};
let external_esm_if_node =
move |context_dir: ResolvedVc<FileSystemPath>, request: &str| match runtime {
NextRuntime::Edge => request_to_import_mapping(context_dir, request),
NextRuntime::NodeJs => external_request_to_esm_import_mapping(request),
NextRuntime::NodeJs => external_request_to_esm_import_mapping(context_dir, request),
};
import_map.insert_exact_alias(
......@@ -1120,12 +1135,30 @@ fn request_to_import_mapping(
/// Creates a direct import mapping to the result of resolving an external
/// request.
fn external_request_to_cjs_import_mapping(request: &str) -> ResolvedVc<ImportMapping> {
ImportMapping::External(Some(request.into()), ExternalType::CommonJs).resolved_cell()
fn external_request_to_cjs_import_mapping(
context_dir: ResolvedVc<FileSystemPath>,
request: &str,
) -> ResolvedVc<ImportMapping> {
ImportMapping::PrimaryAlternativeExternal {
name: Some(request.into()),
ty: ExternalType::CommonJs,
traced: ExternalTraced::Traced,
lookup_dir: context_dir,
}
.resolved_cell()
}
/// Creates a direct import mapping to the result of resolving an external
/// request.
fn external_request_to_esm_import_mapping(request: &str) -> ResolvedVc<ImportMapping> {
ImportMapping::External(Some(request.into()), ExternalType::EcmaScriptModule).resolved_cell()
fn external_request_to_esm_import_mapping(
context_dir: ResolvedVc<FileSystemPath>,
request: &str,
) -> ResolvedVc<ImportMapping> {
ImportMapping::PrimaryAlternativeExternal {
name: Some(request.into()),
ty: ExternalType::EcmaScriptModule,
traced: ExternalTraced::Traced,
lookup_dir: context_dir,
}
.resolved_cell()
}
......@@ -13,8 +13,8 @@ use turbopack_core::{
parse::Request,
pattern::Pattern,
plugin::{AfterResolvePlugin, AfterResolvePluginCondition},
resolve, ExternalType, FindContextFileResult, ResolveResult, ResolveResultItem,
ResolveResultOption,
resolve, ExternalTraced, ExternalType, FindContextFileResult, ResolveResult,
ResolveResultItem, ResolveResultOption,
},
source::Source,
};
......@@ -376,10 +376,12 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin {
(FileType::CommonJs, false) => {
// mark as external
Ok(ResolveResultOption::some(
ResolveResult::primary(ResolveResultItem::External(
request_str.into(),
ExternalType::CommonJs,
))
ResolveResult::primary(ResolveResultItem::External {
name: request_str.into(),
ty: ExternalType::CommonJs,
// TODO(arlyon): wiring up in a follow up PR
traced: ExternalTraced::Untraced,
})
.cell(),
))
}
......@@ -413,14 +415,16 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin {
} else {
// mark as external
Ok(ResolveResultOption::some(
ResolveResult::primary(ResolveResultItem::External(
request_str.into(),
if resolves_equal {
ResolveResult::primary(ResolveResultItem::External {
name: request_str.into(),
ty: if resolves_equal {
ExternalType::CommonJs
} else {
ExternalType::EcmaScriptModule
},
))
// TODO(arlyon): wiring up in a follow up PR
traced: ExternalTraced::Untraced,
})
.cell(),
))
}
......@@ -428,10 +432,12 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin {
(FileType::EcmaScriptModule, true) => {
// mark as external
Ok(ResolveResultOption::some(
ResolveResult::primary(ResolveResultItem::External(
request_str.into(),
ExternalType::EcmaScriptModule,
))
ResolveResult::primary(ResolveResultItem::External {
name: request_str.into(),
ty: ExternalType::EcmaScriptModule,
// TODO(arlyon): wiring up in a follow up PR
traced: ExternalTraced::Untraced,
})
.cell(),
))
}
......
......@@ -16,7 +16,7 @@ use turbopack_core::{
AfterResolvePlugin, AfterResolvePluginCondition, BeforeResolvePlugin,
BeforeResolvePluginCondition,
},
ExternalType, ResolveResult, ResolveResultItem, ResolveResultOption,
ExternalTraced, ExternalType, ResolveResult, ResolveResultItem, ResolveResultOption,
},
};
......@@ -205,14 +205,14 @@ pub(crate) fn get_invalid_styled_jsx_resolve_plugin(
#[turbo_tasks::value]
pub(crate) struct NextExternalResolvePlugin {
root: Vc<FileSystemPath>,
project_path: ResolvedVc<FileSystemPath>,
}
#[turbo_tasks::value_impl]
impl NextExternalResolvePlugin {
#[turbo_tasks::function]
pub fn new(root: Vc<FileSystemPath>) -> Vc<Self> {
NextExternalResolvePlugin { root }.cell()
pub fn new(project_path: ResolvedVc<FileSystemPath>) -> Vc<Self> {
NextExternalResolvePlugin { project_path }.cell()
}
}
......@@ -221,7 +221,7 @@ impl AfterResolvePlugin for NextExternalResolvePlugin {
#[turbo_tasks::function]
fn after_resolve_condition(&self) -> Vc<AfterResolvePluginCondition> {
AfterResolvePluginCondition::new(
self.root.root(),
self.project_path.root(),
Glob::new("**/next/dist/**/*.{external,runtime.dev,runtime.prod}.js".into()),
)
}
......@@ -234,18 +234,20 @@ impl AfterResolvePlugin for NextExternalResolvePlugin {
_reference_type: Value<ReferenceType>,
_request: Vc<Request>,
) -> Result<Vc<ResolveResultOption>> {
let raw_fs_path = &*fs_path.await?;
let path = raw_fs_path.path.to_string();
let path = fs_path.await?.path.to_string();
// Find the starting index of 'next/dist' and slice from that point. It should
// always be found since the glob pattern above is specific enough.
let starting_index = path.find("next/dist").unwrap();
let specifier = &path[starting_index..];
// Replace '/esm/' with '/' to match the CJS version of the file.
let modified_path = path[starting_index..].replace("/esm/", "/");
let specifier: RcStr = specifier.replace("/esm/", "/").into();
Ok(Vc::cell(Some(
ResolveResult::primary(ResolveResultItem::External(
modified_path.into(),
ExternalType::CommonJs,
))
ResolveResult::primary(ResolveResultItem::External {
name: specifier.clone(),
ty: ExternalType::CommonJs,
traced: ExternalTraced::Traced,
})
.into(),
)))
}
......
......@@ -3,10 +3,10 @@ use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack::module_options::WebpackLoadersOptions;
use turbopack_core::resolve::options::ImportMapping;
use turbopack_core::resolve::{options::ImportMapping, ExternalTraced, ExternalType};
use self::{babel::maybe_add_babel_loader, sass::maybe_add_sass_loader};
use crate::{next_build::get_external_next_compiled_package_mapping, next_config::NextConfig};
use crate::next_config::NextConfig;
pub(crate) mod babel;
pub(crate) mod sass;
......@@ -38,6 +38,12 @@ pub async fn webpack_loader_options(
}
#[turbo_tasks::function]
fn loader_runner_package_mapping() -> Vc<ImportMapping> {
get_external_next_compiled_package_mapping(Vc::cell("loader-runner".into()))
async fn loader_runner_package_mapping() -> Result<Vc<ImportMapping>> {
Ok(ImportMapping::Alternatives(vec![ImportMapping::External(
Some("next/dist/compiled/loader-runner".into()),
ExternalType::CommonJs,
ExternalTraced::Untraced,
)
.resolved_cell()])
.cell())
}
......@@ -2,16 +2,20 @@ use std::collections::HashSet;
use anyhow::Result;
use auto_hash_map::AutoSet;
use futures::future::try_join_all;
use turbo_tasks::{
FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc,
};
use turbo_tasks_fs::{FileSystem, VirtualFileSystem};
use super::{
availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItemInfo,
chunk_content, chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem,
ChunkingContext,
};
use crate::{module::Module, output::OutputAssets, reference::ModuleReference};
use crate::{
module::Module, output::OutputAssets, rebase::RebasedAsset, reference::ModuleReference,
};
pub struct MakeChunkGroupResult {
pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
......@@ -27,6 +31,7 @@ pub async fn make_chunk_group(
let ChunkContentResult {
chunk_items,
async_modules,
traced_modules,
external_module_references,
forward_edges_inherit_async,
local_back_edges_inherit_async,
......@@ -143,12 +148,24 @@ pub async fn make_chunk_group(
.flat_map(|references| references.iter().copied())
.collect();
let mut referenced_output_assets = references_to_output_assets(external_module_references)
.await?
.await?
.clone_value();
let rebased_modules = try_join_all(traced_modules.into_iter().map(|module| {
RebasedAsset::new(*module, module.ident().path().root(), traced_fs().root()).to_resolved()
}))
.await?;
referenced_output_assets.extend(rebased_modules.into_iter().map(ResolvedVc::upcast));
// Pass chunk items to chunking algorithm
let mut chunks = make_chunks(
chunking_context,
Vc::cell(chunk_items.into_iter().collect()),
"".into(),
references_to_output_assets(external_module_references).await?,
Vc::cell(referenced_output_assets),
)
.await?
.clone_value();
......@@ -181,6 +198,12 @@ pub async fn make_chunk_group(
})
}
// Without this wrapper, VirtualFileSystem::new_with_name always returns a new filesystem
#[turbo_tasks::function]
fn traced_fs() -> Vc<VirtualFileSystem> {
VirtualFileSystem::new_with_name("traced".into())
}
async fn references_to_output_assets(
references: FxIndexSet<Vc<Box<dyn ModuleReference>>>,
) -> Result<Vc<OutputAssets>> {
......
......@@ -16,7 +16,7 @@ use std::{
hash::Hash,
};
use anyhow::Result;
use anyhow::{bail, Result};
use auto_hash_map::AutoSet;
use serde::{Deserialize, Serialize};
use tracing::{info_span, Span};
......@@ -154,7 +154,17 @@ pub trait OutputChunk: Asset {
/// Specifies how a chunk interacts with other chunks when building a chunk
/// group
#[derive(
Copy, Default, Clone, Hash, TraceRawVcs, Serialize, Deserialize, Eq, PartialEq, ValueDebugFormat,
Copy,
Debug,
Default,
Clone,
Hash,
TraceRawVcs,
Serialize,
Deserialize,
Eq,
PartialEq,
ValueDebugFormat,
)]
pub enum ChunkingType {
/// Module is placed in the same chunk group and is loaded in parallel. It
......@@ -167,15 +177,18 @@ pub enum ChunkingType {
/// An async loader is placed into the referencing chunk and loads the
/// separate chunk group in which the module is placed.
Async,
/// Module not placed in chunk group, but its references are still followed.
/// Module not placed in chunk group, but its references are still followed and placed into the
/// chunk group.
Passthrough,
// Module not placed in chunk group, but its references are still followed.
Traced,
}
#[turbo_tasks::value(transparent)]
pub struct ChunkingTypeOption(Option<ChunkingType>);
/// A [ModuleReference] implementing this trait and returning true for
/// [ChunkableModuleReference::is_chunkable] are considered as potentially
/// A [ModuleReference] implementing this trait and returning Some(_) for
/// [ChunkableModuleReference::chunking_type] are considered as potentially
/// chunkable references. When all [Module]s of such a reference implement
/// [ChunkableModule] they are placed in [Chunk]s during chunking.
/// They are even potentially placed in the same [Chunk] when a chunk type
......@@ -192,6 +205,7 @@ type AsyncInfo = FxIndexMap<Vc<Box<dyn ChunkItem>>, Vec<Vc<Box<dyn ChunkItem>>>>
pub struct ChunkContentResult {
pub chunk_items: FxIndexSet<Vc<Box<dyn ChunkItem>>>,
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
pub external_module_references: FxIndexSet<Vc<Box<dyn ModuleReference>>>,
/// A map from local module to all children from which the async module
/// status is inherited
......@@ -240,6 +254,10 @@ enum ChunkContentGraphNode {
AsyncModule {
module: Vc<Box<dyn ChunkableModule>>,
},
// Module that is referenced as traced and will be turned into a separate RebasedAsset
TracedModule {
module: Vc<Box<dyn Module>>,
},
// ModuleReferences that are not placed in the current chunk group
ExternalModuleReference(ResolvedVc<Box<dyn ModuleReference>>),
/// A list of directly referenced chunk items from which `is_async_module`
......@@ -370,6 +388,20 @@ async fn graph_node_to_referenced_nodes(
.await?
.into_iter()
.map(|&module| async move {
if chunking_type == ChunkingType::Traced {
if *chunking_context.is_tracing_enabled().await? {
return Ok((
Some(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::TracedModule { module: *module },
}),
None,
));
} else {
return Ok((None, None));
}
}
let Some(chunkable_module) =
ResolvedVc::try_sidecast::<Box<dyn ChunkableModule>>(module).await?
else {
......@@ -466,6 +498,9 @@ async fn graph_node_to_referenced_nodes(
))
}
}
ChunkingType::Traced => {
bail!("unreachable ChunkingType::Traced");
}
}
})
.try_join()
......@@ -622,10 +657,15 @@ async fn chunk_content_internal_parallel(
let mut forward_edges_inherit_async = FxIndexMap::default();
let mut local_back_edges_inherit_async = FxIndexMap::default();
let mut available_async_modules_back_edges_inherit_async = FxIndexMap::default();
let mut traced_modules = FxIndexSet::default();
for graph_node in graph_nodes {
match graph_node {
ChunkContentGraphNode::PassthroughChunkItem { .. } => {}
ChunkContentGraphNode::TracedModule { module } => {
let module = module.to_resolved().await?;
traced_modules.insert(module);
}
ChunkContentGraphNode::ChunkItem { item, .. } => {
chunk_items.insert(*item.to_resolved().await?);
}
......@@ -663,6 +703,7 @@ async fn chunk_content_internal_parallel(
Ok(ChunkContentResult {
chunk_items,
async_modules,
traced_modules,
external_module_references,
forward_edges_inherit_async,
local_back_edges_inherit_async,
......
......@@ -38,6 +38,11 @@ fn passthrough_reference_ty() -> Vc<RcStr> {
Vc::cell("passthrough reference".into())
}
#[turbo_tasks::function]
fn traced_reference_ty() -> Vc<RcStr> {
Vc::cell("traced reference".into())
}
#[turbo_tasks::function]
pub async fn content_to_details(content: Vc<AssetContent>) -> Result<Vc<RcStr>> {
Ok(match &*content.await? {
......@@ -77,6 +82,7 @@ pub async fn children_from_module_references(
}
Some(ChunkingType::Async) => key = async_reference_ty(),
Some(ChunkingType::Passthrough) => key = passthrough_reference_ty(),
Some(ChunkingType::Traced) => key = traced_reference_ty(),
}
}
......
......@@ -35,17 +35,17 @@ impl Issue for ResolvingIssue {
#[turbo_tasks::function]
async fn title(&self) -> Result<Vc<StyledString>> {
let module_not_found = StyledString::Strong("Module not found".into());
Ok(match self.request.await?.request() {
Some(request) => StyledString::Line(vec![
module_not_found,
StyledString::Text(": Can't resolve '".into()),
StyledString::Code(request),
StyledString::Text("'".into()),
]),
None => module_not_found,
}
let request = self
.request
.request_pattern()
.to_string()
.await?
.clone_value();
Ok(StyledString::Line(vec![
StyledString::Strong("Module not found".into()),
StyledString::Text(": Can't resolve ".into()),
StyledString::Code(request),
])
.cell())
}
......
......@@ -26,6 +26,7 @@ pub mod package_json;
pub mod proxied_asset;
pub mod raw_module;
pub mod raw_output;
pub mod rebase;
pub mod reference;
pub mod reference_type;
pub mod resolve;
......
......@@ -3,7 +3,8 @@ use std::hash::Hash;
use anyhow::Result;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
use crate::{
asset::{Asset, AssetContent},
ident::AssetIdent,
module::Module,
......
......@@ -199,6 +199,45 @@ pub async fn referenced_modules_and_affecting_sources(
Ok(Vc::cell(resolved_modules.into_iter().collect()))
}
#[turbo_tasks::value]
pub struct TracedModuleReference {
module: ResolvedVc<Box<dyn Module>>,
}
#[turbo_tasks::value_impl]
impl ModuleReference for TracedModuleReference {
#[turbo_tasks::function]
fn resolve_reference(&self) -> Vc<ModuleResolveResult> {
ModuleResolveResult::module(self.module).cell()
}
}
#[turbo_tasks::value_impl]
impl ValueToString for TracedModuleReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("traced {}", self.module.ident().to_string().await?).into(),
))
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for TracedModuleReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl TracedModuleReference {
#[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> {
Self::cell(TracedModuleReference { module })
}
}
/// Aggregates all primary [Module]s referenced by an [Module]. [AssetReference]
/// This does not include transitively references [Module]s, only includes
/// primary [Module]s referenced.
......
......@@ -41,6 +41,7 @@ use crate::{
raw_module::RawModule,
reference_type::ReferenceType,
resolve::{
node::{node_cjs_resolve_options, node_esm_resolve_options},
pattern::{read_matches, PatternMatch},
plugin::AfterResolvePlugin,
},
......@@ -68,7 +69,12 @@ use crate::{error::PrettyPrintError, issue::IssueSeverity};
pub enum ModuleResolveResultItem {
Module(ResolvedVc<Box<dyn Module>>),
OutputAsset(ResolvedVc<Box<dyn OutputAsset>>),
External(RcStr, ExternalType),
External {
/// uri, path, reference, etc.
name: RcStr,
ty: ExternalType,
traced: Option<ResolvedVc<ModuleResolveResult>>,
},
/// A module could not be created (according to the rules, e.g. no module type as assigned)
Unknown(Vc<Box<dyn Source>>),
Ignore,
......@@ -363,6 +369,22 @@ impl ModuleResolveResult {
}
}
#[derive(Copy, Clone)]
#[turbo_tasks::value(shared)]
pub enum ExternalTraced {
Untraced,
Traced,
}
impl Display for ExternalTraced {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
ExternalTraced::Untraced => write!(f, "untraced"),
ExternalTraced::Traced => write!(f, "traced"),
}
}
}
#[derive(
Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, TraceRawVcs, TaskInput,
)]
......@@ -383,10 +405,15 @@ impl Display for ExternalType {
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug)]
#[derive(Clone)]
pub enum ResolveResultItem {
Source(ResolvedVc<Box<dyn Source>>),
External(RcStr, ExternalType),
External {
/// uri, path, reference, etc.
name: RcStr,
ty: ExternalType,
traced: ExternalTraced,
},
Ignore,
Error(Vc<RcStr>),
Empty,
......@@ -437,7 +464,7 @@ impl RequestKey {
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug)]
#[derive(Clone)]
pub struct ResolveResult {
pub primary: FxIndexMap<RequestKey, ResolveResultItem>,
pub affecting_sources: Vec<ResolvedVc<Box<dyn Source>>>,
......@@ -466,10 +493,14 @@ impl ValueToString for ResolveResult {
ResolveResultItem::Source(a) => {
result.push_str(&a.ident().to_string().await?);
}
ResolveResultItem::External(s, ty) => {
ResolveResultItem::External {
name: s,
ty,
traced,
} => {
result.push_str("external ");
result.push_str(s);
write!(result, " ({})", ty)?;
write!(result, " ({}, {})", ty, traced)?;
}
ResolveResultItem::Ignore => {
result.push_str("ignore");
......@@ -667,9 +698,17 @@ impl ResolveResult {
request,
match item {
ResolveResultItem::Source(source) => asset_fn(source).await?,
ResolveResultItem::External(s, ty) => {
ModuleResolveResultItem::External(s, ty)
}
ResolveResultItem::External {
name,
ty,
// TODO remove this whole function? it's easy to drop traced
// externals now
traced: _,
} => ModuleResolveResultItem::External {
name,
ty,
traced: None,
},
ResolveResultItem::Ignore => ModuleResolveResultItem::Ignore,
ResolveResultItem::Empty => ModuleResolveResultItem::Empty,
ResolveResultItem::Error(e) => {
......@@ -690,6 +729,29 @@ impl ResolveResult {
})
}
pub async fn map_primary_items<A, AF>(&self, item_fn: A) -> Result<ModuleResolveResult>
where
A: Fn(ResolveResultItem) -> AF,
AF: Future<Output = Result<ModuleResolveResultItem>>,
{
Ok(ModuleResolveResult {
primary: self
.primary
.iter()
.map(|(request, item)| {
let asset_fn = &item_fn;
let request = request.clone();
let item = item.clone();
async move { Ok((request, asset_fn(item).await?)) }
})
.try_join()
.await?
.into_iter()
.collect(),
affecting_sources: self.affecting_sources.clone(),
})
}
/// Returns a new [ResolveResult] where all [RequestKey]s are set to the
/// passed `request`.
pub fn with_request_ref(&self, request: RcStr) -> Self {
......@@ -1857,7 +1919,11 @@ async fn resolve_internal_inline(
let uri: RcStr = format!("{}{}", protocol, remainder).into();
ResolveResult::primary_with_key(
RequestKey::new(uri.clone()),
ResolveResultItem::External(uri, ExternalType::Url),
ResolveResultItem::External {
name: uri,
ty: ExternalType::Url,
traced: ExternalTraced::Untraced,
},
)
.into()
}
......@@ -2556,6 +2622,58 @@ async fn resolve_import_map_result(
))
}
}
ImportMapResult::External(name, ty, traced) => Some(
ResolveResult::primary(ResolveResultItem::External {
name: name.clone(),
ty: *ty,
traced: *traced,
})
.cell(),
),
ImportMapResult::AliasExternal {
name,
ty,
traced,
lookup_dir: alias_lookup_path,
} => {
let request = Request::parse_string(name.clone());
// We must avoid cycles during resolving
if request.resolve().await? == *original_request
&& alias_lookup_path.to_resolved().await? == original_lookup_path
{
None
} else {
let is_external_resolvable = !resolve_internal(
**alias_lookup_path,
request,
match ty {
ExternalType::Url => options,
// TODO is that root correct?
ExternalType::CommonJs => {
node_cjs_resolve_options(alias_lookup_path.root())
}
ExternalType::EcmaScriptModule => {
node_esm_resolve_options(alias_lookup_path.root())
}
},
)
.await?
.is_unresolvable_ref();
if is_external_resolvable {
Some(
ResolveResult::primary(ResolveResultItem::External {
name: name.clone(),
ty: *ty,
traced: *traced,
})
.cell(),
)
} else {
None
}
}
}
ImportMapResult::Alternatives(list) => {
let results = list
.iter()
......
......@@ -15,7 +15,7 @@ use super::{
plugin::BeforeResolvePlugin,
AliasPattern, ExternalType, ResolveResult, ResolveResultItem,
};
use crate::resolve::{parse::Request, plugin::AfterResolvePlugin};
use crate::resolve::{parse::Request, plugin::AfterResolvePlugin, ExternalTraced};
#[turbo_tasks::value(shared)]
#[derive(Hash, Debug)]
......@@ -94,7 +94,18 @@ pub enum ResolveInPackage {
#[turbo_tasks::value(shared)]
#[derive(Clone)]
pub enum ImportMapping {
External(Option<RcStr>, ExternalType),
/// If specified, the optional name overrides the request, importing that external instead.
External(Option<RcStr>, ExternalType, ExternalTraced),
/// Try to make the request external if it is is resolvable from the specified
/// directory, and fall back to resolving the original request if it fails.
///
/// If specified, the optional name overrides the request, importing that external instead.
PrimaryAlternativeExternal {
name: Option<RcStr>,
ty: ExternalType,
traced: ExternalTraced,
lookup_dir: ResolvedVc<FileSystemPath>,
},
/// An already resolved result that will be returned directly.
Direct(ResolvedVc<ResolveResult>),
/// A request alias that will be resolved first, and fall back to resolving
......@@ -112,7 +123,13 @@ pub enum ImportMapping {
#[turbo_tasks::value(shared)]
#[derive(Clone)]
pub enum ReplacedImportMapping {
External(Option<RcStr>, ExternalType),
External(Option<RcStr>, ExternalType, ExternalTraced),
PrimaryAlternativeExternal {
name: Option<RcStr>,
ty: ExternalType,
traced: ExternalTraced,
lookup_dir: ResolvedVc<FileSystemPath>,
},
Direct(ResolvedVc<ResolveResult>),
PrimaryAlternative(Pattern, Option<ResolvedVc<FileSystemPath>>),
Ignore,
......@@ -160,11 +177,22 @@ impl AliasTemplate for Vc<ImportMapping> {
Box::pin(async move {
let this = &*self.await?;
Ok(match this {
ImportMapping::External(name, ty) => {
ReplacedImportMapping::External(name.clone(), *ty)
ImportMapping::External(name, ty, traced) => {
ReplacedImportMapping::External(name.clone(), *ty, *traced)
}
ImportMapping::PrimaryAlternative(name, context) => {
ReplacedImportMapping::PrimaryAlternative((*name).clone().into(), *context)
ImportMapping::PrimaryAlternativeExternal {
name,
ty,
traced,
lookup_dir,
} => ReplacedImportMapping::PrimaryAlternativeExternal {
name: name.clone(),
ty: *ty,
traced: *traced,
lookup_dir: *lookup_dir,
},
ImportMapping::PrimaryAlternative(name, lookup_dir) => {
ReplacedImportMapping::PrimaryAlternative((*name).clone().into(), *lookup_dir)
}
ImportMapping::Direct(v) => ReplacedImportMapping::Direct(*v),
ImportMapping::Ignore => ReplacedImportMapping::Ignore,
......@@ -189,20 +217,43 @@ impl AliasTemplate for Vc<ImportMapping> {
Box::pin(async move {
let this = &*self.await?;
Ok(match this {
ImportMapping::External(name, ty) => {
ImportMapping::External(name, ty, traced) => {
if let Some(name) = name {
ReplacedImportMapping::External(
capture.spread_into_star(name).as_string().map(|s| s.into()),
*ty,
*traced,
)
} else {
ReplacedImportMapping::External(None, *ty)
ReplacedImportMapping::External(None, *ty, *traced)
}
}
ImportMapping::PrimaryAlternativeExternal {
name,
ty,
traced,
lookup_dir,
} => {
if let Some(name) = name {
ReplacedImportMapping::PrimaryAlternativeExternal {
name: capture.spread_into_star(name).as_string().map(|s| s.into()),
ty: *ty,
traced: *traced,
lookup_dir: *lookup_dir,
}
} else {
ReplacedImportMapping::PrimaryAlternativeExternal {
name: None,
ty: *ty,
traced: *traced,
lookup_dir: *lookup_dir,
}
}
}
ImportMapping::PrimaryAlternative(name, context) => {
ImportMapping::PrimaryAlternative(name, lookup_dir) => {
ReplacedImportMapping::PrimaryAlternative(
capture.spread_into_star(name),
*context,
*lookup_dir,
)
}
ImportMapping::Direct(v) => ReplacedImportMapping::Direct(*v),
......@@ -324,9 +375,16 @@ pub struct ResolvedMap {
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug)]
#[derive(Clone)]
pub enum ImportMapResult {
Result(ResolvedVc<ResolveResult>),
External(RcStr, ExternalType, ExternalTraced),
AliasExternal {
name: RcStr,
ty: ExternalType,
traced: ExternalTraced,
lookup_dir: ResolvedVc<FileSystemPath>,
},
Alias(ResolvedVc<Request>, Option<ResolvedVc<FileSystemPath>>),
Alternatives(Vec<ImportMapResult>),
NoEntry,
......@@ -339,16 +397,34 @@ async fn import_mapping_to_result(
) -> Result<ImportMapResult> {
Ok(match &*mapping.await? {
ReplacedImportMapping::Direct(result) => ImportMapResult::Result(*result),
ReplacedImportMapping::External(name, ty) => ImportMapResult::Result(
ResolveResult::primary(if let Some(name) = name {
ResolveResultItem::External(name.clone(), *ty)
ReplacedImportMapping::External(name, ty, traced) => ImportMapResult::External(
if let Some(name) = name {
name.clone()
} else if let Some(request) = request.await?.request() {
ResolveResultItem::External(request, *ty)
request
} else {
bail!("Cannot resolve external reference without request")
})
.resolved_cell(),
},
*ty,
*traced,
),
ReplacedImportMapping::PrimaryAlternativeExternal {
name,
ty,
traced,
lookup_dir,
} => ImportMapResult::AliasExternal {
name: if let Some(name) = name {
name.clone()
} else if let Some(request) = request.await?.request() {
request
} else {
bail!("Cannot resolve external reference without request")
},
ty: *ty,
traced: *traced,
lookup_dir: *lookup_dir,
},
ReplacedImportMapping::Ignore => ImportMapResult::Result(
ResolveResult::primary(ResolveResultItem::Ignore).resolved_cell(),
),
......@@ -380,6 +456,8 @@ impl ValueToString for ImportMapResult {
async fn to_string(&self) -> Result<Vc<RcStr>> {
match self {
ImportMapResult::Result(_) => Ok(Vc::cell("Resolved by import map".into())),
ImportMapResult::External(_, _, _) => Ok(Vc::cell("TODO external".into())),
ImportMapResult::AliasExternal { .. } => Ok(Vc::cell("TODO external".into())),
ImportMapResult::Alias(request, context) => {
let s = if let Some(path) = context {
let path = path.to_string().await?;
......
......@@ -10,7 +10,9 @@ use regex::Regex;
use serde::{Deserialize, Serialize};
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{trace::TraceRawVcs, ResolvedVc, Value, ValueToString, Vc};
use turbo_tasks::{
debug::ValueDebugFormat, trace::TraceRawVcs, ResolvedVc, Value, ValueToString, Vc,
};
use turbo_tasks_fs::{
util::normalize_path, DirectoryContent, DirectoryEntry, FileSystemEntryType, FileSystemPath,
LinkContent, LinkType,
......@@ -1268,7 +1270,7 @@ impl From<RcStr> for Pattern {
impl Display for Pattern {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Pattern::Constant(c) => write!(f, "\"{c}\""),
Pattern::Constant(c) => write!(f, "'{c}'"),
Pattern::Dynamic => write!(f, "<dynamic>"),
Pattern::Alternatives(list) => write!(
f,
......@@ -1298,7 +1300,7 @@ impl ValueToString for Pattern {
}
}
#[derive(Debug, PartialEq, Eq, Clone, TraceRawVcs, Serialize, Deserialize)]
#[derive(Debug, PartialEq, Eq, Clone, TraceRawVcs, Serialize, Deserialize, ValueDebugFormat)]
pub enum PatternMatch {
File(RcStr, Vc<FileSystemPath>),
Directory(RcStr, Vc<FileSystemPath>),
......
......@@ -4,21 +4,42 @@ use turbo_tasks_fs::FileSystemPath;
use crate::{
asset::{Asset, AssetContent},
ident::AssetIdent,
output::OutputAsset,
output::{OutputAsset, OutputAssets},
};
/// An [OutputAsset] that is created from some passed source code.
/// An [OutputAsset] that is created from some passed source code and can have a list of references
/// to other assets.
#[turbo_tasks::value]
pub struct VirtualOutputAsset {
pub path: Vc<FileSystemPath>,
pub content: Vc<AssetContent>,
pub references: Vc<OutputAssets>,
}
#[turbo_tasks::value_impl]
impl VirtualOutputAsset {
#[turbo_tasks::function]
pub fn new(path: Vc<FileSystemPath>, content: Vc<AssetContent>) -> Vc<Self> {
VirtualOutputAsset { path, content }.cell()
VirtualOutputAsset {
path,
content,
references: OutputAssets::empty(),
}
.cell()
}
#[turbo_tasks::function]
pub fn new_with_references(
path: Vc<FileSystemPath>,
content: Vc<AssetContent>,
references: Vc<OutputAssets>,
) -> Vc<Self> {
VirtualOutputAsset {
path,
content,
references,
}
.cell()
}
}
......@@ -28,6 +49,11 @@ impl OutputAsset for VirtualOutputAsset {
fn ident(&self) -> Vc<AssetIdent> {
AssetIdent::from_path(self.path)
}
#[turbo_tasks::function]
fn references(&self) -> Vc<OutputAssets> {
self.references
}
}
#[turbo_tasks::value_impl]
......
......@@ -80,7 +80,9 @@ impl ReferencedAsset {
}
for (_, result) in result.primary.iter() {
match result {
ModuleResolveResultItem::External(request, ty) => {
ModuleResolveResultItem::External {
name: request, ty, ..
} => {
return Ok(ReferencedAsset::External(request.clone(), *ty).cell());
}
&ModuleResolveResultItem::Module(module) => {
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkableModule, ChunkingContext},
ident::AssetIdent,
module::Module,
reference::ModuleReferences,
reference::{ModuleReference, ModuleReferences},
};
use crate::{
......@@ -51,15 +51,21 @@ impl Display for CachedExternalType {
pub struct CachedExternalModule {
pub request: RcStr,
pub external_type: CachedExternalType,
pub additional_references: Vec<ResolvedVc<Box<dyn ModuleReference>>>,
}
#[turbo_tasks::value_impl]
impl CachedExternalModule {
#[turbo_tasks::function]
pub fn new(request: RcStr, external_type: CachedExternalType) -> Vc<Self> {
pub fn new(
request: RcStr,
external_type: CachedExternalType,
additional_references: Vec<ResolvedVc<Box<dyn ModuleReference>>>,
) -> Vc<Self> {
Self::cell(CachedExternalModule {
request,
external_type,
additional_references,
})
}
......@@ -104,7 +110,7 @@ impl Module for CachedExternalModule {
fn ident(&self) -> Vc<AssetIdent> {
let fs = VirtualFileSystem::new_with_name("externals".into());
AssetIdent::from_path(fs.root())
AssetIdent::from_path(fs.root().join(self.request.clone()))
.with_layer(layer())
.with_modifier(Vc::cell(self.request.clone()))
.with_modifier(Vc::cell(self.external_type.to_string().into()))
......@@ -180,6 +186,12 @@ pub struct CachedExternalModuleChunkItem {
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
// Without this wrapper, VirtualFileSystem::new_with_name always returns a new filesystem
#[turbo_tasks::function]
fn external_fs() -> Vc<VirtualFileSystem> {
VirtualFileSystem::new_with_name("externals".into())
}
#[turbo_tasks::value_impl]
impl ChunkItem for CachedExternalModuleChunkItem {
#[turbo_tasks::function]
......@@ -188,8 +200,15 @@ impl ChunkItem for CachedExternalModuleChunkItem {
}
#[turbo_tasks::function]
fn references(&self) -> Vc<ModuleReferences> {
self.module.references()
async fn references(&self) -> Result<Vc<ModuleReferences>> {
let additional_references = &self.module.await?.additional_references;
if !additional_references.is_empty() {
let mut module_references = self.module.references().await?.clone_value();
module_references.extend(additional_references.iter().map(|rvc| **rvc));
Ok(Vc::cell(module_references))
} else {
Ok(self.module.references())
}
}
#[turbo_tasks::function]
......
......@@ -316,7 +316,7 @@ async fn to_single_pattern_mapping(
) -> Result<SinglePatternMapping> {
let module = match resolve_item {
ModuleResolveResultItem::Module(module) => *module,
ModuleResolveResultItem::External(s, ty) => {
ModuleResolveResultItem::External { name: s, ty, .. } => {
return Ok(SinglePatternMapping::External(s.clone(), *ty));
}
ModuleResolveResultItem::Ignore => return Ok(SinglePatternMapping::Ignored),
......
......@@ -212,9 +212,7 @@ async fn extra_configs_changed(
)
.try_into_module()
.await?
.as_deref()
.copied()
.map(any_content_changed_of_module)
.map(|rvc| any_content_changed_of_module(*rvc))
} else {
None
},
......
......@@ -7,7 +7,7 @@ use turbopack_core::resolve::{
ConditionValue, ImportMap, ImportMapping, ResolutionConditions, ResolveInPackage,
ResolveIntoPackage, ResolveModules, ResolveOptions,
},
AliasMap, AliasPattern, ExternalType, FindContextFileResult,
AliasMap, AliasPattern, ExternalTraced, ExternalType, FindContextFileResult,
};
use crate::{
......@@ -106,11 +106,13 @@ async fn base_resolve_options(
for req in NODE_EXTERNALS {
direct_mappings.insert(
AliasPattern::exact(req),
ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(),
ImportMapping::External(None, ExternalType::CommonJs, ExternalTraced::Untraced)
.resolved_cell(),
);
direct_mappings.insert(
AliasPattern::exact(format!("node:{req}")),
ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(),
ImportMapping::External(None, ExternalType::CommonJs, ExternalTraced::Untraced)
.resolved_cell(),
);
}
}
......@@ -118,12 +120,17 @@ async fn base_resolve_options(
for req in EDGE_NODE_EXTERNALS {
direct_mappings.insert(
AliasPattern::exact(req),
ImportMapping::External(Some(format!("node:{req}").into()), ExternalType::CommonJs)
.resolved_cell(),
ImportMapping::External(
Some(format!("node:{req}").into()),
ExternalType::CommonJs,
ExternalTraced::Untraced,
)
.resolved_cell(),
);
direct_mappings.insert(
AliasPattern::exact(format!("node:{req}")),
ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(),
ImportMapping::External(None, ExternalType::CommonJs, ExternalTraced::Untraced)
.resolved_cell(),
);
}
}
......
......@@ -38,7 +38,7 @@ use turbopack_core::{
reference_type::{InnerAssets, ReferenceType},
resolve::{
options::{ImportMap, ImportMapping},
ExternalType,
ExternalTraced, ExternalType,
},
source::Source,
};
......@@ -288,7 +288,12 @@ async fn run_test(prepared_test: Vc<PreparedTest>) -> Result<Vc<RunTestResult>>
let mut import_map = ImportMap::empty();
import_map.insert_wildcard_alias(
"esm-external/",
ImportMapping::External(Some("*".into()), ExternalType::EcmaScriptModule).resolved_cell(),
ImportMapping::External(
Some("*".into()),
ExternalType::EcmaScriptModule,
ExternalTraced::Untraced,
)
.resolved_cell(),
);
let asset_context: Vc<Box<dyn AssetContext>> = Vc::upcast(ModuleAssetContext::new(
......
......@@ -14,7 +14,7 @@ error - [resolve] [project]/turbopack/crates/turbopack-tests/tests/execution/tur
| It was not possible to find the requested file.
| Parsed request as written in source code: relative "./not-existing-file"
| Parsed request as written in source code: relative './not-existing-file'
| Path where resolving has started: [project]/turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/comptime/input/index.js
| Type of request: commonjs request
|
\ No newline at end of file
......@@ -10,7 +10,7 @@ error - [resolve] [project]/turbopack/crates/turbopack-tests/tests/snapshot/impo
| It was not possible to find the requested file.
| Parsed request as written in source code: module "does-not-exist" with subpath "/path"
| Parsed request as written in source code: module "does-not-exist" with subpath '/path'
| Path where resolving has started: [project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/resolve_error_cjs/input/index.js
| Type of request: commonjs request
|
\ No newline at end of file
......@@ -11,7 +11,7 @@ error - [resolve] [project]/turbopack/crates/turbopack-tests/tests/snapshot/impo
| It was not possible to find the requested file.
| Parsed request as written in source code: module "does-not-exist" with subpath "/path"
| Parsed request as written in source code: module "does-not-exist" with subpath '/path'
| Path where resolving has started: [project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/resolve_error_esm/input/index.js
| Type of request: EcmaScript Modules request
|
\ No newline at end of file
......@@ -9,7 +9,6 @@ use turbo_tasks_memory::MemoryBackend;
use turbopack::{
emit_with_completion,
module_options::{EcmascriptOptionsContext, ModuleOptionsContext},
rebase::RebasedAsset,
register, ModuleAssetContext,
};
use turbopack_core::{
......@@ -17,6 +16,7 @@ use turbopack_core::{
context::AssetContext,
environment::{Environment, ExecutionEnvironment, NodeJsEnvironment},
file_source::FileSource,
rebase::RebasedAsset,
reference_type::ReferenceType,
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......
......@@ -12,12 +12,13 @@ use turbo_rcstr::RcStr;
use turbo_tasks::{util::FormatDuration, ReadConsistency, TurboTasks, UpdateInfo, Value, Vc};
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbo_tasks_memory::MemoryBackend;
use turbopack::{emit_with_completion, rebase::RebasedAsset, register};
use turbopack::{emit_with_completion, register};
use turbopack_core::{
compile_time_info::CompileTimeInfo,
context::AssetContext,
environment::{Environment, ExecutionEnvironment, NodeJsEnvironment},
file_source::FileSource,
rebase::RebasedAsset,
PROJECT_FILESYSTEM_NAME,
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......
......@@ -10,7 +10,6 @@
pub mod evaluate_context;
mod graph;
pub mod module_options;
pub mod rebase;
pub mod transition;
pub(crate) mod unsupported_sass;
......@@ -908,7 +907,10 @@ pub async fn replace_externals(
import_externals: bool,
) -> Result<ModuleResolveResult> {
for item in result.primary.values_mut() {
let ModuleResolveResultItem::External(request, ty) = item else {
let ModuleResolveResultItem::External {
name: request, ty, ..
} = item
else {
continue;
};
......@@ -927,7 +929,7 @@ pub async fn replace_externals(
}
};
let module = CachedExternalModule::new(request.clone(), external_type)
let module = CachedExternalModule::new(request.clone(), external_type, vec![])
.to_resolved()
.await?;
......
......@@ -38,7 +38,6 @@ use turbo_tasks_memory::MemoryBackend;
use turbopack::{
emit_with_completion,
module_options::{CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext},
rebase::RebasedAsset,
register, ModuleAssetContext,
};
use turbopack_core::{
......@@ -47,6 +46,7 @@ use turbopack_core::{
environment::{Environment, ExecutionEnvironment, NodeJsEnvironment},
file_source::FileSource,
output::OutputAsset,
rebase::RebasedAsset,
reference_type::ReferenceType,
};
use turbopack_resolve::resolve_options_context::ResolveOptionsContext;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment