Unverified Commit 90bc934b authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Issue 'too many matches' warning for DirAssets also (#84768)

Same as https://github.com/vercel/next.js/pull/84701 except for DirAssets which are what we create for things like `path.resolve` or `path.join`
parent 87ab87b6
......@@ -104,8 +104,7 @@ use self::{
EsmAssetReference, EsmAsyncAssetReference, EsmExports, EsmModuleItem, ImportMetaBinding,
ImportMetaRef, UrlAssetReference, export::EsmExport,
},
node::DirAssetReference,
raw::FileSourceReference,
raw::{DirAssetReference, FileSourceReference},
typescript::{TsConfigReference, TsReferencePathAssetReference, TsReferenceTypeAssetReference},
};
use super::{
......@@ -1652,6 +1651,8 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
.await
};
let make_issue_source =
|| IssueSource::from_swc_offsets(source, span.lo.to_u32(), span.hi.to_u32());
if new {
match func {
JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor) => {
......@@ -1987,11 +1988,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
context_dir,
Pattern::new(pat),
collect_affecting_sources,
IssueSource::from_swc_offsets(
source,
span.lo.to_u32(),
span.hi.to_u32(),
),
make_issue_source(),
)
.to_resolved()
.await?,
......@@ -2043,7 +2040,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
}
if let Some(context_dir) = get_traced_project_dir().await? {
analysis.add_reference(
DirAssetReference::new(context_dir, Pattern::new(pat))
DirAssetReference::new(context_dir, Pattern::new(pat), make_issue_source())
.to_resolved()
.await?,
);
......@@ -2085,7 +2082,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
}
if let Some(context_dir) = get_traced_project_dir().await? {
analysis.add_reference(
DirAssetReference::new(context_dir, Pattern::new(pat))
DirAssetReference::new(context_dir, Pattern::new(pat), make_issue_source())
.to_resolved()
.await?,
);
......@@ -2376,9 +2373,13 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
};
if let Some(context_dir) = get_traced_project_dir().await? {
analysis.add_reference(
DirAssetReference::new(context_dir, Pattern::new(abs_pattern))
.to_resolved()
.await?,
DirAssetReference::new(
context_dir,
Pattern::new(abs_pattern),
make_issue_source(),
)
.to_resolved()
.await?,
);
}
return Ok(());
......@@ -2441,9 +2442,13 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
};
if let Some(context_dir) = get_traced_project_dir().await? {
analysis.add_reference(
DirAssetReference::new(context_dir, Pattern::new(abs_pattern))
.to_resolved()
.await?,
DirAssetReference::new(
context_dir,
Pattern::new(abs_pattern),
make_issue_source(),
)
.to_resolved()
.await?,
);
}
return Ok(());
......@@ -2510,6 +2515,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
DirAssetReference::new(
context_dir.clone(),
Pattern::new(Pattern::Constant(dir.into())),
make_issue_source(),
)
.to_resolved()
})
......
use anyhow::{Result, bail};
use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr};
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
file_source::FileSource,
raw_module::RawModule,
reference::ModuleReference,
resolve::{
ModuleResolveResult, RequestKey,
pattern::{Pattern, PatternMatch, read_matches},
},
file_source::FileSource, raw_module::RawModule, reference::ModuleReference,
resolve::ModuleResolveResult,
};
#[turbo_tasks::value]
......@@ -53,125 +46,3 @@ impl ValueToString for PackageJsonReference {
))
}
}
#[turbo_tasks::value]
#[derive(Hash, Debug)]
pub struct DirAssetReference {
pub context_dir: FileSystemPath,
pub path: ResolvedVc<Pattern>,
}
#[turbo_tasks::value_impl]
impl DirAssetReference {
#[turbo_tasks::function]
pub fn new(context_dir: FileSystemPath, path: ResolvedVc<Pattern>) -> Vc<Self> {
Self::cell(DirAssetReference { context_dir, path })
}
}
async fn resolve_reference_from_dir(
context_dir: FileSystemPath,
path: Vc<Pattern>,
) -> Result<Vc<ModuleResolveResult>> {
let path_ref = path.await?;
let (abs_path, rel_path) = path_ref.split_could_match("/ROOT/");
if abs_path.is_none() && rel_path.is_none() {
return Ok(*ModuleResolveResult::unresolvable());
}
let abs_matches = if let Some(abs_path) = &abs_path {
Some(
read_matches(
context_dir.root().owned().await?,
rcstr!("/ROOT/"),
true,
Pattern::new(abs_path.or_any_nested_file()),
)
.await?,
)
} else {
None
};
let rel_matches = if let Some(rel_path) = &rel_path {
Some(
read_matches(
context_dir,
rcstr!(""),
true,
Pattern::new(rel_path.or_any_nested_file()),
)
.await?,
)
} else {
None
};
let matches = abs_matches
.into_iter()
.flatten()
.chain(rel_matches.into_iter().flatten());
let mut affecting_sources = Vec::new();
let mut results = Vec::new();
for pat_match in matches {
match pat_match {
PatternMatch::File(matched_path, file) => {
let realpath = file.realpath_with_links().await?;
for symlink in &realpath.symlinks {
affecting_sources.push(ResolvedVc::upcast(
FileSource::new(symlink.clone()).to_resolved().await?,
));
}
let path: FileSystemPath = match &realpath.path_result {
Ok(path) => path.clone(),
Err(e) => bail!(e.as_error_message(file, &realpath)),
};
results.push((
RequestKey::new(matched_path.clone()),
ResolvedVc::upcast(
RawModule::new(Vc::upcast(FileSource::new(path)))
.to_resolved()
.await?,
),
));
}
PatternMatch::Directory(..) => {}
}
}
Ok(*ModuleResolveResult::modules_with_affecting_sources(
results,
affecting_sources,
))
}
#[turbo_tasks::value_impl]
impl ModuleReference for DirAssetReference {
#[turbo_tasks::function]
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let span = tracing::info_span!(
"trace directory",
pattern = display(self.path.to_string().await?)
);
resolve_reference_from_dir(self.context_dir.clone(), *self.path)
.instrument(span)
.await
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for DirAssetReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for DirAssetReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("directory assets {}", self.path.to_string().await?,).into(),
))
}
}
use anyhow::Result;
use anyhow::{Result, bail};
use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
file_source::FileSource,
issue::{
Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, OptionIssueSource,
OptionStyledString, StyledString,
},
raw_module::RawModule,
reference::ModuleReference,
resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw},
resolve::{
ModuleResolveResult, RequestKey,
pattern::{Pattern, PatternMatch, read_matches},
resolve_raw,
},
};
#[turbo_tasks::value]
#[derive(Hash, Debug)]
pub struct FileSourceReference {
......@@ -58,26 +63,202 @@ impl ModuleReference for FileSourceReference {
.as_raw_module_result()
.resolve()
.await?;
let num_matches = result.await?.primary.len();
if num_matches > TOO_MANY_MATCHES_LIMIT {
TooManyMatchesWarning {
source: self.issue_source,
context_dir: self.context_dir.clone(),
num_matches,
pattern: self.path,
check_and_emit_too_many_matches_warning(
result,
self.issue_source,
self.context_dir.clone(),
self.path,
)
.await?;
Ok(result)
}
.instrument(span)
.await
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for FileSourceReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for FileSourceReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("raw asset {}", self.path.to_string().await?,).into(),
))
}
}
#[turbo_tasks::value]
#[derive(Hash, Debug)]
pub struct DirAssetReference {
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
issue_source: IssueSource,
}
#[turbo_tasks::value_impl]
impl DirAssetReference {
#[turbo_tasks::function]
pub fn new(
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
issue_source: IssueSource,
) -> Vc<Self> {
Self::cell(DirAssetReference {
context_dir,
path,
issue_source,
})
}
}
async fn resolve_reference_from_dir(
context_dir: FileSystemPath,
path: Vc<Pattern>,
) -> Result<Vc<ModuleResolveResult>> {
let path_ref = path.await?;
let (abs_path, rel_path) = path_ref.split_could_match("/ROOT/");
if abs_path.is_none() && rel_path.is_none() {
return Ok(*ModuleResolveResult::unresolvable());
}
let abs_matches = if let Some(abs_path) = &abs_path {
Some(
read_matches(
context_dir.root().owned().await?,
rcstr!("/ROOT/"),
true,
Pattern::new(abs_path.or_any_nested_file()),
)
.await?,
)
} else {
None
};
let rel_matches = if let Some(rel_path) = &rel_path {
Some(
read_matches(
context_dir,
rcstr!(""),
true,
Pattern::new(rel_path.or_any_nested_file()),
)
.await?,
)
} else {
None
};
let matches = abs_matches
.into_iter()
.flatten()
.chain(rel_matches.into_iter().flatten());
let mut affecting_sources = Vec::new();
let mut results = Vec::new();
for pat_match in matches {
match pat_match {
PatternMatch::File(matched_path, file) => {
let realpath = file.realpath_with_links().await?;
for symlink in &realpath.symlinks {
affecting_sources.push(ResolvedVc::upcast(
FileSource::new(symlink.clone()).to_resolved().await?,
));
}
.resolved_cell()
.emit();
let path: FileSystemPath = match &realpath.path_result {
Ok(path) => path.clone(),
Err(e) => bail!(e.as_error_message(file, &realpath)),
};
results.push((
RequestKey::new(matched_path.clone()),
ResolvedVc::upcast(
RawModule::new(Vc::upcast(FileSource::new(path)))
.to_resolved()
.await?,
),
));
}
PatternMatch::Directory(..) => {}
}
}
Ok(*ModuleResolveResult::modules_with_affecting_sources(
results,
affecting_sources,
))
}
#[turbo_tasks::value_impl]
impl ModuleReference for DirAssetReference {
#[turbo_tasks::function]
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let span = tracing::info_span!(
"trace directory",
pattern = display(self.path.to_string().await?)
);
async {
let result = resolve_reference_from_dir(self.context_dir.clone(), *self.path).await?;
check_and_emit_too_many_matches_warning(
result,
self.issue_source,
self.context_dir.clone(),
self.path,
)
.await?;
Ok(result)
}
.instrument(span)
.await
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for DirAssetReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for DirAssetReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("directory assets {}", self.path.to_string().await?,).into(),
))
}
}
/// If a pattern resolves to more than 10000 results, it's likely a mistake so issue a warning.
const TOO_MANY_MATCHES_LIMIT: usize = 10000;
async fn check_and_emit_too_many_matches_warning(
result: Vc<ModuleResolveResult>,
issue_source: IssueSource,
context_dir: FileSystemPath,
pattern: ResolvedVc<Pattern>,
) -> Result<()> {
let num_matches = result.await?.primary.len();
if num_matches > TOO_MANY_MATCHES_LIMIT {
TooManyMatchesWarning {
source: issue_source,
context_dir,
num_matches,
pattern,
}
.resolved_cell()
.emit();
}
Ok(())
}
#[turbo_tasks::value(shared)]
struct TooManyMatchesWarning {
source: IssueSource,
......@@ -131,21 +312,3 @@ impl Issue for TooManyMatchesWarning {
Vc::cell(Some(self.source))
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for FileSourceReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for FileSourceReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("raw asset {}", self.path.to_string().await?,).into(),
))
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment