Unverified Commit f09567ea authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Emit a warning when there are too many matches from a FileSourceReference (#84701)

Previously we would `eprintln!` in this case, but that is missing key source information that would help with debugging.


Example warning (i hacked the limit to be lower to make this less tedious to repro):

![image.png](https://app.graphite.dev/user-attachments/assets/084b9b27-2558-4131-8f31-e54e3c188359.png)

If there are more than 10K matches we will issue this warning.


A tricky thing is that because we issue this warning during the module graph construction, but `resolve_reference` won't be called again for a FileSource reference during chunking the issue actually wouldn't get emitted.  The fix is to drop the `take_issues` call inside of `whole_app_module_graphs` in production builds.
parent 8f731e66
...@@ -490,7 +490,7 @@ impl Issue for CssGlobalImportIssue { ...@@ -490,7 +490,7 @@ impl Issue for CssGlobalImportIssue {
} }
fn severity(&self) -> IssueSeverity { fn severity(&self) -> IssueSeverity {
IssueSeverity::Fatal IssueSeverity::Error
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -984,8 +984,15 @@ impl Project { ...@@ -984,8 +984,15 @@ impl Project {
pub async fn whole_app_module_graphs(self: ResolvedVc<Self>) -> Result<Vc<ModuleGraphs>> { pub async fn whole_app_module_graphs(self: ResolvedVc<Self>) -> Result<Vc<ModuleGraphs>> {
async move { async move {
let module_graphs_op = whole_app_module_graph_operation(self); let module_graphs_op = whole_app_module_graph_operation(self);
let module_graphs_vc = module_graphs_op.resolve_strongly_consistent().await?; let module_graphs_vc = if self.next_mode().await?.is_production() {
let _ = module_graphs_op.take_issues(); module_graphs_op.connect()
} else {
// In development mode, we need to to take and drop the issues, otherwise every
// route will report all issues.
let vc = module_graphs_op.resolve_strongly_consistent().await?;
let _ = module_graphs_op.take_issues();
*vc
};
// At this point all modules have been computed and we can get rid of the node.js // At this point all modules have been computed and we can get rid of the node.js
// process pools // process pools
...@@ -995,7 +1002,7 @@ impl Project { ...@@ -995,7 +1002,7 @@ impl Project {
turbopack_node::evaluate::scale_zero(); turbopack_node::evaluate::scale_zero();
} }
Ok(*module_graphs_vc) Ok(module_graphs_vc)
} }
.instrument(tracing::info_span!("module graph for app")) .instrument(tracing::info_span!("module graph for app"))
.await .await
......
...@@ -1467,6 +1467,8 @@ pub async fn resolve_raw( ...@@ -1467,6 +1467,8 @@ pub async fn resolve_raw(
let pat = path.await?; let pat = path.await?;
if let Some(pat) = pat if let Some(pat) = pat
.filter_could_match("/ROOT/") .filter_could_match("/ROOT/")
// Checks if this pattern is more specific than everything, so we test using a random path
// that is unlikely to actually exist
.and_then(|pat| pat.filter_could_not_match("/ROOT/fsd8nz8og54z")) .and_then(|pat| pat.filter_could_not_match("/ROOT/fsd8nz8og54z"))
{ {
let path = Pattern::new(pat); let path = Pattern::new(pat);
...@@ -1477,34 +1479,17 @@ pub async fn resolve_raw( ...@@ -1477,34 +1479,17 @@ pub async fn resolve_raw(
path, path,
) )
.await?; .await?;
if matches.len() > 10000 { results.extend(
let path_str = path.to_string().await?; collect_matches(&matches, collect_affecting_sources)
println!( .await?
"WARN: resolving abs pattern {} in {} leads to {} results", .into_iter(),
path_str, );
lookup_dir.value_to_string().await?,
matches.len()
);
} else {
results.extend(
collect_matches(&matches, collect_affecting_sources)
.await?
.into_iter(),
);
}
} }
{ {
let matches = let matches =
read_matches(lookup_dir.clone(), rcstr!(""), force_in_lookup_dir, path).await?; read_matches(lookup_dir.clone(), rcstr!(""), force_in_lookup_dir, path).await?;
if matches.len() > 10000 {
println!(
"WARN: resolving pattern {} in {} leads to {} results",
pat.describe_as_string(),
lookup_dir.value_to_string().await?,
matches.len()
);
}
results.extend( results.extend(
collect_matches(&matches, collect_affecting_sources) collect_matches(&matches, collect_affecting_sources)
.await? .await?
......
...@@ -1987,6 +1987,11 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -1987,6 +1987,11 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
context_dir, context_dir,
Pattern::new(pat), Pattern::new(pat),
collect_affecting_sources, collect_affecting_sources,
IssueSource::from_swc_offsets(
source,
span.lo.to_u32(),
span.hi.to_u32(),
),
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -2136,6 +2141,11 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -2136,6 +2141,11 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
context_dir, context_dir,
Pattern::new(pat), Pattern::new(pat),
collect_affecting_sources, collect_affecting_sources,
IssueSource::from_swc_offsets(
source,
span.lo.to_u32(),
span.hi.to_u32(),
),
) )
.to_resolved() .to_resolved()
.await?, .await?,
......
use anyhow::Result; use anyhow::Result;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
issue::{
Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, OptionIssueSource,
OptionStyledString, StyledString,
},
reference::ModuleReference, reference::ModuleReference,
resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw}, resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw},
}; };
...@@ -15,6 +19,7 @@ pub struct FileSourceReference { ...@@ -15,6 +19,7 @@ pub struct FileSourceReference {
context_dir: FileSystemPath, context_dir: FileSystemPath,
path: ResolvedVc<Pattern>, path: ResolvedVc<Pattern>,
collect_affecting_sources: bool, collect_affecting_sources: bool,
issue_source: IssueSource,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -24,11 +29,13 @@ impl FileSourceReference { ...@@ -24,11 +29,13 @@ impl FileSourceReference {
context_dir: FileSystemPath, context_dir: FileSystemPath,
path: ResolvedVc<Pattern>, path: ResolvedVc<Pattern>,
collect_affecting_sources: bool, collect_affecting_sources: bool,
issue_source: IssueSource,
) -> Vc<Self> { ) -> Vc<Self> {
Self::cell(FileSourceReference { Self::cell(FileSourceReference {
context_dir, context_dir,
path, path,
collect_affecting_sources, collect_affecting_sources,
issue_source,
}) })
} }
} }
...@@ -42,7 +49,7 @@ impl ModuleReference for FileSourceReference { ...@@ -42,7 +49,7 @@ impl ModuleReference for FileSourceReference {
pattern = display(self.path.to_string().await?) pattern = display(self.path.to_string().await?)
); );
async { async {
resolve_raw( let result = resolve_raw(
self.context_dir.clone(), self.context_dir.clone(),
*self.path, *self.path,
self.collect_affecting_sources, self.collect_affecting_sources,
...@@ -50,12 +57,80 @@ impl ModuleReference for FileSourceReference { ...@@ -50,12 +57,80 @@ impl ModuleReference for FileSourceReference {
) )
.as_raw_module_result() .as_raw_module_result()
.resolve() .resolve()
.await .await?;
let num_matches = result.await?.primary.len();
if num_matches > TOO_MANY_MATCHES_LIMIT {
TooManyMatchesWarning {
source: self.issue_source,
context_dir: self.context_dir.clone(),
num_matches,
pattern: self.path,
}
.resolved_cell()
.emit();
}
Ok(result)
} }
.instrument(span) .instrument(span)
.await .await
} }
} }
/// If a pattern resolves to more than 10000 results, it's likely a mistake so issue a warning.
const TOO_MANY_MATCHES_LIMIT: usize = 10000;
#[turbo_tasks::value(shared)]
struct TooManyMatchesWarning {
source: IssueSource,
context_dir: FileSystemPath,
num_matches: usize,
pattern: ResolvedVc<Pattern>,
}
#[turbo_tasks::value_impl]
impl Issue for TooManyMatchesWarning {
#[turbo_tasks::function]
async fn title(&self) -> Result<Vc<StyledString>> {
Ok(StyledString::Text(
format!(
"The file pattern {pattern} matches {num_matches} files in {context_dir}",
pattern = self.pattern.to_string().await?,
context_dir = self.context_dir.value_to_string().await?,
num_matches = self.num_matches
)
.into(),
)
.cell())
}
#[turbo_tasks::function]
fn description(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(
StyledString::Text(rcstr!(
"Overly broad patterns can lead to build performance issues and over bundling."
))
.resolved_cell(),
))
}
#[turbo_tasks::function]
async fn file_path(&self) -> Vc<FileSystemPath> {
self.source.file_path()
}
#[turbo_tasks::function]
fn stage(&self) -> Vc<IssueStage> {
IssueStage::Resolve.cell()
}
fn severity(&self) -> IssueSeverity {
IssueSeverity::Error
}
#[turbo_tasks::function]
fn source(&self) -> Vc<OptionIssueSource> {
Vc::cell(Some(self.source))
}
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkableModuleReference for FileSourceReference { impl ChunkableModuleReference for FileSourceReference {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment