Unverified Commit a13487fe authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: trace worker_threads worker entry (#85734)

Closes PACK-5688  
Closes https://github.com/vercel/next.js/issues/84766

1. Trace `new require("worker_threads").Worker` calls
2. ignore syntax errors in all traced files

This does lead to a build error currently with cargo run --bin turbopack-nft -- --show-issues --graph bench/app-router-server/node_modules/thread-stream/index.js:

```
error - [analysis] [workspace]/node_modules/.pnpm/thread-stream@3.1.0/node_modules/thread-stream/test/syntax-error.mjs
  [workspace]/node_modules/.pnpm/thread-stream@3.1.0/node_modules/thread-stream/test/syntax-error.mjs:2:6  Parsing ecmascript source code failed
       1 | // this is a syntax error
         +       v
       2 + import
         +       ^
       3 | 
  
  Expected 'from', got '<eof>'
```

Pulled in by this partly dynamic `new Worker()`  call: https://github.com/pinojs/thread-stream/blob/82e011281b8895bf5e8f70008e0242927fcaf0cb/index.js#L53-L55

(Related to https://github.com/vercel/next.js/pull/84408, where this was previously fixed for extensionless files such as `LICENSE`)
parent 18c9f02b
...@@ -290,6 +290,7 @@ pub enum WorkerReferenceSubType { ...@@ -290,6 +290,7 @@ pub enum WorkerReferenceSubType {
WebWorker, WebWorker,
SharedWorker, SharedWorker,
ServiceWorker, ServiceWorker,
NodeWorker,
Custom(u8), Custom(u8),
Undefined, Undefined,
} }
......
...@@ -1788,7 +1788,11 @@ impl JsValue { ...@@ -1788,7 +1788,11 @@ impl JsValue {
"module", "module",
"The Node.js `module` module: https://nodejs.org/api/module.html", "The Node.js `module` module: https://nodejs.org/api/module.html",
), ),
WellKnownObjectKind::ChildProcess | WellKnownObjectKind::ChildProcessDefault => ( WellKnownObjectKind::WorkerThreadsModule | WellKnownObjectKind::WorkerThreadsModuleDefault => (
"worker_threads",
"The Node.js `worker_threads` module: https://nodejs.org/api/worker_threads.html",
),
WellKnownObjectKind::ChildProcessModule | WellKnownObjectKind::ChildProcessModuleDefault => (
"child_process", "child_process",
"The Node.js child_process module: https://nodejs.org/api/child_process.html", "The Node.js child_process module: https://nodejs.org/api/child_process.html",
), ),
...@@ -1796,7 +1800,7 @@ impl JsValue { ...@@ -1796,7 +1800,7 @@ impl JsValue {
"os", "os",
"The Node.js os module: https://nodejs.org/api/os.html", "The Node.js os module: https://nodejs.org/api/os.html",
), ),
WellKnownObjectKind::NodeProcess => ( WellKnownObjectKind::NodeProcessModule => (
"process", "process",
"The Node.js process module: https://nodejs.org/api/process.html", "The Node.js process module: https://nodejs.org/api/process.html",
), ),
...@@ -1954,6 +1958,10 @@ impl JsValue { ...@@ -1954,6 +1958,10 @@ impl JsValue {
"load/loadSync".to_string(), "load/loadSync".to_string(),
"require('@grpc/proto-loader').load(filepath, { includeDirs: [root] }) https://github.com/grpc/grpc-node" "require('@grpc/proto-loader').load(filepath, { includeDirs: [root] }) https://github.com/grpc/grpc-node"
), ),
WellKnownFunctionKind::NodeWorkerConstructor => (
"Worker".to_string(),
"The Node.js worker_threads Worker constructor: https://nodejs.org/api/worker_threads.html#worker_threads_class_worker"
),
WellKnownFunctionKind::WorkerConstructor => ( WellKnownFunctionKind::WorkerConstructor => (
"Worker".to_string(), "Worker".to_string(),
"The standard Worker constructor: https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker" "The standard Worker constructor: https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker"
...@@ -3468,11 +3476,13 @@ pub enum WellKnownObjectKind { ...@@ -3468,11 +3476,13 @@ pub enum WellKnownObjectKind {
ModuleModuleDefault, ModuleModuleDefault,
UrlModule, UrlModule,
UrlModuleDefault, UrlModuleDefault,
ChildProcess, WorkerThreadsModule,
ChildProcessDefault, WorkerThreadsModuleDefault,
ChildProcessModule,
ChildProcessModuleDefault,
OsModule, OsModule,
OsModuleDefault, OsModuleDefault,
NodeProcess, NodeProcessModule,
NodeProcessArgv, NodeProcessArgv,
NodeProcessEnv, NodeProcessEnv,
NodePreGyp, NodePreGyp,
...@@ -3492,9 +3502,10 @@ impl WellKnownObjectKind { ...@@ -3492,9 +3502,10 @@ impl WellKnownObjectKind {
Self::PathModule => Some(&["path"]), Self::PathModule => Some(&["path"]),
Self::FsModule => Some(&["fs"]), Self::FsModule => Some(&["fs"]),
Self::UrlModule => Some(&["url"]), Self::UrlModule => Some(&["url"]),
Self::ChildProcess => Some(&["child_process"]), Self::ChildProcessModule => Some(&["child_process"]),
Self::OsModule => Some(&["os"]), Self::OsModule => Some(&["os"]),
Self::NodeProcess => Some(&["process"]), Self::WorkerThreadsModule => Some(&["worker_threads"]),
Self::NodeProcessModule => Some(&["process"]),
Self::NodeProcessArgv => Some(&["process", "argv"]), Self::NodeProcessArgv => Some(&["process", "argv"]),
Self::NodeProcessEnv => Some(&["process", "env"]), Self::NodeProcessEnv => Some(&["process", "env"]),
Self::NodeBuffer => Some(&["Buffer"]), Self::NodeBuffer => Some(&["Buffer"]),
...@@ -3623,6 +3634,8 @@ pub enum WellKnownFunctionKind { ...@@ -3623,6 +3634,8 @@ pub enum WellKnownFunctionKind {
NodeResolveFrom, NodeResolveFrom,
NodeProtobufLoad, NodeProtobufLoad,
WorkerConstructor, WorkerConstructor,
// The worker_threads Worker class
NodeWorkerConstructor,
URLConstructor, URLConstructor,
} }
...@@ -3797,7 +3810,7 @@ pub mod test_utils { ...@@ -3797,7 +3810,7 @@ pub mod test_utils {
), ),
"define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define), "define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define),
"URL" => JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor), "URL" => JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor),
"process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess), "process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessModule),
"Object" => JsValue::WellKnownObject(WellKnownObjectKind::GlobalObject), "Object" => JsValue::WellKnownObject(WellKnownObjectKind::GlobalObject),
"Buffer" => JsValue::WellKnownObject(WellKnownObjectKind::NodeBuffer), "Buffer" => JsValue::WellKnownObject(WellKnownObjectKind::NodeBuffer),
_ => v.into_unknown(true, "unknown global"), _ => v.into_unknown(true, "unknown global"),
......
...@@ -609,13 +609,18 @@ async fn well_known_object_member( ...@@ -609,13 +609,18 @@ async fn well_known_object_member(
WellKnownObjectKind::UrlModule | WellKnownObjectKind::UrlModuleDefault => { WellKnownObjectKind::UrlModule | WellKnownObjectKind::UrlModuleDefault => {
url_module_member(kind, prop) url_module_member(kind, prop)
} }
WellKnownObjectKind::ChildProcess | WellKnownObjectKind::ChildProcessDefault => { WellKnownObjectKind::WorkerThreadsModule
child_process_module_member(kind, prop) | WellKnownObjectKind::WorkerThreadsModuleDefault => {
worker_threads_module_member(kind, prop)
} }
WellKnownObjectKind::ChildProcessModule
| WellKnownObjectKind::ChildProcessModuleDefault => child_process_module_member(kind, prop),
WellKnownObjectKind::OsModule | WellKnownObjectKind::OsModuleDefault => { WellKnownObjectKind::OsModule | WellKnownObjectKind::OsModuleDefault => {
os_module_member(kind, prop) os_module_member(kind, prop)
} }
WellKnownObjectKind::NodeProcess => node_process_member(prop, compile_time_info).await?, WellKnownObjectKind::NodeProcessModule => {
node_process_member(prop, compile_time_info).await?
}
WellKnownObjectKind::NodePreGyp => node_pre_gyp(prop), WellKnownObjectKind::NodePreGyp => node_pre_gyp(prop),
WellKnownObjectKind::NodeExpressApp => express(prop), WellKnownObjectKind::NodeExpressApp => express(prop),
WellKnownObjectKind::NodeProtobufLoader => protobuf_loader(prop), WellKnownObjectKind::NodeProtobufLoader => protobuf_loader(prop),
...@@ -762,7 +767,7 @@ fn url_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue { ...@@ -762,7 +767,7 @@ fn url_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
(.., Some("pathToFileURL")) => { (.., Some("pathToFileURL")) => {
JsValue::WellKnownFunction(WellKnownFunctionKind::PathToFileUrl) JsValue::WellKnownFunction(WellKnownFunctionKind::PathToFileUrl)
} }
(WellKnownObjectKind::UrlModuleDefault, Some("default")) => { (WellKnownObjectKind::UrlModule, Some("default")) => {
JsValue::WellKnownObject(WellKnownObjectKind::UrlModuleDefault) JsValue::WellKnownObject(WellKnownObjectKind::UrlModuleDefault)
} }
_ => JsValue::unknown( _ => JsValue::unknown(
...@@ -776,6 +781,27 @@ fn url_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue { ...@@ -776,6 +781,27 @@ fn url_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
} }
} }
fn worker_threads_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
match (kind, prop.as_str()) {
(.., Some("Worker")) => {
JsValue::WellKnownFunction(WellKnownFunctionKind::NodeWorkerConstructor)
}
(WellKnownObjectKind::WorkerThreadsModule, Some("default")) => {
JsValue::WellKnownObject(WellKnownObjectKind::WorkerThreadsModuleDefault)
}
_ => JsValue::unknown(
JsValue::member(
Box::new(JsValue::WellKnownObject(
WellKnownObjectKind::WorkerThreadsModule,
)),
Box::new(prop),
),
true,
"unsupported property on Node.js worker_threads module",
),
}
}
fn child_process_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue { fn child_process_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
let prop_str = prop.as_str(); let prop_str = prop.as_str();
match (kind, prop_str) { match (kind, prop_str) {
...@@ -785,13 +811,15 @@ fn child_process_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsVa ...@@ -785,13 +811,15 @@ fn child_process_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsVa
)) ))
} }
(.., Some("fork")) => JsValue::WellKnownFunction(WellKnownFunctionKind::ChildProcessFork), (.., Some("fork")) => JsValue::WellKnownFunction(WellKnownFunctionKind::ChildProcessFork),
(WellKnownObjectKind::ChildProcess, Some("default")) => { (WellKnownObjectKind::ChildProcessModule, Some("default")) => {
JsValue::WellKnownObject(WellKnownObjectKind::ChildProcessDefault) JsValue::WellKnownObject(WellKnownObjectKind::ChildProcessModuleDefault)
} }
_ => JsValue::unknown( _ => JsValue::unknown(
JsValue::member( JsValue::member(
Box::new(JsValue::WellKnownObject(WellKnownObjectKind::ChildProcess)), Box::new(JsValue::WellKnownObject(
WellKnownObjectKind::ChildProcessModule,
)),
Box::new(prop), Box::new(prop),
), ),
true, true,
...@@ -843,7 +871,9 @@ async fn node_process_member( ...@@ -843,7 +871,9 @@ async fn node_process_member(
Some("env") => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessEnv), Some("env") => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessEnv),
_ => JsValue::unknown( _ => JsValue::unknown(
JsValue::member( JsValue::member(
Box::new(JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess)), Box::new(JsValue::WellKnownObject(
WellKnownObjectKind::NodeProcessModule,
)),
Box::new(prop), Box::new(prop),
), ),
true, true,
......
...@@ -176,15 +176,9 @@ pub async fn parse( ...@@ -176,15 +176,9 @@ pub async fn parse(
ty = display(&ty) ty = display(&ty)
); );
match parse_internal( match parse_internal(source, ty, transforms, is_external_tracing, inline_helpers)
source, .instrument(span)
ty, .await
transforms,
is_external_tracing && matches!(ty, EcmascriptModuleAssetType::EcmascriptExtensionless),
inline_helpers,
)
.instrument(span)
.await
{ {
Ok(result) => Ok(result), Ok(result) => Ok(result),
Err(error) => Err(error.context(format!( Err(error) => Err(error.context(format!(
......
...@@ -151,7 +151,7 @@ use crate::{ ...@@ -151,7 +151,7 @@ use crate::{
}, },
ident::IdentReplacement, ident::IdentReplacement,
member::MemberReplacement, member::MemberReplacement,
node::PackageJsonReference, node::{FilePathModuleReference, PackageJsonReference},
require_context::{RequireContextAssetReference, RequireContextMap}, require_context::{RequireContextAssetReference, RequireContextMap},
type_issue::SpecifiedModuleTypeIssue, type_issue::SpecifiedModuleTypeIssue,
}, },
...@@ -1646,8 +1646,27 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -1646,8 +1646,27 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
.await .await
}; };
let make_issue_source = let get_traced_project_dir = async || -> Result<FileSystemPath> {
// readFileSync("./foo") should always be relative to the project root, but this is
// dangerous inside of node_modules as it can cause a lot of false positives in the tracing,
// if some package does `path.join(dynamic)`, it would include everything from the project
// root as well.
//
// Also, when there's no cwd set (i.e. in a tracing-specific module context, as we shouldn't
// assume a `process.cwd()` for all of node_modules), fallback to the source file directory.
// This still allows relative file accesses, just not from the project root.
if allow_project_root_tracing
&& let Some(cwd) = compile_time_info.environment().cwd().owned().await?
{
Ok(cwd)
} else {
Ok(source.ident().path().await?.parent())
}
};
let get_issue_source =
|| IssueSource::from_swc_offsets(source, span.lo.to_u32(), span.hi.to_u32()); || IssueSource::from_swc_offsets(source, span.lo.to_u32(), span.hi.to_u32());
if new { if new {
match func { match func {
JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor) => { JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor) => {
...@@ -1726,9 +1745,54 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -1726,9 +1745,54 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
// Ignore (e.g. dynamic parameter or string literal), just as Webpack does // Ignore (e.g. dynamic parameter or string literal), just as Webpack does
return Ok(()); return Ok(());
} }
JsValue::WellKnownFunction(WellKnownFunctionKind::NodeWorkerConstructor)
if analysis.analyze_mode.is_tracing() =>
{
// Only for tracing, not for bundling (yet?)
let args = linked_args(args).await?;
if !args.is_empty() {
let pat = js_value_to_pattern(&args[0]);
if !pat.has_constant_parts() {
let (args, hints) = explain_args(&args);
handler.span_warn_with_code(
span,
&format!("new Worker({args}) is very dynamic{hints}",),
DiagnosticId::Lint(
errors::failed_to_analyze::ecmascript::NEW_WORKER.to_string(),
),
);
if ignore_dynamic_requests {
return Ok(());
}
}
analysis.add_reference(
FilePathModuleReference::new(
origin.asset_context(),
get_traced_project_dir().await?,
Pattern::new(pat),
collect_affecting_sources,
get_issue_source(),
)
.to_resolved()
.await?,
);
return Ok(());
}
let (args, hints) = explain_args(&args);
handler.span_warn_with_code(
span,
&format!("new Worker({args}) is not statically analyze-able{hints}",),
DiagnosticId::Error(
errors::failed_to_analyze::ecmascript::FS_METHOD.to_string(),
),
);
// Ignore (e.g. dynamic parameter or string literal)
return Ok(());
}
_ => {} _ => {}
} }
// linked_args wasn't called, so manually add the closure effects
for arg in args { for arg in args {
if let EffectArg::Closure(_, block) = arg { if let EffectArg::Closure(_, block) = arg {
add_effects(block.effects); add_effects(block.effects);
...@@ -1737,24 +1801,6 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -1737,24 +1801,6 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
return Ok(()); return Ok(());
} }
let get_traced_project_dir = async || -> Result<FileSystemPath> {
// readFileSync("./foo") should always be relative to the project root, but this is
// dangerous inside of node_modules as it can cause a lot of false positives in the tracing,
// if some package does `path.join(dynamic)`, it would include everything from the project
// root as well.
//
// Also, when there's no cwd set (i.e. in a tracing-specific module context, as we shouldn't
// assume a `process.cwd()` for all of node_modules), fallback to the source file directory.
// This still allows relative file accesses, just not from the project root.
if allow_project_root_tracing
&& let Some(cwd) = compile_time_info.environment().cwd().owned().await?
{
Ok(cwd)
} else {
Ok(source.ident().path().await?.parent())
}
};
match func { match func {
JsValue::Alternatives { JsValue::Alternatives {
total_nodes: _, total_nodes: _,
...@@ -1988,7 +2034,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -1988,7 +2034,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
get_traced_project_dir().await?, get_traced_project_dir().await?,
Pattern::new(pat), Pattern::new(pat),
collect_affecting_sources, collect_affecting_sources,
make_issue_source(), get_issue_source(),
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -2041,7 +2087,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -2041,7 +2087,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
DirAssetReference::new( DirAssetReference::new(
get_traced_project_dir().await?, get_traced_project_dir().await?,
Pattern::new(pat), Pattern::new(pat),
make_issue_source(), get_issue_source(),
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -2085,7 +2131,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -2085,7 +2131,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
DirAssetReference::new( DirAssetReference::new(
get_traced_project_dir().await?, get_traced_project_dir().await?,
Pattern::new(pat), Pattern::new(pat),
make_issue_source(), get_issue_source(),
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -2376,7 +2422,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -2376,7 +2422,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
DirAssetReference::new( DirAssetReference::new(
get_traced_project_dir().await?, get_traced_project_dir().await?,
Pattern::new(abs_pattern), Pattern::new(abs_pattern),
make_issue_source(), get_issue_source(),
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -2443,7 +2489,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -2443,7 +2489,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
DirAssetReference::new( DirAssetReference::new(
get_traced_project_dir().await?, get_traced_project_dir().await?,
Pattern::new(abs_pattern), Pattern::new(abs_pattern),
make_issue_source(), get_issue_source(),
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -2512,7 +2558,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -2512,7 +2558,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
DirAssetReference::new( DirAssetReference::new(
context_dir.clone(), context_dir.clone(),
Pattern::new(Pattern::Constant(dir.into())), Pattern::new(Pattern::Constant(dir.into())),
make_issue_source(), get_issue_source(),
) )
.to_resolved() .to_resolved()
}) })
...@@ -3147,7 +3193,7 @@ async fn value_visitor_inner( ...@@ -3147,7 +3193,7 @@ async fn value_visitor_inner(
), ),
"define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define), "define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define),
"URL" => JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor), "URL" => JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor),
"process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess), "process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessModule),
"Object" => JsValue::WellKnownObject(WellKnownObjectKind::GlobalObject), "Object" => JsValue::WellKnownObject(WellKnownObjectKind::GlobalObject),
"Buffer" => JsValue::WellKnownObject(WellKnownObjectKind::NodeBuffer), "Buffer" => JsValue::WellKnownObject(WellKnownObjectKind::NodeBuffer),
_ => return Ok((v, false)), _ => return Ok((v, false)),
......
use anyhow::Result; use anyhow::Result;
use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
file_source::FileSource, raw_module::RawModule, reference::ModuleReference, chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
resolve::ModuleResolveResult, context::AssetContext,
file_source::FileSource,
issue::IssueSource,
raw_module::RawModule,
reference::ModuleReference,
reference_type::{ReferenceType, WorkerReferenceSubType},
resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw},
}; };
use crate::references::util::check_and_emit_too_many_matches_warning;
#[turbo_tasks::value] #[turbo_tasks::value]
#[derive(Hash, Clone, Debug)] #[derive(Hash, Clone, Debug)]
pub struct PackageJsonReference { pub struct PackageJsonReference {
...@@ -46,3 +55,89 @@ impl ValueToString for PackageJsonReference { ...@@ -46,3 +55,89 @@ impl ValueToString for PackageJsonReference {
)) ))
} }
} }
#[turbo_tasks::value]
#[derive(Hash, Debug)]
pub struct FilePathModuleReference {
asset_context: ResolvedVc<Box<dyn AssetContext>>,
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
}
#[turbo_tasks::value_impl]
impl FilePathModuleReference {
#[turbo_tasks::function]
pub fn new(
asset_context: ResolvedVc<Box<dyn AssetContext>>,
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
) -> Vc<Self> {
Self {
asset_context,
context_dir,
path,
collect_affecting_sources,
issue_source,
}
.cell()
}
}
// A reference to an module by absolute or cwd-relative file path (e.g. for the
// worker-threads `new Worker` which has the resolving behavior of `fs.readFile` but should treat
// the resolve result as an module instead of a raw source).
#[turbo_tasks::value_impl]
impl ModuleReference for FilePathModuleReference {
#[turbo_tasks::function]
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let span = tracing::info_span!(
"trace module",
pattern = display(self.path.to_string().await?)
);
async {
let result = resolve_raw(
self.context_dir.clone(),
*self.path,
self.collect_affecting_sources,
/* force_in_lookup_dir */ false,
);
let result = self.asset_context.process_resolve_result(
result,
ReferenceType::Worker(WorkerReferenceSubType::NodeWorker),
);
check_and_emit_too_many_matches_warning(
result,
self.issue_source,
self.context_dir.clone(),
self.path,
)
.await?;
Ok(result)
}
.instrument(span)
.await
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for FilePathModuleReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for FilePathModuleReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("raw asset {}", self.path.to_string().await?,).into(),
))
}
}
...@@ -6,10 +6,7 @@ use turbo_tasks_fs::FileSystemPath; ...@@ -6,10 +6,7 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
file_source::FileSource, file_source::FileSource,
issue::{ issue::IssueSource,
Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, OptionIssueSource,
OptionStyledString, StyledString,
},
raw_module::RawModule, raw_module::RawModule,
reference::ModuleReference, reference::ModuleReference,
resolve::{ resolve::{
...@@ -18,6 +15,9 @@ use turbopack_core::{ ...@@ -18,6 +15,9 @@ use turbopack_core::{
resolve_raw, resolve_raw,
}, },
}; };
use crate::references::util::check_and_emit_too_many_matches_warning;
#[turbo_tasks::value] #[turbo_tasks::value]
#[derive(Hash, Debug)] #[derive(Hash, Debug)]
pub struct FileSourceReference { pub struct FileSourceReference {
...@@ -235,80 +235,3 @@ impl ValueToString for DirAssetReference { ...@@ -235,80 +235,3 @@ impl ValueToString for DirAssetReference {
)) ))
} }
} }
/// If a pattern resolves to more than 10000 results, it's likely a mistake so issue a warning.
const TOO_MANY_MATCHES_LIMIT: usize = 10000;
async fn check_and_emit_too_many_matches_warning(
result: Vc<ModuleResolveResult>,
issue_source: IssueSource,
context_dir: FileSystemPath,
pattern: ResolvedVc<Pattern>,
) -> Result<()> {
let num_matches = result.await?.primary.len();
if num_matches > TOO_MANY_MATCHES_LIMIT {
TooManyMatchesWarning {
source: issue_source,
context_dir,
num_matches,
pattern,
}
.resolved_cell()
.emit();
}
Ok(())
}
#[turbo_tasks::value(shared)]
struct TooManyMatchesWarning {
source: IssueSource,
context_dir: FileSystemPath,
num_matches: usize,
pattern: ResolvedVc<Pattern>,
}
#[turbo_tasks::value_impl]
impl Issue for TooManyMatchesWarning {
#[turbo_tasks::function]
async fn title(&self) -> Result<Vc<StyledString>> {
Ok(StyledString::Text(
format!(
"The file pattern {pattern} matches {num_matches} files in {context_dir}",
pattern = self.pattern.to_string().await?,
context_dir = self.context_dir.value_to_string().await?,
num_matches = self.num_matches
)
.into(),
)
.cell())
}
#[turbo_tasks::function]
fn description(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(
StyledString::Text(rcstr!(
"Overly broad patterns can lead to build performance issues and over bundling."
))
.resolved_cell(),
))
}
#[turbo_tasks::function]
async fn file_path(&self) -> Vc<FileSystemPath> {
self.source.file_path()
}
#[turbo_tasks::function]
fn stage(&self) -> Vc<IssueStage> {
IssueStage::Resolve.cell()
}
fn severity(&self) -> IssueSeverity {
IssueSeverity::Warning
}
#[turbo_tasks::function]
fn source(&self) -> Vc<OptionIssueSource> {
Vc::cell(Some(self.source))
}
}
use anyhow::Result; use anyhow::Result;
use swc_core::{ecma::ast::Expr, quote}; use swc_core::{ecma::ast::Expr, quote};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::Vc; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbopack_core::{self, resolve::parse::Request}; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
self,
issue::{
Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, OptionIssueSource,
OptionStyledString, StyledString,
},
resolve::{ModuleResolveResult, parse::Request, pattern::Pattern},
};
/// Creates a IIFE expression that throws a "Cannot find module" error for the /// Creates a IIFE expression that throws a "Cannot find module" error for the
/// given request string /// given request string
...@@ -36,3 +44,80 @@ pub async fn request_to_string(request: Vc<Request>) -> Result<Vc<RcStr>> { ...@@ -36,3 +44,80 @@ pub async fn request_to_string(request: Vc<Request>) -> Result<Vc<RcStr>> {
.unwrap_or(rcstr!("unknown")), .unwrap_or(rcstr!("unknown")),
)) ))
} }
/// If a pattern resolves to more than 10000 results, it's likely a mistake so issue a warning.
const TOO_MANY_MATCHES_LIMIT: usize = 10000;
pub async fn check_and_emit_too_many_matches_warning(
result: Vc<ModuleResolveResult>,
issue_source: IssueSource,
context_dir: FileSystemPath,
pattern: ResolvedVc<Pattern>,
) -> Result<()> {
let num_matches = result.await?.primary.len();
if num_matches > TOO_MANY_MATCHES_LIMIT {
TooManyMatchesWarning {
source: issue_source,
context_dir,
num_matches,
pattern,
}
.resolved_cell()
.emit();
}
Ok(())
}
#[turbo_tasks::value(shared)]
struct TooManyMatchesWarning {
source: IssueSource,
context_dir: FileSystemPath,
num_matches: usize,
pattern: ResolvedVc<Pattern>,
}
#[turbo_tasks::value_impl]
impl Issue for TooManyMatchesWarning {
#[turbo_tasks::function]
async fn title(&self) -> Result<Vc<StyledString>> {
Ok(StyledString::Text(
format!(
"The file pattern {pattern} matches {num_matches} files in {context_dir}",
pattern = self.pattern.to_string().await?,
context_dir = self.context_dir.value_to_string().await?,
num_matches = self.num_matches
)
.into(),
)
.cell())
}
#[turbo_tasks::function]
fn description(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(
StyledString::Text(rcstr!(
"Overly broad patterns can lead to build performance issues and over bundling."
))
.resolved_cell(),
))
}
#[turbo_tasks::function]
async fn file_path(&self) -> Vc<FileSystemPath> {
self.source.file_path()
}
#[turbo_tasks::function]
fn stage(&self) -> Vc<IssueStage> {
IssueStage::Resolve.cell()
}
fn severity(&self) -> IssueSeverity {
IssueSeverity::Warning
}
#[turbo_tasks::function]
fn source(&self) -> Vc<OptionIssueSource> {
Vc::cell(Some(self.source))
}
}
...@@ -16,7 +16,6 @@ use crate::EcmascriptAnalyzable; ...@@ -16,7 +16,6 @@ use crate::EcmascriptAnalyzable;
/// the final runtime code, while keeping source map information. /// the final runtime code, while keeping source map information.
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct StaticEcmascriptCode { pub struct StaticEcmascriptCode {
asset_context: ResolvedVc<Box<dyn AssetContext>>,
asset: ResolvedVc<Box<dyn EcmascriptAnalyzable>>, asset: ResolvedVc<Box<dyn EcmascriptAnalyzable>>,
generate_source_map: bool, generate_source_map: bool,
} }
...@@ -42,7 +41,6 @@ impl StaticEcmascriptCode { ...@@ -42,7 +41,6 @@ impl StaticEcmascriptCode {
bail!("asset is not an Ecmascript module") bail!("asset is not an Ecmascript module")
}; };
Ok(Self::cell(StaticEcmascriptCode { Ok(Self::cell(StaticEcmascriptCode {
asset_context,
asset, asset,
generate_source_map, generate_source_map,
})) }))
......
...@@ -197,12 +197,17 @@ pub fn module_value_to_well_known_object(module_value: &ModuleValue) -> Option<J ...@@ -197,12 +197,17 @@ pub fn module_value_to_well_known_object(module_value: &ModuleValue) -> Option<J
} }
"node:fs" | "fs" => JsValue::WellKnownObject(WellKnownObjectKind::FsModule), "node:fs" | "fs" => JsValue::WellKnownObject(WellKnownObjectKind::FsModule),
"node:child_process" | "child_process" => { "node:child_process" | "child_process" => {
JsValue::WellKnownObject(WellKnownObjectKind::ChildProcess) JsValue::WellKnownObject(WellKnownObjectKind::ChildProcessModule)
} }
"node:os" | "os" => JsValue::WellKnownObject(WellKnownObjectKind::OsModule), "node:os" | "os" => JsValue::WellKnownObject(WellKnownObjectKind::OsModule),
"node:process" | "process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess), "node:process" | "process" => {
JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessModule)
}
"node:url" | "url" => JsValue::WellKnownObject(WellKnownObjectKind::UrlModule), "node:url" | "url" => JsValue::WellKnownObject(WellKnownObjectKind::UrlModule),
"node:module" | "module" => JsValue::WellKnownObject(WellKnownObjectKind::ModuleModule), "node:module" | "module" => JsValue::WellKnownObject(WellKnownObjectKind::ModuleModule),
"node:worker_threads" | "worker_threads" => {
JsValue::WellKnownObject(WellKnownObjectKind::WorkerThreadsModule)
}
"node-pre-gyp" | "@mapbox/node-pre-gyp" => { "node-pre-gyp" | "@mapbox/node-pre-gyp" => {
JsValue::WellKnownObject(WellKnownObjectKind::NodePreGyp) JsValue::WellKnownObject(WellKnownObjectKind::NodePreGyp)
} }
......
...@@ -76,7 +76,7 @@ async fn node_file_trace_operation( ...@@ -76,7 +76,7 @@ async fn node_file_trace_operation(
let environment = Environment::new(ExecutionEnvironment::NodeJsLambda( let environment = Environment::new(ExecutionEnvironment::NodeJsLambda(
NodeJsEnvironment::default().resolved_cell(), NodeJsEnvironment::default().resolved_cell(),
)); ));
let module_asset_context = ModuleAssetContext::new( let module_asset_context = ModuleAssetContext::new_without_replace_externals(
Default::default(), Default::default(),
// This config should be kept in sync with // This config should be kept in sync with
// turbopack/crates/turbopack-tracing/tests/node-file-trace.rs and // turbopack/crates/turbopack-tracing/tests/node-file-trace.rs and
...@@ -109,6 +109,7 @@ async fn node_file_trace_operation( ...@@ -109,6 +109,7 @@ async fn node_file_trace_operation(
enable_node_native_modules: true, enable_node_native_modules: true,
enable_node_modules: Some(input_dir), enable_node_modules: Some(input_dir),
custom_conditions: vec![rcstr!("node")], custom_conditions: vec![rcstr!("node")],
enable_node_externals: true,
loose_errors: true, loose_errors: true,
collect_affecting_sources: true, collect_affecting_sources: true,
..Default::default() ..Default::default()
......
...@@ -149,6 +149,7 @@ static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc; ...@@ -149,6 +149,7 @@ static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
#[case::pdf2json("integration/pdf2json.mjs")] #[case::pdf2json("integration/pdf2json.mjs")]
#[case::pdfkit("integration/pdfkit.js")] #[case::pdfkit("integration/pdfkit.js")]
#[case::pg("integration/pg.js")] #[case::pg("integration/pg.js")]
#[case::pino("integration/pino.js")]
#[case::playwright_core("integration/playwright-core.js")] #[case::playwright_core("integration/playwright-core.js")]
#[case::pnpm_like("integration/pnpm/pnpm-like.js")] #[case::pnpm_like("integration/pnpm/pnpm-like.js")]
#[case::polyfill_library("integration/polyfill-library.js")] #[case::polyfill_library("integration/polyfill-library.js")]
...@@ -357,7 +358,7 @@ async fn node_file_trace_operation( ...@@ -357,7 +358,7 @@ async fn node_file_trace_operation(
} }
.resolved_cell(), .resolved_cell(),
)); ));
let module_asset_context = ModuleAssetContext::new( let module_asset_context = ModuleAssetContext::new_without_replace_externals(
Default::default(), Default::default(),
// TODO These test cases should move into the `node-file-trace` crate and use the same // TODO These test cases should move into the `node-file-trace` crate and use the same
// config. // config.
......
// for transport({target: 'pino-pretty'}) to work
require('pino-pretty')
const { pino, transport } = require('pino')
const loggerOptions = {
level: 'info',
timestamp: false,
}
const loggerTransport = transport({
target: 'pino-pretty',
options: {
ignore: 'pid,hostname',
},
})
const logger = pino(loggerOptions, loggerTransport)
logger.info('foo')
...@@ -86,6 +86,8 @@ ...@@ -86,6 +86,8 @@
"pg": "^7.11.0", "pg": "^7.11.0",
"phantomjs-prebuilt": "^2.1.16", "phantomjs-prebuilt": "^2.1.16",
"pixelmatch": "^5.3.0", "pixelmatch": "^5.3.0",
"pino": "^10.1.0",
"pino-pretty": "^13.1.2",
"playwright-core": "^1.17.1", "playwright-core": "^1.17.1",
"polyfill-library": "3.93.0", "polyfill-library": "3.93.0",
"prismjs": "^1.25.0", "prismjs": "^1.25.0",
......
...@@ -194,7 +194,7 @@ async fn node_file_trace_operation(package_root: RcStr, input: RcStr) -> Result< ...@@ -194,7 +194,7 @@ async fn node_file_trace_operation(package_root: RcStr, input: RcStr) -> Result<
} }
.resolved_cell(), .resolved_cell(),
)); ));
let module_asset_context = ModuleAssetContext::new( let module_asset_context = ModuleAssetContext::new_without_replace_externals(
Default::default(), Default::default(),
// TODO These test cases should move into the `node-file-trace` crate and use the same // TODO These test cases should move into the `node-file-trace` crate and use the same
// config. // config.
......
...@@ -391,8 +391,9 @@ impl ModuleAssetContext { ...@@ -391,8 +391,9 @@ impl ModuleAssetContext {
}) })
} }
/// Doesn't replace external resolve results with a CachedExternalModule.
#[turbo_tasks::function] #[turbo_tasks::function]
fn new_without_replace_externals( pub fn new_without_replace_externals(
transitions: ResolvedVc<TransitionOptions>, transitions: ResolvedVc<TransitionOptions>,
compile_time_info: ResolvedVc<CompileTimeInfo>, compile_time_info: ResolvedVc<CompileTimeInfo>,
module_options_context: ResolvedVc<ModuleOptionsContext>, module_options_context: ResolvedVc<ModuleOptionsContext>,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment