Unverified Commit ab7744e4 authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

Respect webpackIgnore and turboIgnore comments in the parser (#68451)

## What

This is step one of PACK-3046, setting up the necessary parser-side changes to detect and store the webpackIgnore and turbopackIgnore comment directives. This is followed by another PR that implements the bundler side. 

## Why

Webpack supports special directives to prevent bundle-time linking of specific imports. Some libraries need this (mapbox) and so we should support it.

## How

The directives are stored in comments within `require` and `import` calls. We parse the comments alongside traversal and look for comments that exist inside the function calls. These are then collected and used in later stages when performing linking to opt out of linking packages that have been explicitly excluded.
parent 678b4879
...@@ -56,7 +56,7 @@ pub fn benchmark(c: &mut Criterion) { ...@@ -56,7 +56,7 @@ pub fn benchmark(c: &mut Criterion) {
program.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false)); program.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
let eval_context = let eval_context =
EvalContext::new(&program, unresolved_mark, top_level_mark, None); EvalContext::new(&program, unresolved_mark, top_level_mark, None, None);
let var_graph = create_graph(&program, &eval_context); let var_graph = create_graph(&program, &eval_context);
let input = BenchInput { let input = BenchInput {
......
# turbopack-ecmascript
## Adding new parser tests
We use a snapshot-based testing system to ensure that changes to the parser don't break existing code.
To add a new test, you need to create a new directory in `tests/analyzer/graph` with an 'input.js' file
inside.
The snapshot tests are done with the `testing` crate. You can upate them by passing the env var
`UPDATE=1` to the test runner.
```sh
UPDATE=1 cargo test -p turbopack-ecmascript
```
...@@ -6,7 +6,7 @@ use std::{ ...@@ -6,7 +6,7 @@ use std::{
use swc_core::{ use swc_core::{
atoms::Atom, atoms::Atom,
common::{pass::AstNodePath, Mark, Span, Spanned, SyntaxContext, GLOBALS}, common::{comments::Comments, pass::AstNodePath, Mark, Span, Spanned, SyntaxContext, GLOBALS},
ecma::{ ecma::{
ast::*, ast::*,
atoms::js_word, atoms::js_word,
...@@ -294,6 +294,8 @@ pub fn create_graph(m: &Program, eval_context: &EvalContext) -> VarGraph { ...@@ -294,6 +294,8 @@ pub fn create_graph(m: &Program, eval_context: &EvalContext) -> VarGraph {
graph graph
} }
/// A context used for assembling the evaluation graph.
#[derive(Debug)]
pub struct EvalContext { pub struct EvalContext {
pub(crate) unresolved_mark: Mark, pub(crate) unresolved_mark: Mark,
pub(crate) top_level_mark: Mark, pub(crate) top_level_mark: Mark,
...@@ -301,16 +303,20 @@ pub struct EvalContext { ...@@ -301,16 +303,20 @@ pub struct EvalContext {
} }
impl EvalContext { impl EvalContext {
/// Produce a new [EvalContext] from a [Program]. If you wish to support
/// webpackIgnore or turbopackIgnore comments, you must pass those in,
/// since the AST does not include comments by default.
pub fn new( pub fn new(
module: &Program, module: &Program,
unresolved_mark: Mark, unresolved_mark: Mark,
top_level_mark: Mark, top_level_mark: Mark,
comments: Option<&dyn Comments>,
source: Option<Vc<Box<dyn Source>>>, source: Option<Vc<Box<dyn Source>>>,
) -> Self { ) -> Self {
Self { Self {
unresolved_mark, unresolved_mark,
top_level_mark, top_level_mark,
imports: ImportMap::analyze(module, source), imports: ImportMap::analyze(module, source, comments),
} }
} }
......
use std::{collections::BTreeMap, fmt::Display}; use std::{
collections::{BTreeMap, HashMap},
fmt::Display,
};
use indexmap::{IndexMap, IndexSet}; use indexmap::{IndexMap, IndexSet};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use swc_core::{ use swc_core::{
common::{source_map::SmallPos, Span}, common::{comments::Comments, source_map::SmallPos, Span, Spanned},
ecma::{ ecma::{
ast::*, ast::*,
atoms::{js_word, JsWord}, atoms::{js_word, JsWord},
...@@ -136,6 +139,17 @@ pub(crate) struct ImportMap { ...@@ -136,6 +139,17 @@ pub(crate) struct ImportMap {
/// True if the module is an ESM module due to top-level await. /// True if the module is an ESM module due to top-level await.
has_top_level_await: bool, has_top_level_await: bool,
/// Locations of webpackIgnore or turbopackIgnore comments
/// This is a webpack feature that allows opting out of static
/// imports, which we should respect.
///
/// Example:
/// ```js
/// const a = import(/* webpackIgnore: true */ "a");
/// const b = import(/* turbopackIgnore: true */ "b");
/// ```
turbopack_ignores: HashMap<Span, bool>,
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
...@@ -200,12 +214,17 @@ impl ImportMap { ...@@ -200,12 +214,17 @@ impl ImportMap {
} }
/// Analyze ES import /// Analyze ES import
pub(super) fn analyze(m: &Program, source: Option<Vc<Box<dyn Source>>>) -> Self { pub(super) fn analyze(
m: &Program,
source: Option<Vc<Box<dyn Source>>>,
comments: Option<&dyn Comments>,
) -> Self {
let mut data = ImportMap::default(); let mut data = ImportMap::default();
m.visit_with(&mut Analyzer { m.visit_with(&mut Analyzer {
data: &mut data, data: &mut data,
source, source,
comments,
}); });
data data
...@@ -215,6 +234,7 @@ impl ImportMap { ...@@ -215,6 +234,7 @@ impl ImportMap {
struct Analyzer<'a> { struct Analyzer<'a> {
data: &'a mut ImportMap, data: &'a mut ImportMap,
source: Option<Vc<Box<dyn Source>>>, source: Option<Vc<Box<dyn Source>>>,
comments: Option<&'a dyn Comments>,
} }
impl<'a> Analyzer<'a> { impl<'a> Analyzer<'a> {
...@@ -409,8 +429,63 @@ impl Visit for Analyzer<'_> { ...@@ -409,8 +429,63 @@ impl Visit for Analyzer<'_> {
fn visit_export_default_expr(&mut self, _: &ExportDefaultExpr) { fn visit_export_default_expr(&mut self, _: &ExportDefaultExpr) {
self.data.has_exports = true; self.data.has_exports = true;
} }
fn visit_stmt(&mut self, _: &Stmt) {
// don't visit children fn visit_stmt(&mut self, n: &Stmt) {
if self.comments.is_some() {
// only visit children if we potentially need to mark import / requires
n.visit_children_with(self);
}
}
/// check if import or require contains an ignore comment
///
/// We are checking for the following cases:
/// - import(/* webpackIgnore: true */ "a")
/// - require(/* webpackIgnore: true */ "a")
///
/// We can do this by checking if any of the comment spans are between the
/// callee and the first argument.
fn visit_call_expr(&mut self, n: &CallExpr) {
// we can actually unwrap thanks to the optimisation above
// but it can't hurt to be safe...
if let Some(comments) = self.comments {
let callee_span = match &n.callee {
Callee::Import(Import { span, .. }) => Some(span),
// this assumes you cannot reassign `require`
Callee::Expr(box Expr::Ident(Ident { span, sym, .. })) if sym == "require" => {
Some(span)
}
_ => None,
};
// we are interested here in the last comment with a valid directive
let ignore_statement = n
.args
.first()
.map(|arg| arg.span_lo())
.and_then(|comment_pos| comments.get_leading(comment_pos))
.iter()
.flatten()
.rev()
.filter_map(|comment| {
let (directive, value) = comment.text.trim().split_once(':')?;
// support whitespace between the colon
match (directive.trim(), value.trim()) {
("webpackIgnore" | "turbopackIgnore", "true") => Some(true),
("webpackIgnore" | "turbopackIgnore", "false") => Some(false),
_ => None, // ignore anything else
}
})
.next();
if let Some((callee_span, ignore_statement)) = callee_span.zip(ignore_statement) {
self.data
.turbopack_ignores
.insert(*callee_span, ignore_statement);
};
}
n.visit_children_with(self);
} }
fn visit_program(&mut self, m: &Program) { fn visit_program(&mut self, m: &Program) {
......
...@@ -3822,7 +3822,8 @@ mod tests { ...@@ -3822,7 +3822,8 @@ mod tests {
let top_level_mark = Mark::new(); let top_level_mark = Mark::new();
m.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false)); m.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
let eval_context = EvalContext::new(&m, unresolved_mark, top_level_mark, None); let eval_context =
EvalContext::new(&m, unresolved_mark, top_level_mark, None, None);
let mut var_graph = create_graph(&m, &eval_context); let mut var_graph = create_graph(&m, &eval_context);
......
...@@ -366,7 +366,7 @@ impl EcmascriptParsable for EcmascriptModuleAsset { ...@@ -366,7 +366,7 @@ impl EcmascriptParsable for EcmascriptModuleAsset {
impl EcmascriptAnalyzable for EcmascriptModuleAsset { impl EcmascriptAnalyzable for EcmascriptModuleAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult> { fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult> {
analyse_ecmascript_module(self, None) analyse_ecmascript_module(self, None, None)
} }
/// Generates module contents without an analysis pass. This is useful for /// Generates module contents without an analysis pass. This is useful for
......
...@@ -433,6 +433,7 @@ async fn parse_content( ...@@ -433,6 +433,7 @@ async fn parse_content(
&parsed_program, &parsed_program,
unresolved_mark, unresolved_mark,
top_level_mark, top_level_mark,
None,
Some(source), Some(source),
); );
......
...@@ -17,7 +17,7 @@ pub mod util; ...@@ -17,7 +17,7 @@ pub mod util;
use std::{ use std::{
borrow::Cow, borrow::Cow,
collections::{BTreeMap, HashMap}, collections::{BTreeMap, HashMap, HashSet},
future::Future, future::Future,
mem::take, mem::take,
pin::Pin, pin::Pin,
...@@ -393,16 +393,23 @@ where ...@@ -393,16 +393,23 @@ where
HANDLER.set(handler, || GLOBALS.set(globals, f)) HANDLER.set(handler, || GLOBALS.set(globals, f))
} }
/// Analyse a provided [EcmascriptModuleAsset] and return a
/// [AnalyzeEcmascriptModuleResult].
///
/// # Arguments
/// * ignored_spans - A set of spans to ignore when analysing the module. This is useful for example
/// to respect turbopackIgnore directives on ignores.
#[turbo_tasks::function] #[turbo_tasks::function]
pub(crate) async fn analyse_ecmascript_module( pub(crate) async fn analyse_ecmascript_module(
module: Vc<EcmascriptModuleAsset>, module: Vc<EcmascriptModuleAsset>,
part: Option<Vc<ModulePart>>, part: Option<Vc<ModulePart>>,
ignored_spans: Option<Vc<HashSet<Span>>>,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> { ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let span = { let span = {
let module = module.ident().to_string().await?.to_string(); let module = module.ident().to_string().await?.to_string();
tracing::info_span!("analyse ecmascript module", module = module) tracing::info_span!("analyse ecmascript module", module = module)
}; };
let result = analyse_ecmascript_module_internal(module, part) let result = analyse_ecmascript_module_internal(module, part, ignored_spans)
.instrument(span) .instrument(span)
.await; .await;
...@@ -418,6 +425,7 @@ pub(crate) async fn analyse_ecmascript_module( ...@@ -418,6 +425,7 @@ pub(crate) async fn analyse_ecmascript_module(
pub(crate) async fn analyse_ecmascript_module_internal( pub(crate) async fn analyse_ecmascript_module_internal(
module: Vc<EcmascriptModuleAsset>, module: Vc<EcmascriptModuleAsset>,
part: Option<Vc<ModulePart>>, part: Option<Vc<ModulePart>>,
_webpack_ignored_effects: Option<Vc<HashSet<Span>>>,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> { ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let raw_module = module.await?; let raw_module = module.await?;
......
use std::collections::HashSet;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use swc_core::common::Span;
use turbo_tasks::Vc; use turbo_tasks::Vc;
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -59,7 +62,11 @@ impl EcmascriptAnalyzable for EcmascriptModulePartAsset { ...@@ -59,7 +62,11 @@ impl EcmascriptAnalyzable for EcmascriptModulePartAsset {
async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> { async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let this = self.await?; let this = self.await?;
let part = this.part; let part = this.part;
Ok(analyse_ecmascript_module(this.full_module, Some(part))) Ok(analyse_ecmascript_module(
this.full_module,
Some(part),
None,
))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -131,7 +138,7 @@ impl Module for EcmascriptModulePartAsset { ...@@ -131,7 +138,7 @@ impl Module for EcmascriptModulePartAsset {
async fn references(&self) -> Result<Vc<ModuleReferences>> { async fn references(&self) -> Result<Vc<ModuleReferences>> {
let split_data = split_module(self.full_module).await?; let split_data = split_module(self.full_module).await?;
let analyze = analyze(self.full_module, self.part).await?; let analyze = analyze(self.full_module, self.part, None).await?;
let (deps, entrypoints) = match &*split_data { let (deps, entrypoints) = match &*split_data {
SplitResult::Ok { SplitResult::Ok {
...@@ -264,16 +271,17 @@ impl EcmascriptModulePartAsset { ...@@ -264,16 +271,17 @@ impl EcmascriptModulePartAsset {
pub(super) async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> { pub(super) async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let this = self.await?; let this = self.await?;
Ok(analyze(this.full_module, this.part)) Ok(analyze(this.full_module, this.part, None))
} }
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn analyze( fn analyze(
module: Vc<EcmascriptModuleAsset>, module: Vc<EcmascriptModuleAsset>,
part: Vc<ModulePart>, part: Vc<ModulePart>,
ignored_spans: Option<Vc<HashSet<Span>>>,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> { ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
Ok(analyse_ecmascript_module(module, Some(part))) Ok(analyse_ecmascript_module(module, Some(part), ignored_spans))
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
......
...@@ -505,6 +505,7 @@ pub(super) async fn split( ...@@ -505,6 +505,7 @@ pub(super) async fn split(
&program, &program,
eval_context.unresolved_mark, eval_context.unresolved_mark,
eval_context.top_level_mark, eval_context.top_level_mark,
None,
Some(source), Some(source),
); );
...@@ -629,6 +630,7 @@ pub(super) async fn part_of_module( ...@@ -629,6 +630,7 @@ pub(super) async fn part_of_module(
eval_context.unresolved_mark, eval_context.unresolved_mark,
eval_context.top_level_mark, eval_context.top_level_mark,
None, None,
None,
); );
return Ok(ParseResult::Ok { return Ok(ParseResult::Ok {
...@@ -706,6 +708,7 @@ pub(super) async fn part_of_module( ...@@ -706,6 +708,7 @@ pub(super) async fn part_of_module(
eval_context.unresolved_mark, eval_context.unresolved_mark,
eval_context.top_level_mark, eval_context.top_level_mark,
None, None,
None,
); );
return Ok(ParseResult::Ok { return Ok(ParseResult::Ok {
program, program,
......
t_i = FreeVar(import)("t_i")
t_r = (
| FreeVar(require)("w_i_f")
| FreeVar(require)("t_i_f")
| FreeVar(require)("t_r_t")
| FreeVar(require)("t_r_f")
)
w_i = FreeVar(import)("w_i")
w_r = (FreeVar(require)("w_r_t") | FreeVar(require)("w_r_f"))
[
(
"t_i",
Call(
3,
FreeVar(
"import",
),
[
Constant(
Str(
Word(
"t_i",
),
),
),
],
),
),
(
"t_r",
Alternatives {
total_nodes: 13,
values: [
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"w_i_f",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"t_i_f",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"t_r_t",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"t_r_f",
),
),
),
],
),
],
logical_property: None,
},
),
(
"w_i",
Call(
3,
FreeVar(
"import",
),
[
Constant(
Str(
Word(
"w_i",
),
),
),
],
),
),
(
"w_r",
Alternatives {
total_nodes: 7,
values: [
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"w_r_t",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"w_r_f",
),
),
),
],
),
],
logical_property: None,
},
),
]
const w_i = import(/* webpackIgnore: true */ 'w_i')
const t_r = require(/* webpackIgnore: false */ 'w_i_f')
const t_i = import(/* turbopackIgnore: true */ 't_i')
const t_r = require(/* turbopackIgnore: false */ 't_i_f')
const w_r = require(/* webpackIgnore: true */ 'w_r_t')
const w_r = require(/* webpackIgnore: false */ 'w_r_f')
const t_r = require(/* turbopackIgnore: true */ 't_r_t')
const t_r = require(/* turbopackIgnore: false */ 't_r_f')
0 -> 1 call = import*0*("w_i")
- *0* import: The dynamic import() method from the ESM specification: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#dynamic_imports
0 -> 2 free var = FreeVar(require)
0 -> 3 call = require*0*("w_i_f")
- *0* require: The require method from CommonJS
0 -> 4 call = import*0*("t_i")
- *0* import: The dynamic import() method from the ESM specification: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#dynamic_imports
0 -> 5 free var = FreeVar(require)
0 -> 6 call = require*0*("t_i_f")
- *0* require: The require method from CommonJS
0 -> 7 free var = FreeVar(require)
0 -> 8 call = require*0*("w_r_t")
- *0* require: The require method from CommonJS
0 -> 9 free var = FreeVar(require)
0 -> 10 call = require*0*("w_r_f")
- *0* require: The require method from CommonJS
0 -> 11 free var = FreeVar(require)
0 -> 12 call = require*0*("t_r_t")
- *0* require: The require method from CommonJS
0 -> 13 free var = FreeVar(require)
0 -> 14 call = require*0*("t_r_f")
- *0* require: The require method from CommonJS
t_i = module<"t_i", {}>
t_r = (module<w_i_f, {}> | module<t_i_f, {}> | module<t_r_t, {}> | module<t_r_f, {}>)
w_i = module<"w_i", {}>
w_r = (module<w_r_t, {}> | module<w_r_f, {}>)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment