Unverified Commit ab7744e4 authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

Respect webpackIgnore and turboIgnore comments in the parser (#68451)

## What

This is step one of PACK-3046, setting up the necessary parser-side changes to detect and store the webpackIgnore and turbopackIgnore comment directives. This is followed by another PR that implements the bundler side. 

## Why

Webpack supports special directives to prevent bundle-time linking of specific imports. Some libraries need this (mapbox) and so we should support it.

## How

The directives are stored in comments within `require` and `import` calls. We parse the comments alongside traversal and look for comments that exist inside the function calls. These are then collected and used in later stages when performing linking to opt out of linking packages that have been explicitly excluded.
parent 678b4879
......@@ -56,7 +56,7 @@ pub fn benchmark(c: &mut Criterion) {
program.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
let eval_context =
EvalContext::new(&program, unresolved_mark, top_level_mark, None);
EvalContext::new(&program, unresolved_mark, top_level_mark, None, None);
let var_graph = create_graph(&program, &eval_context);
let input = BenchInput {
......
# turbopack-ecmascript
## Adding new parser tests
We use a snapshot-based testing system to ensure that changes to the parser don't break existing code.
To add a new test, you need to create a new directory in `tests/analyzer/graph` with an 'input.js' file
inside.
The snapshot tests are done with the `testing` crate. You can upate them by passing the env var
`UPDATE=1` to the test runner.
```sh
UPDATE=1 cargo test -p turbopack-ecmascript
```
......@@ -6,7 +6,7 @@ use std::{
use swc_core::{
atoms::Atom,
common::{pass::AstNodePath, Mark, Span, Spanned, SyntaxContext, GLOBALS},
common::{comments::Comments, pass::AstNodePath, Mark, Span, Spanned, SyntaxContext, GLOBALS},
ecma::{
ast::*,
atoms::js_word,
......@@ -294,6 +294,8 @@ pub fn create_graph(m: &Program, eval_context: &EvalContext) -> VarGraph {
graph
}
/// A context used for assembling the evaluation graph.
#[derive(Debug)]
pub struct EvalContext {
pub(crate) unresolved_mark: Mark,
pub(crate) top_level_mark: Mark,
......@@ -301,16 +303,20 @@ pub struct EvalContext {
}
impl EvalContext {
/// Produce a new [EvalContext] from a [Program]. If you wish to support
/// webpackIgnore or turbopackIgnore comments, you must pass those in,
/// since the AST does not include comments by default.
pub fn new(
module: &Program,
unresolved_mark: Mark,
top_level_mark: Mark,
comments: Option<&dyn Comments>,
source: Option<Vc<Box<dyn Source>>>,
) -> Self {
Self {
unresolved_mark,
top_level_mark,
imports: ImportMap::analyze(module, source),
imports: ImportMap::analyze(module, source, comments),
}
}
......
use std::{collections::BTreeMap, fmt::Display};
use std::{
collections::{BTreeMap, HashMap},
fmt::Display,
};
use indexmap::{IndexMap, IndexSet};
use once_cell::sync::Lazy;
use swc_core::{
common::{source_map::SmallPos, Span},
common::{comments::Comments, source_map::SmallPos, Span, Spanned},
ecma::{
ast::*,
atoms::{js_word, JsWord},
......@@ -136,6 +139,17 @@ pub(crate) struct ImportMap {
/// True if the module is an ESM module due to top-level await.
has_top_level_await: bool,
/// Locations of webpackIgnore or turbopackIgnore comments
/// This is a webpack feature that allows opting out of static
/// imports, which we should respect.
///
/// Example:
/// ```js
/// const a = import(/* webpackIgnore: true */ "a");
/// const b = import(/* turbopackIgnore: true */ "b");
/// ```
turbopack_ignores: HashMap<Span, bool>,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
......@@ -200,12 +214,17 @@ impl ImportMap {
}
/// Analyze ES import
pub(super) fn analyze(m: &Program, source: Option<Vc<Box<dyn Source>>>) -> Self {
pub(super) fn analyze(
m: &Program,
source: Option<Vc<Box<dyn Source>>>,
comments: Option<&dyn Comments>,
) -> Self {
let mut data = ImportMap::default();
m.visit_with(&mut Analyzer {
data: &mut data,
source,
comments,
});
data
......@@ -215,6 +234,7 @@ impl ImportMap {
struct Analyzer<'a> {
data: &'a mut ImportMap,
source: Option<Vc<Box<dyn Source>>>,
comments: Option<&'a dyn Comments>,
}
impl<'a> Analyzer<'a> {
......@@ -409,8 +429,63 @@ impl Visit for Analyzer<'_> {
fn visit_export_default_expr(&mut self, _: &ExportDefaultExpr) {
self.data.has_exports = true;
}
fn visit_stmt(&mut self, _: &Stmt) {
// don't visit children
fn visit_stmt(&mut self, n: &Stmt) {
if self.comments.is_some() {
// only visit children if we potentially need to mark import / requires
n.visit_children_with(self);
}
}
/// check if import or require contains an ignore comment
///
/// We are checking for the following cases:
/// - import(/* webpackIgnore: true */ "a")
/// - require(/* webpackIgnore: true */ "a")
///
/// We can do this by checking if any of the comment spans are between the
/// callee and the first argument.
fn visit_call_expr(&mut self, n: &CallExpr) {
// we can actually unwrap thanks to the optimisation above
// but it can't hurt to be safe...
if let Some(comments) = self.comments {
let callee_span = match &n.callee {
Callee::Import(Import { span, .. }) => Some(span),
// this assumes you cannot reassign `require`
Callee::Expr(box Expr::Ident(Ident { span, sym, .. })) if sym == "require" => {
Some(span)
}
_ => None,
};
// we are interested here in the last comment with a valid directive
let ignore_statement = n
.args
.first()
.map(|arg| arg.span_lo())
.and_then(|comment_pos| comments.get_leading(comment_pos))
.iter()
.flatten()
.rev()
.filter_map(|comment| {
let (directive, value) = comment.text.trim().split_once(':')?;
// support whitespace between the colon
match (directive.trim(), value.trim()) {
("webpackIgnore" | "turbopackIgnore", "true") => Some(true),
("webpackIgnore" | "turbopackIgnore", "false") => Some(false),
_ => None, // ignore anything else
}
})
.next();
if let Some((callee_span, ignore_statement)) = callee_span.zip(ignore_statement) {
self.data
.turbopack_ignores
.insert(*callee_span, ignore_statement);
};
}
n.visit_children_with(self);
}
fn visit_program(&mut self, m: &Program) {
......
......@@ -3822,7 +3822,8 @@ mod tests {
let top_level_mark = Mark::new();
m.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
let eval_context = EvalContext::new(&m, unresolved_mark, top_level_mark, None);
let eval_context =
EvalContext::new(&m, unresolved_mark, top_level_mark, None, None);
let mut var_graph = create_graph(&m, &eval_context);
......
......@@ -366,7 +366,7 @@ impl EcmascriptParsable for EcmascriptModuleAsset {
impl EcmascriptAnalyzable for EcmascriptModuleAsset {
#[turbo_tasks::function]
fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult> {
analyse_ecmascript_module(self, None)
analyse_ecmascript_module(self, None, None)
}
/// Generates module contents without an analysis pass. This is useful for
......
......@@ -433,6 +433,7 @@ async fn parse_content(
&parsed_program,
unresolved_mark,
top_level_mark,
None,
Some(source),
);
......
......@@ -17,7 +17,7 @@ pub mod util;
use std::{
borrow::Cow,
collections::{BTreeMap, HashMap},
collections::{BTreeMap, HashMap, HashSet},
future::Future,
mem::take,
pin::Pin,
......@@ -393,16 +393,23 @@ where
HANDLER.set(handler, || GLOBALS.set(globals, f))
}
/// Analyse a provided [EcmascriptModuleAsset] and return a
/// [AnalyzeEcmascriptModuleResult].
///
/// # Arguments
/// * ignored_spans - A set of spans to ignore when analysing the module. This is useful for example
/// to respect turbopackIgnore directives on ignores.
#[turbo_tasks::function]
pub(crate) async fn analyse_ecmascript_module(
module: Vc<EcmascriptModuleAsset>,
part: Option<Vc<ModulePart>>,
ignored_spans: Option<Vc<HashSet<Span>>>,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let span = {
let module = module.ident().to_string().await?.to_string();
tracing::info_span!("analyse ecmascript module", module = module)
};
let result = analyse_ecmascript_module_internal(module, part)
let result = analyse_ecmascript_module_internal(module, part, ignored_spans)
.instrument(span)
.await;
......@@ -418,6 +425,7 @@ pub(crate) async fn analyse_ecmascript_module(
pub(crate) async fn analyse_ecmascript_module_internal(
module: Vc<EcmascriptModuleAsset>,
part: Option<Vc<ModulePart>>,
_webpack_ignored_effects: Option<Vc<HashSet<Span>>>,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let raw_module = module.await?;
......
use std::collections::HashSet;
use anyhow::{Context, Result};
use swc_core::common::Span;
use turbo_tasks::Vc;
use turbopack_core::{
asset::{Asset, AssetContent},
......@@ -59,7 +62,11 @@ impl EcmascriptAnalyzable for EcmascriptModulePartAsset {
async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let this = self.await?;
let part = this.part;
Ok(analyse_ecmascript_module(this.full_module, Some(part)))
Ok(analyse_ecmascript_module(
this.full_module,
Some(part),
None,
))
}
#[turbo_tasks::function]
......@@ -131,7 +138,7 @@ impl Module for EcmascriptModulePartAsset {
async fn references(&self) -> Result<Vc<ModuleReferences>> {
let split_data = split_module(self.full_module).await?;
let analyze = analyze(self.full_module, self.part).await?;
let analyze = analyze(self.full_module, self.part, None).await?;
let (deps, entrypoints) = match &*split_data {
SplitResult::Ok {
......@@ -264,16 +271,17 @@ impl EcmascriptModulePartAsset {
pub(super) async fn analyze(self: Vc<Self>) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
let this = self.await?;
Ok(analyze(this.full_module, this.part))
Ok(analyze(this.full_module, this.part, None))
}
}
#[turbo_tasks::function]
async fn analyze(
fn analyze(
module: Vc<EcmascriptModuleAsset>,
part: Vc<ModulePart>,
ignored_spans: Option<Vc<HashSet<Span>>>,
) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
Ok(analyse_ecmascript_module(module, Some(part)))
Ok(analyse_ecmascript_module(module, Some(part), ignored_spans))
}
#[turbo_tasks::value_impl]
......
......@@ -505,6 +505,7 @@ pub(super) async fn split(
&program,
eval_context.unresolved_mark,
eval_context.top_level_mark,
None,
Some(source),
);
......@@ -629,6 +630,7 @@ pub(super) async fn part_of_module(
eval_context.unresolved_mark,
eval_context.top_level_mark,
None,
None,
);
return Ok(ParseResult::Ok {
......@@ -706,6 +708,7 @@ pub(super) async fn part_of_module(
eval_context.unresolved_mark,
eval_context.top_level_mark,
None,
None,
);
return Ok(ParseResult::Ok {
program,
......
t_i = FreeVar(import)("t_i")
t_r = (
| FreeVar(require)("w_i_f")
| FreeVar(require)("t_i_f")
| FreeVar(require)("t_r_t")
| FreeVar(require)("t_r_f")
)
w_i = FreeVar(import)("w_i")
w_r = (FreeVar(require)("w_r_t") | FreeVar(require)("w_r_f"))
[
(
"t_i",
Call(
3,
FreeVar(
"import",
),
[
Constant(
Str(
Word(
"t_i",
),
),
),
],
),
),
(
"t_r",
Alternatives {
total_nodes: 13,
values: [
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"w_i_f",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"t_i_f",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"t_r_t",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"t_r_f",
),
),
),
],
),
],
logical_property: None,
},
),
(
"w_i",
Call(
3,
FreeVar(
"import",
),
[
Constant(
Str(
Word(
"w_i",
),
),
),
],
),
),
(
"w_r",
Alternatives {
total_nodes: 7,
values: [
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"w_r_t",
),
),
),
],
),
Call(
3,
FreeVar(
"require",
),
[
Constant(
Str(
Word(
"w_r_f",
),
),
),
],
),
],
logical_property: None,
},
),
]
const w_i = import(/* webpackIgnore: true */ 'w_i')
const t_r = require(/* webpackIgnore: false */ 'w_i_f')
const t_i = import(/* turbopackIgnore: true */ 't_i')
const t_r = require(/* turbopackIgnore: false */ 't_i_f')
const w_r = require(/* webpackIgnore: true */ 'w_r_t')
const w_r = require(/* webpackIgnore: false */ 'w_r_f')
const t_r = require(/* turbopackIgnore: true */ 't_r_t')
const t_r = require(/* turbopackIgnore: false */ 't_r_f')
0 -> 1 call = import*0*("w_i")
- *0* import: The dynamic import() method from the ESM specification: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#dynamic_imports
0 -> 2 free var = FreeVar(require)
0 -> 3 call = require*0*("w_i_f")
- *0* require: The require method from CommonJS
0 -> 4 call = import*0*("t_i")
- *0* import: The dynamic import() method from the ESM specification: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#dynamic_imports
0 -> 5 free var = FreeVar(require)
0 -> 6 call = require*0*("t_i_f")
- *0* require: The require method from CommonJS
0 -> 7 free var = FreeVar(require)
0 -> 8 call = require*0*("w_r_t")
- *0* require: The require method from CommonJS
0 -> 9 free var = FreeVar(require)
0 -> 10 call = require*0*("w_r_f")
- *0* require: The require method from CommonJS
0 -> 11 free var = FreeVar(require)
0 -> 12 call = require*0*("t_r_t")
- *0* require: The require method from CommonJS
0 -> 13 free var = FreeVar(require)
0 -> 14 call = require*0*("t_r_f")
- *0* require: The require method from CommonJS
t_i = module<"t_i", {}>
t_r = (module<w_i_f, {}> | module<t_i_f, {}> | module<t_r_t, {}> | module<t_r_f, {}>)
w_i = module<"w_i", {}>
w_r = (module<w_r_t, {}> | module<w_r_f, {}>)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment