Unverified Commit 565d224b authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Turbopack: Support ECMAScript styleregex with negative lookahead in…

Turbopack: Support ECMAScript styleregex with negative lookahead in `require.context` calls (#78251)

Add support for more ECMASCRIPT style regular expressions via the `regress` regex engine in order to support `require.context` calls with complex filters.

We still prefer the `regex` crate when possible as it performs better but fall back to `regress` when the regular expression uses unsupported features.

This should improve webpack compatibility for `require.context` calls.

Closes PACK-3895
parent 61abc3fa
...@@ -2365,6 +2365,12 @@ version = "1.0.7" ...@@ -2365,6 +2365,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]] [[package]]
name = "foreign-types" name = "foreign-types"
version = "0.3.2" version = "0.3.2"
...@@ -2772,6 +2778,11 @@ name = "hashbrown" ...@@ -2772,6 +2778,11 @@ name = "hashbrown"
version = "0.15.2" version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]] [[package]]
name = "hdrhistogram" name = "hdrhistogram"
...@@ -5928,6 +5939,16 @@ dependencies = [ ...@@ -5928,6 +5939,16 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "regress"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366"
dependencies = [
"hashbrown 0.15.2",
"memchr",
]
[[package]] [[package]]
name = "relative-path" name = "relative-path"
version = "1.9.0" version = "1.9.0"
...@@ -9838,6 +9859,7 @@ dependencies = [ ...@@ -9838,6 +9859,7 @@ dependencies = [
"parking_lot", "parking_lot",
"petgraph 0.6.3", "petgraph 0.6.3",
"regex", "regex",
"regress",
"rustc-hash 2.1.0", "rustc-hash 2.1.0",
"serde", "serde",
"serde_json", "serde_json",
......
...@@ -388,6 +388,7 @@ quote = "1.0.23" ...@@ -388,6 +388,7 @@ quote = "1.0.23"
rand = "0.9.0" rand = "0.9.0"
rayon = "1.10.0" rayon = "1.10.0"
regex = "1.10.6" regex = "1.10.6"
regress = "0.10.3"
reqwest = { version = "=0.11.17", default-features = false } reqwest = { version = "=0.11.17", default-features = false }
rstest = "0.16.0" rstest = "0.16.0"
rustc-hash = "2.1.0" rustc-hash = "2.1.0"
......
...@@ -26,6 +26,7 @@ once_cell = { workspace = true } ...@@ -26,6 +26,7 @@ once_cell = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
petgraph = { workspace = true } petgraph = { workspace = true }
regex = { workspace = true } regex = { workspace = true }
regress = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
......
use anyhow::{bail, Result};
/// A simple regular expression implementation following ecmascript semantics
///
/// Delegates to the `regex` crate when possible and `regress` otherwise.
#[derive(Debug, Clone)]
#[turbo_tasks::value(eq = "manual", shared)]
#[serde(into = "RegexForm", try_from = "RegexForm")]
pub struct EsRegex {
#[turbo_tasks(trace_ignore)]
delegate: EsRegexImpl,
// Store the original arguments used to construct
// this regex to support equality and serialization.
pattern: String,
flags: String,
}
#[derive(Debug, Clone)]
enum EsRegexImpl {
Regex(regex::Regex),
Regress(regress::Regex),
}
/// Equality uses the source inputs since our delegate regex impls don't support
/// equality natively.
/// NOTE: there are multiple 'equivalent' ways to write a regex and this
/// approach does _not_ attempt to equate them.
impl PartialEq for EsRegex {
fn eq(&self, other: &Self) -> bool {
self.pattern == other.pattern && self.flags == other.flags
}
}
impl Eq for EsRegex {}
impl TryFrom<RegexForm> for EsRegex {
type Error = anyhow::Error;
fn try_from(value: RegexForm) -> std::result::Result<Self, Self::Error> {
EsRegex::new(&value.pattern, &value.pattern)
}
}
/// This is the serializable form for the `EsRegex` struct
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
struct RegexForm {
pattern: String,
flags: String,
}
impl From<EsRegex> for RegexForm {
fn from(value: EsRegex) -> Self {
Self {
pattern: value.pattern,
flags: value.flags,
}
}
}
impl EsRegex {
/// Support ecmascript style regular expressions by selecting the `regex` crate when possible
/// and using regress when not.
pub fn new(pattern: &str, flags: &str) -> Result<Self> {
// rust regex doesn't allow escaped slashes, but they are necessary in js
let pattern = pattern.replace("\\/", "/");
let mut applied_flags = String::new();
for flag in flags.chars() {
match flag {
// indices for substring matches: not relevant for the regex itself
'd' => {}
// global: default in rust, ignore
'g' => {}
// case-insensitive: letters match both upper and lower case
'i' => applied_flags.push('i'),
// multi-line mode: ^ and $ match begin/end of line
'm' => applied_flags.push('m'),
// allow . to match \n
's' => applied_flags.push('s'),
// Unicode support (enabled by default)
'u' => applied_flags.push('u'),
// sticky search: not relevant for the regex itself
'y' => {}
_ => bail!("unsupported flag `{}` in regex", flag),
}
}
let regex = if !applied_flags.is_empty() {
regex::Regex::new(&format!("(?{}){}", applied_flags, pattern))
} else {
regex::Regex::new(&pattern)
};
let delegate = match regex {
Ok(reg) => Ok(EsRegexImpl::Regex(reg)),
Err(_e) => {
// We failed to parse as an regex:Regex, try using regress. Regress uses the es
// flags format so we can pass the original flags value.
match regress::Regex::with_flags(&pattern, regress::Flags::from(flags)) {
Ok(reg) => Ok(EsRegexImpl::Regress(reg)),
// Propogate the error as is, regress has useful error messages.
Err(e) => Err(e),
}
}
}?;
Ok(Self {
delegate,
pattern,
flags: flags.to_string(),
})
}
/// Returns true if there is any match for this regex in the `haystac`
pub fn is_match(&self, haystack: &str) -> bool {
match &self.delegate {
EsRegexImpl::Regex(r) => r.is_match(haystack),
EsRegexImpl::Regress(r) => r.find(haystack).is_some(),
}
}
}
#[cfg(test)]
mod tests {
use super::{EsRegex, EsRegexImpl};
#[test]
fn es_regex_matches_simple() {
let regex = EsRegex::new("a", "").unwrap();
assert!(matches!(regex.delegate, EsRegexImpl::Regex { .. }));
assert!(regex.is_match("a"));
}
#[test]
fn es_regex_matches_negative_lookahead() {
// This feature is not supported by the regex crate
let regex = EsRegex::new("a(?!b)", "").unwrap();
assert!(matches!(regex.delegate, EsRegexImpl::Regress { .. }));
assert!(!regex.is_match("ab"));
assert!(regex.is_match("ac"));
}
#[test]
fn invalid_regex() {
// This is invalid since there is nothing being repeated
// Don't bother asserting on the message since we delegate
// that to the underlying implementations.
assert!(matches!(EsRegex::new("*", ""), Err { .. }))
}
}
...@@ -11,12 +11,11 @@ use std::{ ...@@ -11,12 +11,11 @@ use std::{
sync::Arc, sync::Arc,
}; };
use anyhow::{bail, Context, Result}; use anyhow::{bail, Result};
use graph::VarGraph; use graph::VarGraph;
use num_bigint::BigInt; use num_bigint::BigInt;
use num_traits::identities::Zero; use num_traits::identities::Zero;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use swc_core::{ use swc_core::{
common::Mark, common::Mark,
...@@ -33,9 +32,12 @@ use turbopack_core::compile_time_info::{ ...@@ -33,9 +32,12 @@ use turbopack_core::compile_time_info::{
use self::imports::ImportAnnotations; use self::imports::ImportAnnotations;
pub(crate) use self::imports::ImportMap; pub(crate) use self::imports::ImportMap;
use crate::{references::require_context::RequireContextMap, utils::StringifyJs}; use crate::{
analyzer::es_regex::EsRegex, references::require_context::RequireContextMap, utils::StringifyJs,
};
pub mod builtin; pub mod builtin;
pub mod es_regex;
pub mod graph; pub mod graph;
pub mod imports; pub mod imports;
pub mod linker; pub mod linker;
...@@ -3722,42 +3724,7 @@ pub struct RequireContextOptions { ...@@ -3722,42 +3724,7 @@ pub struct RequireContextOptions {
pub dir: RcStr, pub dir: RcStr,
pub include_subdirs: bool, pub include_subdirs: bool,
/// this is a regex (pattern, flags) /// this is a regex (pattern, flags)
pub filter: Regex, pub filter: EsRegex,
}
/// Convert an ECMAScript regex to a Rust regex.
fn regex_from_js(pattern: &str, flags: &str) -> Result<Regex> {
// rust regex doesn't allow escaped slashes, but they are necessary in js
let pattern = pattern.replace("\\/", "/");
let mut applied_flags = String::new();
for flag in flags.chars() {
match flag {
// indices for substring matches: not relevant for the regex itself
'd' => {}
// global: default in rust, ignore
'g' => {}
// case-insensitive: letters match both upper and lower case
'i' => applied_flags.push('i'),
// multi-line mode: ^ and $ match begin/end of line
'm' => applied_flags.push('m'),
// allow . to match \n
's' => applied_flags.push('s'),
// Unicode support (enabled by default)
'u' => applied_flags.push('u'),
// sticky search: not relevant for the regex itself
'y' => {}
_ => bail!("unsupported flag `{}` in regex", flag),
}
}
let regex = if !applied_flags.is_empty() {
format!("(?{}){}", applied_flags, pattern)
} else {
pattern
};
Regex::new(&regex).context("could not convert ECMAScript regex to Rust regex")
} }
/// Parse the arguments passed to a require.context invocation, validate them /// Parse the arguments passed to a require.context invocation, validate them
...@@ -3787,14 +3754,14 @@ pub fn parse_require_context(args: &[JsValue]) -> Result<RequireContextOptions> ...@@ -3787,14 +3754,14 @@ pub fn parse_require_context(args: &[JsValue]) -> Result<RequireContextOptions>
let filter = if let Some(filter) = args.get(2) { let filter = if let Some(filter) = args.get(2) {
if let JsValue::Constant(ConstantValue::Regex(box (pattern, flags))) = filter { if let JsValue::Constant(ConstantValue::Regex(box (pattern, flags))) = filter {
regex_from_js(pattern, flags)? EsRegex::new(pattern, flags)?
} else { } else {
bail!("require.context(..., ..., filter) requires filter to be a regex"); bail!("require.context(..., ..., filter) requires filter to be a regex");
} }
} else { } else {
// https://webpack.js.org/api/module-methods/#requirecontext // https://webpack.js.org/api/module-methods/#requirecontext
// > optional, default /^\.\/.*$/, any file // > optional, default /^\.\/.*$/, any file
static DEFAULT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^\\./.*$").unwrap()); static DEFAULT_REGEX: Lazy<EsRegex> = Lazy::new(|| EsRegex::new(r"^\\./.*$", "").unwrap());
DEFAULT_REGEX.clone() DEFAULT_REGEX.clone()
}; };
......
...@@ -1857,7 +1857,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>( ...@@ -1857,7 +1857,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
origin, origin,
options.dir, options.dir,
options.include_subdirs, options.include_subdirs,
Vc::cell(options.filter), options.filter.cell(),
Some(issue_source(source, span)), Some(issue_source(source, span)),
in_try, in_try,
) )
...@@ -3025,7 +3025,7 @@ async fn require_context_visitor( ...@@ -3025,7 +3025,7 @@ async fn require_context_visitor(
origin, origin,
dir, dir,
options.include_subdirs, options.include_subdirs,
Vc::cell(options.filter), options.filter.cell(),
None, None,
true, true,
); );
......
...@@ -15,8 +15,8 @@ use swc_core::{ ...@@ -15,8 +15,8 @@ use swc_core::{
}; };
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
debug::ValueDebugFormat, primitives::Regex, trace::TraceRawVcs, FxIndexMap, NonLocalValue, debug::ValueDebugFormat, trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, Value,
ResolvedVc, Value, ValueToString, Vc, ValueToString, Vc,
}; };
use turbo_tasks_fs::{DirectoryContent, DirectoryEntry, FileSystemPath}; use turbo_tasks_fs::{DirectoryContent, DirectoryEntry, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
...@@ -36,6 +36,7 @@ use turbopack_core::{ ...@@ -36,6 +36,7 @@ use turbopack_core::{
use turbopack_resolve::ecmascript::cjs_resolve; use turbopack_resolve::ecmascript::cjs_resolve;
use crate::{ use crate::{
analyzer::es_regex::EsRegex,
chunk::{ chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkType, EcmascriptExports, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkType, EcmascriptExports,
}, },
...@@ -63,7 +64,7 @@ pub(crate) struct DirList(FxIndexMap<RcStr, DirListEntry>); ...@@ -63,7 +64,7 @@ pub(crate) struct DirList(FxIndexMap<RcStr, DirListEntry>);
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl DirList { impl DirList {
#[turbo_tasks::function] #[turbo_tasks::function]
pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<Regex>) -> Vc<Self> { pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<EsRegex>) -> Vc<Self> {
Self::read_internal(dir, dir, recursive, filter) Self::read_internal(dir, dir, recursive, filter)
} }
...@@ -72,7 +73,7 @@ impl DirList { ...@@ -72,7 +73,7 @@ impl DirList {
root: Vc<FileSystemPath>, root: Vc<FileSystemPath>,
dir: Vc<FileSystemPath>, dir: Vc<FileSystemPath>,
recursive: bool, recursive: bool,
filter: Vc<Regex>, filter: Vc<EsRegex>,
) -> Result<Vc<Self>> { ) -> Result<Vc<Self>> {
let root_val = &root.await?; let root_val = &root.await?;
let dir_val = &dir.await?; let dir_val = &dir.await?;
...@@ -148,7 +149,7 @@ pub(crate) struct FlatDirList(FxIndexMap<RcStr, ResolvedVc<FileSystemPath>>); ...@@ -148,7 +149,7 @@ pub(crate) struct FlatDirList(FxIndexMap<RcStr, ResolvedVc<FileSystemPath>>);
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl FlatDirList { impl FlatDirList {
#[turbo_tasks::function] #[turbo_tasks::function]
pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<Regex>) -> Vc<Self> { pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<EsRegex>) -> Vc<Self> {
DirList::read(dir, recursive, filter).flatten() DirList::read(dir, recursive, filter).flatten()
} }
} }
...@@ -172,7 +173,7 @@ impl RequireContextMap { ...@@ -172,7 +173,7 @@ impl RequireContextMap {
origin: Vc<Box<dyn ResolveOrigin>>, origin: Vc<Box<dyn ResolveOrigin>>,
dir: Vc<FileSystemPath>, dir: Vc<FileSystemPath>,
recursive: bool, recursive: bool,
filter: Vc<Regex>, filter: Vc<EsRegex>,
issue_source: Option<IssueSource>, issue_source: Option<IssueSource>,
is_optional: bool, is_optional: bool,
) -> Result<Vc<Self>> { ) -> Result<Vc<Self>> {
...@@ -227,7 +228,7 @@ impl RequireContextAssetReference { ...@@ -227,7 +228,7 @@ impl RequireContextAssetReference {
origin: ResolvedVc<Box<dyn ResolveOrigin>>, origin: ResolvedVc<Box<dyn ResolveOrigin>>,
dir: RcStr, dir: RcStr,
include_subdirs: bool, include_subdirs: bool,
filter: Vc<Regex>, filter: Vc<EsRegex>,
issue_source: Option<IssueSource>, issue_source: Option<IssueSource>,
in_try: bool, in_try: bool,
) -> Result<Self> { ) -> Result<Self> {
......
// PACK-3895: ensure that negative lookaround works.
it('ensure require.context respects filters with look-around assertions', ()=> {
let ctx = require.context("./deps", true, /foo(?!_test)/);
// import all the matches, should just get foo.js
expect(ctx.keys().map(k => Object.values(ctx(k))).flat()).toEqual(['You found me']);
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment