Unverified Commit 565d224b authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Turbopack: Support ECMAScript styleregex with negative lookahead in…

Turbopack: Support ECMAScript styleregex with negative lookahead in `require.context` calls (#78251)

Add support for more ECMASCRIPT style regular expressions via the `regress` regex engine in order to support `require.context` calls with complex filters.

We still prefer the `regex` crate when possible as it performs better but fall back to `regress` when the regular expression uses unsupported features.

This should improve webpack compatibility for `require.context` calls.

Closes PACK-3895
parent 61abc3fa
......@@ -2365,6 +2365,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foreign-types"
version = "0.3.2"
......@@ -2772,6 +2778,11 @@ name = "hashbrown"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "hdrhistogram"
......@@ -5928,6 +5939,16 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "regress"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366"
dependencies = [
"hashbrown 0.15.2",
"memchr",
]
[[package]]
name = "relative-path"
version = "1.9.0"
......@@ -9838,6 +9859,7 @@ dependencies = [
"parking_lot",
"petgraph 0.6.3",
"regex",
"regress",
"rustc-hash 2.1.0",
"serde",
"serde_json",
......
......@@ -388,6 +388,7 @@ quote = "1.0.23"
rand = "0.9.0"
rayon = "1.10.0"
regex = "1.10.6"
regress = "0.10.3"
reqwest = { version = "=0.11.17", default-features = false }
rstest = "0.16.0"
rustc-hash = "2.1.0"
......
......@@ -26,6 +26,7 @@ once_cell = { workspace = true }
parking_lot = { workspace = true }
petgraph = { workspace = true }
regex = { workspace = true }
regress = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
......
use anyhow::{bail, Result};
/// A simple regular expression implementation following ecmascript semantics
///
/// Delegates to the `regex` crate when possible and `regress` otherwise.
#[derive(Debug, Clone)]
#[turbo_tasks::value(eq = "manual", shared)]
#[serde(into = "RegexForm", try_from = "RegexForm")]
pub struct EsRegex {
#[turbo_tasks(trace_ignore)]
delegate: EsRegexImpl,
// Store the original arguments used to construct
// this regex to support equality and serialization.
pattern: String,
flags: String,
}
#[derive(Debug, Clone)]
enum EsRegexImpl {
Regex(regex::Regex),
Regress(regress::Regex),
}
/// Equality uses the source inputs since our delegate regex impls don't support
/// equality natively.
/// NOTE: there are multiple 'equivalent' ways to write a regex and this
/// approach does _not_ attempt to equate them.
impl PartialEq for EsRegex {
fn eq(&self, other: &Self) -> bool {
self.pattern == other.pattern && self.flags == other.flags
}
}
impl Eq for EsRegex {}
impl TryFrom<RegexForm> for EsRegex {
type Error = anyhow::Error;
fn try_from(value: RegexForm) -> std::result::Result<Self, Self::Error> {
EsRegex::new(&value.pattern, &value.pattern)
}
}
/// This is the serializable form for the `EsRegex` struct
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
struct RegexForm {
pattern: String,
flags: String,
}
impl From<EsRegex> for RegexForm {
fn from(value: EsRegex) -> Self {
Self {
pattern: value.pattern,
flags: value.flags,
}
}
}
impl EsRegex {
/// Support ecmascript style regular expressions by selecting the `regex` crate when possible
/// and using regress when not.
pub fn new(pattern: &str, flags: &str) -> Result<Self> {
// rust regex doesn't allow escaped slashes, but they are necessary in js
let pattern = pattern.replace("\\/", "/");
let mut applied_flags = String::new();
for flag in flags.chars() {
match flag {
// indices for substring matches: not relevant for the regex itself
'd' => {}
// global: default in rust, ignore
'g' => {}
// case-insensitive: letters match both upper and lower case
'i' => applied_flags.push('i'),
// multi-line mode: ^ and $ match begin/end of line
'm' => applied_flags.push('m'),
// allow . to match \n
's' => applied_flags.push('s'),
// Unicode support (enabled by default)
'u' => applied_flags.push('u'),
// sticky search: not relevant for the regex itself
'y' => {}
_ => bail!("unsupported flag `{}` in regex", flag),
}
}
let regex = if !applied_flags.is_empty() {
regex::Regex::new(&format!("(?{}){}", applied_flags, pattern))
} else {
regex::Regex::new(&pattern)
};
let delegate = match regex {
Ok(reg) => Ok(EsRegexImpl::Regex(reg)),
Err(_e) => {
// We failed to parse as an regex:Regex, try using regress. Regress uses the es
// flags format so we can pass the original flags value.
match regress::Regex::with_flags(&pattern, regress::Flags::from(flags)) {
Ok(reg) => Ok(EsRegexImpl::Regress(reg)),
// Propogate the error as is, regress has useful error messages.
Err(e) => Err(e),
}
}
}?;
Ok(Self {
delegate,
pattern,
flags: flags.to_string(),
})
}
/// Returns true if there is any match for this regex in the `haystac`
pub fn is_match(&self, haystack: &str) -> bool {
match &self.delegate {
EsRegexImpl::Regex(r) => r.is_match(haystack),
EsRegexImpl::Regress(r) => r.find(haystack).is_some(),
}
}
}
#[cfg(test)]
mod tests {
use super::{EsRegex, EsRegexImpl};
#[test]
fn es_regex_matches_simple() {
let regex = EsRegex::new("a", "").unwrap();
assert!(matches!(regex.delegate, EsRegexImpl::Regex { .. }));
assert!(regex.is_match("a"));
}
#[test]
fn es_regex_matches_negative_lookahead() {
// This feature is not supported by the regex crate
let regex = EsRegex::new("a(?!b)", "").unwrap();
assert!(matches!(regex.delegate, EsRegexImpl::Regress { .. }));
assert!(!regex.is_match("ab"));
assert!(regex.is_match("ac"));
}
#[test]
fn invalid_regex() {
// This is invalid since there is nothing being repeated
// Don't bother asserting on the message since we delegate
// that to the underlying implementations.
assert!(matches!(EsRegex::new("*", ""), Err { .. }))
}
}
......@@ -11,12 +11,11 @@ use std::{
sync::Arc,
};
use anyhow::{bail, Context, Result};
use anyhow::{bail, Result};
use graph::VarGraph;
use num_bigint::BigInt;
use num_traits::identities::Zero;
use once_cell::sync::Lazy;
use regex::Regex;
use rustc_hash::FxHasher;
use swc_core::{
common::Mark,
......@@ -33,9 +32,12 @@ use turbopack_core::compile_time_info::{
use self::imports::ImportAnnotations;
pub(crate) use self::imports::ImportMap;
use crate::{references::require_context::RequireContextMap, utils::StringifyJs};
use crate::{
analyzer::es_regex::EsRegex, references::require_context::RequireContextMap, utils::StringifyJs,
};
pub mod builtin;
pub mod es_regex;
pub mod graph;
pub mod imports;
pub mod linker;
......@@ -3722,42 +3724,7 @@ pub struct RequireContextOptions {
pub dir: RcStr,
pub include_subdirs: bool,
/// this is a regex (pattern, flags)
pub filter: Regex,
}
/// Convert an ECMAScript regex to a Rust regex.
fn regex_from_js(pattern: &str, flags: &str) -> Result<Regex> {
// rust regex doesn't allow escaped slashes, but they are necessary in js
let pattern = pattern.replace("\\/", "/");
let mut applied_flags = String::new();
for flag in flags.chars() {
match flag {
// indices for substring matches: not relevant for the regex itself
'd' => {}
// global: default in rust, ignore
'g' => {}
// case-insensitive: letters match both upper and lower case
'i' => applied_flags.push('i'),
// multi-line mode: ^ and $ match begin/end of line
'm' => applied_flags.push('m'),
// allow . to match \n
's' => applied_flags.push('s'),
// Unicode support (enabled by default)
'u' => applied_flags.push('u'),
// sticky search: not relevant for the regex itself
'y' => {}
_ => bail!("unsupported flag `{}` in regex", flag),
}
}
let regex = if !applied_flags.is_empty() {
format!("(?{}){}", applied_flags, pattern)
} else {
pattern
};
Regex::new(&regex).context("could not convert ECMAScript regex to Rust regex")
pub filter: EsRegex,
}
/// Parse the arguments passed to a require.context invocation, validate them
......@@ -3787,14 +3754,14 @@ pub fn parse_require_context(args: &[JsValue]) -> Result<RequireContextOptions>
let filter = if let Some(filter) = args.get(2) {
if let JsValue::Constant(ConstantValue::Regex(box (pattern, flags))) = filter {
regex_from_js(pattern, flags)?
EsRegex::new(pattern, flags)?
} else {
bail!("require.context(..., ..., filter) requires filter to be a regex");
}
} else {
// https://webpack.js.org/api/module-methods/#requirecontext
// > optional, default /^\.\/.*$/, any file
static DEFAULT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^\\./.*$").unwrap());
static DEFAULT_REGEX: Lazy<EsRegex> = Lazy::new(|| EsRegex::new(r"^\\./.*$", "").unwrap());
DEFAULT_REGEX.clone()
};
......
......@@ -1857,7 +1857,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
origin,
options.dir,
options.include_subdirs,
Vc::cell(options.filter),
options.filter.cell(),
Some(issue_source(source, span)),
in_try,
)
......@@ -3025,7 +3025,7 @@ async fn require_context_visitor(
origin,
dir,
options.include_subdirs,
Vc::cell(options.filter),
options.filter.cell(),
None,
true,
);
......
......@@ -15,8 +15,8 @@ use swc_core::{
};
use turbo_rcstr::RcStr;
use turbo_tasks::{
debug::ValueDebugFormat, primitives::Regex, trace::TraceRawVcs, FxIndexMap, NonLocalValue,
ResolvedVc, Value, ValueToString, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, Value,
ValueToString, Vc,
};
use turbo_tasks_fs::{DirectoryContent, DirectoryEntry, FileSystemPath};
use turbopack_core::{
......@@ -36,6 +36,7 @@ use turbopack_core::{
use turbopack_resolve::ecmascript::cjs_resolve;
use crate::{
analyzer::es_regex::EsRegex,
chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkType, EcmascriptExports,
},
......@@ -63,7 +64,7 @@ pub(crate) struct DirList(FxIndexMap<RcStr, DirListEntry>);
#[turbo_tasks::value_impl]
impl DirList {
#[turbo_tasks::function]
pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<Regex>) -> Vc<Self> {
pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<EsRegex>) -> Vc<Self> {
Self::read_internal(dir, dir, recursive, filter)
}
......@@ -72,7 +73,7 @@ impl DirList {
root: Vc<FileSystemPath>,
dir: Vc<FileSystemPath>,
recursive: bool,
filter: Vc<Regex>,
filter: Vc<EsRegex>,
) -> Result<Vc<Self>> {
let root_val = &root.await?;
let dir_val = &dir.await?;
......@@ -148,7 +149,7 @@ pub(crate) struct FlatDirList(FxIndexMap<RcStr, ResolvedVc<FileSystemPath>>);
#[turbo_tasks::value_impl]
impl FlatDirList {
#[turbo_tasks::function]
pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<Regex>) -> Vc<Self> {
pub(crate) fn read(dir: Vc<FileSystemPath>, recursive: bool, filter: Vc<EsRegex>) -> Vc<Self> {
DirList::read(dir, recursive, filter).flatten()
}
}
......@@ -172,7 +173,7 @@ impl RequireContextMap {
origin: Vc<Box<dyn ResolveOrigin>>,
dir: Vc<FileSystemPath>,
recursive: bool,
filter: Vc<Regex>,
filter: Vc<EsRegex>,
issue_source: Option<IssueSource>,
is_optional: bool,
) -> Result<Vc<Self>> {
......@@ -227,7 +228,7 @@ impl RequireContextAssetReference {
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
dir: RcStr,
include_subdirs: bool,
filter: Vc<Regex>,
filter: Vc<EsRegex>,
issue_source: Option<IssueSource>,
in_try: bool,
) -> Result<Self> {
......
// PACK-3895: ensure that negative lookaround works.
it('ensure require.context respects filters with look-around assertions', ()=> {
let ctx = require.context("./deps", true, /foo(?!_test)/);
// import all the matches, should just get foo.js
expect(ctx.keys().map(k => Object.values(ctx(k))).flat()).toEqual(['You found me']);
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment