Unverified Commit 64e9c804 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Add support for partial glob matches (#82906)

### What

Add a `contains` option to our Glob API that is compatible with the [`picomatch`](https://www.npmjs.com/package/picomatch/v/4.0.3) `contains` option

### Why

This is the default behavior of the globs passed to NFT configuration options in next, and so we need this feature to be compatible in our rewrite.

### How

Just drop `$` and `^` from the generated regexes when `contains` is set.

For the nft globs we set `contains:true` which is compatible with the current behavior in [`collect-build-traces.ts`](https://github.com/vercel/next.js/blob/b56d1e1af1ad37f96e1905662b15ac5f4707726b/packages/next/src/build/collect-build-traces.ts#L189-L192)

Closes PACK-5305
parent 46864926
...@@ -7,7 +7,10 @@ use turbo_tasks::{ ...@@ -7,7 +7,10 @@ use turbo_tasks::{
FxIndexMap, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc, FxIndexMap, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc,
graph::{AdjacencyMap, GraphTraversal}, graph::{AdjacencyMap, GraphTraversal},
}; };
use turbo_tasks_fs::{DirectoryEntry, File, FileSystem, FileSystemPath, glob::Glob}; use turbo_tasks_fs::{
DirectoryEntry, File, FileSystem, FileSystemPath,
glob::{Glob, GlobOptions},
};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
...@@ -174,7 +177,11 @@ impl Asset for NftJsonAsset { ...@@ -174,7 +177,11 @@ impl Asset for NftJsonAsset {
if let Some(excludes_obj) = excludes_config.as_object() { if let Some(excludes_obj) = excludes_config.as_object() {
for (glob_pattern, exclude_patterns) in excludes_obj { for (glob_pattern, exclude_patterns) in excludes_obj {
// Check if the route matches the glob pattern // Check if the route matches the glob pattern
let glob = Glob::new(RcStr::from(glob_pattern.clone())).await?; let glob = Glob::new(
RcStr::from(glob_pattern.clone()),
GlobOptions { contains: true },
)
.await?;
if glob.matches(route) if glob.matches(route)
&& let Some(patterns) = exclude_patterns.as_array() && let Some(patterns) = exclude_patterns.as_array()
{ {
...@@ -207,6 +214,7 @@ impl Asset for NftJsonAsset { ...@@ -207,6 +214,7 @@ impl Asset for NftJsonAsset {
.join(",") .join(",")
) )
.into(), .into(),
GlobOptions { contains: true },
); );
Some(glob) Some(glob)
...@@ -289,7 +297,9 @@ impl Asset for NftJsonAsset { ...@@ -289,7 +297,9 @@ impl Asset for NftJsonAsset {
{ {
for (glob_pattern, include_patterns) in includes_obj { for (glob_pattern, include_patterns) in includes_obj {
// Check if the route matches the glob pattern // Check if the route matches the glob pattern
let glob = Glob::new(glob_pattern.as_str().into()).await?; let glob =
Glob::new(glob_pattern.as_str().into(), GlobOptions { contains: true })
.await?;
if glob.matches(route) if glob.matches(route)
&& let Some(patterns) = include_patterns.as_array() && let Some(patterns) = include_patterns.as_array()
{ {
...@@ -311,7 +321,10 @@ impl Asset for NftJsonAsset { ...@@ -311,7 +321,10 @@ impl Asset for NftJsonAsset {
let includes = combined_includes_by_root let includes = combined_includes_by_root
.into_iter() .into_iter()
.map(|(root, globs)| { .map(|(root, globs)| {
let glob = Glob::new(format!("{{{}}}", globs.join(",")).into()); let glob = Glob::new(
format!("{{{}}}", globs.join(",")).into(),
GlobOptions { contains: true },
);
apply_includes(root, glob, &ident_folder_in_project_fs) apply_includes(root, glob, &ident_folder_in_project_fs)
}) })
.try_join() .try_join()
......
...@@ -5,7 +5,9 @@ use serde::{Deserialize, Serialize}; ...@@ -5,7 +5,9 @@ use serde::{Deserialize, Serialize};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
FileContent, FileSystemPath, glob::Glob, json::parse_json_with_source_context, FileContent, FileSystemPath,
glob::{Glob, GlobOptions},
json::parse_json_with_source_context,
}; };
use turbopack_core::{ use turbopack_core::{
asset::AssetContent, asset::AssetContent,
...@@ -68,9 +70,10 @@ impl NextFontLocalResolvePlugin { ...@@ -68,9 +70,10 @@ impl NextFontLocalResolvePlugin {
impl BeforeResolvePlugin for NextFontLocalResolvePlugin { impl BeforeResolvePlugin for NextFontLocalResolvePlugin {
#[turbo_tasks::function] #[turbo_tasks::function]
fn before_resolve_condition(&self) -> Vc<BeforeResolvePluginCondition> { fn before_resolve_condition(&self) -> Vc<BeforeResolvePluginCondition> {
BeforeResolvePluginCondition::from_request_glob(Glob::new(rcstr!( BeforeResolvePluginCondition::from_request_glob(Glob::new(
"{next,@vercel/turbopack-next/internal}/font/local/*" rcstr!("{next,@vercel/turbopack-next/internal}/font/local/*"),
))) GlobOptions::default(),
))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -5,7 +5,10 @@ use regex::Regex; ...@@ -5,7 +5,10 @@ use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{NonLocalValue, ResolvedVc, Vc, trace::TraceRawVcs}; use turbo_tasks::{NonLocalValue, ResolvedVc, Vc, trace::TraceRawVcs};
use turbo_tasks_fs::{self, FileJsonContent, FileSystemPath, glob::Glob}; use turbo_tasks_fs::{
self, FileJsonContent, FileSystemPath,
glob::{Glob, GlobOptions},
};
use turbopack_core::{ use turbopack_core::{
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString}, issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
reference_type::{EcmaScriptModulesReferenceSubType, ReferenceType}, reference_type::{EcmaScriptModulesReferenceSubType, ReferenceType},
...@@ -67,7 +70,10 @@ impl ExternalCjsModulesResolvePlugin { ...@@ -67,7 +70,10 @@ impl ExternalCjsModulesResolvePlugin {
#[turbo_tasks::function] #[turbo_tasks::function]
fn condition(root: FileSystemPath) -> Vc<AfterResolvePluginCondition> { fn condition(root: FileSystemPath) -> Vc<AfterResolvePluginCondition> {
AfterResolvePluginCondition::new(root, Glob::new(rcstr!("**/node_modules/**"))) AfterResolvePluginCondition::new(
root,
Glob::new(rcstr!("**/node_modules/**"), GlobOptions::default()),
)
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -441,9 +447,14 @@ async fn packages_glob(packages: Vc<Vec<RcStr>>) -> Result<Vc<OptionPackagesGlob ...@@ -441,9 +447,14 @@ async fn packages_glob(packages: Vc<Vec<RcStr>>) -> Result<Vc<OptionPackagesGlob
if packages.is_empty() { if packages.is_empty() {
return Ok(Vc::cell(None)); return Ok(Vc::cell(None));
} }
let path_glob = Glob::new(format!("**/node_modules/{{{}}}/**", packages.join(",")).into()); let path_glob = Glob::new(
let request_glob = format!("**/node_modules/{{{}}}/**", packages.join(",")).into(),
Glob::new(format!("{{{},{}/**}}", packages.join(","), packages.join("/**,")).into()); GlobOptions::default(),
);
let request_glob = Glob::new(
format!("{{{},{}/**}}", packages.join(","), packages.join("/**,")).into(),
GlobOptions::default(),
);
Ok(Vc::cell(Some(PackagesGlobs { Ok(Vc::cell(Some(PackagesGlobs {
path_glob: path_glob.to_resolved().await?, path_glob: path_glob.to_resolved().await?,
request_glob: request_glob.to_resolved().await?, request_glob: request_glob.to_resolved().await?,
......
...@@ -4,7 +4,10 @@ use anyhow::Result; ...@@ -4,7 +4,10 @@ use anyhow::Result;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{FileSystemPath, glob::Glob}; use turbo_tasks_fs::{
FileSystemPath,
glob::{Glob, GlobOptions},
};
use turbopack_core::{ use turbopack_core::{
diagnostics::DiagnosticExt, diagnostics::DiagnosticExt,
file_source::FileSource, file_source::FileSource,
...@@ -223,9 +226,10 @@ impl AfterResolvePlugin for NextExternalResolvePlugin { ...@@ -223,9 +226,10 @@ impl AfterResolvePlugin for NextExternalResolvePlugin {
async fn after_resolve_condition(&self) -> Result<Vc<AfterResolvePluginCondition>> { async fn after_resolve_condition(&self) -> Result<Vc<AfterResolvePluginCondition>> {
Ok(AfterResolvePluginCondition::new( Ok(AfterResolvePluginCondition::new(
self.project_path.root().owned().await?, self.project_path.root().owned().await?,
Glob::new(rcstr!( Glob::new(
"**/next/dist/**/*.{external,runtime.dev,runtime.prod}.js" rcstr!("**/next/dist/**/*.{external,runtime.dev,runtime.prod}.js"),
)), GlobOptions::default(),
),
)) ))
} }
...@@ -279,7 +283,10 @@ impl AfterResolvePlugin for NextNodeSharedRuntimeResolvePlugin { ...@@ -279,7 +283,10 @@ impl AfterResolvePlugin for NextNodeSharedRuntimeResolvePlugin {
async fn after_resolve_condition(&self) -> Result<Vc<AfterResolvePluginCondition>> { async fn after_resolve_condition(&self) -> Result<Vc<AfterResolvePluginCondition>> {
Ok(AfterResolvePluginCondition::new( Ok(AfterResolvePluginCondition::new(
self.root.root().owned().await?, self.root.root().owned().await?,
Glob::new(rcstr!("**/next/dist/**/*.shared-runtime.js")), Glob::new(
rcstr!("**/next/dist/**/*.shared-runtime.js"),
GlobOptions::default(),
),
)) ))
} }
...@@ -404,7 +411,10 @@ impl AfterResolvePlugin for NextSharedRuntimeResolvePlugin { ...@@ -404,7 +411,10 @@ impl AfterResolvePlugin for NextSharedRuntimeResolvePlugin {
async fn after_resolve_condition(&self) -> Result<Vc<AfterResolvePluginCondition>> { async fn after_resolve_condition(&self) -> Result<Vc<AfterResolvePluginCondition>> {
Ok(AfterResolvePluginCondition::new( Ok(AfterResolvePluginCondition::new(
self.root.root().owned().await?, self.root.root().owned().await?,
Glob::new(rcstr!("**/next/dist/esm/**/*.shared-runtime.js")), Glob::new(
rcstr!("**/next/dist/esm/**/*.shared-runtime.js"),
GlobOptions::default(),
),
)) ))
} }
......
...@@ -15,7 +15,8 @@ use turbo_tasks::{ReadConsistency, TurboTasks, UpdateInfo, Vc, util::FormatDurat ...@@ -15,7 +15,8 @@ use turbo_tasks::{ReadConsistency, TurboTasks, UpdateInfo, Vc, util::FormatDurat
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath, ReadGlobResult, DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath, ReadGlobResult,
glob::Glob, register, glob::{Glob, GlobOptions},
register,
}; };
#[tokio::main] #[tokio::main]
...@@ -38,7 +39,7 @@ async fn main() -> Result<()> { ...@@ -38,7 +39,7 @@ async fn main() -> Result<()> {
// Smart Pointer cast // Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs); let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
let input = fs.root().await?.join("crates")?; let input = fs.root().await?.join("crates")?;
let glob = Glob::new(rcstr!("**/*.rs")); let glob = Glob::new(rcstr!("**/*.rs"), GlobOptions::default());
let glob_result = input.read_glob(glob); let glob_result = input.read_glob(glob);
let dir_hash = hash_glob_result(glob_result); let dir_hash = hash_glob_result(glob_result);
print_hash(dir_hash).await?; print_hash(dir_hash).await?;
......
use std::fmt::Display; use std::fmt::Display;
use anyhow::Result; use anyhow::{Result, bail};
use regex::bytes::{Regex, RegexBuilder}; use regex::bytes::{Regex, RegexBuilder};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::Vc; use turbo_tasks::{TaskInput, Vc, trace::TraceRawVcs};
use crate::globset::parse; use crate::globset::parse;
...@@ -26,6 +26,8 @@ use crate::globset::parse; ...@@ -26,6 +26,8 @@ use crate::globset::parse;
pub struct Glob { pub struct Glob {
glob: String, glob: String,
#[turbo_tasks(trace_ignore)] #[turbo_tasks(trace_ignore)]
opts: GlobOptions,
#[turbo_tasks(trace_ignore)]
regex: Regex, regex: Regex,
#[turbo_tasks(trace_ignore)] #[turbo_tasks(trace_ignore)]
directory_match_regex: Regex, directory_match_regex: Regex,
...@@ -42,22 +44,35 @@ impl Display for Glob { ...@@ -42,22 +44,35 @@ impl Display for Glob {
write!(f, "Glob({})", self.glob) write!(f, "Glob({})", self.glob)
} }
} }
#[derive(
Serialize, Deserialize, Copy, Clone, PartialEq, Eq, Hash, Default, TaskInput, TraceRawVcs, Debug,
)]
pub struct GlobOptions {
/// Whether the glob is a partial match.
/// Allows glob to match any part of the given string(s).
/// NOTE: this means that a pattern like `node_modules/package_name` with `contains:true` will
/// match `foo_node_modules/package_name_bar` If you want to match a _directory_ named
/// `node_modules/package_name` you should use `**/node_modules/package_name/**`
pub contains: bool,
}
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
#[serde(transparent)]
#[repr(transparent)]
struct GlobForm { struct GlobForm {
glob: String, glob: String,
opts: GlobOptions,
} }
impl From<Glob> for GlobForm { impl From<Glob> for GlobForm {
fn from(value: Glob) -> Self { fn from(value: Glob) -> Self {
Self { glob: value.glob } Self {
glob: value.glob,
opts: value.opts,
}
} }
} }
impl TryFrom<GlobForm> for Glob { impl TryFrom<GlobForm> for Glob {
type Error = anyhow::Error; type Error = anyhow::Error;
fn try_from(value: GlobForm) -> Result<Self, Self::Error> { fn try_from(value: GlobForm) -> Result<Self, Self::Error> {
Glob::parse(&value.glob) Glob::parse(&value.glob, value.opts)
} }
} }
...@@ -77,13 +92,14 @@ impl Glob { ...@@ -77,13 +92,14 @@ impl Glob {
self.directory_match_regex.is_match(path.as_bytes()) self.directory_match_regex.is_match(path.as_bytes())
} }
pub fn parse(input: &str) -> Result<Glob> { pub fn parse(input: &str, opts: GlobOptions) -> Result<Glob> {
let (glob_re, directory_match_re) = parse(input)?; let (glob_re, directory_match_re) = parse(input, opts)?;
let regex = new_regex(glob_re.as_str()); let regex = new_regex(glob_re.as_str());
let directory_match_regex = new_regex(directory_match_re.as_str()); let directory_match_regex = new_regex(directory_match_re.as_str());
Ok(Glob { Ok(Glob {
glob: input.to_string(), glob: input.to_string(),
opts,
regex, regex,
directory_match_regex, directory_match_regex,
}) })
...@@ -94,33 +110,46 @@ impl TryFrom<&str> for Glob { ...@@ -94,33 +110,46 @@ impl TryFrom<&str> for Glob {
type Error = anyhow::Error; type Error = anyhow::Error;
fn try_from(value: &str) -> Result<Self, Self::Error> { fn try_from(value: &str) -> Result<Self, Self::Error> {
Glob::parse(value) Glob::parse(value, GlobOptions::default())
} }
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Glob { impl Glob {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(glob: RcStr) -> Result<Vc<Self>> { pub fn new(glob: RcStr, opts: GlobOptions) -> Result<Vc<Self>> {
Ok(Self::cell(Glob::parse(glob.as_str())?)) Ok(Self::cell(Glob::parse(glob.as_str(), opts)?))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn alternatives(globs: Vec<Vc<Glob>>) -> Result<Vc<Self>> { pub async fn alternatives(globs: Vec<Vc<Glob>>) -> Result<Vc<Self>> {
match globs.len() { match globs.len() {
0 => Ok(Glob::new(rcstr!(""))), 0 => Ok(Glob::new(rcstr!(""), GlobOptions::default())),
1 => Ok(globs.into_iter().next().unwrap()), 1 => Ok(globs.into_iter().next().unwrap()),
_ => { _ => {
let mut new_glob = String::new(); let mut new_glob = String::new();
new_glob.push('{'); new_glob.push('{');
let mut opts = None;
for (index, glob) in globs.iter().enumerate() { for (index, glob) in globs.iter().enumerate() {
if index > 0 { if index > 0 {
new_glob.push(','); new_glob.push(',');
} }
new_glob.push_str(&glob.await?.glob); let glob = &*glob.await?;
if let Some(old_opts) = opts {
if old_opts != glob.opts {
bail!(
"Cannot compose globs with different options via the \
`alternatives` function."
)
}
} else {
opts = Some(glob.opts);
}
new_glob.push_str(&glob.glob);
} }
new_glob.push('}'); new_glob.push('}');
Ok(Glob::new(new_glob.into())) // The loop must have iterated at least once, so the options must be initialized.
Ok(Glob::new(new_glob.into(), opts.unwrap()))
} }
} }
} }
...@@ -138,6 +167,7 @@ mod tests { ...@@ -138,6 +167,7 @@ mod tests {
use rstest::*; use rstest::*;
use super::Glob; use super::Glob;
use crate::glob::GlobOptions;
#[rstest] #[rstest]
#[case::file("file.js", "file.js")] #[case::file("file.js", "file.js")]
...@@ -153,6 +183,9 @@ mod tests { ...@@ -153,6 +183,9 @@ mod tests {
#[case::globstar("**/*.js", "dir/sub/file.js")] #[case::globstar("**/*.js", "dir/sub/file.js")]
#[case::globstar("**/**/*.js", "file.js")] #[case::globstar("**/**/*.js", "file.js")]
#[case::globstar("**/**/*.js", "dir/sub/file.js")] #[case::globstar("**/**/*.js", "dir/sub/file.js")]
#[case::globstar("**", "/foo")]
#[case::globstar("**", "foo")]
#[case::star("*", "foo")]
#[case::globstar_in_dir("dir/**/sub/file.js", "dir/sub/file.js")] #[case::globstar_in_dir("dir/**/sub/file.js", "dir/sub/file.js")]
#[case::globstar_in_dir("dir/**/sub/file.js", "dir/a/sub/file.js")] #[case::globstar_in_dir("dir/**/sub/file.js", "dir/a/sub/file.js")]
#[case::globstar_in_dir("dir/**/sub/file.js", "dir/a/b/sub/file.js")] #[case::globstar_in_dir("dir/**/sub/file.js", "dir/a/b/sub/file.js")]
...@@ -205,7 +238,7 @@ mod tests { ...@@ -205,7 +238,7 @@ mod tests {
#[case::alternatives_empty2("react{,-dom}", "react-dom")] #[case::alternatives_empty2("react{,-dom}", "react-dom")]
#[case::alternatives_chars("[abc]", "b")] #[case::alternatives_chars("[abc]", "b")]
fn glob_match(#[case] glob: &str, #[case] path: &str) { fn glob_match(#[case] glob: &str, #[case] path: &str) {
let glob = Glob::parse(glob).unwrap(); let glob = Glob::parse(glob, GlobOptions::default()).unwrap();
println!("{glob:?} {path}"); println!("{glob:?} {path}");
...@@ -218,8 +251,9 @@ mod tests { ...@@ -218,8 +251,9 @@ mod tests {
"**/next/dist/esm/*.shared-runtime.js", "**/next/dist/esm/*.shared-runtime.js",
"next/dist/shared/lib/app-router-context.shared-runtime.js" "next/dist/shared/lib/app-router-context.shared-runtime.js"
)] )]
#[case::star("*", "/foo")]
fn glob_not_matching(#[case] glob: &str, #[case] path: &str) { fn glob_not_matching(#[case] glob: &str, #[case] path: &str) {
let glob = Glob::parse(glob).unwrap(); let glob = Glob::parse(glob, GlobOptions::default()).unwrap();
println!("{glob:?} {path}"); println!("{glob:?} {path}");
...@@ -238,7 +272,7 @@ mod tests { ...@@ -238,7 +272,7 @@ mod tests {
#[case::globstar_in_dir_partial("dir/**/sub/file.js", "dir/a/b/sub")] #[case::globstar_in_dir_partial("dir/**/sub/file.js", "dir/a/b/sub")]
#[case::globstar_in_dir_partial("dir/**/sub/file.js", "dir/a/b/sub/file.js")] #[case::globstar_in_dir_partial("dir/**/sub/file.js", "dir/a/b/sub/file.js")]
fn glob_can_match_directory(#[case] glob: &str, #[case] path: &str) { fn glob_can_match_directory(#[case] glob: &str, #[case] path: &str) {
let glob = Glob::parse(glob).unwrap(); let glob = Glob::parse(glob, GlobOptions::default()).unwrap();
println!("{glob:?} {path}"); println!("{glob:?} {path}");
...@@ -248,10 +282,39 @@ mod tests { ...@@ -248,10 +282,39 @@ mod tests {
#[case::dir_and_file_partial("dir/file.js", "dir/file.js")] // even if there was a dir, named `file.js` we know the glob wasn't intended to match it. #[case::dir_and_file_partial("dir/file.js", "dir/file.js")] // even if there was a dir, named `file.js` we know the glob wasn't intended to match it.
#[case::alternatives_chars("[abc]", "b")] #[case::alternatives_chars("[abc]", "b")]
fn glob_not_can_match_directory(#[case] glob: &str, #[case] path: &str) { fn glob_not_can_match_directory(#[case] glob: &str, #[case] path: &str) {
let glob = Glob::parse(glob).unwrap(); let glob = Glob::parse(glob, GlobOptions::default()).unwrap();
println!("{glob:?} {path}"); println!("{glob:?} {path}");
assert!(!glob.can_match_in_directory(path)); assert!(!glob.can_match_in_directory(path));
} }
#[rstest]
#[case::star("*", "/foo")]
#[case::star("*", "foo")]
#[case::star("*", "foo/bar")]
#[case::prefix("foo/*", "bar/foo/baz")]
// This is a possibly surprising case.
#[case::dir_match("node_modules/foo", "my_node_modules/foobar")]
fn partial_glob_match(#[case] glob: &str, #[case] path: &str) {
let glob = Glob::parse(glob, GlobOptions { contains: true }).unwrap();
println!("{glob:?} {path}");
assert!(glob.matches(path));
}
#[rstest]
#[case::literal("foo", "bar")]
#[case::suffix("*.js", "foo.ts")]
#[case::prefix("foo/*", "bar")]
// This is a possibly surprising case
#[case::dir_match("/node_modules/", "node_modules/")]
fn partial_glob_not_matching(#[case] glob: &str, #[case] path: &str) {
let glob = Glob::parse(glob, GlobOptions { contains: true }).unwrap();
println!("{glob:?} {path}");
assert!(!glob.matches(path));
}
} }
...@@ -28,6 +28,8 @@ ...@@ -28,6 +28,8 @@
/// of regexes and those are mostly preserved. /// of regexes and those are mostly preserved.
use anyhow::Error; use anyhow::Error;
use crate::glob::GlobOptions;
/// The parsed tokens of a glob pattern. /// The parsed tokens of a glob pattern.
#[derive(Clone, Debug, Default, Eq, PartialEq)] #[derive(Clone, Debug, Default, Eq, PartialEq)]
#[repr(transparent)] #[repr(transparent)]
...@@ -72,8 +74,8 @@ impl Tokens { ...@@ -72,8 +74,8 @@ impl Tokens {
/// Convert this pattern to a string that is guaranteed to be a valid /// Convert this pattern to a string that is guaranteed to be a valid
/// regular expression and will represent the matching semantics of this /// regular expression and will represent the matching semantics of this
/// glob pattern and the options given. /// glob pattern and the options given.
fn to_regex(&self) -> String { fn to_regex(&self, opts: GlobOptions) -> String {
self.to_regex_impl(Self::tokens_to_regex) self.to_regex_impl(Self::tokens_to_regex, opts)
} }
/// Convert this pattern to a string that is guaranteed to be a valid /// Convert this pattern to a string that is guaranteed to be a valid
...@@ -84,24 +86,34 @@ impl Tokens { ...@@ -84,24 +86,34 @@ impl Tokens {
/// The basic strategy is to add 'early exits' to the regex at `/` boundaries. /// The basic strategy is to add 'early exits' to the regex at `/` boundaries.
/// e.g. `foo/bar/baz` -> `foo(?:/bar(?:/baz(?:/.*)?)?)?` /// e.g. `foo/bar/baz` -> `foo(?:/bar(?:/baz(?:/.*)?)?)?`
/// that way 'foo' will match but foo/baz will not /// that way 'foo' will match but foo/baz will not
fn to_directory_match_regex(&self) -> String { fn to_directory_match_regex(&self, opts: GlobOptions) -> String {
self.to_regex_impl(Self::tokens_to_directory_match_regex) self.to_regex_impl(Self::tokens_to_directory_match_regex, opts)
} }
fn to_regex_impl(&self, tokens_to_regex_fn: fn(&[Token], &mut String)) -> String { fn to_regex_impl(
&self,
tokens_to_regex_fn: fn(&[Token], &mut String),
opts: GlobOptions,
) -> String {
let mut re = String::new(); let mut re = String::new();
// Our patterns are always anchored to the beginning and end of the string and we care not // We don't care care about for unicode correctness. Paths do not require this and if the
// for unicode correctness. Paths do not require this and if the caller does they can // caller does they can simply take care to pass us valid utf8 themselves.
// simply take care to pass us valid utf8 themselves. re.push_str("(?-u)");
re.push_str("(?-u)^");
// Enable anchored matches unless
if !opts.contains {
re.push('^');
}
// Special case. If the entire glob is just `**`, then it should match // Special case. If the entire glob is just `**`, then it should match
// everything. // everything.
if self.len() == 1 && self[0] == Token::RecursivePrefix { if self.len() == 1 && self[0] == Token::RecursivePrefix {
re.push_str(".*$"); re.push_str(".*");
return re; } else {
tokens_to_regex_fn(self, &mut re);
}
if !opts.contains {
re.push('$');
} }
tokens_to_regex_fn(self, &mut re);
re.push('$');
re re
} }
...@@ -340,9 +352,9 @@ impl ErrorKind { ...@@ -340,9 +352,9 @@ impl ErrorKind {
// The directory match regex is guaranteed to be valid and will match the same // The directory match regex is guaranteed to be valid and will match the same
// semantics as the glob when used to check if a directory path might contain files that match // semantics as the glob when used to check if a directory path might contain files that match
// this glob. This means we care about matching prefixes that are bounded by `/` characters. // this glob. This means we care about matching prefixes that are bounded by `/` characters.
pub(crate) fn parse(glob: &str) -> Result<(String, String), Error> { pub(crate) fn parse(glob: &str, opts: GlobOptions) -> Result<(String, String), Error> {
let tokens = Parser::new(glob).parse()?; let tokens = Parser::new(glob).parse()?;
Ok((tokens.to_regex(), tokens.to_directory_match_regex())) Ok((tokens.to_regex(opts), tokens.to_directory_match_regex(opts)))
} }
struct Parser<'a> { struct Parser<'a> {
...@@ -624,6 +636,7 @@ mod tests { ...@@ -624,6 +636,7 @@ mod tests {
use rstest::*; use rstest::*;
use super::parse; use super::parse;
use crate::glob::GlobOptions;
#[rstest] #[rstest]
#[case::literal("dir/file.js", "dir/file\\.js", "dir")] #[case::literal("dir/file.js", "dir/file\\.js", "dir")]
...@@ -644,7 +657,7 @@ mod tests { ...@@ -644,7 +657,7 @@ mod tests {
#[case] glob_regex: &str, #[case] glob_regex: &str,
#[case] directory_match_regex: &str, #[case] directory_match_regex: &str,
) { ) {
let (glob_re, directory_match_re) = parse(glob).unwrap(); let (glob_re, directory_match_re) = parse(glob, GlobOptions::default()).unwrap();
// All our regexes come with a fixed prefix and suffix, just assert and drop them // All our regexes come with a fixed prefix and suffix, just assert and drop them
fn strip_overhead(s: String) -> String { fn strip_overhead(s: String) -> String {
assert!(s.starts_with("(?-u)^")); assert!(s.starts_with("(?-u)^"));
......
...@@ -237,7 +237,8 @@ pub mod tests { ...@@ -237,7 +237,8 @@ pub mod tests {
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use crate::{ use crate::{
DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath, glob::Glob, DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath,
glob::{Glob, GlobOptions},
}; };
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
...@@ -265,7 +266,10 @@ pub mod tests { ...@@ -265,7 +266,10 @@ pub mod tests {
tt.run_once(async { tt.run_once(async {
let fs = DiskFileSystem::new(rcstr!("temp"), path); let fs = DiskFileSystem::new(rcstr!("temp"), path);
let root = fs.root().await?; let root = fs.root().await?;
let read_dir = root.read_glob(Glob::new(rcstr!("**"))).await.unwrap(); let read_dir = root
.read_glob(Glob::new(rcstr!("**"), GlobOptions::default()))
.await
.unwrap();
assert_eq!(read_dir.results.len(), 2); assert_eq!(read_dir.results.len(), 2);
assert_eq!( assert_eq!(
read_dir.results.get("foo"), read_dir.results.get("foo"),
...@@ -285,7 +289,10 @@ pub mod tests { ...@@ -285,7 +289,10 @@ pub mod tests {
assert_eq!(inner.inner.len(), 0); assert_eq!(inner.inner.len(), 0);
// Now with a more specific pattern // Now with a more specific pattern
let read_dir = root.read_glob(Glob::new(rcstr!("**/bar"))).await.unwrap(); let read_dir = root
.read_glob(Glob::new(rcstr!("**/bar"), GlobOptions::default()))
.await
.unwrap();
assert_eq!(read_dir.results.len(), 0); assert_eq!(read_dir.results.len(), 0);
assert_eq!(read_dir.inner.len(), 1); assert_eq!(read_dir.inner.len(), 1);
let inner = &*read_dir.inner.get("sub").unwrap().await?; let inner = &*read_dir.inner.get("sub").unwrap().await?;
...@@ -340,7 +347,10 @@ pub mod tests { ...@@ -340,7 +347,10 @@ pub mod tests {
let fs = DiskFileSystem::new(rcstr!("temp"), path); let fs = DiskFileSystem::new(rcstr!("temp"), path);
let root = fs.root().await?; let root = fs.root().await?;
// Symlinked files // Symlinked files
let read_dir = root.read_glob(Glob::new(rcstr!("sub/*.js"))).await.unwrap(); let read_dir = root
.read_glob(Glob::new(rcstr!("sub/*.js"), GlobOptions::default()))
.await
.unwrap();
assert_eq!(read_dir.results.len(), 0); assert_eq!(read_dir.results.len(), 0);
let inner = &*read_dir.inner.get("sub").unwrap().await?; let inner = &*read_dir.inner.get("sub").unwrap().await?;
assert_eq!( assert_eq!(
...@@ -364,7 +374,7 @@ pub mod tests { ...@@ -364,7 +374,7 @@ pub mod tests {
// A symlinked folder // A symlinked folder
let read_dir = root let read_dir = root
.read_glob(Glob::new(rcstr!("sub/dir/*"))) .read_glob(Glob::new(rcstr!("sub/dir/*"), GlobOptions::default()))
.await .await
.unwrap(); .unwrap();
assert_eq!(read_dir.results.len(), 0); assert_eq!(read_dir.results.len(), 0);
...@@ -403,7 +413,7 @@ pub mod tests { ...@@ -403,7 +413,7 @@ pub mod tests {
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
pub fn track_star_star_glob(path: FileSystemPath) -> Vc<Completion> { pub fn track_star_star_glob(path: FileSystemPath) -> Vc<Completion> {
path.track_glob(Glob::new(rcstr!("**")), false) path.track_glob(Glob::new(rcstr!("**"), GlobOptions::default()), false)
} }
#[cfg(unix)] #[cfg(unix)]
...@@ -523,7 +533,7 @@ pub mod tests { ...@@ -523,7 +533,7 @@ pub mod tests {
let err = fs let err = fs
.root() .root()
.await? .await?
.track_glob(Glob::new(rcstr!("**")), false) .track_glob(Glob::new(rcstr!("**"), GlobOptions::default()), false)
.await .await
.expect_err("Should have detected an infinite loop"); .expect_err("Should have detected an infinite loop");
...@@ -536,7 +546,7 @@ pub mod tests { ...@@ -536,7 +546,7 @@ pub mod tests {
let err = fs let err = fs
.root() .root()
.await? .await?
.track_glob(Glob::new(rcstr!("**")), false) .track_glob(Glob::new(rcstr!("**"), GlobOptions::default()), false)
.await .await
.expect_err("Should have detected an infinite loop"); .expect_err("Should have detected an infinite loop");
...@@ -579,7 +589,7 @@ pub mod tests { ...@@ -579,7 +589,7 @@ pub mod tests {
let err = fs let err = fs
.root() .root()
.await? .await?
.read_glob(Glob::new(rcstr!("**"))) .read_glob(Glob::new(rcstr!("**"), GlobOptions::default()))
.await .await
.expect_err("Should have detected an infinite loop"); .expect_err("Should have detected an infinite loop");
...@@ -592,7 +602,7 @@ pub mod tests { ...@@ -592,7 +602,7 @@ pub mod tests {
let err = fs let err = fs
.root() .root()
.await? .await?
.track_glob(Glob::new(rcstr!("**")), false) .track_glob(Glob::new(rcstr!("**"), GlobOptions::default()), false)
.await .await
.expect_err("Should have detected an infinite loop"); .expect_err("Should have detected an infinite loop");
......
use anyhow::Result; use anyhow::Result;
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, Vc}; use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, Vc};
use turbo_tasks_fs::{FileJsonContent, FileSystemPath, glob::Glob}; use turbo_tasks_fs::{
FileJsonContent, FileSystemPath,
glob::{Glob, GlobOptions},
};
use turbopack_core::{ use turbopack_core::{
asset::Asset, asset::Asset,
chunk::ChunkableModule, chunk::ChunkableModule,
...@@ -66,9 +69,13 @@ async fn side_effects_from_package_json( ...@@ -66,9 +69,13 @@ async fn side_effects_from_package_json(
if side_effect.contains('/') { if side_effect.contains('/') {
Some(Glob::new( Some(Glob::new(
side_effect.strip_prefix("./").unwrap_or(side_effect).into(), side_effect.strip_prefix("./").unwrap_or(side_effect).into(),
GlobOptions::default(),
)) ))
} else { } else {
Some(Glob::new(format!("**/{side_effect}").into())) Some(Glob::new(
format!("**/{side_effect}").into(),
GlobOptions::default(),
))
} }
} else { } else {
SideEffectsInPackageJsonIssue { SideEffectsInPackageJsonIssue {
......
...@@ -15,7 +15,10 @@ use turbo_tasks::{ ...@@ -15,7 +15,10 @@ use turbo_tasks::{
use turbo_tasks_bytes::stream::SingleValue; use turbo_tasks_bytes::stream::SingleValue;
use turbo_tasks_env::ProcessEnv; use turbo_tasks_env::ProcessEnv;
use turbo_tasks_fs::{ use turbo_tasks_fs::{
File, FileContent, FileSystemPath, glob::Glob, json::parse_json_with_source_context, rope::Rope, File, FileContent, FileSystemPath,
glob::{Glob, GlobOptions},
json::parse_json_with_source_context,
rope::Rope,
}; };
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -513,7 +516,7 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -513,7 +516,7 @@ impl EvaluateContext for WebpackLoaderContext {
.map(|(dir, glob)| async move { .map(|(dir, glob)| async move {
self.cwd self.cwd
.join(dir)? .join(dir)?
.track_glob(Glob::new(glob.clone()), false) .track_glob(Glob::new(glob.clone(), GlobOptions::default()), false)
.await .await
}) })
.try_join(); .try_join();
......
...@@ -26,7 +26,10 @@ use module_options::{ModuleOptions, ModuleOptionsContext, ModuleRuleEffect, Modu ...@@ -26,7 +26,10 @@ use module_options::{ModuleOptions, ModuleOptionsContext, ModuleRuleEffect, Modu
use tracing::{Instrument, field::Empty}; use tracing::{Instrument, field::Empty};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{FileSystemPath, glob::Glob}; use turbo_tasks_fs::{
FileSystemPath,
glob::{Glob, GlobOptions},
};
pub use turbopack_core::condition; pub use turbopack_core::condition;
use turbopack_core::{ use turbopack_core::{
asset::Asset, asset::Asset,
...@@ -895,15 +898,14 @@ impl AssetContext for ModuleAssetContext { ...@@ -895,15 +898,14 @@ impl AssetContext for ModuleAssetContext {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn side_effect_free_packages(&self) -> Result<Vc<Glob>> { async fn side_effect_free_packages(&self) -> Result<Vc<Glob>> {
let pkgs = &*self.module_options_context.await?.side_effect_free_packages; let pkgs = &self.module_options_context.await?.side_effect_free_packages;
let mut globs = Vec::with_capacity(pkgs.len());
for pkg in pkgs { let mut globs = String::new();
globs.push(Glob::new(format!("**/node_modules/{{{pkg}}}/**").into())); globs.push_str("**/node_modules/{");
} globs.push_str(&pkgs.join(","));
globs.push_str("}/**");
Ok(Glob::alternatives(globs)) Ok(Glob::new(globs.into(), GlobOptions::default()))
} }
} }
......
...@@ -12,7 +12,10 @@ pub use module_rule::*; ...@@ -12,7 +12,10 @@ pub use module_rule::*;
pub use rule_condition::*; pub use rule_condition::*;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{FileSystemPath, glob::Glob}; use turbo_tasks_fs::{
FileSystemPath,
glob::{Glob, GlobOptions},
};
use turbopack_core::{ use turbopack_core::{
chunk::SourceMapsType, chunk::SourceMapsType,
ident::Layer, ident::Layer,
...@@ -510,11 +513,11 @@ impl ModuleOptions { ...@@ -510,11 +513,11 @@ impl ModuleOptions {
if glob.contains('/') { if glob.contains('/') {
rule_conditions.push(RuleCondition::ResourcePathGlob { rule_conditions.push(RuleCondition::ResourcePathGlob {
base: execution_context.project_path().owned().await?, base: execution_context.project_path().owned().await?,
glob: Glob::new(glob.clone()).await?, glob: Glob::new(glob.clone(), GlobOptions::default()).await?,
}); });
} else { } else {
rule_conditions.push(RuleCondition::ResourceBasePathGlob( rule_conditions.push(RuleCondition::ResourceBasePathGlob(
Glob::new(glob.clone()).await?, Glob::new(glob.clone(), GlobOptions::default()).await?,
)); ));
} }
} }
...@@ -529,11 +532,11 @@ impl ModuleOptions { ...@@ -529,11 +532,11 @@ impl ModuleOptions {
} else if key.contains('/') { } else if key.contains('/') {
rule_conditions.push(RuleCondition::ResourcePathGlob { rule_conditions.push(RuleCondition::ResourcePathGlob {
base: execution_context.project_path().owned().await?, base: execution_context.project_path().owned().await?,
glob: Glob::new(key.clone()).await?, glob: Glob::new(key.clone(), GlobOptions::default()).await?,
}); });
} else { } else {
rule_conditions.push(RuleCondition::ResourceBasePathGlob( rule_conditions.push(RuleCondition::ResourceBasePathGlob(
Glob::new(key.clone()).await?, Glob::new(key.clone(), GlobOptions::default()).await?,
)); ));
}; };
rule_conditions.push(RuleCondition::not(RuleCondition::ResourceIsVirtualSource)); rule_conditions.push(RuleCondition::not(RuleCondition::ResourceIsVirtualSource));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment