Unverified Commit dd7fed07 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Fork the globset strategy for turbopack (#79333)

## What 
After discussion with [upstream](https://github.com/BurntSushi/ripgrep/issues/3049) it was determined that upstreaming a function like `can_skip_directory` wasn't aligned. It turns out the needs of ripgrep and ours are somewhat divergent.  Instead they encouraged me to just fork and modify which is what we do here.

This copies the glob->regex translation logic and drops much of the other functionality that is optimized for large sets of globs.   Then I have added our `can_skip_directory` function following a similar pattern.

## Why

This approach allows us to easily support for character ranges (e.g. `[A-Za-z0-9_-]`) and improves performance.  I also suspect that this will be easier to maintain as debugging the generated regular expressions should be quite straightforward.

## Alternatives

We could just adopt globset directly as proposed in #79116 however we would lose the ability to skip directories during traversal which can reduce the cost of setting up turbotasks dependencies and directory watchers.  The other alternative that i pursued was a direct NFA implementation (see #78976).  This would be more memory efficient than regular expressions since both queries can share the same NFA, but the regex crate is much faster and better maintained.  If memory overhead becomes a concern we can configure the regexes to use less, however, i suspect many of our generated regexes will be quite simple and thus generate small automata.

Closes PACK-4401
Fixes #72196
parent e06d2d87
...@@ -9736,6 +9736,7 @@ dependencies = [ ...@@ -9736,6 +9736,7 @@ dependencies = [
"notify", "notify",
"parking_lot", "parking_lot",
"rayon", "rayon",
"regex",
"rstest", "rstest",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
......
...@@ -119,8 +119,8 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { ...@@ -119,8 +119,8 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin {
request_glob, request_glob,
}) = *exception_glob }) = *exception_glob
{ {
let path_match = path_glob.await?.execute(&raw_fs_path.path); let path_match = path_glob.await?.matches(&raw_fs_path.path);
let request_match = request_glob.await?.execute(&request_str); let request_match = request_glob.await?.matches(&request_str);
if path_match || request_match { if path_match || request_match {
return Ok(ResolveResultOption::none()); return Ok(ResolveResultOption::none());
} }
...@@ -135,8 +135,8 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { ...@@ -135,8 +135,8 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin {
request_glob, request_glob,
}) = *external_glob }) = *external_glob
{ {
let path_match = path_glob.await?.execute(&raw_fs_path.path); let path_match = path_glob.await?.matches(&raw_fs_path.path);
let request_match = request_glob.await?.execute(&request_str); let request_match = request_glob.await?.matches(&request_str);
if !path_match && !request_match { if !path_match && !request_match {
return Ok(ResolveResultOption::none()); return Ok(ResolveResultOption::none());
......
...@@ -41,6 +41,7 @@ mime = { workspace = true } ...@@ -41,6 +41,7 @@ mime = { workspace = true }
notify = { workspace = true } notify = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
rayon = { workspace = true } rayon = { workspace = true }
regex = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true, features = ["rc"] } serde = { workspace = true, features = ["rc"] }
serde_bytes = { workspace = true } serde_bytes = { workspace = true }
......
This diff is collapsed.
This diff is collapsed.
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
pub mod attach; pub mod attach;
pub mod embed; pub mod embed;
pub mod glob; pub mod glob;
mod globset;
pub mod invalidation; pub mod invalidation;
mod invalidator_map; mod invalidator_map;
pub mod json; pub mod json;
......
...@@ -83,7 +83,7 @@ async fn track_glob_internal( ...@@ -83,7 +83,7 @@ async fn track_glob_internal(
match resolve_symlink_safely(entry).await? { match resolve_symlink_safely(entry).await? {
DirectoryEntry::Directory(path) => { DirectoryEntry::Directory(path) => {
if glob_value.match_in_directory(&entry_path) { if glob_value.can_match_in_directory(&entry_path) {
completions.push(track_glob_inner( completions.push(track_glob_inner(
entry_path, entry_path,
*path, *path,
...@@ -93,13 +93,13 @@ async fn track_glob_internal( ...@@ -93,13 +93,13 @@ async fn track_glob_internal(
} }
} }
DirectoryEntry::File(path) => { DirectoryEntry::File(path) => {
if glob_value.execute(&entry_path) { if glob_value.matches(&entry_path) {
reads.push(fs.read(*path)) reads.push(fs.read(*path))
} }
} }
DirectoryEntry::Symlink(_) => panic!("we already resolved symlinks"), DirectoryEntry::Symlink(_) => unreachable!("we already resolved symlinks"),
DirectoryEntry::Other(path) => { DirectoryEntry::Other(path) => {
if glob_value.execute(&entry_path) { if glob_value.matches(&entry_path) {
types.push(path.get_type()) types.push(path.get_type())
} }
} }
......
...@@ -557,7 +557,7 @@ impl ResolvedMap { ...@@ -557,7 +557,7 @@ impl ResolvedMap {
for (root, glob, mapping) in self.by_glob.iter() { for (root, glob, mapping) in self.by_glob.iter() {
let root = root.await?; let root = root.await?;
if let Some(path) = root.get_path_to(&resolved) { if let Some(path) = root.get_path_to(&resolved) {
if glob.await?.execute(path) { if glob.await?.matches(path) {
return Ok(import_mapping_to_result( return Ok(import_mapping_to_result(
*mapping.convert().await?, *mapping.convert().await?,
lookup_path, lookup_path,
......
...@@ -31,7 +31,7 @@ impl AfterResolvePluginCondition { ...@@ -31,7 +31,7 @@ impl AfterResolvePluginCondition {
let path = fs_path.await?; let path = fs_path.await?;
if let Some(path) = root.get_path_to(&path) { if let Some(path) = root.get_path_to(&path) {
if glob.execute(path) { if glob.matches(path) {
return Ok(Vc::cell(true)); return Ok(Vc::cell(true));
} }
} }
...@@ -66,7 +66,7 @@ impl BeforeResolvePluginCondition { ...@@ -66,7 +66,7 @@ impl BeforeResolvePluginCondition {
pub async fn matches(&self, request: Vc<Request>) -> Result<Vc<bool>> { pub async fn matches(&self, request: Vc<Request>) -> Result<Vc<bool>> {
Ok(Vc::cell(match self { Ok(Vc::cell(match self {
BeforeResolvePluginCondition::Request(glob) => match request.await?.request() { BeforeResolvePluginCondition::Request(glob) => match request.await?.request() {
Some(request) => glob.await?.execute(request.as_str()), Some(request) => glob.await?.matches(request.as_str()),
None => false, None => false,
}, },
BeforeResolvePluginCondition::Modules(modules) => { BeforeResolvePluginCondition::Modules(modules) => {
......
...@@ -164,7 +164,7 @@ pub async fn is_marked_as_side_effect_free( ...@@ -164,7 +164,7 @@ pub async fn is_marked_as_side_effect_free(
path: Vc<FileSystemPath>, path: Vc<FileSystemPath>,
side_effect_free_packages: Vc<Glob>, side_effect_free_packages: Vc<Glob>,
) -> Result<Vc<bool>> { ) -> Result<Vc<bool>> {
if side_effect_free_packages.await?.execute(&path.await?.path) { if side_effect_free_packages.await?.matches(&path.await?.path) {
return Ok(Vc::cell(true)); return Ok(Vc::cell(true));
} }
...@@ -181,7 +181,7 @@ pub async fn is_marked_as_side_effect_free( ...@@ -181,7 +181,7 @@ pub async fn is_marked_as_side_effect_free(
.get_relative_path_to(&*path.await?) .get_relative_path_to(&*path.await?)
{ {
let rel_path = rel_path.strip_prefix("./").unwrap_or(&rel_path); let rel_path = rel_path.strip_prefix("./").unwrap_or(&rel_path);
return Ok(Vc::cell(!glob.await?.execute(rel_path))); return Ok(Vc::cell(!glob.await?.matches(rel_path)));
} }
} }
} }
......
...@@ -148,9 +148,9 @@ impl RuleCondition { ...@@ -148,9 +148,9 @@ impl RuleCondition {
} }
RuleCondition::ResourcePathGlob { glob, base } => { RuleCondition::ResourcePathGlob { glob, base } => {
return Ok(if let Some(rel_path) = base.get_relative_path_to(path) { return Ok(if let Some(rel_path) = base.get_relative_path_to(path) {
glob.execute(&rel_path) glob.matches(&rel_path)
} else { } else {
glob.execute(&path.path) glob.matches(&path.path)
}); });
} }
RuleCondition::ResourceBasePathGlob(glob) => { RuleCondition::ResourceBasePathGlob(glob) => {
...@@ -158,7 +158,7 @@ impl RuleCondition { ...@@ -158,7 +158,7 @@ impl RuleCondition {
.path .path
.rsplit_once('/') .rsplit_once('/')
.map_or(path.path.as_str(), |(_, b)| b); .map_or(path.path.as_str(), |(_, b)| b);
return Ok(glob.execute(basename)); return Ok(glob.matches(basename));
} }
RuleCondition::ResourcePathRegex(_) => { RuleCondition::ResourcePathRegex(_) => {
bail!("ResourcePathRegex not implemented yet"); bail!("ResourcePathRegex not implemented yet");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment