Unverified Commit 83dc2ebd authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Turbopack: Implement regex support for matching webpack loaders (#78733)

Inspired by https://x.com/wSokra/status/1907794635914612777, this allows you to declare a ‘marker’ in place of a glob, allowing you to express more conditions than simply just a glob pattern. To start, this allows us to implement regexes. For example, to match either `.svg` or `.svgr` files, use this `next.config.ts`:

```
const nextConfig: NextConfig = {
  turbopack: {
    rules: {
      '#svg': {
        as: '*.js',
        loaders: ['@svgr/webpack'],
      }
    },
    conditions: {
      '#svg': {
        path: /\.svgr?$/,
      }
    }
  }
};
```
parent e72a76e4
...@@ -4583,10 +4583,12 @@ dependencies = [ ...@@ -4583,10 +4583,12 @@ dependencies = [
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"serde_json", "serde_json",
"serde_path_to_error",
"swc_core", "swc_core",
"swc_relay", "swc_relay",
"thiserror 1.0.69", "thiserror 1.0.69",
"tracing", "tracing",
"turbo-esregex",
"turbo-rcstr", "turbo-rcstr",
"turbo-tasks", "turbo-tasks",
"turbo-tasks-build", "turbo-tasks-build",
...@@ -9690,6 +9692,7 @@ dependencies = [ ...@@ -9690,6 +9692,7 @@ dependencies = [
"smallvec", "smallvec",
"tokio", "tokio",
"tracing", "tracing",
"turbo-esregex",
"turbo-rcstr", "turbo-rcstr",
"turbo-tasks", "turbo-tasks",
"turbo-tasks-build", "turbo-tasks-build",
......
...@@ -37,6 +37,7 @@ remove_console = "0.37.0" ...@@ -37,6 +37,7 @@ remove_console = "0.37.0"
itertools = { workspace = true } itertools = { workspace = true }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
percent-encoding = "2.3.1" percent-encoding = "2.3.1"
serde_path_to_error = { workspace = true }
swc_core = { workspace = true, features = [ swc_core = { workspace = true, features = [
"base", "base",
...@@ -60,6 +61,7 @@ modularize_imports = { workspace = true } ...@@ -60,6 +61,7 @@ modularize_imports = { workspace = true }
swc_relay = { workspace = true } swc_relay = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbo-esregex = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-bytes = { workspace = true } turbo-tasks-bytes = { workspace = true }
turbo-tasks-env = { workspace = true } turbo-tasks-env = { workspace = true }
......
...@@ -10,7 +10,10 @@ use turbo_tasks::{ ...@@ -10,7 +10,10 @@ use turbo_tasks::{
use turbo_tasks_env::EnvMap; use turbo_tasks_env::EnvMap;
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack::module_options::{ use turbopack::module_options::{
module_options_context::MdxTransformOptions, LoaderRuleItem, OptionWebpackRules, module_options_context::{
ConditionItem, ConditionPath, MdxTransformOptions, OptionWebpackConditions,
},
LoaderRuleItem, OptionWebpackRules,
}; };
use turbopack_core::{ use turbopack_core::{
issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString}, issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString},
...@@ -541,11 +544,57 @@ pub struct TurbopackConfig { ...@@ -541,11 +544,57 @@ pub struct TurbopackConfig {
/// This option has been replaced by `rules`. /// This option has been replaced by `rules`.
pub loaders: Option<JsonValue>, pub loaders: Option<JsonValue>,
pub rules: Option<FxIndexMap<RcStr, RuleConfigItemOrShortcut>>, pub rules: Option<FxIndexMap<RcStr, RuleConfigItemOrShortcut>>,
#[turbo_tasks(trace_ignore)]
pub conditions: Option<FxIndexMap<RcStr, ConfigConditionItem>>,
pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>, pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
pub resolve_extensions: Option<Vec<RcStr>>, pub resolve_extensions: Option<Vec<RcStr>>,
pub module_ids: Option<ModuleIds>, pub module_ids: Option<ModuleIds>,
} }
#[derive(Clone, Debug, PartialEq, Serialize, TraceRawVcs, NonLocalValue)]
pub struct ConfigConditionItem(ConditionItem);
impl<'de> Deserialize<'de> for ConfigConditionItem {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
struct RegexComponents {
source: RcStr,
flags: RcStr,
}
#[derive(Deserialize)]
struct ConfigPath {
path: RegexOrGlob,
}
#[derive(Deserialize)]
#[serde(tag = "type", rename_all = "lowercase")]
enum RegexOrGlob {
Regexp { value: RegexComponents },
Glob { value: String },
}
let config_path = ConfigPath::deserialize(deserializer)?;
let condition_item = match config_path.path {
RegexOrGlob::Regexp { value } => {
let regex = turbo_esregex::EsRegex::new(&value.source, &value.flags)
.map_err(serde::de::Error::custom)?;
ConditionItem {
path: ConditionPath::Regex(regex.resolved_cell()),
}
}
RegexOrGlob::Glob { value } => ConditionItem {
path: ConditionPath::Glob(value.into()),
},
};
Ok(ConfigConditionItem(condition_item))
}
}
#[derive( #[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue, OperationValue, Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue, OperationValue,
)] )]
...@@ -1065,7 +1114,8 @@ impl NextConfig { ...@@ -1065,7 +1114,8 @@ impl NextConfig {
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn from_string(string: Vc<RcStr>) -> Result<Vc<Self>> { pub async fn from_string(string: Vc<RcStr>) -> Result<Vc<Self>> {
let string = string.await?; let string = string.await?;
let config: NextConfig = serde_json::from_str(&string) let mut jdeserializer = serde_json::Deserializer::from_str(&string);
let config: NextConfig = serde_path_to_error::deserialize(&mut jdeserializer)
.with_context(|| format!("failed to parse next.config.js: {}", string))?; .with_context(|| format!("failed to parse next.config.js: {}", string))?;
Ok(config.cell()) Ok(config.cell())
} }
...@@ -1227,6 +1277,22 @@ impl NextConfig { ...@@ -1227,6 +1277,22 @@ impl NextConfig {
Vc::cell(Some(ResolvedVc::cell(rules))) Vc::cell(Some(ResolvedVc::cell(rules)))
} }
#[turbo_tasks::function]
pub fn webpack_conditions(&self) -> Vc<OptionWebpackConditions> {
let Some(config_conditions) = self.turbopack.as_ref().and_then(|t| t.conditions.as_ref())
else {
return Vc::cell(None);
};
let conditions = FxIndexMap::from_iter(
config_conditions
.iter()
.map(|(k, v)| (k.clone(), v.0.clone())),
);
Vc::cell(Some(ResolvedVc::cell(conditions)))
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn persistent_caching_enabled(&self) -> Result<Vc<bool>> { pub fn persistent_caching_enabled(&self) -> Result<Vc<bool>> {
Ok(Vc::cell( Ok(Vc::cell(
......
...@@ -15,19 +15,22 @@ pub async fn webpack_loader_options( ...@@ -15,19 +15,22 @@ pub async fn webpack_loader_options(
project_path: ResolvedVc<FileSystemPath>, project_path: ResolvedVc<FileSystemPath>,
next_config: Vc<NextConfig>, next_config: Vc<NextConfig>,
foreign: bool, foreign: bool,
conditions: Vec<RcStr>, condition_strs: Vec<RcStr>,
) -> Result<Option<ResolvedVc<WebpackLoadersOptions>>> { ) -> Result<Option<ResolvedVc<WebpackLoadersOptions>>> {
let rules = *next_config.webpack_rules(conditions).await?; let rules = *next_config.webpack_rules(condition_strs).await?;
let rules = *maybe_add_sass_loader(next_config.sass_config(), rules.map(|v| *v)).await?; let rules = *maybe_add_sass_loader(next_config.sass_config(), rules.map(|v| *v)).await?;
let rules = if foreign { let rules = if foreign {
rules rules
} else { } else {
*maybe_add_babel_loader(*project_path, rules.map(|v| *v)).await? *maybe_add_babel_loader(*project_path, rules.map(|v| *v)).await?
}; };
let conditions = next_config.webpack_conditions().to_resolved().await?;
Ok(if let Some(rules) = rules { Ok(if let Some(rules) = rules {
Some( Some(
WebpackLoadersOptions { WebpackLoadersOptions {
rules, rules,
conditions,
loader_runner_package: Some(loader_runner_package_mapping().to_resolved().await?), loader_runner_package: Some(loader_runner_package_mapping().to_resolved().await?),
} }
.resolved_cell(), .resolved_cell(),
......
...@@ -874,6 +874,33 @@ function bindingToApi( ...@@ -874,6 +874,33 @@ function bindingToApi(
} }
} }
const conditions: (typeof nextConfig)['turbopack']['conditions'] =
nextConfigSerializable.turbopack?.conditions
if (conditions) {
type SerializedConditions = {
[key: string]: {
path:
| { type: 'regexp'; value: { source: string; flags: string } }
| { type: 'glob'; value: string }
}
}
const serializedConditions: SerializedConditions = {}
for (const [key, value] of Object.entries(conditions)) {
serializedConditions[key] = {
...value,
path:
value.path instanceof RegExp
? {
type: 'regexp',
value: { source: value.path.source, flags: value.path.flags },
}
: { type: 'glob', value: value.path },
}
}
nextConfigSerializable.turbopack.conditions = serializedConditions
}
return JSON.stringify(nextConfigSerializable, null, 2) return JSON.stringify(nextConfigSerializable, null, 2)
} }
......
...@@ -13,6 +13,7 @@ import type { ...@@ -13,6 +13,7 @@ import type {
TurbopackRuleConfigItem, TurbopackRuleConfigItem,
TurbopackRuleConfigItemOptions, TurbopackRuleConfigItemOptions,
TurbopackRuleConfigItemOrShortcut, TurbopackRuleConfigItemOrShortcut,
TurbopackRuleCondition,
} from './config-shared' } from './config-shared'
import type { import type {
Header, Header,
...@@ -128,8 +129,13 @@ const zTurboRuleConfigItem: zod.ZodType<TurbopackRuleConfigItem> = z.union([ ...@@ -128,8 +129,13 @@ const zTurboRuleConfigItem: zod.ZodType<TurbopackRuleConfigItem> = z.union([
const zTurboRuleConfigItemOrShortcut: zod.ZodType<TurbopackRuleConfigItemOrShortcut> = const zTurboRuleConfigItemOrShortcut: zod.ZodType<TurbopackRuleConfigItemOrShortcut> =
z.union([z.array(zTurboLoaderItem), zTurboRuleConfigItem]) z.union([z.array(zTurboLoaderItem), zTurboRuleConfigItem])
const zTurboCondition: zod.ZodType<TurbopackRuleCondition> = z.object({
path: z.union([z.string(), z.instanceof(RegExp)]),
})
const zTurbopackConfig: zod.ZodType<TurbopackOptions> = z.strictObject({ const zTurbopackConfig: zod.ZodType<TurbopackOptions> = z.strictObject({
rules: z.record(z.string(), zTurboRuleConfigItemOrShortcut).optional(), rules: z.record(z.string(), zTurboRuleConfigItemOrShortcut).optional(),
conditions: z.record(z.string(), zTurboCondition).optional(),
resolveAlias: z resolveAlias: z
.record( .record(
z.string(), z.string(),
......
...@@ -105,6 +105,10 @@ export type TurbopackLoaderItem = ...@@ -105,6 +105,10 @@ export type TurbopackLoaderItem =
options: Record<string, JSONValue> options: Record<string, JSONValue>
} }
export type TurbopackRuleCondition = {
path: string | RegExp
}
export type TurbopackRuleConfigItemOrShortcut = export type TurbopackRuleConfigItemOrShortcut =
| TurbopackLoaderItem[] | TurbopackLoaderItem[]
| TurbopackRuleConfigItem | TurbopackRuleConfigItem
...@@ -144,6 +148,13 @@ export interface TurbopackOptions { ...@@ -144,6 +148,13 @@ export interface TurbopackOptions {
*/ */
rules?: Record<string, TurbopackRuleConfigItemOrShortcut> rules?: Record<string, TurbopackRuleConfigItemOrShortcut>
/**
* (`next --turbopack` only) A list of conditions to apply when running webpack loaders with Turbopack.
*
* @see [Turbopack Loaders](https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders)
*/
conditions?: Record<string, TurbopackRuleCondition>
/** /**
* The module ID strategy to use for Turbopack. * The module ID strategy to use for Turbopack.
* If not set, the default is `'named'` for development and `'deterministic'` * If not set, the default is `'named'` for development and `'deterministic'`
......
...@@ -29,6 +29,7 @@ tokio = { workspace = true } ...@@ -29,6 +29,7 @@ tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbo-esregex = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-env = { workspace = true } turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
......
...@@ -627,16 +627,40 @@ impl ModuleOptions { ...@@ -627,16 +627,40 @@ impl ModuleOptions {
path.context("need_path in ModuleOptions::new is incorrect")?, path.context("need_path in ModuleOptions::new is incorrect")?,
) )
}; };
for (glob, rule) in webpack_loaders_options.rules.await?.iter() { for (key, rule) in webpack_loaders_options.rules.await?.iter() {
rules.push(ModuleRule::new( rules.push(ModuleRule::new(
RuleCondition::All(vec![ RuleCondition::All(vec![
if !glob.contains('/') { if key.starts_with("#") {
RuleCondition::ResourceBasePathGlob(Glob::new(glob.clone()).await?) // This is a custom marker requiring a corresponding condition entry
} else { let conditions = (*webpack_loaders_options.conditions.await?)
.context(
"Expected a condition entry for the webpack loader rule \
matching {key}. Create a `conditions` mapping in your \
next.config.js",
)?
.await?;
let condition = conditions.get(key).context(
"Expected a condition entry for the webpack loader rule matching \
{key}.",
)?;
match &condition.path {
ConditionPath::Glob(glob) => RuleCondition::ResourcePathGlob {
base: execution_context.project_path().await?,
glob: Glob::new(glob.clone()).await?,
},
ConditionPath::Regex(regex) => {
RuleCondition::ResourcePathEsRegex(regex.await?)
}
}
} else if key.contains('/') {
RuleCondition::ResourcePathGlob { RuleCondition::ResourcePathGlob {
base: execution_context.project_path().await?, base: execution_context.project_path().await?,
glob: Glob::new(glob.clone()).await?, glob: Glob::new(key.clone()).await?,
} }
} else {
RuleCondition::ResourceBasePathGlob(Glob::new(key.clone()).await?)
}, },
RuleCondition::not(RuleCondition::ResourceIsVirtualSource), RuleCondition::not(RuleCondition::ResourceIsVirtualSource),
]), ]),
......
use std::fmt::Debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_esregex::EsRegex;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, ValueDefault, Vc}; use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, ValueDefault, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
...@@ -31,10 +34,31 @@ pub struct WebpackRules(FxIndexMap<RcStr, LoaderRuleItem>); ...@@ -31,10 +34,31 @@ pub struct WebpackRules(FxIndexMap<RcStr, LoaderRuleItem>);
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct OptionWebpackRules(Option<ResolvedVc<WebpackRules>>); pub struct OptionWebpackRules(Option<ResolvedVc<WebpackRules>>);
#[derive(Default)]
#[turbo_tasks::value(transparent)]
pub struct WebpackConditions(pub FxIndexMap<RcStr, ConditionItem>);
#[derive(Default)]
#[turbo_tasks::value(transparent)]
pub struct OptionWebpackConditions(Option<ResolvedVc<WebpackConditions>>);
#[derive(Clone, PartialEq, Eq, Debug, TraceRawVcs, Serialize, Deserialize, NonLocalValue)]
pub enum ConditionPath {
Glob(RcStr),
Regex(ResolvedVc<EsRegex>),
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug)]
pub struct ConditionItem {
pub path: ConditionPath,
}
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct WebpackLoadersOptions { pub struct WebpackLoadersOptions {
pub rules: ResolvedVc<WebpackRules>, pub rules: ResolvedVc<WebpackRules>,
pub conditions: ResolvedVc<OptionWebpackConditions>,
pub loader_runner_package: Option<ResolvedVc<ImportMapping>>, pub loader_runner_package: Option<ResolvedVc<ImportMapping>>,
} }
......
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_esregex::EsRegex;
use turbo_tasks::{primitives::Regex, trace::TraceRawVcs, NonLocalValue, ReadRef, ResolvedVc}; use turbo_tasks::{primitives::Regex, trace::TraceRawVcs, NonLocalValue, ReadRef, ResolvedVc};
use turbo_tasks_fs::{glob::Glob, FileSystemPath}; use turbo_tasks_fs::{glob::Glob, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
...@@ -21,6 +22,7 @@ pub enum RuleCondition { ...@@ -21,6 +22,7 @@ pub enum RuleCondition {
ContentTypeStartsWith(String), ContentTypeStartsWith(String),
ContentTypeEmpty, ContentTypeEmpty,
ResourcePathRegex(#[turbo_tasks(trace_ignore)] Regex), ResourcePathRegex(#[turbo_tasks(trace_ignore)] Regex),
ResourcePathEsRegex(#[turbo_tasks(trace_ignore)] ReadRef<EsRegex>),
/// For paths that are within the same filesystem as the `base`, it need to /// For paths that are within the same filesystem as the `base`, it need to
/// match the relative path from base to resource. This includes `./` or /// match the relative path from base to resource. This includes `./` or
/// `../` prefix. For paths in a different filesystem, it need to match /// `../` prefix. For paths in a different filesystem, it need to match
...@@ -125,6 +127,7 @@ impl RuleCondition { ...@@ -125,6 +127,7 @@ impl RuleCondition {
RuleCondition::ResourcePathRegex(_) => { RuleCondition::ResourcePathRegex(_) => {
bail!("ResourcePathRegex not implemented yet") bail!("ResourcePathRegex not implemented yet")
} }
RuleCondition::ResourcePathEsRegex(regex) => regex.is_match(&path.path),
}) })
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment