Unverified Commit 83dc2ebd authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Turbopack: Implement regex support for matching webpack loaders (#78733)

Inspired by https://x.com/wSokra/status/1907794635914612777, this allows you to declare a ‘marker’ in place of a glob, allowing you to express more conditions than simply just a glob pattern. To start, this allows us to implement regexes. For example, to match either `.svg` or `.svgr` files, use this `next.config.ts`:

```
const nextConfig: NextConfig = {
  turbopack: {
    rules: {
      '#svg': {
        as: '*.js',
        loaders: ['@svgr/webpack'],
      }
    },
    conditions: {
      '#svg': {
        path: /\.svgr?$/,
      }
    }
  }
};
```
parent e72a76e4
......@@ -4583,10 +4583,12 @@ dependencies = [
"rustc-hash 2.1.1",
"serde",
"serde_json",
"serde_path_to_error",
"swc_core",
"swc_relay",
"thiserror 1.0.69",
"tracing",
"turbo-esregex",
"turbo-rcstr",
"turbo-tasks",
"turbo-tasks-build",
......@@ -9690,6 +9692,7 @@ dependencies = [
"smallvec",
"tokio",
"tracing",
"turbo-esregex",
"turbo-rcstr",
"turbo-tasks",
"turbo-tasks-build",
......
......@@ -37,6 +37,7 @@ remove_console = "0.37.0"
itertools = { workspace = true }
auto-hash-map = { workspace = true }
percent-encoding = "2.3.1"
serde_path_to_error = { workspace = true }
swc_core = { workspace = true, features = [
"base",
......@@ -60,6 +61,7 @@ modularize_imports = { workspace = true }
swc_relay = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-esregex = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-bytes = { workspace = true }
turbo-tasks-env = { workspace = true }
......
......@@ -10,7 +10,10 @@ use turbo_tasks::{
use turbo_tasks_env::EnvMap;
use turbo_tasks_fs::FileSystemPath;
use turbopack::module_options::{
module_options_context::MdxTransformOptions, LoaderRuleItem, OptionWebpackRules,
module_options_context::{
ConditionItem, ConditionPath, MdxTransformOptions, OptionWebpackConditions,
},
LoaderRuleItem, OptionWebpackRules,
};
use turbopack_core::{
issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString},
......@@ -541,11 +544,57 @@ pub struct TurbopackConfig {
/// This option has been replaced by `rules`.
pub loaders: Option<JsonValue>,
pub rules: Option<FxIndexMap<RcStr, RuleConfigItemOrShortcut>>,
#[turbo_tasks(trace_ignore)]
pub conditions: Option<FxIndexMap<RcStr, ConfigConditionItem>>,
pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
pub resolve_extensions: Option<Vec<RcStr>>,
pub module_ids: Option<ModuleIds>,
}
#[derive(Clone, Debug, PartialEq, Serialize, TraceRawVcs, NonLocalValue)]
pub struct ConfigConditionItem(ConditionItem);
impl<'de> Deserialize<'de> for ConfigConditionItem {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
struct RegexComponents {
source: RcStr,
flags: RcStr,
}
#[derive(Deserialize)]
struct ConfigPath {
path: RegexOrGlob,
}
#[derive(Deserialize)]
#[serde(tag = "type", rename_all = "lowercase")]
enum RegexOrGlob {
Regexp { value: RegexComponents },
Glob { value: String },
}
let config_path = ConfigPath::deserialize(deserializer)?;
let condition_item = match config_path.path {
RegexOrGlob::Regexp { value } => {
let regex = turbo_esregex::EsRegex::new(&value.source, &value.flags)
.map_err(serde::de::Error::custom)?;
ConditionItem {
path: ConditionPath::Regex(regex.resolved_cell()),
}
}
RegexOrGlob::Glob { value } => ConditionItem {
path: ConditionPath::Glob(value.into()),
},
};
Ok(ConfigConditionItem(condition_item))
}
}
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue, OperationValue,
)]
......@@ -1065,7 +1114,8 @@ impl NextConfig {
#[turbo_tasks::function]
pub async fn from_string(string: Vc<RcStr>) -> Result<Vc<Self>> {
let string = string.await?;
let config: NextConfig = serde_json::from_str(&string)
let mut jdeserializer = serde_json::Deserializer::from_str(&string);
let config: NextConfig = serde_path_to_error::deserialize(&mut jdeserializer)
.with_context(|| format!("failed to parse next.config.js: {}", string))?;
Ok(config.cell())
}
......@@ -1227,6 +1277,22 @@ impl NextConfig {
Vc::cell(Some(ResolvedVc::cell(rules)))
}
#[turbo_tasks::function]
pub fn webpack_conditions(&self) -> Vc<OptionWebpackConditions> {
let Some(config_conditions) = self.turbopack.as_ref().and_then(|t| t.conditions.as_ref())
else {
return Vc::cell(None);
};
let conditions = FxIndexMap::from_iter(
config_conditions
.iter()
.map(|(k, v)| (k.clone(), v.0.clone())),
);
Vc::cell(Some(ResolvedVc::cell(conditions)))
}
#[turbo_tasks::function]
pub fn persistent_caching_enabled(&self) -> Result<Vc<bool>> {
Ok(Vc::cell(
......
......@@ -15,19 +15,22 @@ pub async fn webpack_loader_options(
project_path: ResolvedVc<FileSystemPath>,
next_config: Vc<NextConfig>,
foreign: bool,
conditions: Vec<RcStr>,
condition_strs: Vec<RcStr>,
) -> Result<Option<ResolvedVc<WebpackLoadersOptions>>> {
let rules = *next_config.webpack_rules(conditions).await?;
let rules = *next_config.webpack_rules(condition_strs).await?;
let rules = *maybe_add_sass_loader(next_config.sass_config(), rules.map(|v| *v)).await?;
let rules = if foreign {
rules
} else {
*maybe_add_babel_loader(*project_path, rules.map(|v| *v)).await?
};
let conditions = next_config.webpack_conditions().to_resolved().await?;
Ok(if let Some(rules) = rules {
Some(
WebpackLoadersOptions {
rules,
conditions,
loader_runner_package: Some(loader_runner_package_mapping().to_resolved().await?),
}
.resolved_cell(),
......
......@@ -874,6 +874,33 @@ function bindingToApi(
}
}
const conditions: (typeof nextConfig)['turbopack']['conditions'] =
nextConfigSerializable.turbopack?.conditions
if (conditions) {
type SerializedConditions = {
[key: string]: {
path:
| { type: 'regexp'; value: { source: string; flags: string } }
| { type: 'glob'; value: string }
}
}
const serializedConditions: SerializedConditions = {}
for (const [key, value] of Object.entries(conditions)) {
serializedConditions[key] = {
...value,
path:
value.path instanceof RegExp
? {
type: 'regexp',
value: { source: value.path.source, flags: value.path.flags },
}
: { type: 'glob', value: value.path },
}
}
nextConfigSerializable.turbopack.conditions = serializedConditions
}
return JSON.stringify(nextConfigSerializable, null, 2)
}
......
......@@ -13,6 +13,7 @@ import type {
TurbopackRuleConfigItem,
TurbopackRuleConfigItemOptions,
TurbopackRuleConfigItemOrShortcut,
TurbopackRuleCondition,
} from './config-shared'
import type {
Header,
......@@ -128,8 +129,13 @@ const zTurboRuleConfigItem: zod.ZodType<TurbopackRuleConfigItem> = z.union([
const zTurboRuleConfigItemOrShortcut: zod.ZodType<TurbopackRuleConfigItemOrShortcut> =
z.union([z.array(zTurboLoaderItem), zTurboRuleConfigItem])
const zTurboCondition: zod.ZodType<TurbopackRuleCondition> = z.object({
path: z.union([z.string(), z.instanceof(RegExp)]),
})
const zTurbopackConfig: zod.ZodType<TurbopackOptions> = z.strictObject({
rules: z.record(z.string(), zTurboRuleConfigItemOrShortcut).optional(),
conditions: z.record(z.string(), zTurboCondition).optional(),
resolveAlias: z
.record(
z.string(),
......
......@@ -105,6 +105,10 @@ export type TurbopackLoaderItem =
options: Record<string, JSONValue>
}
export type TurbopackRuleCondition = {
path: string | RegExp
}
export type TurbopackRuleConfigItemOrShortcut =
| TurbopackLoaderItem[]
| TurbopackRuleConfigItem
......@@ -144,6 +148,13 @@ export interface TurbopackOptions {
*/
rules?: Record<string, TurbopackRuleConfigItemOrShortcut>
/**
* (`next --turbopack` only) A list of conditions to apply when running webpack loaders with Turbopack.
*
* @see [Turbopack Loaders](https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders)
*/
conditions?: Record<string, TurbopackRuleCondition>
/**
* The module ID strategy to use for Turbopack.
* If not set, the default is `'named'` for development and `'deterministic'`
......
......@@ -29,6 +29,7 @@ tokio = { workspace = true }
tracing = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-esregex = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true }
......
......@@ -627,16 +627,40 @@ impl ModuleOptions {
path.context("need_path in ModuleOptions::new is incorrect")?,
)
};
for (glob, rule) in webpack_loaders_options.rules.await?.iter() {
for (key, rule) in webpack_loaders_options.rules.await?.iter() {
rules.push(ModuleRule::new(
RuleCondition::All(vec![
if !glob.contains('/') {
RuleCondition::ResourceBasePathGlob(Glob::new(glob.clone()).await?)
} else {
if key.starts_with("#") {
// This is a custom marker requiring a corresponding condition entry
let conditions = (*webpack_loaders_options.conditions.await?)
.context(
"Expected a condition entry for the webpack loader rule \
matching {key}. Create a `conditions` mapping in your \
next.config.js",
)?
.await?;
let condition = conditions.get(key).context(
"Expected a condition entry for the webpack loader rule matching \
{key}.",
)?;
match &condition.path {
ConditionPath::Glob(glob) => RuleCondition::ResourcePathGlob {
base: execution_context.project_path().await?,
glob: Glob::new(glob.clone()).await?,
},
ConditionPath::Regex(regex) => {
RuleCondition::ResourcePathEsRegex(regex.await?)
}
}
} else if key.contains('/') {
RuleCondition::ResourcePathGlob {
base: execution_context.project_path().await?,
glob: Glob::new(glob.clone()).await?,
glob: Glob::new(key.clone()).await?,
}
} else {
RuleCondition::ResourceBasePathGlob(Glob::new(key.clone()).await?)
},
RuleCondition::not(RuleCondition::ResourceIsVirtualSource),
]),
......
use std::fmt::Debug;
use serde::{Deserialize, Serialize};
use turbo_esregex::EsRegex;
use turbo_rcstr::RcStr;
use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, ValueDefault, Vc};
use turbo_tasks_fs::FileSystemPath;
......@@ -31,10 +34,31 @@ pub struct WebpackRules(FxIndexMap<RcStr, LoaderRuleItem>);
#[turbo_tasks::value(transparent)]
pub struct OptionWebpackRules(Option<ResolvedVc<WebpackRules>>);
#[derive(Default)]
#[turbo_tasks::value(transparent)]
pub struct WebpackConditions(pub FxIndexMap<RcStr, ConditionItem>);
#[derive(Default)]
#[turbo_tasks::value(transparent)]
pub struct OptionWebpackConditions(Option<ResolvedVc<WebpackConditions>>);
#[derive(Clone, PartialEq, Eq, Debug, TraceRawVcs, Serialize, Deserialize, NonLocalValue)]
pub enum ConditionPath {
Glob(RcStr),
Regex(ResolvedVc<EsRegex>),
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug)]
pub struct ConditionItem {
pub path: ConditionPath,
}
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug)]
pub struct WebpackLoadersOptions {
pub rules: ResolvedVc<WebpackRules>,
pub conditions: ResolvedVc<OptionWebpackConditions>,
pub loader_runner_package: Option<ResolvedVc<ImportMapping>>,
}
......
use anyhow::{bail, Result};
use serde::{Deserialize, Serialize};
use turbo_esregex::EsRegex;
use turbo_tasks::{primitives::Regex, trace::TraceRawVcs, NonLocalValue, ReadRef, ResolvedVc};
use turbo_tasks_fs::{glob::Glob, FileSystemPath};
use turbopack_core::{
......@@ -21,6 +22,7 @@ pub enum RuleCondition {
ContentTypeStartsWith(String),
ContentTypeEmpty,
ResourcePathRegex(#[turbo_tasks(trace_ignore)] Regex),
ResourcePathEsRegex(#[turbo_tasks(trace_ignore)] ReadRef<EsRegex>),
/// For paths that are within the same filesystem as the `base`, it need to
/// match the relative path from base to resource. This includes `./` or
/// `../` prefix. For paths in a different filesystem, it need to match
......@@ -125,6 +127,7 @@ impl RuleCondition {
RuleCondition::ResourcePathRegex(_) => {
bail!("ResourcePathRegex not implemented yet")
}
RuleCondition::ResourcePathEsRegex(regex) => regex.is_match(&path.path),
})
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment