Unverified Commit b067991e authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Add support for with `type: "text"` under a new experimental flag, following…

Add support for with `type: "text"` under a new experimental flag, following what webpack did (#89560)

## Summary

Adds support for the TC39 `import ... with { type: "text" }` proposal to Turbopack, allowing files to be imported as strings.

This is gated behind a new experimental flag `turbopackImportTypeText`.

```js
// next.config.js
export default {
  experimental: {
    turbopackImportTypeText: true,
  },
}
```

```js
import text from './file.txt' with { type: 'text' }
// text is a string containing the file contents
```

### Related TC39 Proposals

- **[`type: "text"`](https://github.com/tc39/proposal-import-text)** - Stage 2 - Import file contents as a string
- **[`type: "bytes"`](https://github.com/nicolo-ribaudo/proposal-import-bytes)** - Stage 2.7 - Import file contents as a Uint8Array (already supported via `turbopackImportTypeBytes`)

### Implementation Details

- Created `TextSourceTransform` which transforms any file into an ES module exporting the file contents as a string
- Added `ResourceHasModifier` rule condition to match on ident modifiers, enabling proper module type resolution after source transforms
- Refactored `BytesSourceTransform` to use the same pattern for consistency
- Both transforms add a modifier to the ident (`text_module` / `bytes_module`) for uniqueness, allowing the same file to be imported both normally and with a type attribute

Using a source transform is simpler and allows the modules to participate in other bundler optimizations more easily (scope hoisting in particular)

This also fixes a few bugs where `with` imports would conflict with other module type rules:
  * if you attempted to import a `.js` file with `type:'bytes'` didn't work since the file extension matching would trigger first.
  * if you set a module type rule in your next config it would override the `with` attribute


## Test Plan

- Expanded e2e tests at `test/e2e/turbopack-import-with-type/`
parent adcac5df
...@@ -341,6 +341,7 @@ pub async fn get_client_module_options_context( ...@@ -341,6 +341,7 @@ pub async fn get_client_module_options_context(
esm_url_rewrite_behavior: Some(UrlRewriteBehavior::Relative), esm_url_rewrite_behavior: Some(UrlRewriteBehavior::Relative),
enable_typeof_window_inlining: Some(TypeofWindow::Object), enable_typeof_window_inlining: Some(TypeofWindow::Object),
enable_import_as_bytes: *next_config.turbopack_import_type_bytes().await?, enable_import_as_bytes: *next_config.turbopack_import_type_bytes().await?,
enable_import_as_text: *next_config.turbopack_import_type_text().await?,
source_maps, source_maps,
infer_module_side_effects: *next_config.turbopack_infer_module_side_effects().await?, infer_module_side_effects: *next_config.turbopack_infer_module_side_effects().await?,
..Default::default() ..Default::default()
......
...@@ -1152,6 +1152,7 @@ pub struct ExperimentalConfig { ...@@ -1152,6 +1152,7 @@ pub struct ExperimentalConfig {
turbopack_client_side_nested_async_chunking: Option<bool>, turbopack_client_side_nested_async_chunking: Option<bool>,
turbopack_server_side_nested_async_chunking: Option<bool>, turbopack_server_side_nested_async_chunking: Option<bool>,
turbopack_import_type_bytes: Option<bool>, turbopack_import_type_bytes: Option<bool>,
turbopack_import_type_text: Option<bool>,
/// Disable automatic configuration of the sass loader. /// Disable automatic configuration of the sass loader.
#[serde(default)] #[serde(default)]
turbopack_use_builtin_sass: Option<bool>, turbopack_use_builtin_sass: Option<bool>,
...@@ -2093,6 +2094,15 @@ impl NextConfig { ...@@ -2093,6 +2094,15 @@ impl NextConfig {
) )
} }
#[turbo_tasks::function]
pub async fn turbopack_import_type_text(&self) -> Vc<bool> {
Vc::cell(
self.experimental
.turbopack_import_type_text
.unwrap_or(false),
)
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn client_source_maps(&self, mode: Vc<NextMode>) -> Result<Vc<SourceMapsType>> { pub async fn client_source_maps(&self, mode: Vc<NextMode>) -> Result<Vc<SourceMapsType>> {
let input_source_maps = self let input_source_maps = self
......
...@@ -592,6 +592,7 @@ pub async fn get_server_module_options_context( ...@@ -592,6 +592,7 @@ pub async fn get_server_module_options_context(
ecmascript: EcmascriptOptionsContext { ecmascript: EcmascriptOptionsContext {
enable_typeof_window_inlining: Some(TypeofWindow::Undefined), enable_typeof_window_inlining: Some(TypeofWindow::Undefined),
enable_import_as_bytes: *next_config.turbopack_import_type_bytes().await?, enable_import_as_bytes: *next_config.turbopack_import_type_bytes().await?,
enable_import_as_text: *next_config.turbopack_import_type_text().await?,
import_externals: *next_config.import_externals().await?, import_externals: *next_config.import_externals().await?,
ignore_dynamic_requests: true, ignore_dynamic_requests: true,
source_maps, source_maps,
......
...@@ -325,6 +325,7 @@ export const experimentalSchema = { ...@@ -325,6 +325,7 @@ export const experimentalSchema = {
turbopackClientSideNestedAsyncChunking: z.boolean().optional(), turbopackClientSideNestedAsyncChunking: z.boolean().optional(),
turbopackServerSideNestedAsyncChunking: z.boolean().optional(), turbopackServerSideNestedAsyncChunking: z.boolean().optional(),
turbopackImportTypeBytes: z.boolean().optional(), turbopackImportTypeBytes: z.boolean().optional(),
turbopackImportTypeText: z.boolean().optional(),
turbopackUseBuiltinBabel: z.boolean().optional(), turbopackUseBuiltinBabel: z.boolean().optional(),
turbopackUseBuiltinSass: z.boolean().optional(), turbopackUseBuiltinSass: z.boolean().optional(),
turbopackModuleIds: z.enum(['named', 'deterministic']).optional(), turbopackModuleIds: z.enum(['named', 'deterministic']).optional(),
......
...@@ -148,6 +148,7 @@ export type TurbopackModuleType = ...@@ -148,6 +148,7 @@ export type TurbopackModuleType =
| 'raw' | 'raw'
| 'node' | 'node'
| 'bytes' | 'bytes'
| 'text'
export type TurbopackRuleConfigItem = { export type TurbopackRuleConfigItem = {
/** Loaders to apply to matched files. */ /** Loaders to apply to matched files. */
...@@ -486,6 +487,11 @@ export interface ExperimentalConfig { ...@@ -486,6 +487,11 @@ export interface ExperimentalConfig {
*/ */
turbopackImportTypeBytes?: boolean turbopackImportTypeBytes?: boolean
/**
* Enable support for `with {type: "text"}` for ESM imports.
*/
turbopackImportTypeText?: boolean
/** /**
* Enable scope hoisting. Defaults to true in build mode. Always disabled in development mode. * Enable scope hoisting. Defaults to true in build mode. Always disabled in development mode.
*/ */
......
import bytes from './data.bin' with { type: 'bytes' }
export async function GET(_req) {
return Response.json(
{
instanceofUint8Array: bytes instanceof Uint8Array,
length: bytes.length,
content: new TextDecoder().decode(bytes),
},
{ status: 200 }
)
}
import { nextTestSetup } from 'e2e-utils'
// Not supported by Webpack
;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)(
'turbopack-import-type-bytes',
() => {
const { next, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
})
if (skipped) {
return
}
it('support import type: bytes', async () => {
const response = JSON.parse(await next.render('/api'))
expect(response).toEqual({
instanceofUint8Array: true,
length: 18,
content: 'this is some data\n',
})
})
}
)
export default {
experimental: {
turbopackImportTypeBytes: true,
},
}
throw new Error('this file is configured as ecmascript but imported as bytes')
import text from './data.txt' with { type: 'text' }
import jsAsText from './some.js' with { type: 'text' }
import bytes from './data.bin' with { type: 'bytes' }
import jsAsBytes from './some.js' with { type: 'bytes' }
import configuredAsJsAsBytes from './configured-as-ecmascript.txt' with { type: 'bytes' }
export async function GET(_req) {
return Response.json(
{
text: {
typeofString: typeof text === 'string',
length: text.length,
content: text,
},
jsAsText: {
typeofString: typeof jsAsText === 'string',
content: jsAsText,
},
bytes: {
instanceofUint8Array: bytes instanceof Uint8Array,
length: bytes.length,
content: new TextDecoder().decode(bytes),
},
jsAsBytes: {
instanceofUint8Array: jsAsBytes instanceof Uint8Array,
content: new TextDecoder().decode(jsAsBytes),
},
configuredAsJsAsBytes: {
instanceofUint8Array: configuredAsJsAsBytes instanceof Uint8Array,
content: new TextDecoder().decode(configuredAsJsAsBytes),
},
},
{ status: 200 }
)
}
export const nope = 'nope'
throw new Error('please dont execute me')
import { nextTestSetup } from 'e2e-utils'
const jsContent = `export const nope = 'nope'
throw new Error('please dont execute me')
`
;(process.env.TURBOPACK ? describe : describe.skip)(
'turbopack-import-with-type',
() => {
const { next, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
})
if (skipped) {
return
}
// Testing this together on one route ensures we also avoid weird duplicate module ident things
it('supports import with type: text and type: bytes', async () => {
const response = JSON.parse(await next.render('/api'))
expect(response).toEqual({
text: {
typeofString: true,
length: 12,
content: 'hello world\n',
},
jsAsText: {
typeofString: true,
content: jsContent,
},
bytes: {
instanceofUint8Array: true,
length: 18,
content: 'this is some data\n',
},
jsAsBytes: {
instanceofUint8Array: true,
content: jsContent,
},
configuredAsJsAsBytes: {
instanceofUint8Array: true,
content:
"throw new Error('this file is configured as ecmascript but imported as bytes')\n",
},
})
})
}
)
import type { NextConfig } from 'next'
const config: NextConfig = {
experimental: {
turbopackImportTypeBytes: true,
turbopackImportTypeText: true,
},
turbopack: {
rules: {
// This rule configures a .txt file as ecmascript to test that
// import attributes (with { type: 'bytes' }) take priority
'**/configured-as-ecmascript.txt': {
type: 'ecmascript',
},
},
},
}
export default config
use std::io::Read;
use anyhow::{Result, bail};
use turbo_tasks::Vc;
use turbo_tasks_fs::{File, FileContent};
use turbopack_core::{
asset::{Asset, AssetContent},
source::Source,
source_transform::SourceTransform,
virtual_source::VirtualSource,
};
use crate::utils::{StringifyJs, inline_source_map_comment};
/// A source transform that converts any file into an ES module that exports
/// the file's content as a default Uint8Array export.
///
/// This is used for `import bytes from './file.bin' with { type: 'bytes' }`.
#[turbo_tasks::value]
pub struct BytesSourceTransform;
#[turbo_tasks::value_impl]
impl BytesSourceTransform {
#[turbo_tasks::function]
pub fn new() -> Vc<Self> {
BytesSourceTransform.cell()
}
}
#[turbo_tasks::value_impl]
impl SourceTransform for BytesSourceTransform {
#[turbo_tasks::function]
async fn transform(self: Vc<Self>, source: Vc<Box<dyn Source>>) -> Result<Vc<Box<dyn Source>>> {
let ident = source.ident();
let path = ident.path().await?;
let content = source.content().file_content().await?;
let bytes = match &*content {
FileContent::Content(data) => {
data.read().bytes().collect::<std::io::Result<Vec<u8>>>()?
}
FileContent::NotFound => {
bail!("File not found: {:?}", path);
}
};
let encoded = data_encoding::BASE64_NOPAD.encode(&bytes);
// Generate ES module that decodes base64 to Uint8Array with inline source map.
// Uses Uint8Array.fromBase64 (ES2024+) with atob fallback for older environments.
// The /*#__PURE__*/ annotation marks the decode call as side-effect free for tree shaking.
// For binary files, we use an empty string as sourcesContent since the
// original content isn't meaningful text.
let code = format!(
r#"
"use turbopack no side effects";
const decode = Uint8Array.fromBase64 || function(base64) {{
const binaryString = atob(base64);
const buffer = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {{
buffer[i] = binaryString.charCodeAt(i)
}}
return buffer
}};
export default decode({});
{}"#,
StringifyJs(&encoded),
inline_source_map_comment(&path.path, "")
);
// Rename to .mjs so module rules recognize it as ESM.
// The inline source map ensures debuggers show the original file.
let new_ident = ident.rename_as(format!("{}.[bytes].mjs", path.path).into());
Ok(Vc::upcast(VirtualSource::new_with_ident(
new_ident,
AssetContent::file(FileContent::Content(File::from(code)).cell()),
)))
}
}
use std::io::{Read, Write};
use anyhow::{Result, bail};
use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{FileContent, rope::RopeBuilder};
use turbopack_core::{
asset::Asset,
chunk::{ChunkItem, ChunkType, ChunkableModule, ChunkingContext},
ident::AssetIdent,
module::{Module, ModuleSideEffects},
module_graph::ModuleGraph,
output::OutputAssetsReference,
source::Source,
};
use crate::{
chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable,
EcmascriptChunkType, EcmascriptExports,
},
runtime_functions::TURBOPACK_EXPORT_VALUE,
utils::StringifyJs,
};
#[turbo_tasks::value]
pub struct InlinedBytesJsModule {
source: ResolvedVc<Box<dyn Source>>,
}
#[turbo_tasks::value_impl]
impl InlinedBytesJsModule {
#[turbo_tasks::function]
pub fn new(source: ResolvedVc<Box<dyn Source>>) -> Vc<Self> {
Self::cell(InlinedBytesJsModule { source })
}
}
#[turbo_tasks::value_impl]
impl Module for InlinedBytesJsModule {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
self.source
.ident()
.with_modifier(rcstr!("static bytes in ecmascript"))
}
#[turbo_tasks::function]
fn source(&self) -> Vc<turbopack_core::source::OptionSource> {
Vc::cell(Some(self.source))
}
#[turbo_tasks::function]
fn side_effects(self: Vc<Self>) -> Vc<ModuleSideEffects> {
ModuleSideEffects::SideEffectFree.cell()
}
}
#[turbo_tasks::value_impl]
impl ChunkableModule for InlinedBytesJsModule {
#[turbo_tasks::function]
fn as_chunk_item(
self: ResolvedVc<Self>,
_module_graph: Vc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
) -> Vc<Box<dyn turbopack_core::chunk::ChunkItem>> {
Vc::upcast(InlinedBytesJsChunkItem::cell(InlinedBytesJsChunkItem {
module: self,
chunking_context,
}))
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkPlaceable for InlinedBytesJsModule {
#[turbo_tasks::function]
fn get_exports(&self) -> Vc<EcmascriptExports> {
EcmascriptExports::Value.cell()
}
}
#[turbo_tasks::value]
struct InlinedBytesJsChunkItem {
module: ResolvedVc<InlinedBytesJsModule>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for InlinedBytesJsChunkItem {}
#[turbo_tasks::value_impl]
impl ChunkItem for InlinedBytesJsChunkItem {
#[turbo_tasks::function]
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
}
#[turbo_tasks::function]
async fn ty(&self) -> Result<Vc<Box<dyn ChunkType>>> {
Ok(Vc::upcast(
Vc::<EcmascriptChunkType>::default().resolve().await?,
))
}
#[turbo_tasks::function]
fn module(&self) -> Vc<Box<dyn Module>> {
Vc::upcast(*self.module)
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for InlinedBytesJsChunkItem {
#[turbo_tasks::function]
async fn content(&self) -> Result<Vc<EcmascriptChunkItemContent>> {
let content = self.module.await?.source.content().file_content().await?;
match &*content {
FileContent::Content(data) => {
let mut inner_code = RopeBuilder::default();
inner_code += "
var decode = Uint8Array.fromBase64 || function Uint8Array_fromBase64(base64) {
var binaryString = atob(base64);
var buffer = new Uint8Array(binaryString.length);
for (var i = 0; i < binaryString.length; i++) {
buffer[i] = binaryString.charCodeAt(i)
}
return buffer
};\n";
let encoded = data_encoding::BASE64_NOPAD
.encode(&data.read().bytes().collect::<std::io::Result<Vec<u8>>>()?);
write!(
inner_code,
"{TURBOPACK_EXPORT_VALUE}(decode({}));",
StringifyJs(&encoded)
)?;
Ok(EcmascriptChunkItemContent {
inner_code: inner_code.build(),
..Default::default()
}
.cell())
}
FileContent::NotFound => {
bail!("File not found: {}", self.module.ident().to_string().await?);
}
}
}
}
...@@ -11,10 +11,10 @@ ...@@ -11,10 +11,10 @@
pub mod analyzer; pub mod analyzer;
pub mod annotations; pub mod annotations;
pub mod async_chunk; pub mod async_chunk;
pub mod bytes_source_transform;
pub mod chunk; pub mod chunk;
pub mod code_gen; pub mod code_gen;
mod errors; mod errors;
pub mod inlined_bytes_module;
pub mod magic_identifier; pub mod magic_identifier;
pub mod manifest; pub mod manifest;
mod merged_module; mod merged_module;
...@@ -29,6 +29,7 @@ pub mod source_map; ...@@ -29,6 +29,7 @@ pub mod source_map;
pub(crate) mod static_code; pub(crate) mod static_code;
mod swc_comments; mod swc_comments;
pub mod text; pub mod text;
pub mod text_source_transform;
pub(crate) mod transform; pub(crate) mod transform;
pub mod tree_shake; pub mod tree_shake;
pub mod typescript; pub mod typescript;
......
use anyhow::{Result, bail};
use turbo_tasks::Vc;
use turbo_tasks_fs::{File, FileContent};
use turbopack_core::{
asset::{Asset, AssetContent},
source::Source,
source_transform::SourceTransform,
virtual_source::VirtualSource,
};
use crate::utils::{StringifyJs, inline_source_map_comment};
/// A source transform that converts any file into an ES module that exports
/// the file's content as a default string export.
///
/// This is used for `import text from './file.txt' with { type: 'text' }`.
#[turbo_tasks::value]
pub struct TextSourceTransform;
#[turbo_tasks::value_impl]
impl TextSourceTransform {
#[turbo_tasks::function]
pub fn new() -> Vc<Self> {
TextSourceTransform.cell()
}
}
#[turbo_tasks::value_impl]
impl SourceTransform for TextSourceTransform {
#[turbo_tasks::function]
async fn transform(self: Vc<Self>, source: Vc<Box<dyn Source>>) -> Result<Vc<Box<dyn Source>>> {
let ident = source.ident();
let path = ident.path().await?;
let content = source.content().file_content().await?;
let text = match &*content {
FileContent::Content(data) => data.content().to_str()?,
FileContent::NotFound => {
// This shouldn't happen because the import was already resolved
bail!("File not found: {:?}", path);
}
};
// Generate ES module with inline source map pointing back to the original file
let code = format!(
"\"use turbopack no side effects\";\nexport default {};\n{}",
StringifyJs(&text),
inline_source_map_comment(&path.path, &text)
);
// Rename to .mjs so module rules recognize it as ESM.
// The inline source map ensures debuggers show the original file.
let new_ident = ident.rename_as(format!("{}.[text].mjs", path.path).into());
Ok(Vc::upcast(VirtualSource::new_with_ident(
new_ident,
AssetContent::file(FileContent::Content(File::from(code)).cell()),
)))
}
}
...@@ -263,6 +263,37 @@ impl From<SyntaxContext> for AstSyntaxContext { ...@@ -263,6 +263,37 @@ impl From<SyntaxContext> for AstSyntaxContext {
} }
} }
/// Generates an inline source map comment that maps back to the original file in a trivial way
///
/// This is useful for source transforms that convert non-JS files (like text or binary files)
/// into ES modules. The inline source map ensures that debuggers and error stacks show
/// the original file path rather than the generated code.
///
/// # Arguments
/// * `original_path` - The path to the original file (used in source map's "sources" field)
/// * `original_content` - The original file content (used in source map's "sourcesContent" field)
///
/// # Returns
/// An inline source map comment (e.g., `//# sourceMappingURL=data:application/json;base64,...`)
pub fn inline_source_map_comment(original_path: &str, original_content: &str) -> String {
let source_map = serde_json::json!({
"version": 3,
"sources": [format!("turbopack:///{}", original_path)],
"sourcesContent": [original_content],
"names": [],
// Maps 0:0 in the output code to 0:0 in the original. Sufficient for
// bundle analyzers to attribute the bytes in the output chunks
"mappings": "AAAA",
});
let source_map_base64 = data_encoding::BASE64.encode(source_map.to_string().as_bytes());
format!(
"//# sourceMappingURL=data:application/json;base64,{}",
source_map_base64
)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
...@@ -294,4 +325,29 @@ mod tests { ...@@ -294,4 +325,29 @@ mod tests {
)) ))
); );
} }
#[test]
fn test_inline_source_map_comment() {
use super::inline_source_map_comment;
let comment = inline_source_map_comment("test.txt", "hello");
assert!(comment.starts_with("//# sourceMappingURL=data:application/json;base64,"));
// Verify the source map is valid JSON when decoded
let source_map_part = comment
.split("base64,")
.nth(1)
.expect("should have base64 part");
let decoded = data_encoding::BASE64
.decode(source_map_part.as_bytes())
.expect("should decode");
let json: serde_json::Value =
serde_json::from_slice(&decoded).expect("should be valid JSON");
assert_eq!(json["version"], 3);
assert_eq!(json["sources"][0], "turbopack:///test.txt");
assert_eq!(json["sourcesContent"][0], "hello");
assert_eq!(json["mappings"], "AAAA");
}
} }
...@@ -46,7 +46,6 @@ use turbopack_ecmascript::{ ...@@ -46,7 +46,6 @@ use turbopack_ecmascript::{
AnalyzeMode, EcmascriptInputTransforms, EcmascriptModuleAsset, EcmascriptModuleAssetType, AnalyzeMode, EcmascriptInputTransforms, EcmascriptModuleAsset, EcmascriptModuleAssetType,
TreeShakingMode, TreeShakingMode,
chunk::EcmascriptChunkPlaceable, chunk::EcmascriptChunkPlaceable,
inlined_bytes_module::InlinedBytesJsModule,
references::{ references::{
FollowExportsResult, FollowExportsResult,
external_module::{CachedExternalModule, CachedExternalTracingMode, CachedExternalType}, external_module::{CachedExternalModule, CachedExternalTracingMode, CachedExternalType},
...@@ -275,9 +274,6 @@ async fn apply_module_type( ...@@ -275,9 +274,6 @@ async fn apply_module_type(
.to_resolved() .to_resolved()
.await?, .await?,
), ),
ModuleType::InlinedBytesJs => {
ResolvedVc::upcast(InlinedBytesJsModule::new(*source).to_resolved().await?)
}
ModuleType::WebAssembly { source_ty } => ResolvedVc::upcast( ModuleType::WebAssembly { source_ty } => ResolvedVc::upcast(
WebAssemblyModuleAsset::new( WebAssemblyModuleAsset::new(
WebAssemblySource::new(*source, *source_ty), WebAssemblySource::new(*source, *source_ty),
......
...@@ -27,6 +27,7 @@ use turbopack_core::{ ...@@ -27,6 +27,7 @@ use turbopack_core::{
use turbopack_css::CssModuleAssetType; use turbopack_css::CssModuleAssetType;
use turbopack_ecmascript::{ use turbopack_ecmascript::{
EcmascriptInputTransform, EcmascriptInputTransforms, EcmascriptOptions, SpecifiedModuleType, EcmascriptInputTransform, EcmascriptInputTransforms, EcmascriptOptions, SpecifiedModuleType,
bytes_source_transform::BytesSourceTransform, text_source_transform::TextSourceTransform,
}; };
use turbopack_mdx::MdxTransform; use turbopack_mdx::MdxTransform;
use turbopack_node::{ use turbopack_node::{
...@@ -237,6 +238,7 @@ impl ModuleOptions { ...@@ -237,6 +238,7 @@ impl ModuleOptions {
enable_typeof_window_inlining, enable_typeof_window_inlining,
enable_exports_info_inlining, enable_exports_info_inlining,
enable_import_as_bytes, enable_import_as_bytes,
enable_import_as_text,
source_maps: ecmascript_source_maps, source_maps: ecmascript_source_maps,
inline_helpers, inline_helpers,
infer_module_side_effects, infer_module_side_effects,
...@@ -359,6 +361,31 @@ impl ModuleOptions { ...@@ -359,6 +361,31 @@ impl ModuleOptions {
let mut rules = vec![]; let mut rules = vec![];
// Import attribute rules (bytes/text) must come BEFORE config rules.
// Import attributes have a stronger API contract - they're explicit in the source code
// and should override any file-pattern-based config rules.
if enable_import_as_bytes {
rules.push(ModuleRule::new(
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType("bytes".into()),
)),
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![
ResolvedVc::upcast(BytesSourceTransform::new().to_resolved().await?),
]))],
));
}
if enable_import_as_text {
rules.push(ModuleRule::new(
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType("text".into()),
)),
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![
ResolvedVc::upcast(TextSourceTransform::new().to_resolved().await?),
]))],
));
}
if let Some(webpack_loaders_options) = enable_webpack_loaders { if let Some(webpack_loaders_options) = enable_webpack_loaders {
let webpack_loaders_options = webpack_loaders_options.await?; let webpack_loaders_options = webpack_loaders_options.await?;
let execution_context = let execution_context =
...@@ -435,15 +462,17 @@ impl ModuleOptions { ...@@ -435,15 +462,17 @@ impl ModuleOptions {
// Add module type if specified // Add module type if specified
if let Some(type_str) = rule.module_type.as_ref() { if let Some(type_str) = rule.module_type.as_ref() {
let module_type = ModuleType::from_str_with_defaults( effects.push(
type_str, ConfiguredModuleType::parse(type_str)?
.into_effect(
ecma_preprocess, ecma_preprocess,
main, main,
postprocess, postprocess,
ecmascript_options_vc, ecmascript_options_vc,
environment, environment,
)?; )
effects.push(ModuleRuleEffect::ModuleType(module_type)); .await?,
)
} }
if !effects.is_empty() { if !effects.is_empty() {
...@@ -527,15 +556,6 @@ impl ModuleOptions { ...@@ -527,15 +556,6 @@ impl ModuleOptions {
), ),
]); ]);
if enable_import_as_bytes {
rules.push(ModuleRule::new(
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType("bytes".into()),
)),
vec![ModuleRuleEffect::ModuleType(ModuleType::InlinedBytesJs)],
));
}
// Rules that apply based on file extension or content type // Rules that apply based on file extension or content type
rules.extend([ rules.extend([
ModuleRule::new_all( ModuleRule::new_all(
......
...@@ -261,9 +261,12 @@ pub struct EcmascriptOptionsContext { ...@@ -261,9 +261,12 @@ pub struct EcmascriptOptionsContext {
/// Whether to allow accessing exports info via `__webpack_exports_info__`. /// Whether to allow accessing exports info via `__webpack_exports_info__`.
pub enable_exports_info_inlining: bool, pub enable_exports_info_inlining: bool,
/// Whether to enable `import bytes from 'module' as { type: "bytes }` syntax. /// Whether to enable `import bytes from 'module' with { type: "bytes" }` syntax.
pub enable_import_as_bytes: bool, pub enable_import_as_bytes: bool,
/// Whether to enable `import text from 'module' with { type: "text" }` syntax.
pub enable_import_as_text: bool,
// TODO should this be a part of Environment instead? // TODO should this be a part of Environment instead?
pub inline_helpers: bool, pub inline_helpers: bool,
......
...@@ -10,7 +10,9 @@ use turbopack_core::{ ...@@ -10,7 +10,9 @@ use turbopack_core::{
source_transform::SourceTransforms, source_transform::SourceTransforms,
}; };
use turbopack_css::CssModuleAssetType; use turbopack_css::CssModuleAssetType;
use turbopack_ecmascript::{EcmascriptInputTransforms, EcmascriptOptions}; use turbopack_ecmascript::{
EcmascriptInputTransforms, EcmascriptOptions, bytes_source_transform::BytesSourceTransform,
};
use turbopack_wasm::source::WebAssemblySourceType; use turbopack_wasm::source::WebAssemblySourceType;
use crate::module_options::{CustomModuleType, RuleCondition, match_mode::MatchMode}; use crate::module_options::{CustomModuleType, RuleCondition, match_mode::MatchMode};
...@@ -149,7 +151,6 @@ pub enum ModuleType { ...@@ -149,7 +151,6 @@ pub enum ModuleType {
/// The tag that is passed to ChunkingContext::asset_url /// The tag that is passed to ChunkingContext::asset_url
tag: Option<RcStr>, tag: Option<RcStr>,
}, },
InlinedBytesJs,
WebAssembly { WebAssembly {
source_ty: WebAssemblySourceType, source_ty: WebAssemblySourceType,
}, },
...@@ -170,55 +171,106 @@ impl Display for ModuleType { ...@@ -170,55 +171,106 @@ impl Display for ModuleType {
ModuleType::Css { .. } => write!(f, "Css"), ModuleType::Css { .. } => write!(f, "Css"),
ModuleType::StaticUrlJs { .. } => write!(f, "StaticUrlJs"), ModuleType::StaticUrlJs { .. } => write!(f, "StaticUrlJs"),
ModuleType::StaticUrlCss { .. } => write!(f, "StaticUrlCss"), ModuleType::StaticUrlCss { .. } => write!(f, "StaticUrlCss"),
ModuleType::InlinedBytesJs => write!(f, "InlinedBytesJs"),
ModuleType::WebAssembly { .. } => write!(f, "WebAssembly"), ModuleType::WebAssembly { .. } => write!(f, "WebAssembly"),
ModuleType::Custom(_) => write!(f, "Custom"), ModuleType::Custom(_) => write!(f, "Custom"),
} }
} }
} }
impl ModuleType { /// User-facing module type names used in configuration.
/// Creates a ModuleType from a string identifier. ///
/// This is used for user-configured module types in turbopack rules. /// This enum represents the semantic module types that users can specify in their config
pub fn from_str_with_defaults( /// (e.g., next.config.js turbopack rules). Some of these map directly to internal `ModuleType`
type_str: &str, /// variants, while others (like `Bytes`) are implemented via source transforms.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ConfiguredModuleType {
Asset,
Ecmascript,
Typescript,
Css,
CssModule,
Json,
Wasm,
Raw,
Node,
/// Converts any file to an ES module exporting its contents as a Uint8Array.
/// Implemented as a source transform, not a ModuleType.
Bytes,
}
impl ConfiguredModuleType {
/// Parse a module type string from user configuration.
pub fn parse(type_str: &str) -> Result<Self> {
Ok(match type_str {
"asset" => ConfiguredModuleType::Asset,
"ecmascript" => ConfiguredModuleType::Ecmascript,
"typescript" => ConfiguredModuleType::Typescript,
"css" => ConfiguredModuleType::Css,
"css-module" => ConfiguredModuleType::CssModule,
"json" => ConfiguredModuleType::Json,
"wasm" => ConfiguredModuleType::Wasm,
"raw" => ConfiguredModuleType::Raw,
"node" => ConfiguredModuleType::Node,
"bytes" => ConfiguredModuleType::Bytes,
_ => bail!(
"Unknown module type: {type_str:?}. Valid types are: asset, ecmascript, \
typescript, css, css-module, json, wasm, raw, node, bytes"
),
})
}
/// Convert this configured module type into module rule effects.
///
/// Some module types (like `Bytes`) are implemented as source transforms rather than
/// `ModuleType` variants, allowing them to compose with the standard Ecmascript pipeline.
pub async fn into_effect(
self,
preprocess: ResolvedVc<EcmascriptInputTransforms>, preprocess: ResolvedVc<EcmascriptInputTransforms>,
main: ResolvedVc<EcmascriptInputTransforms>, main: ResolvedVc<EcmascriptInputTransforms>,
postprocess: ResolvedVc<EcmascriptInputTransforms>, postprocess: ResolvedVc<EcmascriptInputTransforms>,
options: ResolvedVc<EcmascriptOptions>, options: ResolvedVc<EcmascriptOptions>,
environment: Option<ResolvedVc<Environment>>, environment: Option<ResolvedVc<Environment>>,
) -> Result<Self> { ) -> Result<ModuleRuleEffect> {
Ok(match type_str { Ok(match self {
"asset" => ModuleType::StaticUrlJs { tag: None }, ConfiguredModuleType::Bytes => {
"ecmascript" => ModuleType::Ecmascript { // Use source transform instead of ModuleType - the transform produces .mjs
// which gets picked up by the standard Ecmascript rules
ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![ResolvedVc::upcast(
BytesSourceTransform::new().to_resolved().await?,
)]))
}
ConfiguredModuleType::Asset => {
ModuleRuleEffect::ModuleType(ModuleType::StaticUrlJs { tag: None })
}
ConfiguredModuleType::Ecmascript => {
ModuleRuleEffect::ModuleType(ModuleType::Ecmascript {
preprocess, preprocess,
main, main,
postprocess, postprocess,
options, options,
}, })
"typescript" => ModuleType::Typescript { }
ConfiguredModuleType::Typescript => {
ModuleRuleEffect::ModuleType(ModuleType::Typescript {
preprocess, preprocess,
main, main,
postprocess, postprocess,
tsx: false, tsx: false,
analyze_types: false, analyze_types: false,
options, options,
}, })
"css" => ModuleType::Css { }
ConfiguredModuleType::Css => ModuleRuleEffect::ModuleType(ModuleType::Css {
ty: CssModuleAssetType::Default, ty: CssModuleAssetType::Default,
environment, environment,
}, }),
"css-module" => ModuleType::CssModule, ConfiguredModuleType::CssModule => ModuleRuleEffect::ModuleType(ModuleType::CssModule),
"wasm" => ModuleType::WebAssembly { ConfiguredModuleType::Json => ModuleRuleEffect::ModuleType(ModuleType::Json),
ConfiguredModuleType::Wasm => ModuleRuleEffect::ModuleType(ModuleType::WebAssembly {
source_ty: WebAssemblySourceType::Binary, source_ty: WebAssemblySourceType::Binary,
}, }),
"raw" => ModuleType::Raw, ConfiguredModuleType::Raw => ModuleRuleEffect::ModuleType(ModuleType::Raw),
"node" => ModuleType::NodeAddon, ConfiguredModuleType::Node => ModuleRuleEffect::ModuleType(ModuleType::NodeAddon),
"bytes" => ModuleType::InlinedBytesJs,
_ => bail!(
"Unknown module type: {type_str:?}. Valid types are: asset, ecmascript, \
typescript, css, css-module, wasm, raw, node, bytes"
),
}) })
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment