Unverified Commit c0a1a32c authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: support `import ... with {type: "bytes"}` (#83896)

- [x] The big question is: do we
	- put this behind a flag (`nextConfig.experimental.importTypeBytes`?)
	- ~~put this behind a non-standard `type: "turbopack-bytes"` (currently implemented)~~
	- ~~just ship it?~~
	- the only "problem" is that type: "bytes" is now silently ignored if the flag is not enabled
- [x] [Furthermore, `Uint8Array.fromBase64` isn't that widely supported](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array/fromBase64#browser_compatibility). So emitting that always is questionable from a browser support standpoint.

This is the output in the test:
```js
module.exports = [
  "[project]/turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/type-bytes/input/data.bin (static bytes in ecmascript)",
  (ctx) => {
    ctx.v(Uint8Array.fromBase64("dGhpcyBpcyBzb21lIGRhdGEK"));
  },

  "[project]/turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/type-bytes/input/index.js [test] (ecmascript)",
  (ctx) => {
    "use strict";
    var bytes = ctx.i(
      "[project]/turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/type-bytes/input/data.bin (static bytes in ecmascript)"
    );
    it("import type:bytes should work", () => {
      expect(bytes["default"]).toBeInstanceOf(Uint8Array);
      expect(new TextDecoder().decode(bytes["default"])).toBe(
        "this is some data\n"
      );
    });
  },
];

```
parent 82e1eb1f
......@@ -9,10 +9,11 @@ use crate::{
next_client::context::ClientContextType,
next_config::NextConfig,
next_shared::transforms::{
debug_fn_name::get_debug_fn_name_rule, get_next_dynamic_transform_rule,
get_next_font_transform_rule, get_next_image_rule, get_next_lint_transform_rule,
get_next_modularize_imports_rule, get_next_pages_transforms_rule,
get_server_actions_transform_rule, next_cjs_optimizer::get_next_cjs_optimizer_rule,
debug_fn_name::get_debug_fn_name_rule, get_import_type_bytes_rule,
get_next_dynamic_transform_rule, get_next_font_transform_rule, get_next_image_rule,
get_next_lint_transform_rule, get_next_modularize_imports_rule,
get_next_pages_transforms_rule, get_server_actions_transform_rule,
next_cjs_optimizer::get_next_cjs_optimizer_rule,
next_disallow_re_export_all_in_page::get_next_disallow_export_all_in_page_rule,
next_pure::get_next_pure_rule, server_actions::ActionsTransform,
},
......@@ -94,5 +95,9 @@ pub async fn get_next_client_transforms_rules(
rules.push(get_next_image_rule().await?);
}
if *next_config.turbopack_import_type_bytes().await? {
rules.push(get_import_type_bytes_rule());
}
Ok(rules)
}
......@@ -874,6 +874,7 @@ pub struct ExperimentalConfig {
turbopack_source_maps: Option<bool>,
turbopack_tree_shaking: Option<bool>,
turbopack_scope_hoisting: Option<bool>,
turbopack_import_type_bytes: Option<bool>,
turbopack_use_system_tls_certs: Option<bool>,
/// Disable automatic configuration of the sass loader.
#[serde(default)]
......@@ -1809,6 +1810,15 @@ impl NextConfig {
}))
}
#[turbo_tasks::function]
pub async fn turbopack_import_type_bytes(&self) -> Vc<bool> {
Vc::cell(
self.experimental
.turbopack_import_type_bytes
.unwrap_or(false),
)
}
#[turbo_tasks::function]
pub async fn client_source_maps(&self, mode: Vc<NextMode>) -> Result<Vc<bool>> {
let source_maps = self.experimental.turbopack_source_maps;
......
......@@ -12,8 +12,8 @@ use crate::{
next_config::NextConfig,
next_server::context::ServerContextType,
next_shared::transforms::{
get_next_dynamic_transform_rule, get_next_font_transform_rule, get_next_image_rule,
get_next_lint_transform_rule, get_next_modularize_imports_rule,
get_import_type_bytes_rule, get_next_dynamic_transform_rule, get_next_font_transform_rule,
get_next_image_rule, get_next_lint_transform_rule, get_next_modularize_imports_rule,
get_next_pages_transforms_rule, get_next_track_dynamic_imports_transform_rule,
get_server_actions_transform_rule, next_cjs_optimizer::get_next_cjs_optimizer_rule,
next_disallow_re_export_all_in_page::get_next_disallow_export_all_in_page_rule,
......@@ -209,6 +209,10 @@ pub async fn get_next_server_transforms_rules(
}
}
if *next_config.turbopack_import_type_bytes().await? {
rules.push(get_import_type_bytes_rule());
}
Ok(rules)
}
......
......@@ -32,7 +32,9 @@ pub use server_actions::get_server_actions_transform_rule;
use turbo_tasks::ResolvedVc;
use turbo_tasks_fs::FileSystemPath;
use turbopack::module_options::{ModuleRule, ModuleRuleEffect, ModuleType, RuleCondition};
use turbopack_core::reference_type::{ReferenceType, UrlReferenceSubType};
use turbopack_core::reference_type::{
EcmaScriptModulesReferenceSubType, ImportWithType, ReferenceType, UrlReferenceSubType,
};
use turbopack_ecmascript::{CustomTransformer, EcmascriptInputTransform};
use crate::next_image::{StructuredImageModuleType, module::BlurPlaceholderMode};
......@@ -123,6 +125,16 @@ pub(crate) fn module_rule_match_pages_page_file(
])
}
pub(crate) fn get_import_type_bytes_rule() -> ModuleRule {
// Move this into turbopack once the feature is standardized
ModuleRule::new(
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType(ImportWithType::Bytes),
)),
vec![ModuleRuleEffect::ModuleType(ModuleType::InlinedBytesJs)],
)
}
pub(crate) enum EcmascriptTransformStage {
Preprocess,
Main,
......
......@@ -473,6 +473,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
turbopackTreeShaking: z.boolean().optional(),
turbopackRemoveUnusedExports: z.boolean().optional(),
turbopackScopeHoisting: z.boolean().optional(),
turbopackImportTypeBytes: z.boolean().optional(),
turbopackUseSystemTlsCerts: z.boolean().optional(),
turbopackUseBuiltinBabel: z.boolean().optional(),
turbopackUseBuiltinSass: z.boolean().optional(),
......
......@@ -438,6 +438,11 @@ export interface ExperimentalConfig {
*/
turbopackMinify?: boolean
/**
* Enable support for `with {type: "module"}` for ESM imports.
*/
turbopackImportTypeBytes?: boolean
/**
* Enable scope hoisting. Defaults to true in build mode. Always disabled in development mode.
*/
......
import bytes from './data.bin' with { type: 'bytes' }
export async function GET(_req) {
return Response.json(
{
instanceofUint8Array: bytes instanceof Uint8Array,
length: bytes.length,
content: new TextDecoder().decode(bytes),
},
{ status: 200 }
)
}
import { nextTestSetup } from 'e2e-utils'
// Not supported by Webpack
;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)(
'turbopack-import-type-bytes',
() => {
const { next, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
})
if (skipped) {
return
}
it('support import type: bytes', async () => {
const response = JSON.parse(await next.render('/api'))
expect(response).toEqual({
instanceofUint8Array: true,
length: 18,
content: 'this is some data\n',
})
})
}
)
export default {
experimental: {
turbopackImportTypeBytes: true,
},
}
import { nextTestSetup } from 'e2e-utils'
describe('turbopack-loader-config', () => {
const { next, isTurbopack, isNextDev } = nextTestSetup({
const { next, isTurbopack, isNextDev, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
// we can't set `nextConfig` inline because it contains regexes that fail to serialize, it needs
// to be set in a separate module (`next.config.ts`)
})
if (skipped) {
return
}
if (!isTurbopack) {
it('should only run the test in turbopack', () => {})
return
......
......@@ -62,6 +62,7 @@ pub enum CommonJsReferenceSubType {
)]
pub enum ImportWithType {
Json,
Bytes,
}
#[derive(
......
use std::io::{Read, Write};
use anyhow::{Result, bail};
use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{FileContent, glob::Glob, rope::RopeBuilder};
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::{ChunkItem, ChunkType, ChunkableModule, ChunkingContext},
ident::AssetIdent,
module::Module,
module_graph::ModuleGraph,
source::Source,
};
use crate::{
chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable,
EcmascriptChunkType, EcmascriptExports,
},
runtime_functions::TURBOPACK_EXPORT_VALUE,
utils::StringifyJs,
};
#[turbo_tasks::value]
pub struct InlinedBytesJsModule {
source: ResolvedVc<Box<dyn Source>>,
}
#[turbo_tasks::value_impl]
impl InlinedBytesJsModule {
#[turbo_tasks::function]
pub fn new(source: ResolvedVc<Box<dyn Source>>) -> Vc<Self> {
Self::cell(InlinedBytesJsModule { source })
}
}
#[turbo_tasks::value_impl]
impl Module for InlinedBytesJsModule {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
self.source
.ident()
.with_modifier(rcstr!("static bytes in ecmascript"))
}
}
#[turbo_tasks::value_impl]
impl Asset for InlinedBytesJsModule {
#[turbo_tasks::function]
fn content(&self) -> Vc<AssetContent> {
self.source.content()
}
}
#[turbo_tasks::value_impl]
impl ChunkableModule for InlinedBytesJsModule {
#[turbo_tasks::function]
fn as_chunk_item(
self: ResolvedVc<Self>,
_module_graph: Vc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
) -> Vc<Box<dyn turbopack_core::chunk::ChunkItem>> {
Vc::upcast(InlinedBytesJsChunkItem::cell(InlinedBytesJsChunkItem {
module: self,
chunking_context,
}))
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkPlaceable for InlinedBytesJsModule {
#[turbo_tasks::function]
fn get_exports(&self) -> Vc<EcmascriptExports> {
EcmascriptExports::Value.cell()
}
#[turbo_tasks::function]
fn is_marked_as_side_effect_free(&self, _side_effect_free_packages: Vc<Glob>) -> Vc<bool> {
Vc::cell(true)
}
}
#[turbo_tasks::value]
struct InlinedBytesJsChunkItem {
module: ResolvedVc<InlinedBytesJsModule>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
}
#[turbo_tasks::value_impl]
impl ChunkItem for InlinedBytesJsChunkItem {
#[turbo_tasks::function]
fn asset_ident(&self) -> Vc<AssetIdent> {
self.module.ident()
}
#[turbo_tasks::function]
fn chunking_context(&self) -> Vc<Box<dyn ChunkingContext>> {
*self.chunking_context
}
#[turbo_tasks::function]
async fn ty(&self) -> Result<Vc<Box<dyn ChunkType>>> {
Ok(Vc::upcast(
Vc::<EcmascriptChunkType>::default().resolve().await?,
))
}
#[turbo_tasks::function]
fn module(&self) -> Vc<Box<dyn Module>> {
Vc::upcast(*self.module)
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for InlinedBytesJsChunkItem {
#[turbo_tasks::function]
async fn content(&self) -> Result<Vc<EcmascriptChunkItemContent>> {
let content = self.module.content().file_content().await?;
match &*content {
FileContent::Content(data) => {
let mut inner_code = RopeBuilder::default();
inner_code += "
var decode = Uint8Array.fromBase64 || function Uint8Array_fromBase64(base64) {
var binaryString = atob(base64);
var buffer = new Uint8Array(binaryString.length);
for (var i = 0; i < binaryString.length; i++) {
buffer[i] = binaryString.charCodeAt(i)
}
return buffer
};\n";
let encoded = data_encoding::BASE64_NOPAD
.encode(&data.read().bytes().collect::<std::io::Result<Vec<u8>>>()?);
write!(
inner_code,
"{TURBOPACK_EXPORT_VALUE}(decode({}));",
StringifyJs(&encoded)
)?;
Ok(EcmascriptChunkItemContent {
inner_code: inner_code.build(),
..Default::default()
}
.into())
}
FileContent::NotFound => {
bail!("File not found: {}", self.module.ident().to_string().await?);
}
}
}
}
......@@ -14,6 +14,7 @@ pub mod async_chunk;
pub mod chunk;
pub mod code_gen;
mod errors;
pub mod inlined_bytes_module;
pub mod magic_identifier;
pub mod manifest;
mod merged_module;
......
......@@ -391,6 +391,8 @@ impl ModuleReference for EsmAssetReference {
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let ty = if matches!(self.annotations.module_type(), Some("json")) {
EcmaScriptModulesReferenceSubType::ImportWithType(ImportWithType::Json)
} else if matches!(self.annotations.module_type(), Some("bytes")) {
EcmaScriptModulesReferenceSubType::ImportWithType(ImportWithType::Bytes)
} else if let Some(part) = &self.export_name {
EcmaScriptModulesReferenceSubType::ImportPart(part.clone())
} else {
......
......@@ -56,6 +56,7 @@ use turbopack_core::{
pub use turbopack_css as css;
pub use turbopack_ecmascript as ecmascript;
use turbopack_ecmascript::{
inlined_bytes_module::InlinedBytesJsModule,
references::external_module::{
CachedExternalModule, CachedExternalTracingMode, CachedExternalType,
},
......@@ -264,6 +265,9 @@ async fn apply_module_type(
ModuleType::StaticUrlCss => {
ResolvedVc::upcast(StaticUrlCssModule::new(*source).to_resolved().await?)
}
ModuleType::InlinedBytesJs => {
ResolvedVc::upcast(InlinedBytesJsModule::new(*source).to_resolved().await?)
}
ModuleType::WebAssembly { source_ty } => ResolvedVc::upcast(
WebAssemblyModuleAsset::new(
WebAssemblySource::new(*source, *source_ty),
......@@ -518,6 +522,8 @@ async fn process_default_internal(
match ty {
ImportWithType::Json => Some(ModuleType::Json),
// Reenable this once `import {type: "bytes"}` is stabilized
ImportWithType::Bytes => None,
}
}
_ => None,
......
......@@ -130,6 +130,7 @@ pub enum ModuleType {
},
StaticUrlJs,
StaticUrlCss,
InlinedBytesJs,
WebAssembly {
source_ty: WebAssemblySourceType,
},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment