Unverified Commit 0d613062 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Simplify how `json` imports work by turning it into a source transform (#89631)

# Refactor JSON module handling in Turbopack

## What?
Moves the JSON module handling from a separate crate (`turbopack-json`) into the `turbopack-ecmascript` crate as a source transform, similar to how text and binary files are handled.

## Why?
This simplifies the codebase by:
1. Removing a separate crate dependency
2. Using a consistent pattern for handling non-JavaScript assets
    * notably by sending them through the normal asset pipeline we unlock other behaviors in turbopack like scope-hoisting
3. Improving error handling for invalid JSON files

## How?
- Removed the `turbopack-json` crate
- Added a new `json_source_transform.rs` module to `turbopack-ecmascript`
- Implemented two modes for JSON handling:
  - Spec-compliant ESM mode (for `import ... with { type: 'json' }`)
  - Webpack-compatible CommonJS mode (for regular imports)
- Added "no side effects" directive to JSON, text, and binary modules to preserve tree shaking
- Updated error handling to provide better feedback for invalid JSON
- Opt'd out traced modules from json/bytes/text processing, this is more correct and should be a bit faster.  On the other hand this exposed a subtle issue with traced dependencies
parent 445d880e
...@@ -9730,7 +9730,6 @@ dependencies = [ ...@@ -9730,7 +9730,6 @@ dependencies = [
"turbopack-css", "turbopack-css",
"turbopack-ecmascript", "turbopack-ecmascript",
"turbopack-env", "turbopack-env",
"turbopack-json",
"turbopack-mdx", "turbopack-mdx",
"turbopack-node", "turbopack-node",
"turbopack-resolve", "turbopack-resolve",
...@@ -10124,19 +10123,6 @@ dependencies = [ ...@@ -10124,19 +10123,6 @@ dependencies = [
"turbopack-core", "turbopack-core",
] ]
[[package]]
name = "turbopack-json"
version = "0.1.0"
dependencies = [
"anyhow",
"serde_json",
"turbo-rcstr",
"turbo-tasks",
"turbo-tasks-fs",
"turbopack-core",
"turbopack-ecmascript",
]
[[package]] [[package]]
name = "turbopack-mdx" name = "turbopack-mdx"
version = "0.1.0" version = "0.1.0"
......
...@@ -324,7 +324,6 @@ turbopack-ecmascript-runtime = { path = "turbopack/crates/turbopack-ecmascript-r ...@@ -324,7 +324,6 @@ turbopack-ecmascript-runtime = { path = "turbopack/crates/turbopack-ecmascript-r
turbopack-ecmascript-hmr-protocol = { path = "turbopack/crates/turbopack-ecmascript-hmr-protocol" } turbopack-ecmascript-hmr-protocol = { path = "turbopack/crates/turbopack-ecmascript-hmr-protocol" }
turbopack-env = { path = "turbopack/crates/turbopack-env" } turbopack-env = { path = "turbopack/crates/turbopack-env" }
turbopack-image = { path = "turbopack/crates/turbopack-image" } turbopack-image = { path = "turbopack/crates/turbopack-image" }
turbopack-json = { path = "turbopack/crates/turbopack-json" }
turbopack-mdx = { path = "turbopack/crates/turbopack-mdx" } turbopack-mdx = { path = "turbopack/crates/turbopack-mdx" }
turbopack-node = { path = "turbopack/crates/turbopack-node" } turbopack-node = { path = "turbopack/crates/turbopack-node" }
turbopack-resolve = { path = "turbopack/crates/turbopack-resolve" } turbopack-resolve = { path = "turbopack/crates/turbopack-resolve" }
......
...@@ -3,6 +3,8 @@ import jsAsText from './some.js' with { type: 'text' } ...@@ -3,6 +3,8 @@ import jsAsText from './some.js' with { type: 'text' }
import bytes from './data.bin' with { type: 'bytes' } import bytes from './data.bin' with { type: 'bytes' }
import jsAsBytes from './some.js' with { type: 'bytes' } import jsAsBytes from './some.js' with { type: 'bytes' }
import configuredAsJsAsBytes from './configured-as-ecmascript.txt' with { type: 'bytes' } import configuredAsJsAsBytes from './configured-as-ecmascript.txt' with { type: 'bytes' }
import json from './data.json' with { type: 'json' }
import jsonAsText from './data.json' with { type: 'text' }
export async function GET(_req) { export async function GET(_req) {
return Response.json( return Response.json(
...@@ -29,6 +31,14 @@ export async function GET(_req) { ...@@ -29,6 +31,14 @@ export async function GET(_req) {
instanceofUint8Array: configuredAsJsAsBytes instanceof Uint8Array, instanceofUint8Array: configuredAsJsAsBytes instanceof Uint8Array,
content: new TextDecoder().decode(configuredAsJsAsBytes), content: new TextDecoder().decode(configuredAsJsAsBytes),
}, },
json: {
typeofObject: typeof json === 'object',
content: json,
},
jsonAsText: {
typeofString: typeof jsonAsText === 'string',
content: jsonAsText,
},
}, },
{ status: 200 } { status: 200 }
) )
......
...@@ -5,7 +5,7 @@ const jsContent = `export const nope = 'nope' ...@@ -5,7 +5,7 @@ const jsContent = `export const nope = 'nope'
throw new Error('please dont execute me') throw new Error('please dont execute me')
` `
;(process.env.TURBOPACK ? describe : describe.skip)( ;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)(
'turbopack-import-with-type', 'turbopack-import-with-type',
() => { () => {
const { next, skipped } = nextTestSetup({ const { next, skipped } = nextTestSetup({
...@@ -18,7 +18,7 @@ throw new Error('please dont execute me') ...@@ -18,7 +18,7 @@ throw new Error('please dont execute me')
} }
// Testing this together on one route ensures we also avoid weird duplicate module ident things // Testing this together on one route ensures we also avoid weird duplicate module ident things
it('supports import with type: text and type: bytes', async () => { it('supports import with type: text, type: bytes, and type: json', async () => {
const response = JSON.parse(await next.render('/api')) const response = JSON.parse(await next.render('/api'))
expect(response).toEqual({ expect(response).toEqual({
text: { text: {
...@@ -44,6 +44,14 @@ throw new Error('please dont execute me') ...@@ -44,6 +44,14 @@ throw new Error('please dont execute me')
content: content:
"throw new Error('this file is configured as ecmascript but imported as bytes')\n", "throw new Error('this file is configured as ecmascript but imported as bytes')\n",
}, },
json: {
typeofObject: true,
content: { hello: 'world' },
},
jsonAsText: {
typeofString: true,
content: '{ "hello": "world" }\n',
},
}) })
}) })
} }
......
...@@ -21,6 +21,7 @@ use turbo_tasks::{ ...@@ -21,6 +21,7 @@ use turbo_tasks::{
}; };
use turbo_tasks_fs::{ use turbo_tasks_fs::{
FileContent, FileLine, FileLinesContent, FileSystem, FileSystemPath, glob::Glob, FileContent, FileLine, FileLinesContent, FileSystem, FileSystemPath, glob::Glob,
json::UnparsableJson,
}; };
use turbo_tasks_hash::{DeterministicHash, Xxh3Hash64Hasher}; use turbo_tasks_hash::{DeterministicHash, Xxh3Hash64Hasher};
...@@ -540,6 +541,35 @@ impl IssueSource { ...@@ -540,6 +541,35 @@ impl IssueSource {
}) })
} }
/// Create an [`IssueSource`] from an [`UnparsableJson`] error, using its
/// start/end location if available.
pub fn from_unparsable_json(
source: ResolvedVc<Box<dyn Source>>,
error: &UnparsableJson,
) -> Self {
match (error.start_location, error.end_location) {
(None, None) => Self::from_source_only(source),
(Some((line, column)), None) | (None, Some((line, column))) => Self::from_line_col(
source,
SourcePos { line, column },
SourcePos { line, column },
),
(Some((start_line, start_column)), Some((end_line, end_column))) => {
Self::from_line_col(
source,
SourcePos {
line: start_line,
column: start_column,
},
SourcePos {
line: end_line,
column: end_column,
},
)
}
}
}
/// Create a [`IssueSource`] from byte offsets given by an swc ast node /// Create a [`IssueSource`] from byte offsets given by an swc ast node
/// span. /// span.
/// ///
......
...@@ -352,19 +352,62 @@ impl SingleModuleGraph { ...@@ -352,19 +352,62 @@ impl SingleModuleGraph {
} }
}); });
if *CHECK_FOR_DUPLICATE_MODULES { if *CHECK_FOR_DUPLICATE_MODULES {
let mut duplicates = Vec::new(); let mut duplicates = FxHashSet::default();
let mut set = FxHashSet::default(); let mut set = FxHashSet::default();
for &module in modules.keys() { for &module in modules.keys() {
let ident = module.ident().to_string().await?; let ident = module.ident().to_string().await?;
if !set.insert(ident.clone()) { if !set.insert(ident.clone()) {
duplicates.push(ident) duplicates.insert(ident);
} }
} }
if !duplicates.is_empty() { if !duplicates.is_empty() {
panic!( use turbo_tasks::TryFlatJoinIterExt;
"Duplicate module idents in graph: {duplicates:#?}\n{:#?}",
modules let duplicates_clone = duplicates.clone();
); let duplicate_modules = modules
.iter()
.map(async |(&m, &idx)| {
let id = m.ident().to_string().await?;
if duplicates_clone.contains(&id) {
// 3 is arbitrary but it is enough to reveal a little bit
// of detail.
let debug = m.value_debug_format(3).try_to_string().await?;
// Collect reverse dependencies (parents) to help
// diagnose how this module entered the graph.
let parent_modules: Vec<_> = graph
.edges_directed(idx, petgraph::Direction::Incoming)
.filter_map(|edge| match graph.node_weight(edge.source()) {
Some(SingleModuleGraphNode::Module(m)) => Some(*m),
Some(SingleModuleGraphNode::VisitedModule {
module,
..
}) => Some(*module),
None => None,
})
.collect();
let parents: Vec<String> = parent_modules
.iter()
.map(async |p| {
let ident = p.ident().to_string().await?;
Ok((*ident).to_string())
})
.try_join()
.await?;
Ok(Some((id, debug, parents)))
} else {
Ok(None)
}
})
.try_flat_join()
.await?;
// group by ident
let mut map: FxHashMap<_, Vec<(String, Vec<String>)>> = FxHashMap::default();
for (key, debug, parents) in duplicate_modules {
map.entry(key).or_default().push((debug, parents));
}
panic!("Duplicate module idents in graph: {map:#?}",);
} }
} }
} }
......
...@@ -15,7 +15,6 @@ use crate::{ ...@@ -15,7 +15,6 @@ use crate::{
IssueExt, IssueSource, IssueStage, OptionIssueSource, OptionStyledString, StyledString, IssueExt, IssueSource, IssueStage, OptionIssueSource, OptionStyledString, StyledString,
}, },
source::Source, source::Source,
source_pos::SourcePos,
}; };
/// PackageJson wraps the parsed JSON content of a `package.json` file. The /// PackageJson wraps the parsed JSON content of a `package.json` file. The
...@@ -51,29 +50,7 @@ pub async fn read_package_json(path: ResolvedVc<Box<dyn Source>>) -> Result<Vc<O ...@@ -51,29 +50,7 @@ pub async fn read_package_json(path: ResolvedVc<Box<dyn Source>>) -> Result<Vc<O
e.message e.message
)); ));
let source = match (e.start_location, e.end_location) { let source = IssueSource::from_unparsable_json(path, e);
(None, None) => IssueSource::from_source_only(path),
(Some((line, column)), None) | (None, Some((line, column))) => {
IssueSource::from_line_col(
path,
SourcePos { line, column },
SourcePos { line, column },
)
}
(Some((start_line, start_column)), Some((end_line, end_column))) => {
IssueSource::from_line_col(
path,
SourcePos {
line: start_line,
column: start_column,
},
SourcePos {
line: end_line,
column: end_column,
},
)
}
};
PackageJsonIssue { PackageJsonIssue {
error_message, error_message,
source, source,
......
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use crate::{ use crate::{
...@@ -11,13 +12,17 @@ use crate::{ ...@@ -11,13 +12,17 @@ use crate::{
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct RawModule { pub struct RawModule {
source: ResolvedVc<Box<dyn Source>>, source: ResolvedVc<Box<dyn Source>>,
modifier: Option<RcStr>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Module for RawModule { impl Module for RawModule {
#[turbo_tasks::function] #[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> { fn ident(&self) -> Vc<AssetIdent> {
self.source.ident() match &self.modifier {
Some(modifier) => self.source.ident().with_modifier(modifier.clone()),
None => self.source.ident(),
}
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -30,10 +35,20 @@ impl Module for RawModule { ...@@ -30,10 +35,20 @@ impl Module for RawModule {
} }
} }
impl RawModule {
pub fn new(source: Vc<Box<dyn Source>>) -> Vc<RawModule> {
Self::new_inner(source, None)
}
pub fn new_with_modifier(source: Vc<Box<dyn Source>>, modifier: RcStr) -> Vc<RawModule> {
Self::new_inner(source, Some(modifier))
}
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl RawModule { impl RawModule {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(source: ResolvedVc<Box<dyn Source>>) -> Vc<RawModule> { fn new_inner(source: ResolvedVc<Box<dyn Source>>, modifier: Option<RcStr>) -> Vc<RawModule> {
RawModule { source }.cell() RawModule { source, modifier }.cell()
} }
} }
...@@ -47,9 +47,6 @@ impl SourceTransform for BytesSourceTransform { ...@@ -47,9 +47,6 @@ impl SourceTransform for BytesSourceTransform {
// Generate ES module that decodes base64 to Uint8Array with inline source map. // Generate ES module that decodes base64 to Uint8Array with inline source map.
// Uses Uint8Array.fromBase64 (ES2024+) with atob fallback for older environments. // Uses Uint8Array.fromBase64 (ES2024+) with atob fallback for older environments.
// The /*#__PURE__*/ annotation marks the decode call as side-effect free for tree shaking.
// For binary files, we use an empty string as sourcesContent since the
// original content isn't meaningful text.
let code = format!( let code = format!(
r#" r#"
"use turbopack no side effects"; "use turbopack no side effects";
...@@ -66,6 +63,8 @@ const decode = Uint8Array.fromBase64 || function(base64) {{ ...@@ -66,6 +63,8 @@ const decode = Uint8Array.fromBase64 || function(base64) {{
export default decode({}); export default decode({});
{}"#, {}"#,
StringifyJs(&encoded), StringifyJs(&encoded),
// For binary files, we use an empty string as sourcesContent since the
// original content isn't meaningful text.
inline_source_map_comment(&path.path, "") inline_source_map_comment(&path.path, "")
); );
......
use anyhow::{Result, bail};
use turbo_rcstr::rcstr;
use turbo_tasks::Vc;
use turbo_tasks_fs::{File, FileContent, FileJsonContent};
use turbopack_core::{
asset::{Asset, AssetContent},
issue::{IssueExt, IssueSeverity, IssueSource, StyledString, code_gen::CodeGenerationIssue},
source::Source,
source_transform::SourceTransform,
virtual_source::VirtualSource,
};
use crate::utils::inline_source_map_comment;
/// A source transform that converts a JSON file into a JavaScript module.
///
/// Two modes are supported:
///
/// - **Spec-compliant mode** (`use_esm = true`): Generates an ESM module with only a default
/// export, per the JSON Modules spec. Used for `import ... with { type: 'json' }`.
///
/// - **Webpack-compatible mode** (`use_esm = false`): Generates a CommonJS module via
/// `module.exports`, which allows both default imports and named property imports (e.g., `import
/// { a } from './data.json'`). This matches webpack's historical behavior.
///
/// For JSON files larger than 10KB, we use `JSON.parse()` for better performance.
/// See: https://v8.dev/blog/cost-of-javascript-2019#json
#[turbo_tasks::value]
pub struct JsonSourceTransform {
/// If true, generate spec-compliant ESM with only an esm default export.
/// If false, generate CommonJS for webpack compatibility.
use_esm: bool,
}
#[turbo_tasks::value_impl]
impl JsonSourceTransform {
/// Create a new JSON transform with webpack-compatible CommonJS output.
#[turbo_tasks::function]
pub fn new_cjs() -> Vc<Self> {
JsonSourceTransform { use_esm: false }.cell()
}
/// Create a new JSON transform with spec-compliant ESM output.
/// Use this for `import ... with { type: 'json' }` imports.
#[turbo_tasks::function]
pub fn new_esm() -> Vc<Self> {
JsonSourceTransform { use_esm: true }.cell()
}
}
#[turbo_tasks::value_impl]
impl SourceTransform for JsonSourceTransform {
#[turbo_tasks::function]
async fn transform(self: Vc<Self>, source: Vc<Box<dyn Source>>) -> Result<Vc<Box<dyn Source>>> {
let this = self.await?;
let ident = source.ident();
let path = ident.path().await?;
let content = source.content().file_content();
// Parse the JSON to validate it and get the data
let data = content.parse_json().await?;
let (code, extension) = match &*data {
FileJsonContent::Content(data) => {
let data_str = data.to_string();
// The "use turbopack no side effects" directive marks this module as
// side-effect free for tree shaking
let mut code = String::with_capacity(
data_str.len() + 100, /* estimate to account for our `use` comment, export
* overhead and sourcemap comment */
);
code.push_str("\"use turbopack no side effects\";\n");
let extension = if this.use_esm {
// Spec-compliant ESM: only default export
code.push_str("export default ");
"mjs"
} else {
// Webpack-compatible CommonJS: allows named property imports
code.push_str("module.exports = ");
"cjs"
};
// For large JSON files, wrap in JSON.parse for better performance
// https://v8.dev/blog/cost-of-javascript-2019#json
if data_str.len() > 10_000 {
code.push_str("JSON.parse(");
code.push_str(&serde_json::to_string(&data_str)?);
code.push(')');
} else {
code.push_str(&data_str);
}
code.push_str(";\n");
code.push_str(&inline_source_map_comment(&path.path, &data_str));
(code, extension)
}
FileJsonContent::Unparsable(e) => {
let resolved_source = source.to_resolved().await?;
let issue_source = IssueSource::from_unparsable_json(resolved_source, e);
CodeGenerationIssue {
severity: IssueSeverity::Error,
path: ident.path().owned().await?,
title: StyledString::Text(rcstr!("Unable to make a module from invalid JSON"))
.resolved_cell(),
message: StyledString::Text(e.message.clone()).resolved_cell(),
source: Some(issue_source),
}
.resolved_cell()
.emit();
let js_error_message = serde_json::to_string(&format!(
"Unable to make a module from invalid JSON: {}",
e.message
))?;
(format!("throw new Error({js_error_message});"), "js")
}
FileJsonContent::NotFound => {
// This is basically impossible since we wouldn't be called if the module
// doesn't exist but some kind of eventual consistency situation is
// possible where we resolve the file and then it disappears, so bail is appropriate
bail!("JSON file not found: {:?}", path);
}
};
let new_ident = ident.rename_as(format!("{}.[json].{}", path.path, extension).into());
Ok(Vc::upcast(VirtualSource::new_with_ident(
new_ident,
AssetContent::file(FileContent::Content(File::from(code)).cell()),
)))
}
}
...@@ -15,6 +15,7 @@ pub mod bytes_source_transform; ...@@ -15,6 +15,7 @@ pub mod bytes_source_transform;
pub mod chunk; pub mod chunk;
pub mod code_gen; pub mod code_gen;
mod errors; mod errors;
pub mod json_source_transform;
pub mod magic_identifier; pub mod magic_identifier;
pub mod manifest; pub mod manifest;
mod merged_module; mod merged_module;
......
...@@ -323,7 +323,15 @@ impl Module for CachedExternalModule { ...@@ -323,7 +323,15 @@ impl Module for CachedExternalModule {
let references = external_result let references = external_result
.affecting_sources .affecting_sources
.iter() .iter()
.map(|s| Vc::upcast::<Box<dyn Module>>(RawModule::new(**s))) .map(|s| {
// Add a modifier
// it is possible to reference a module as an affecting source and as Module
// so this will distinguish them
Vc::upcast::<Box<dyn Module>>(RawModule::new_with_modifier(
**s,
rcstr!("affecting source"),
))
})
.chain( .chain(
external_result external_result
.primary_modules_raw_iter() .primary_modules_raw_iter()
......
...@@ -40,7 +40,7 @@ impl SourceTransform for TextSourceTransform { ...@@ -40,7 +40,7 @@ impl SourceTransform for TextSourceTransform {
} }
}; };
// Generate ES module with inline source map pointing back to the original file // Generate ES module with inline source map pointing back to the original file.
let code = format!( let code = format!(
"\"use turbopack no side effects\";\nexport default {};\n{}", "\"use turbopack no side effects\";\nexport default {};\n{}",
StringifyJs(&text), StringifyJs(&text),
......
[package]
name = "turbopack-json"
version = "0.1.0"
description = "TBD"
license = "MIT"
edition = "2024"
autobenches = false
[lib]
bench = false
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbopack-core = { workspace = true }
turbopack-ecmascript = { workspace = true }
serde_json = { workspace = true }
//! JSON asset support for turbopack.
//!
//! JSON assets are parsed to ensure they contain valid JSON.
//!
//! When imported from ES modules, they produce a module that exports the
//! JSON value as an object.
#![feature(min_specialization)]
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
use std::fmt::Write;
use anyhow::{Error, Result, bail};
use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{FileContent, FileJsonContent};
use turbopack_core::{
asset::Asset,
chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext},
code_builder::CodeBuilder,
ident::AssetIdent,
module::{Module, ModuleSideEffects},
module_graph::ModuleGraph,
source::Source,
};
use turbopack_ecmascript::{
chunk::{
EcmascriptChunkItemContent, EcmascriptChunkPlaceable, EcmascriptExports,
ecmascript_chunk_item,
},
runtime_functions::TURBOPACK_EXPORT_VALUE,
};
#[turbo_tasks::value]
pub struct JsonModuleAsset {
source: ResolvedVc<Box<dyn Source>>,
}
#[turbo_tasks::value_impl]
impl JsonModuleAsset {
#[turbo_tasks::function]
pub fn new(source: ResolvedVc<Box<dyn Source>>) -> Vc<Self> {
Self::cell(JsonModuleAsset { source })
}
}
#[turbo_tasks::value_impl]
impl Module for JsonModuleAsset {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
self.source.ident().with_modifier(rcstr!("json"))
}
#[turbo_tasks::function]
fn source(&self) -> Vc<turbopack_core::source::OptionSource> {
Vc::cell(Some(self.source))
}
#[turbo_tasks::function]
fn side_effects(self: Vc<Self>) -> Vc<ModuleSideEffects> {
ModuleSideEffects::SideEffectFree.cell()
}
}
#[turbo_tasks::value_impl]
impl ChunkableModule for JsonModuleAsset {
#[turbo_tasks::function]
fn as_chunk_item(
self: ResolvedVc<Self>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
) -> Vc<Box<dyn turbopack_core::chunk::ChunkItem>> {
ecmascript_chunk_item(ResolvedVc::upcast(self), module_graph, chunking_context)
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkPlaceable for JsonModuleAsset {
#[turbo_tasks::function]
fn get_exports(&self) -> Vc<EcmascriptExports> {
EcmascriptExports::Value.cell()
}
#[turbo_tasks::function]
async fn chunk_item_content(
self: Vc<Self>,
_chunking_context: Vc<Box<dyn ChunkingContext>>,
_module_graph: Vc<ModuleGraph>,
_async_module_info: Option<Vc<AsyncModuleInfo>>,
_estimated: bool,
) -> Result<Vc<EcmascriptChunkItemContent>> {
// We parse to JSON and then stringify again to ensure that the
// JSON is valid.
let this = self.await?;
let content = this.source.content().file_content();
let data = content.parse_json().await?;
match &*data {
FileJsonContent::Content(data) => {
let data_str = data.to_string();
let mut code = CodeBuilder::default();
let source_code = if data_str.len() > 10_000 {
// Only use JSON.parse if the content is larger than 10kb
// https://v8.dev/blog/cost-of-javascript-2019#json
let js_str_content = serde_json::to_string(&data_str)?;
format!("{TURBOPACK_EXPORT_VALUE}(JSON.parse({js_str_content}));")
} else {
format!("{TURBOPACK_EXPORT_VALUE}({data_str});")
};
let source_code = source_code.into();
let source_map = serde_json::json!({
"version": 3,
// TODO: Encode using `urlencoding`, so that these
// are valid URLs. However, `project_trace_source_operation` (and
// `uri_from_file`) need to handle percent encoding correctly first.
//
// See turbopack/crates/turbopack-core/src/source_map/utils.rs as well
"sources": [format!("turbopack:///{}", self.ident().path().to_string().await?)],
"sourcesContent": [&data_str],
"names": [],
// Maps 0:0 in the output code to 0:0 in the `source_code`. Sufficient for
// bundle analyzers to attribute the bytes in the output chunks
"mappings": "AAAA",
})
.to_string()
.into();
code.push_source(&source_code, Some(source_map));
let code = code.build();
Ok(EcmascriptChunkItemContent {
source_map: Some(code.generate_source_map_ref(None)),
inner_code: code.into_source_code(),
..Default::default()
}
.cell())
}
FileJsonContent::Unparsable(e) => {
let mut message = "Unable to make a module from invalid JSON: ".to_string();
if let FileContent::Content(content) = &*content.await? {
let text = content.content().to_str()?;
e.write_with_content(&mut message, text.as_ref())?;
} else {
write!(message, "{e}")?;
}
Err(Error::msg(message))
}
FileJsonContent::NotFound => {
bail!("JSON file not found: {}", self.ident().to_string().await?);
}
}
}
}
...@@ -28,7 +28,6 @@ use turbopack_core::{ ...@@ -28,7 +28,6 @@ use turbopack_core::{
resolve, resolve,
}, },
source::{OptionSource, Source}, source::{OptionSource, Source},
source_pos::SourcePos,
}; };
use crate::ecmascript::get_condition_maps; use crate::ecmascript::get_condition_maps;
...@@ -69,29 +68,7 @@ pub async fn read_tsconfigs( ...@@ -69,29 +68,7 @@ pub async fn read_tsconfigs(
match &*parsed_data.await? { match &*parsed_data.await? {
FileJsonContent::Unparsable(e) => { FileJsonContent::Unparsable(e) => {
let message = format!("tsconfig is not parseable: invalid JSON: {}", e.message); let message = format!("tsconfig is not parseable: invalid JSON: {}", e.message);
let source = match (e.start_location, e.end_location) { let source = IssueSource::from_unparsable_json(tsconfig, e);
(None, None) => IssueSource::from_source_only(tsconfig),
(Some((line, column)), None) | (None, Some((line, column))) => {
IssueSource::from_line_col(
tsconfig,
SourcePos { line, column },
SourcePos { line, column },
)
}
(Some((start_line, start_column)), Some((end_line, end_column))) => {
IssueSource::from_line_col(
tsconfig,
SourcePos {
line: start_line,
column: start_column,
},
SourcePos {
line: end_line,
column: end_column,
},
)
}
};
TsConfigIssue { TsConfigIssue {
severity: IssueSeverity::Error, severity: IssueSeverity::Error,
source, source,
......
error - [code gen] /turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/invalid.json Code generation for chunk item errored
An error occurred while generating the chunk item [project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/invalid.json (json)
Caused by:
- Unable to make a module from invalid JSON: expected `,` or `}` at line 3 column 26
Debug info:
- An error occurred while generating the chunk item [project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/invalid.json (json)
- Execution of <JsonModuleAsset as EcmascriptChunkPlaceable>::chunk_item_content failed
- Unable to make a module from invalid JSON: expected `,` or `}` at line 3 column 26
at nested.?
1 | {
2 | "nested": {
| v
3 + "this-is": "invalid" // lint-staged will remove trailing commas, so here's a comment
| ^
4 | }
5 | }
Import trace:
test:
./turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/invalid.json
./turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/index.js
\ No newline at end of file
error - [code gen] /turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/invalid.json:3:25 Unable to make a module from invalid JSON
1 | {
2 | "nested": {
+ v
3 + "this-is": "invalid" // lint-staged will remove trailing commas, so here's a comment
+ ^
4 | }
5 | }
6 |
expected `,` or `}` at line 3 column 26
\ No newline at end of file
(globalThis["TURBOPACK"] || (globalThis["TURBOPACK"] = [])).push([ (globalThis["TURBOPACK"] || (globalThis["TURBOPACK"] = [])).push([
"output/ba425_crates_turbopack-tests_tests_snapshot_imports_json_input_index_dfc4b8ed.js", "output/ba425_crates_turbopack-tests_tests_snapshot_imports_json_input_index_45c6ceb7.js",
{"otherChunks":["output/turbopack_crates_turbopack-tests_tests_snapshot_imports_json_input_fc5f1049._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/index.js [test] (ecmascript)"]} {"otherChunks":["output/turbopack_crates_turbopack-tests_tests_snapshot_imports_json_input_f0f01d68._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/json/input/index.js [test] (ecmascript)"]}
]); ]);
// Dummy runtime // Dummy runtime
\ No newline at end of file
...@@ -34,7 +34,6 @@ turbopack-core = { workspace = true } ...@@ -34,7 +34,6 @@ turbopack-core = { workspace = true }
turbopack-css = { workspace = true } turbopack-css = { workspace = true }
turbopack-ecmascript = { workspace = true } turbopack-ecmascript = { workspace = true }
turbopack-env = { workspace = true } turbopack-env = { workspace = true }
turbopack-json = { workspace = true }
turbopack-mdx = { workspace = true } turbopack-mdx = { workspace = true }
turbopack-node = { workspace = true } turbopack-node = { workspace = true }
turbopack-resolve = { workspace = true } turbopack-resolve = { workspace = true }
......
...@@ -59,7 +59,6 @@ use turbopack_ecmascript::{ ...@@ -59,7 +59,6 @@ use turbopack_ecmascript::{
}, },
tree_shake::asset::EcmascriptModulePartAsset, tree_shake::asset::EcmascriptModulePartAsset,
}; };
use turbopack_json::JsonModuleAsset;
use turbopack_node::transforms::webpack::{WebpackLoaderItem, WebpackLoaderItems, WebpackLoaders}; use turbopack_node::transforms::webpack::{WebpackLoaderItem, WebpackLoaderItems, WebpackLoaders};
use turbopack_resolve::{ use turbopack_resolve::{
resolve::resolve_options, resolve_options_context::ResolveOptionsContext, resolve::resolve_options, resolve_options_context::ResolveOptionsContext,
...@@ -248,7 +247,6 @@ async fn apply_module_type( ...@@ -248,7 +247,6 @@ async fn apply_module_type(
.await? .await?
} }
} }
ModuleType::Json => ResolvedVc::upcast(JsonModuleAsset::new(*source).to_resolved().await?),
ModuleType::Raw => ResolvedVc::upcast(RawModule::new(*source).to_resolved().await?), ModuleType::Raw => ResolvedVc::upcast(RawModule::new(*source).to_resolved().await?),
ModuleType::NodeAddon => { ModuleType::NodeAddon => {
ResolvedVc::upcast(NodeAddonModule::new(*source).to_resolved().await?) ResolvedVc::upcast(NodeAddonModule::new(*source).to_resolved().await?)
......
...@@ -26,8 +26,9 @@ use turbopack_core::{ ...@@ -26,8 +26,9 @@ use turbopack_core::{
}; };
use turbopack_css::CssModuleAssetType; use turbopack_css::CssModuleAssetType;
use turbopack_ecmascript::{ use turbopack_ecmascript::{
EcmascriptInputTransform, EcmascriptInputTransforms, EcmascriptOptions, SpecifiedModuleType, AnalyzeMode, EcmascriptInputTransform, EcmascriptInputTransforms, EcmascriptOptions,
bytes_source_transform::BytesSourceTransform, text_source_transform::TextSourceTransform, SpecifiedModuleType, bytes_source_transform::BytesSourceTransform,
json_source_transform::JsonSourceTransform, text_source_transform::TextSourceTransform,
}; };
use turbopack_mdx::MdxTransform; use turbopack_mdx::MdxTransform;
use turbopack_node::{ use turbopack_node::{
...@@ -361,6 +362,13 @@ impl ModuleOptions { ...@@ -361,6 +362,13 @@ impl ModuleOptions {
let mut rules = vec![]; let mut rules = vec![];
// In tracing mode, we only need to record file dependencies — not transform them.
// Source transforms rename the file identity (e.g., foo.json -> foo.json.[json].cjs),
// which produces virtual paths that don't exist on disk. This breaks NFT file tracing
// and standalone build file copying. Use Raw module type instead so the original
// filesystem path is preserved in the trace.
let is_tracing = analyze_mode == AnalyzeMode::Tracing;
// Import attribute rules (bytes/text) must come BEFORE config rules. // Import attribute rules (bytes/text) must come BEFORE config rules.
// Import attributes have a stronger API contract - they're explicit in the source code // Import attributes have a stronger API contract - they're explicit in the source code
// and should override any file-pattern-based config rules. // and should override any file-pattern-based config rules.
...@@ -369,9 +377,13 @@ impl ModuleOptions { ...@@ -369,9 +377,13 @@ impl ModuleOptions {
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules( RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType("bytes".into()), EcmaScriptModulesReferenceSubType::ImportWithType("bytes".into()),
)), )),
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![ if is_tracing {
ResolvedVc::upcast(BytesSourceTransform::new().to_resolved().await?), vec![ModuleRuleEffect::ModuleType(ModuleType::Raw)]
]))], } else {
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![
ResolvedVc::upcast(BytesSourceTransform::new().to_resolved().await?),
]))]
},
)); ));
} }
...@@ -380,9 +392,13 @@ impl ModuleOptions { ...@@ -380,9 +392,13 @@ impl ModuleOptions {
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules( RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType("text".into()), EcmaScriptModulesReferenceSubType::ImportWithType("text".into()),
)), )),
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![ if is_tracing {
ResolvedVc::upcast(TextSourceTransform::new().to_resolved().await?), vec![ModuleRuleEffect::ModuleType(ModuleType::Raw)]
]))], } else {
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![
ResolvedVc::upcast(TextSourceTransform::new().to_resolved().await?),
]))]
},
)); ));
} }
...@@ -552,7 +568,14 @@ impl ModuleOptions { ...@@ -552,7 +568,14 @@ impl ModuleOptions {
RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules( RuleCondition::ReferenceType(ReferenceType::EcmaScriptModules(
EcmaScriptModulesReferenceSubType::ImportWithType("json".into()), EcmaScriptModulesReferenceSubType::ImportWithType("json".into()),
)), )),
vec![ModuleRuleEffect::ModuleType(ModuleType::Json)], if is_tracing {
vec![ModuleRuleEffect::ModuleType(ModuleType::Raw)]
} else {
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![
// Use spec-compliant ESM for import attributes
ResolvedVc::upcast(JsonSourceTransform::new_esm().to_resolved().await?),
]))]
},
), ),
]); ]);
...@@ -563,7 +586,14 @@ impl ModuleOptions { ...@@ -563,7 +586,14 @@ impl ModuleOptions {
RuleCondition::ResourcePathEndsWith(".json".to_string()), RuleCondition::ResourcePathEndsWith(".json".to_string()),
RuleCondition::ContentTypeStartsWith("application/json".to_string()), RuleCondition::ContentTypeStartsWith("application/json".to_string()),
]), ]),
vec![ModuleRuleEffect::ModuleType(ModuleType::Json)], if is_tracing {
vec![ModuleRuleEffect::ModuleType(ModuleType::Raw)]
} else {
vec![ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![
// For backcompat with webpack we generate a cjs style export
ResolvedVc::upcast(JsonSourceTransform::new_cjs().to_resolved().await?),
]))]
},
), ),
ModuleRule::new_all( ModuleRule::new_all(
RuleCondition::any(vec![ RuleCondition::any(vec![
......
...@@ -12,6 +12,7 @@ use turbopack_core::{ ...@@ -12,6 +12,7 @@ use turbopack_core::{
use turbopack_css::CssModuleAssetType; use turbopack_css::CssModuleAssetType;
use turbopack_ecmascript::{ use turbopack_ecmascript::{
EcmascriptInputTransforms, EcmascriptOptions, bytes_source_transform::BytesSourceTransform, EcmascriptInputTransforms, EcmascriptOptions, bytes_source_transform::BytesSourceTransform,
json_source_transform::JsonSourceTransform,
}; };
use turbopack_wasm::source::WebAssemblySourceType; use turbopack_wasm::source::WebAssemblySourceType;
...@@ -135,7 +136,6 @@ pub enum ModuleType { ...@@ -135,7 +136,6 @@ pub enum ModuleType {
#[turbo_tasks(trace_ignore)] #[turbo_tasks(trace_ignore)]
options: ResolvedVc<EcmascriptOptions>, options: ResolvedVc<EcmascriptOptions>,
}, },
Json,
Raw, Raw,
NodeAddon, NodeAddon,
CssModule, CssModule,
...@@ -164,7 +164,6 @@ impl Display for ModuleType { ...@@ -164,7 +164,6 @@ impl Display for ModuleType {
ModuleType::Typescript { .. } => write!(f, "Typescript"), ModuleType::Typescript { .. } => write!(f, "Typescript"),
ModuleType::TypescriptDeclaration { .. } => write!(f, "TypescriptDeclaration"), ModuleType::TypescriptDeclaration { .. } => write!(f, "TypescriptDeclaration"),
ModuleType::EcmascriptExtensionless { .. } => write!(f, "EcmascriptExtensionless"), ModuleType::EcmascriptExtensionless { .. } => write!(f, "EcmascriptExtensionless"),
ModuleType::Json => write!(f, "Json"),
ModuleType::Raw => write!(f, "Raw"), ModuleType::Raw => write!(f, "Raw"),
ModuleType::NodeAddon => write!(f, "NodeAddon"), ModuleType::NodeAddon => write!(f, "NodeAddon"),
ModuleType::CssModule => write!(f, "CssModule"), ModuleType::CssModule => write!(f, "CssModule"),
...@@ -189,6 +188,8 @@ pub enum ConfiguredModuleType { ...@@ -189,6 +188,8 @@ pub enum ConfiguredModuleType {
Typescript, Typescript,
Css, Css,
CssModule, CssModule,
/// Parses JSON and exports it as an ES module default export.
/// Implemented as a source transform, not a ModuleType.
Json, Json,
Wasm, Wasm,
Raw, Raw,
...@@ -265,7 +266,12 @@ impl ConfiguredModuleType { ...@@ -265,7 +266,12 @@ impl ConfiguredModuleType {
environment, environment,
}), }),
ConfiguredModuleType::CssModule => ModuleRuleEffect::ModuleType(ModuleType::CssModule), ConfiguredModuleType::CssModule => ModuleRuleEffect::ModuleType(ModuleType::CssModule),
ConfiguredModuleType::Json => ModuleRuleEffect::ModuleType(ModuleType::Json), ConfiguredModuleType::Json => {
ModuleRuleEffect::SourceTransforms(ResolvedVc::cell(vec![ResolvedVc::upcast(
// TODO: can we switch this to `new_esm`?
JsonSourceTransform::new_cjs().to_resolved().await?,
)]))
}
ConfiguredModuleType::Wasm => ModuleRuleEffect::ModuleType(ModuleType::WebAssembly { ConfiguredModuleType::Wasm => ModuleRuleEffect::ModuleType(ModuleType::WebAssembly {
source_ty: WebAssemblySourceType::Binary, source_ty: WebAssemblySourceType::Binary,
}), }),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment