Unverified Commit c1f4e3e7 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: fix sourcemaps of scopehoisted comments (#80987)

There was a bug in the source mappings for scope hoisted comments. They actually contained invalid mappings, which was only caught by the invalid mapping landing exactly on an invalid multibyte character boundary, which triggered a debug assertion:
![Bildschirmfoto 2025-06-27 um 10 01 15](https://github.com/user-attachments/assets/90cd7771-72c2-41ea-bc54-05b3be5f17f7)

What's happening is:
1. You do `comments.add_leading` with a `Comment` containing the real span
2. Traverse the AST and encode all bytepos in the soans
3. codegen, the bytepos in the AST are encoded, but the ones in the comments (Comment.span) are not (i.e. via `comments.take_leading`

Now, they are all consistently encoded and correctly mapped:
![Bildschirmfoto 2025-06-27 um 10 02 47](https://github.com/user-attachments/assets/90b4bb58-e946-4781-b498-d88668193107)
parent c910be03
...@@ -2416,6 +2416,21 @@ enum CodeGenResultCommentsConsumable<'a> { ...@@ -2416,6 +2416,21 @@ enum CodeGenResultCommentsConsumable<'a> {
unsafe impl Send for CodeGenResultComments {} unsafe impl Send for CodeGenResultComments {}
unsafe impl Sync for CodeGenResultComments {} unsafe impl Sync for CodeGenResultComments {}
/// All BytePos in Spans in the AST are encoded correctly in [`merge_modules`], but the Comments
/// also contain spans. These also need to be encoded so that all pos in `mappings` are consistently
/// encoded.
fn encode_module_into_comment_span(
modules_header_width: u32,
module: usize,
mut comment: Comment,
) -> Comment {
comment.span.lo =
CodeGenResultComments::encode_bytepos(modules_header_width, module as u32, comment.span.lo);
comment.span.hi =
CodeGenResultComments::encode_bytepos(modules_header_width, module as u32, comment.span.hi);
comment
}
impl Comments for CodeGenResultCommentsConsumable<'_> { impl Comments for CodeGenResultCommentsConsumable<'_> {
fn add_leading(&self, _pos: BytePos, _cmt: Comment) { fn add_leading(&self, _pos: BytePos, _cmt: Comment) {
unimplemented!() unimplemented!()
...@@ -2459,7 +2474,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> { ...@@ -2459,7 +2474,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
} => { } => {
let (module, pos) = let (module, pos) =
CodeGenResultComments::decode_bytepos(*modules_header_width, pos); CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
comments[module].take_leading(pos) comments[module].take_leading(pos).map(|comments| {
comments
.into_iter()
.map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
.collect()
})
} }
Self::Empty => None, Self::Empty => None,
} }
...@@ -2477,7 +2497,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> { ...@@ -2477,7 +2497,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
} => { } => {
let (module, pos) = let (module, pos) =
CodeGenResultComments::decode_bytepos(*modules_header_width, pos); CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
comments[module].get_leading(pos) comments[module].get_leading(pos).map(|comments| {
comments
.into_iter()
.map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
.collect()
})
} }
Self::Empty => None, Self::Empty => None,
} }
...@@ -2528,7 +2553,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> { ...@@ -2528,7 +2553,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
} => { } => {
let (module, pos) = let (module, pos) =
CodeGenResultComments::decode_bytepos(*modules_header_width, pos); CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
comments[module].take_trailing(pos) comments[module].take_trailing(pos).map(|comments| {
comments
.into_iter()
.map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
.collect()
})
} }
Self::Empty => None, Self::Empty => None,
} }
...@@ -2546,7 +2576,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> { ...@@ -2546,7 +2576,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
} => { } => {
let (module, pos) = let (module, pos) =
CodeGenResultComments::decode_bytepos(*modules_header_width, pos); CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
comments[module].get_leading(pos) comments[module].get_leading(pos).map(|comments| {
comments
.into_iter()
.map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
.collect()
})
} }
Self::Empty => None, Self::Empty => None,
} }
......
import { jsx as _jsx } from "./jsx-runtime";
// eslint-disable-next-line import/no-extraneous-dependencies
const createFromReadableStream = 123; //import { createFromReadableStream } from 'react-server-dom-webpack/client.edge';
// eslint-disable-next-line import/no-extraneous-dependencies
const prerender = 123; // import { unstable_prerender as prerender } from 'react-server-dom-webpack/static.edge';
const streamFromBuffer=1,streamToBuffer=1; // import { streamFromBuffer, streamToBuffer } from '../stream-utils/node-web-streams-helper';
const waitAtLeastOneReactRenderTask=1; //import { waitAtLeastOneReactRenderTask } from '../../lib/scheduler';
// import './segment-value-encoding';
// "entry-base.js [app-rsc] (ecmascript) <module evaluation>" with
// ["reflect-utils.js [app-rsc] (ecmascript)",
// "params.js [app-rsc] (ecmascript)",
// "segment-value-encoding.js [app-rsc] (ecmascript)",
// "collect-segment-data.js [app-rsc] (ecmascript)",
// "entry-base.js [app-rsc] (ecmascript) <locals>"]
// "entry-base.js [test] (ecmascript) <module evaluation>" with
// ["reflect-utils.js [test] (ecmascript)",
// "params.js [test] (ecmascript)",
// "segment-value-encoding.js [test] (ecmascript)",
// "collect-segment-data.js [test] (ecmascript)",
// "entry-base.js [test] (ecmascript) <locals>"]
if (Date.now() > 0) {
require('./index1.js')
}
if (Date.now() > 0) {
require('./index2.js')
}
// This regex will have fast negatives meaning valid identifiers may not pass
// this test. However this is only used during static generation to provide hints
// about why a page bailed out of some or all prerendering and we can use bracket notation
// for example while `ಠ_ಠ` is a valid identifier it's ok to print `searchParams['ಠ_ಠ']`
// even if this would have been fine too `searchParams.ಠ_ಠ`
const isDefinitelyAValidIdentifier = /s/;
export function describeStringPropertyAccess(target, prop) {}
export function describeHasCheckingStringProperty(target, prop) {}
export const wellKnownProperties = new Set([])
{
"treeShakingMode": "reexports-only",
"scopeHoisting": true
}
(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push(["output/4e721_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_795dc95a._.js", {
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/jsx-runtime.js [test] (ecmascript)": ((__turbopack_context__) => {
var { m: module, e: exports } = __turbopack_context__;
{
module.exports = {};
}}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/entry-base.js [test] (ecmascript)": ((__turbopack_context__) => {
"use strict";
// MERGED MODULE: [project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/entry-base.js [test] (ecmascript)
;
__turbopack_context__.s({}, "[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/entry-base.js [test] (ecmascript)");
// MERGED MODULE: [project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/params.js [test] (ecmascript)
;
// MERGED MODULE: [project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/reflect-utils.js [test] (ecmascript)
;
// This regex will have fast negatives meaning valid identifiers may not pass
// this test. However this is only used during static generation to provide hints
// about why a page bailed out of some or all prerendering and we can use bracket notation
// for example while `ಠ_ಠ` is a valid identifier it's ok to print `searchParams['ಠ_ಠ']`
// even if this would have been fine too `searchParams.ಠ_ಠ`
const isDefinitelyAValidIdentifier = /s/;
function describeStringPropertyAccess(target, prop) {}
function describeHasCheckingStringProperty(target, prop) {}
const wellKnownProperties = new Set([]);
;
// MERGED MODULE: [project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/collect-segment-data.js [test] (ecmascript)
;
var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$source_maps$2f$merged$2d$unicode$2f$input$2f$jsx$2d$runtime$2e$js__$5b$test$5d$__$28$ecmascript$29$__ = __turbopack_context__.i("[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/jsx-runtime.js [test] (ecmascript)");
;
// eslint-disable-next-line import/no-extraneous-dependencies
const createFromReadableStream = 123; //import { createFromReadableStream } from 'react-server-dom-webpack/client.edge';
// eslint-disable-next-line import/no-extraneous-dependencies
const prerender = 123; // import { unstable_prerender as prerender } from 'react-server-dom-webpack/static.edge';
const streamFromBuffer = 1, streamToBuffer = 1; // import { streamFromBuffer, streamToBuffer } from '../stream-utils/node-web-streams-helper';
const waitAtLeastOneReactRenderTask = 1; //import { waitAtLeastOneReactRenderTask } from '../../lib/scheduler';
// import './segment-value-encoding';
;
;
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index1.js [test] (ecmascript)": ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s({});
var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$source_maps$2f$merged$2d$unicode$2f$input$2f$entry$2d$base$2e$js__$5b$test$5d$__$28$ecmascript$29$__ = __turbopack_context__.i("[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/entry-base.js [test] (ecmascript)");
;
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index2.js [test] (ecmascript)": ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s({});
var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$source_maps$2f$merged$2d$unicode$2f$input$2f$entry$2d$base$2e$js__$5b$test$5d$__$28$ecmascript$29$__ = __turbopack_context__.i("[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/entry-base.js [test] (ecmascript)");
;
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index.js [test] (ecmascript)": ((__turbopack_context__) => {
var { m: module, e: exports } = __turbopack_context__;
{
// "entry-base.js [app-rsc] (ecmascript) <module evaluation>" with
// ["reflect-utils.js [app-rsc] (ecmascript)",
// "params.js [app-rsc] (ecmascript)",
// "segment-value-encoding.js [app-rsc] (ecmascript)",
// "collect-segment-data.js [app-rsc] (ecmascript)",
// "entry-base.js [app-rsc] (ecmascript) <locals>"]
// "entry-base.js [test] (ecmascript) <module evaluation>" with
// ["reflect-utils.js [test] (ecmascript)",
// "params.js [test] (ecmascript)",
// "segment-value-encoding.js [test] (ecmascript)",
// "collect-segment-data.js [test] (ecmascript)",
// "entry-base.js [test] (ecmascript) <locals>"]
if (Date.now() > 0) {
__turbopack_context__.r("[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index1.js [test] (ecmascript)");
}
if (Date.now() > 0) {
__turbopack_context__.r("[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index2.js [test] (ecmascript)");
}
}}),
}]);
//# sourceMappingURL=4e721_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_795dc95a._.js.map
\ No newline at end of file
{
"version": 3,
"sources": [],
"sections": [
{"offset": {"line": 6, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/jsx-runtime.js"],"sourcesContent":["module.exports = {};\n"],"names":[],"mappings":"AAAA,OAAO,OAAO,GAAG,CAAC"}},
{"offset": {"line": 11, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/reflect-utils.js","turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/collect-segment-data.js"],"sourcesContent":["// This regex will have fast negatives meaning valid identifiers may not pass\n// this test. However this is only used during static generation to provide hints\n// about why a page bailed out of some or all prerendering and we can use bracket notation\n// for example while `ಠ_ಠ` is a valid identifier it's ok to print `searchParams['ಠ_ಠ']`\n// even if this would have been fine too `searchParams.ಠ_ಠ`\nconst isDefinitelyAValidIdentifier = /s/;\nexport function describeStringPropertyAccess(target, prop) {}\nexport function describeHasCheckingStringProperty(target, prop) {}\nexport const wellKnownProperties = new Set([])\n","import { jsx as _jsx } from \"./jsx-runtime\";\n// eslint-disable-next-line import/no-extraneous-dependencies\nconst createFromReadableStream = 123; //import { createFromReadableStream } from 'react-server-dom-webpack/client.edge';\n// eslint-disable-next-line import/no-extraneous-dependencies\nconst prerender = 123; // import { unstable_prerender as prerender } from 'react-server-dom-webpack/static.edge';\nconst streamFromBuffer=1,streamToBuffer=1; // import { streamFromBuffer, streamToBuffer } from '../stream-utils/node-web-streams-helper';\nconst waitAtLeastOneReactRenderTask=1; //import { waitAtLeastOneReactRenderTask } from '../../lib/scheduler';\n// import './segment-value-encoding';\n"],"names":[],"mappings":";;;;;;;AAAA,6EAA6E;AAC7E,iFAAiF;AACjF,0FAA0F;AAC1F,uFAAuF;AACvF,2DAA2D;AAC3D,MAAM,+BAA+B;AAC9B,SAAS,6BAA6B,MAAM,EAAE,IAAI,GAAG;AACrD,SAAS,kCAAkC,MAAM,EAAE,IAAI,GAAG;AAC1D,MAAM,sBAAsB,IAAI,IAAI,EAAE;;;;ACR7C;;AACA,6DAA6D;AAC7D,MAAM,2BAA2B,KAAK,kFAAkF;AACxH,6DAA6D;AAC7D,MAAM,YAAY,KAAK,0FAA0F;AACjH,MAAM,mBAAiB,GAAE,iBAAe,GAAG,8FAA8F;AACzI,MAAM,gCAA8B,GAAG,sEAAsE;CAC7G,sCAAsC"}},
{"offset": {"line": 45, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index1.js"],"sourcesContent":["import \"./entry-base.js\";\n"],"names":[],"mappings":";AAAA"}},
{"offset": {"line": 52, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index2.js"],"sourcesContent":["import \"./entry-base.js\";\n"],"names":[],"mappings":";AAAA"}},
{"offset": {"line": 60, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index.js"],"sourcesContent":["// \"entry-base.js [app-rsc] (ecmascript) <module evaluation>\" with\n// [\"reflect-utils.js [app-rsc] (ecmascript)\",\n// \"params.js [app-rsc] (ecmascript)\",\n// \"segment-value-encoding.js [app-rsc] (ecmascript)\",\n// \"collect-segment-data.js [app-rsc] (ecmascript)\",\n// \"entry-base.js [app-rsc] (ecmascript) <locals>\"]\n\n// \"entry-base.js [test] (ecmascript) <module evaluation>\" with\n// [\"reflect-utils.js [test] (ecmascript)\",\n// \"params.js [test] (ecmascript)\",\n// \"segment-value-encoding.js [test] (ecmascript)\",\n// \"collect-segment-data.js [test] (ecmascript)\",\n// \"entry-base.js [test] (ecmascript) <locals>\"]\n\nif (Date.now() > 0) {\n require('./index1.js')\n}\nif (Date.now() > 0) {\n require('./index2.js')\n}\n\n"],"names":[],"mappings":"AAAA,kEAAkE;AAClE,8CAA8C;AAC9C,sCAAsC;AACtC,sDAAsD;AACtD,oDAAoD;AACpD,mDAAmD;AAEnD,+DAA+D;AAC/D,2CAA2C;AAC3C,mCAAmC;AACnC,mDAAmD;AACnD,kDAAkD;AAClD,iDAAiD;AAEjD,IAAI,KAAK,GAAG,KAAK,GAAG;;AAEpB;AACA,IAAI,KAAK,GAAG,KAAK,GAAG;;AAEpB"}}]
}
\ No newline at end of file
(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
"output/b1abf_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_index_a93cc521.js",
{},
{"otherChunks":["output/4e721_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_795dc95a._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index.js [test] (ecmascript)"]}
]);
// Dummy runtime
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment