Unverified Commit e448f681 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Use memchr3 for Turbopack source position scanning

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 4ebb2b7a
...@@ -9688,6 +9688,7 @@ dependencies = [ ...@@ -9688,6 +9688,7 @@ dependencies = [
"fixedbitset", "fixedbitset",
"indexmap 2.14.0", "indexmap 2.14.0",
"itertools 0.10.5", "itertools 0.10.5",
"memchr",
"num-bigint", "num-bigint",
"patricia_tree", "patricia_tree",
"petgraph 0.8.3", "petgraph 0.8.3",
......
...@@ -27,6 +27,7 @@ erased-serde = { workspace = true } ...@@ -27,6 +27,7 @@ erased-serde = { workspace = true }
fixedbitset = { workspace = true } fixedbitset = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
memchr = { workspace = true }
num-bigint = "0.4" num-bigint = "0.4"
patricia_tree = { version = "0.10.1", features = ["serde"] } patricia_tree = { version = "0.10.1", features = ["serde"] }
petgraph = { workspace = true, features = ["serde-1"] } petgraph = { workspace = true, features = ["serde-1"] }
......
...@@ -409,3 +409,47 @@ impl Code { ...@@ -409,3 +409,47 @@ impl Code {
} }
} }
} }
#[cfg(test)]
mod tests {
use serde_json::Value;
use turbo_tasks_fs::rope::Rope;
use super::CodeBuilder;
const SECTION_MAP: &str = r#"{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}"#;
fn section_offsets(code: &super::Code) -> Vec<(u64, u64)> {
let map = code.generate_source_map_ref(None);
let map: Value = serde_json::from_str(&map.to_str().unwrap()).unwrap();
map["sections"]
.as_array()
.unwrap()
.iter()
.map(|section| {
let offset = &section["offset"];
(
offset["line"].as_u64().unwrap(),
offset["column"].as_u64().unwrap(),
)
})
.collect()
}
#[test]
fn section_offsets_count_utf16_code_units() {
let mut builder = CodeBuilder::default();
// `🦀` is a 4-byte UTF-8 sequence and two UTF-16 code units, `é` is a single unit.
let prefix = "line0\n\"🦀é\"; ";
builder += prefix;
builder.push_source(
&Rope::from("mapped();".to_string()),
Some(Rope::from(SECTION_MAP.to_string())),
);
let code = builder.build();
let expected_column = "\"🦀é\"; ".encode_utf16().count() as u64;
assert_eq!(expected_column, 7);
assert_eq!(section_offsets(&code), vec![(1, expected_column)]);
}
}
...@@ -6,6 +6,9 @@ use turbo_tasks_hash::DeterministicHash; ...@@ -6,6 +6,9 @@ use turbo_tasks_hash::DeterministicHash;
const U8_LF: u8 = 0x0A; const U8_LF: u8 = 0x0A;
/// CARRIAGE RETURN (CR), one of the basic JS line terminators. /// CARRIAGE RETURN (CR), one of the basic JS line terminators.
const U8_CR: u8 = 0x0D; const U8_CR: u8 = 0x0D;
/// The leading byte of the UTF-8 encodings of LINE SEPARATOR (`E2 80 A8`) and PARAGRAPH
/// SEPARATOR (`E2 80 A9`), and of every other char in U+2000..=U+2FFF.
const U8_LS_PS_LEAD: u8 = 0xE2;
#[turbo_tasks::task_input] #[turbo_tasks::task_input]
#[derive( #[derive(
...@@ -26,7 +29,8 @@ const U8_CR: u8 = 0x0D; ...@@ -26,7 +29,8 @@ const U8_CR: u8 = 0x0D;
pub struct SourcePos { pub struct SourcePos {
/// The line, 0-indexed. /// The line, 0-indexed.
pub line: u32, pub line: u32,
/// The byte index of the column, 0-indexed. /// The column, 0-indexed. When computed by [`SourcePos::update`], this is
/// counted in UTF-16 code units, as JavaScript source maps expect.
pub column: u32, pub column: u32,
} }
...@@ -47,94 +51,149 @@ impl SourcePos { ...@@ -47,94 +51,149 @@ impl SourcePos {
/// Increments the line/column position to account for new source code. /// Increments the line/column position to account for new source code.
/// Line terminators are the classic "\n", "\r", "\r\n" (which counts as /// Line terminators are the classic "\n", "\r", "\r\n" (which counts as
/// a single terminator), and JSON LINE/PARAGRAPH SEPARATORs. /// a single terminator), and JS LINE/PARAGRAPH SEPARATORs.
/// ///
/// See <https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#sec-line-terminators> /// See <https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#sec-line-terminators>
pub fn update(&mut self, code: &[u8]) { pub fn update(&mut self, code: &[u8]) {
// JS source text is interpreted as UCS-2, which is basically UTF-16 with less // Only the line terminators need to be found, which `memchr` does with SIMD. The
// restrictions. We cannot iterate UTF-8 bytes here, 2-byte UTF-8 octets // columns of all but the last line don't matter, so only the final line segment is
// should count as a 1 char and not 2. // counted.
let &mut SourcePos { let mut skip_lf_at = None;
mut line, let mut line_start = None;
mut column, for i in memchr::memchr3_iter(U8_LF, U8_CR, U8_LS_PS_LEAD, code) {
} = self;
let mut i = 0;
while i < code.len() {
// This is not a UTF-8 validator, but it's likely close enough. It's assumed
// that the input is valid (and if it isn't than what are you doing trying to
// embed it into source code anyways?). The important part is that we process in
// order, and use the first octet's bit pattern to decode the octet length of
// the char.
match code[i] { match code[i] {
U8_LF => { U8_LF => {
i += 1;
line += 1;
column = 0;
}
U8_CR => {
// Count "\r\n" as a single terminator. // Count "\r\n" as a single terminator.
if code.get(i + 1) == Some(&U8_LF) { if skip_lf_at != Some(i) {
i += 2; self.line += 1;
} else {
i += 1;
} }
line += 1; line_start = Some(i + 1);
column = 0;
}
// 1 octet chars do not have the high bit set. If it's not a LF or CR, then it's
// just a regular ASCII.
b if b & 0b10000000 == 0 => {
i += 1;
column += 1;
}
// 2 octet chars have a leading `110` bit pattern. None are considered line
// terminators.
b if b & 0b11100000 == 0b11000000 => {
// eat this byte and the next.
i += 2;
column += 1;
} }
U8_CR => {
// 3 octet chars have a leading `1110` bit pattern. Both the LINE/PARAGRAPH self.line += 1;
// SEPARATOR exist in 3 octets. line_start = Some(i + 1);
b if b & 0b11110000 == 0b11100000 => { skip_lf_at = Some(i + 1);
// The LINE and PARAGRAPH have the bits `11100010 10000000 1010100X`, with the X
// denoting either line or paragraph.
let mut separator = false;
if b == 0b11100010 && code.get(i + 1) == Some(&0b10000000) {
let last = code.get(i + 2).cloned().unwrap_or_default();
separator = (last & 0b11111110) == 0b10101000
}
// eat this byte and the next 2.
i += 3;
if separator {
line += 1;
column = 0;
} else {
column += 1;
}
} }
// 4 octet chars have a leading `11110` pattern, but we don't need to check because
// none of the other patterns matched.
_ => { _ => {
// eat this byte and the next 3. // LINE SEPARATOR and PARAGRAPH SEPARATOR are `E2 80 A8` and `E2 80 A9`. Any
i += 4; // other char with this lead byte is not a line terminator.
column += 1; if code.get(i + 1) == Some(&0x80)
&& code
.get(i + 2)
.is_some_and(|&b| b & 0b11111110 == 0b10101000)
{
self.line += 1;
line_start = Some(i + 3);
}
} }
} }
} }
self.line = line; match line_start {
self.column = column; Some(start) => self.column = utf16_len(&code[start..]) as u32,
None => self.column += utf16_len(code) as u32,
}
} }
} }
/// Counts the UTF-16 code units of UTF-8 encoded text, as JavaScript string lengths and source
/// map columns do.
///
/// Every byte is counted on its own: continuation bytes (`10xxxxxx`) count 0, the leading byte
/// of a 4-byte sequence (a non-BMP char, which UTF-16 encodes as a surrogate pair) counts 2, and
/// all other bytes count 1. This is exact for valid UTF-8 and, as it doesn't decode chars, it is
/// safe for arbitrary bytes and adds up across arbitrary splits of the input.
//
// TODO: Use the SIMD implementation of `simd-utf16-len` once it supports byte slices
// (https://github.com/SyMind/simd-utf16-len/pull/22).
pub(crate) fn utf16_len(bytes: &[u8]) -> usize {
bytes
.iter()
.map(|&b| 1 + (b >= 0xF0) as usize - ((b & 0xC0) == 0x80) as usize)
.sum()
}
impl std::cmp::PartialEq<(u32, u32)> for SourcePos { impl std::cmp::PartialEq<(u32, u32)> for SourcePos {
fn eq(&self, other: &(u32, u32)) -> bool { fn eq(&self, other: &(u32, u32)) -> bool {
&(self.line, self.column) == other &(self.line, self.column) == other
} }
} }
#[cfg(test)]
mod tests {
use super::{SourcePos, utf16_len};
fn pos_after(code: &str) -> SourcePos {
let mut pos = SourcePos::new(0);
pos.update(code.as_bytes());
pos
}
#[test]
fn utf16_len_matches_std() {
for s in [
"",
"abc",
"é€",
"🦀",
"x🦀é€😀y\n\r\u{2028}",
"\u{0}\u{7f}\u{80}\u{7ff}\u{800}\u{ffff}\u{10000}\u{10ffff}",
] {
assert_eq!(utf16_len(s.as_bytes()), s.encode_utf16().count(), "{s:?}");
}
}
#[test]
fn counts_ascii_as_one_column() {
assert_eq!(pos_after("abc"), (0, 3));
}
#[test]
fn counts_bmp_chars_as_one_column() {
// 2-byte (U+00E9) and 3-byte (U+20AC) UTF-8 sequences are a single UTF-16 code unit.
assert_eq!(pos_after("é"), (0, 1));
assert_eq!(pos_after("€"), (0, 1));
assert_eq!(pos_after("aé€b"), (0, 4));
}
#[test]
fn counts_astral_chars_as_two_columns() {
// 4-byte UTF-8 sequences are a surrogate pair in UTF-16.
assert_eq!(pos_after("🦀"), (0, 2));
assert_eq!(pos_after("a🦀b"), (0, 4));
assert_eq!(pos_after("🦀🦀"), (0, 4));
let mixed = "x🦀é€😀y";
assert_eq!(pos_after(mixed), (0, mixed.encode_utf16().count() as u32));
}
#[test]
fn handles_line_terminators() {
assert_eq!(pos_after("ab\ncd"), (1, 2));
assert_eq!(pos_after("ab\rcd"), (1, 2));
// "\r\n" is a single line terminator.
assert_eq!(pos_after("ab\r\ncd"), (1, 2));
assert_eq!(pos_after("\r\r\n\n"), (3, 0));
}
#[test]
fn handles_line_and_paragraph_separators() {
assert_eq!(pos_after("ab\u{2028}c"), (1, 1));
assert_eq!(pos_after("ab\u{2029}c"), (1, 1));
// Neighbouring 3-byte chars with the same lead byte are not separators.
assert_eq!(pos_after("\u{2027}\u{202A}"), (0, 2));
assert_eq!(pos_after("“a” — …\u{2028}“b”"), (1, 3));
}
#[test]
fn resets_column_after_astral_line() {
assert_eq!(pos_after("🦀\n🦀a"), (1, 3));
}
#[test]
fn accumulates_across_updates() {
let mut pos = SourcePos::new(2);
pos.update("a🦀".as_bytes());
assert_eq!(pos, (2, 3));
pos.update("é\nb".as_bytes());
assert_eq!(pos, (3, 1));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment