Unverified Commit 23345b85 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Use simd-utf16-len for SourcePos column counting

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent bd824641
......@@ -6664,6 +6664,12 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]]
name = "simd-utf16-len"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5bd2186ecc8a43c3e9d0114a554beeb5708cc3b7969a8a5866b0d3ec56e5347"
[[package]]
name = "simd_helpers"
version = "0.1.0"
......@@ -9698,6 +9704,7 @@ dependencies = [
"rustc-hash 2.1.1",
"serde",
"serde_json",
"simd-utf16-len",
"smallvec",
"swc_core 82.2.0",
"swc_sourcemap",
......
......@@ -344,6 +344,7 @@ serde_path_to_error = "0.1.20"
serde_qs = "1.1.1"
serde_with = "3.18.0"
sha2 = "0.10.2"
simd-utf16-len = "0.3.1"
smallvec = { version = "1.15.1", features = [
"const_generics",
"union",
......
......@@ -36,6 +36,7 @@ rustc-hash = { workspace = true }
regex = { workspace = true }
serde = { workspace = true, features = ["rc"] }
serde_json = { workspace = true, features = ["preserve_order", "raw_value"] }
simd-utf16-len = { workspace = true }
smallvec = { workspace = true }
swc_sourcemap = { workspace = true }
swc_core = { workspace = true, features = ["ecma_preset_env", "common"] }
......
......@@ -99,15 +99,20 @@ impl SourcePos {
/// Every byte is counted on its own: continuation bytes (`10xxxxxx`) count 0, the leading byte
/// of a 4-byte sequence (a non-BMP char, which UTF-16 encodes as a surrogate pair) counts 2, and
/// all other bytes count 1. This is exact for valid UTF-8 and, as it doesn't decode chars, it is
/// safe for arbitrary bytes and adds up across arbitrary splits of the input.
/// safe for arbitrary bytes and adds up across arbitrary splits of the input. The count is done
/// with the SIMD kernels of `simd-utf16-len`.
//
// TODO: Use the SIMD implementation of `simd-utf16-len` once it supports byte slices
// (https://github.com/SyMind/simd-utf16-len/pull/22).
// TODO: Switch to the byte-slice API of `simd-utf16-len` once
// https://github.com/SyMind/simd-utf16-len/pull/22 lands, and remove the `unsafe` below.
pub(crate) fn utf16_len(bytes: &[u8]) -> usize {
bytes
.iter()
.map(|&b| 1 + (b >= 0xF0) as usize - ((b & 0xC0) == 0x80) as usize)
.sum()
// SAFETY: `simd_utf16_len::utf16_len` does not depend on UTF-8 validity for memory safety: it
// applies the per-byte formula above and only reads within the slice. Despite the safety
// comments inside the crate saying that `utf16_len` requires UTF-8 for safety, that is not
// true, it is only needed for correctness (confirmed by the crate's maintainer). The same
// holds for its scalar fallback, `str::encode_utf16().count()`, which std implements as a
// byte-wise fold. This was checked for `simd-utf16-len` 0.3.1 and the pinned Rust toolchain;
// re-check before upgrading either.
simd_utf16_len::utf16_len(unsafe { std::str::from_utf8_unchecked(bytes) })
}
impl std::cmp::PartialEq<(u32, u32)> for SourcePos {
......@@ -140,6 +145,32 @@ mod tests {
}
}
#[test]
fn utf16_len_counts_invalid_utf8_per_byte() {
fn per_byte(bytes: &[u8]) -> usize {
bytes
.iter()
.map(|&b| 1 + (b >= 0xF0) as usize - ((b & 0xC0) == 0x80) as usize)
.sum()
}
// Lone continuation bytes, truncated sequences and bytes that never appear in UTF-8,
// both short and long enough to go through the vectorized loops.
let long = [b"x\x80\xC3\xE2\x80\xF0\x9F\xA6\xFF\xFE".as_slice(); 32].concat();
for bytes in [
b"\x80".as_slice(),
b"\xC3",
b"\xE2\x80",
b"\xF0\x9F\xA6",
b"\xFF\xFE",
b"abc\xF0",
&long,
&long[1..],
] {
assert_eq!(utf16_len(bytes), per_byte(bytes), "{bytes:?}");
}
}
#[test]
fn counts_ascii_as_one_column() {
assert_eq!(pos_after("abc"), (0, 3));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment