Unverified Commit cf710377 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: add Xxh3Hash128Hasher (#90584)

- turbo-persistence used XxHash64, which is supposed to be slower than XxHash3_64
- refactor debug_id computation to just use `rope.content_hash()` which was added in the meantime
- update and adjust feature flags of `twox_hash`

https://github.com/Cyan4973/xxHash says:
| Hash Name     | Width | Bandwidth (GB/s) | Small Data Velocity | Quality | Comment |
| ---------     | ----- | ---------------- | ----- | --- | --- |
| __XXH3__ (SSE2) |  64 | 31.5 GB/s        | 133.1 | 10
| __XXH128__ (SSE2) | 128 | 29.6 GB/s      | 118.1 | 10
| __XXH64__     |    64 | 19.4 GB/s        |  71.0 | 10
| __XXH32__     |    32 |  9.7 GB/s        |  71.9 | 10
parent 2307bf6e
...@@ -9387,7 +9387,7 @@ dependencies = [ ...@@ -9387,7 +9387,7 @@ dependencies = [
"tracing", "tracing",
"turbo-bincode", "turbo-bincode",
"turbo-tasks-malloc", "turbo-tasks-malloc",
"twox-hash 2.1.0", "twox-hash 2.1.2",
"zstd", "zstd",
] ]
...@@ -9650,7 +9650,7 @@ dependencies = [ ...@@ -9650,7 +9650,7 @@ dependencies = [
"data-encoding", "data-encoding",
"sha2", "sha2",
"turbo-tasks-macros", "turbo-tasks-macros",
"twox-hash 2.1.0", "twox-hash 2.1.2",
] ]
[[package]] [[package]]
...@@ -9912,7 +9912,6 @@ dependencies = [ ...@@ -9912,7 +9912,6 @@ dependencies = [
"turbo-tasks-hash", "turbo-tasks-hash",
"turbo-tasks-testing", "turbo-tasks-testing",
"turbo-unix-path", "turbo-unix-path",
"twox-hash 2.1.0",
"url", "url",
"urlencoding", "urlencoding",
"uuid", "uuid",
...@@ -10418,11 +10417,11 @@ dependencies = [ ...@@ -10418,11 +10417,11 @@ dependencies = [
[[package]] [[package]]
name = "twox-hash" name = "twox-hash"
version = "2.1.0" version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
dependencies = [ dependencies = [
"rand 0.8.5", "rand 0.9.0",
] ]
[[package]] [[package]]
......
...@@ -468,7 +468,7 @@ tokio-util = { version = "0.7.13", features = ["io", "rt"] } ...@@ -468,7 +468,7 @@ tokio-util = { version = "0.7.13", features = ["io", "rt"] }
tracing = "0.1.37" tracing = "0.1.37"
tracing-subscriber = "0.3.16" tracing-subscriber = "0.3.16"
triomphe = { git = "https://github.com/sokra/triomphe", branch = "sokra/unstable" } triomphe = { git = "https://github.com/sokra/triomphe", branch = "sokra/unstable" }
twox-hash = { version = "2.1.0", features = ["xxhash64", "xxhash3_128"] } twox-hash = { version = "2.1.2", features = ["std", "xxhash3_64", "xxhash3_128"] }
unsize = "1.1.0" unsize = "1.1.0"
unty = "0.0.4" unty = "0.0.4"
url = "2.2.2" url = "2.2.2"
......
...@@ -230,7 +230,7 @@ impl<T: StoreKey> StoreKey for &'_ T { ...@@ -230,7 +230,7 @@ impl<T: StoreKey> StoreKey for &'_ T {
/// Hashes a key with a fast, deterministic hash function. /// Hashes a key with a fast, deterministic hash function.
pub fn hash_key(key: &impl KeyBase) -> u64 { pub fn hash_key(key: &impl KeyBase) -> u64 {
let mut hasher = twox_hash::XxHash64::with_seed(0); let mut hasher = twox_hash::XxHash3_64::with_seed(0);
key.hash(&mut hasher); key.hash(&mut hasher);
hasher.finish() hasher.finish()
} }
......
...@@ -2,3 +2,8 @@ ...@@ -2,3 +2,8 @@
pub fn encode_hex(n: u64) -> String { pub fn encode_hex(n: u64) -> String {
format!("{:01$x}", n, std::mem::size_of::<u64>() * 2) format!("{:01$x}", n, std::mem::size_of::<u64>() * 2)
} }
/// Encodes a 128-bit unsigned integer into a hex string.
pub fn encode_hex_128(n: u128) -> String {
format!("{:01$x}", n, std::mem::size_of::<u128>() * 2)
}
...@@ -7,15 +7,17 @@ ...@@ -7,15 +7,17 @@
mod deterministic_hash; mod deterministic_hash;
mod hex; mod hex;
mod sha; mod sha;
mod xxh3_hash128;
mod xxh3_hash64; mod xxh3_hash64;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Decode, Encode)] #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Decode, Encode)]
pub enum HashAlgorithm { pub enum HashAlgorithm {
/// The default hash algorithm, use this when the exact hashing algorithm doesn't matter. /// The default hash algorithm is using xxh3, which is a fast non-cryptographic hash function.
#[default] #[default]
Xxh3Hash64Hex, Xxh3Hash64Hex,
Xxh3Hash128Hex,
/// Used for https://nextjs.org/docs/app/guides/content-security-policy#enabling-sri /// Used for https://nextjs.org/docs/app/guides/content-security-policy#enabling-sri
Sha256Base64, Sha256Base64,
/// Used for https://nextjs.org/docs/app/guides/content-security-policy#enabling-sri /// Used for https://nextjs.org/docs/app/guides/content-security-policy#enabling-sri
...@@ -31,6 +33,11 @@ pub fn deterministic_hash<T: DeterministicHash>(input: T, algorithm: HashAlgorit ...@@ -31,6 +33,11 @@ pub fn deterministic_hash<T: DeterministicHash>(input: T, algorithm: HashAlgorit
input.deterministic_hash(&mut hasher); input.deterministic_hash(&mut hasher);
encode_hex(hasher.finish()) encode_hex(hasher.finish())
} }
HashAlgorithm::Xxh3Hash128Hex => {
let mut hasher = Xxh3Hash128Hasher::new();
input.deterministic_hash(&mut hasher);
encode_hex_128(hasher.finish())
}
HashAlgorithm::Sha256Base64 => { HashAlgorithm::Sha256Base64 => {
let mut hasher = ShaHasher::new_sha256(); let mut hasher = ShaHasher::new_sha256();
input.deterministic_hash(&mut hasher); input.deterministic_hash(&mut hasher);
...@@ -51,7 +58,8 @@ pub fn deterministic_hash<T: DeterministicHash>(input: T, algorithm: HashAlgorit ...@@ -51,7 +58,8 @@ pub fn deterministic_hash<T: DeterministicHash>(input: T, algorithm: HashAlgorit
pub use crate::{ pub use crate::{
deterministic_hash::{DeterministicHash, DeterministicHasher}, deterministic_hash::{DeterministicHash, DeterministicHasher},
hex::encode_hex, hex::{encode_hex, encode_hex_128},
sha::ShaHasher, sha::ShaHasher,
xxh3_hash64::{Xxh3Hash64Hasher, hash_xxh3_hash64}, xxh3_hash64::{Xxh3Hash64Hasher, hash_xxh3_hash64},
xxh3_hash128::{Xxh3Hash128Hasher, hash_xxh3_hash128},
}; };
use twox_hash::XxHash3_128;
use crate::{DeterministicHash, DeterministicHasher};
/// Hash some content with the Xxh3Hash128 non-cryptographic hash function.
pub fn hash_xxh3_hash128<T: DeterministicHash>(input: T) -> u128 {
let mut hasher = Xxh3Hash128Hasher::new();
input.deterministic_hash(&mut hasher);
hasher.finish()
}
/// Xxh3Hash128 hasher.
pub struct Xxh3Hash128Hasher(XxHash3_128);
impl Xxh3Hash128Hasher {
/// Create a new hasher.
pub fn new() -> Self {
Self(XxHash3_128::with_seed(0))
}
/// Uses the DeterministicHash trait to hash the input in a
/// cross-platform way.
pub fn write_value<T: DeterministicHash>(&mut self, input: T) {
input.deterministic_hash(self);
}
/// Uses the DeterministicHash trait to hash the input in a
/// cross-platform way.
pub fn write_ref<T: DeterministicHash>(&mut self, input: &T) {
input.deterministic_hash(self);
}
/// Finish the hash computation and return the digest.
pub fn finish(&self) -> u128 {
self.0.finish_128()
}
}
impl DeterministicHasher for Xxh3Hash128Hasher {
fn finish(&self) -> u64 {
panic!("use the Xxh3Hash128Hasher non-trait function instead");
}
fn write_bytes(&mut self, bytes: &[u8]) {
self.0.write(bytes);
}
}
impl Default for Xxh3Hash128Hasher {
fn default() -> Self {
Self::new()
}
}
...@@ -47,7 +47,6 @@ turbo-tasks-env = { workspace = true } ...@@ -47,7 +47,6 @@ turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true } turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true } turbo-unix-path = { workspace = true }
twox-hash = { workspace = true }
url = { workspace = true } url = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
uuid = { workspace = true} uuid = { workspace = true}
......
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks_fs::rope::Rope; use turbo_tasks_fs::rope::Rope;
use twox_hash::xxhash3_128; use turbo_tasks_hash::Xxh3Hash128Hasher;
/// Generate a deterministic debug ID from content using hash-based UUID generation /// Generate a deterministic debug ID from content using hash-based UUID generation
/// ///
...@@ -8,11 +8,9 @@ use twox_hash::xxhash3_128; ...@@ -8,11 +8,9 @@ use twox_hash::xxhash3_128;
/// based on the content, ensuring reproducible builds while maintaining uniqueness. /// based on the content, ensuring reproducible builds while maintaining uniqueness.
/// Uses xxHash3-128 for fast, stable, and collision-resistant hashing. /// Uses xxHash3-128 for fast, stable, and collision-resistant hashing.
pub fn generate_debug_id(content: &Rope) -> RcStr { pub fn generate_debug_id(content: &Rope) -> RcStr {
let mut hasher = xxhash3_128::Hasher::new(); let mut hasher = Xxh3Hash128Hasher::new();
for bytes in content.read() { hasher.write_value(content.content_hash());
hasher.write(bytes.as_ref()); let hash = hasher.finish();
}
let hash = hasher.finish_128();
uuid::Uuid::from_u128(hash) uuid::Uuid::from_u128(hash)
.as_hyphenated() .as_hyphenated()
.to_string() .to_string()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment