Commit a7dff59d authored by Luke Sandberg's avatar Luke Sandberg

but what if we interned rcstr

parent 8e9f9514
...@@ -9487,11 +9487,13 @@ dependencies = [ ...@@ -9487,11 +9487,13 @@ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"bytes-str", "bytes-str",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"dashmap 6.1.0",
"napi", "napi",
"new_debug_unreachable", "new_debug_unreachable",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"shrink-to-fit", "shrink-to-fit",
"smallvec",
"triomphe 0.1.12", "triomphe 0.1.12",
"turbo-tasks-hash", "turbo-tasks-hash",
] ]
......
...@@ -18,6 +18,8 @@ new_debug_unreachable = "1.0.6" ...@@ -18,6 +18,8 @@ new_debug_unreachable = "1.0.6"
shrink-to-fit = { workspace = true } shrink-to-fit = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
bytes-str = { workspace = true } bytes-str = { workspace = true }
dashmap = { workspace = true }
smallvec = { workspace = true }
[target.'cfg(not(target_family = "wasm"))'.dependencies] [target.'cfg(not(target_family = "wasm"))'.dependencies]
napi = { workspace = true, optional = true } napi = { workspace = true, optional = true }
......
...@@ -47,9 +47,40 @@ fn bench_construct(c: &mut Criterion) { ...@@ -47,9 +47,40 @@ fn bench_construct(c: &mut Criterion) {
f.iter(|| RcStr::from("this is a long string that will take time to copy")); f.iter(|| RcStr::from("this is a long string that will take time to copy"));
}); });
} }
/// Benchmark intern table hit: repeatedly create RcStr from the same string.
/// With interning this is a table lookup + Arc clone instead of a fresh allocation.
fn bench_intern_hit(c: &mut Criterion) {
let mut g = c.benchmark_group("intern_hit");
let long = "this is a long string that exercises the intern table lookup path";
// Pre-populate the intern table
let _keep = RcStr::from(long);
g.bench_function("from/interned", |b| {
b.iter(|| {
let s = RcStr::from(long);
std::hint::black_box(&s);
});
});
g.finish();
}
/// Benchmark equality of interned strings (pointer identity fast path).
fn bench_eq_interned(c: &mut Criterion) {
let mut g = c.benchmark_group("eq");
let long = "benchmark equality: a long string for comparing interned values";
let a = RcStr::from(long);
let b = RcStr::from(long);
g.bench_function("interned", |bench| {
bench.iter(|| std::hint::black_box(&a) == std::hint::black_box(&b));
});
g.finish();
}
criterion_group!( criterion_group!(
name = benches; name = benches;
config = Criterion::default(); config = Criterion::default();
targets = bench_map,bench_construct, targets = bench_map,bench_construct,bench_intern_hit,bench_eq_interned,
); );
criterion_main!(benches); criterion_main!(benches);
use std::{num::NonZeroU8, ptr::NonNull}; use std::{num::NonZeroU8, ptr::NonNull, sync::LazyLock};
use dashmap::DashMap;
use smallvec::SmallVec;
use triomphe::Arc; use triomphe::Arc;
use crate::{ use crate::{
...@@ -7,6 +9,140 @@ use crate::{ ...@@ -7,6 +9,140 @@ use crate::{
tagged_value::{MAX_INLINE_LEN, TaggedValue}, tagged_value::{MAX_INLINE_LEN, TaggedValue},
}; };
/// A wrapper around `TaggedValue` that implements `Send` and `Sync`.
///
/// This is safe because the `TaggedValue` in the intern table always points to
/// either:
/// - A `triomphe::Arc<PrehashedString>` (DYNAMIC_TAG), which is `Send + Sync`
/// - A `&'static PrehashedString` (STATIC_TAG), which is `Send + Sync`
///
/// The wrapper is `Copy` and has no `Drop`, avoiding recursive drop issues.
#[derive(Copy, Clone, PartialEq, Eq)]
struct InternEntry(TaggedValue);
// Safety: InternEntry wraps either an Arc pointer (Send+Sync via Arc) or a
// static reference (inherently Send+Sync). See the doc comment above.
unsafe impl Send for InternEntry {}
unsafe impl Sync for InternEntry {}
/// Global intern table for deduplicating RcStr values.
///
/// Keyed by the precomputed u64 hash of the string content. Values are SmallVec
/// of `InternEntry`s to handle the (extremely rare) case of hash collisions.
/// Using `InternEntry` (a `Copy` wrapper around `TaggedValue`) instead of
/// `RcStr` avoids recursive Drop calls when evicting entries.
///
/// The table entries may be either DYNAMIC-tagged (heap-allocated Arc) or
/// STATIC-tagged (compile-time constant). When a static string is registered,
/// subsequent `RcStr::from()` calls for the same content return a static clone,
/// avoiding heap allocation and refcounting entirely.
///
/// Dynamic entries are self-evicting: when an RcStr is dropped and the only
/// remaining reference is the intern table's copy (refcount == 2), the entry is
/// removed.
static INTERN_TABLE: LazyLock<DashMap<u64, SmallVec<[InternEntry; 1]>>> =
LazyLock::new(DashMap::new);
/// Returns the number of unique strings currently in the intern table.
pub fn interned_count() -> usize {
INTERN_TABLE.iter().map(|e| e.value().len()).sum()
}
/// Look up a string in the intern table by hash and content.
/// Returns a clone of the existing entry as an RcStr if found.
fn intern_lookup(hash: u64, text: &str) -> Option<RcStr> {
let entry = INTERN_TABLE.get(&hash)?;
for &InternEntry(tv) in entry.value().iter() {
// Safety: all entries in the intern table point to valid
// PrehashedString allocations (either Arc-managed or static).
let phs = unsafe { deref_from(tv) };
if phs.value.as_str() == text {
// Clone the entry — for DYNAMIC entries this increments the Arc
// refcount, for STATIC entries it's free.
return Some(clone_intern_entry(tv));
}
}
None
}
/// Clone an interned TaggedValue into a new RcStr, incrementing the Arc
/// refcount for DYNAMIC entries.
fn clone_intern_entry(tv: TaggedValue) -> RcStr {
let tag = tv.tag_byte() & TAG_MASK;
if tag == crate::DYNAMIC_TAG {
unsafe {
let arc = restore_arc(tv);
std::mem::forget(arc.clone()); // increment refcount
std::mem::forget(arc); // don't decrement
}
}
// For STATIC and INLINE (though INLINE should never be in the table),
// just copy the TaggedValue.
RcStr { unsafe_data: tv }
}
/// Register a static RcStr in the intern table so that future `RcStr::from()`
/// calls for the same content return the static version (avoiding heap
/// allocation and refcounting).
pub(crate) fn intern_static(tv: TaggedValue, hash: u64) {
debug_assert!(tv.tag_byte() & TAG_MASK == STATIC_TAG);
let new_entry = InternEntry(tv);
INTERN_TABLE
.entry(hash)
.and_modify(|entries| {
let text = unsafe { deref_from(tv) }.value.as_str();
// Replace any existing dynamic entry with the static one
if let Some(pos) = entries
.iter()
.position(|&InternEntry(e)| unsafe { deref_from(e) }.value.as_str() == text)
{
let InternEntry(old) = entries[pos];
entries[pos] = new_entry;
// If the old entry was DYNAMIC, drop the table's Arc reference
drop_intern_entry(old);
} else {
entries.push(new_entry);
}
})
.or_insert_with(|| SmallVec::from_elem(new_entry, 1));
}
/// Remove a dynamic entry from the intern table by pointer identity.
/// Called from RcStr::Drop or into_owned when the refcount indicates only
/// the caller and the intern table hold references.
///
/// This also drops the table's Arc reference (decrementing the refcount).
pub(crate) fn intern_remove(tagged_value: TaggedValue) {
let phs = unsafe { deref_from(tagged_value) };
let hash = phs.hash;
let needle = InternEntry(tagged_value);
if let Some(mut entry) = INTERN_TABLE.get_mut(&hash) {
let entries = entry.value_mut();
if let Some(pos) = entries.iter().position(|&e| e == needle) {
let InternEntry(removed) = entries.swap_remove(pos);
if entries.is_empty() {
drop(entry);
INTERN_TABLE.remove_if(&hash, |_, entries| entries.is_empty());
} else {
drop(entry);
}
// Drop the table's Arc reference — InternEntry is Copy so removing
// from the SmallVec didn't decrement the refcount.
drop_intern_entry(removed);
}
}
}
/// Drop the Arc refcount for a DYNAMIC TaggedValue. No-op for STATIC/INLINE.
fn drop_intern_entry(tv: TaggedValue) {
let tag = tv.tag_byte() & TAG_MASK;
if tag == crate::DYNAMIC_TAG {
unsafe {
drop(restore_arc(tv));
}
}
}
pub enum Payload { pub enum Payload {
String(String), String(String),
Ref(&'static str), Ref(&'static str),
...@@ -55,6 +191,10 @@ pub unsafe fn restore_arc(v: TaggedValue) -> Arc<PrehashedString> { ...@@ -55,6 +191,10 @@ pub unsafe fn restore_arc(v: TaggedValue) -> Arc<PrehashedString> {
/// This can create any kind of [Atom], although this lives in the `dynamic` /// This can create any kind of [Atom], although this lives in the `dynamic`
/// module. /// module.
///
/// For strings longer than the inline threshold, this checks the global intern
/// table and returns a clone of an existing Arc if the string is already
/// interned, avoiding a new heap allocation.
pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr { pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr {
let len = text.as_ref().len(); let len = text.as_ref().len();
...@@ -70,12 +210,66 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr { ...@@ -70,12 +210,66 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr {
let hash = hash_bytes(text.as_ref().as_bytes()); let hash = hash_bytes(text.as_ref().as_bytes());
let entry: Arc<PrehashedString> = Arc::new(PrehashedString { // Fast path: read lock only. Most hits go through here.
value: Payload::String(text.into()), if let Some(existing) = intern_lookup(hash, text.as_ref()) {
hash, return existing;
}); }
let entry = Arc::into_raw(entry);
// Slow path: take the write lock via entry() and double-check.
// This eliminates the race where two threads both miss the read lock
// and both insert duplicate entries.
intern_insert_or_lookup(hash, text)
}
/// Slow path for `new_atom`: acquires the write lock on the DashMap shard via
/// `entry()`, double-checks for an existing entry (another thread may have
/// inserted between our read-lock miss and this write-lock acquisition), and
/// either returns the existing entry or inserts a new one.
fn intern_insert_or_lookup<T: AsRef<str> + Into<String>>(hash: u64, text: T) -> RcStr {
match INTERN_TABLE.entry(hash) {
dashmap::Entry::Occupied(entry) => {
// Double-check: another thread may have inserted while we were
// waiting for the write lock.
for &InternEntry(tv) in entry.get().iter() {
let phs = unsafe { deref_from(tv) };
if phs.value.as_str() == text.as_ref() {
return clone_intern_entry(tv);
}
}
// Hash bucket exists but no content match (hash collision).
// Insert our new entry alongside the existing ones.
let arc: Arc<PrehashedString> = Arc::new(PrehashedString {
value: Payload::String(text.into()),
hash,
});
let table_arc = arc.clone();
let rcstr = arc_to_rcstr(arc);
let table_rcstr = arc_to_rcstr(table_arc);
entry
.into_ref()
.value_mut()
.push(InternEntry(table_rcstr.unsafe_data));
std::mem::forget(table_rcstr);
rcstr
}
dashmap::Entry::Vacant(entry) => {
let arc: Arc<PrehashedString> = Arc::new(PrehashedString {
value: Payload::String(text.into()),
hash,
});
let table_arc = arc.clone();
let rcstr = arc_to_rcstr(arc);
let table_rcstr = arc_to_rcstr(table_arc);
entry.insert(SmallVec::from_elem(InternEntry(table_rcstr.unsafe_data), 1));
std::mem::forget(table_rcstr);
rcstr
}
}
}
/// Convert a `triomphe::Arc<PrehashedString>` into an `RcStr` with DYNAMIC_TAG.
fn arc_to_rcstr(arc: Arc<PrehashedString>) -> RcStr {
let entry = Arc::into_raw(arc);
let ptr: NonNull<PrehashedString> = unsafe { let ptr: NonNull<PrehashedString> = unsafe {
// Safety: Arc::into_raw returns a non-null pointer // Safety: Arc::into_raw returns a non-null pointer
NonNull::new_unchecked(entry as *mut _) NonNull::new_unchecked(entry as *mut _)
...@@ -97,9 +291,11 @@ pub(crate) fn new_static_atom(string: &'static PrehashedString) -> RcStr { ...@@ -97,9 +291,11 @@ pub(crate) fn new_static_atom(string: &'static PrehashedString) -> RcStr {
NonNull::new_unchecked(entry as *mut _) NonNull::new_unchecked(entry as *mut _)
}; };
RcStr { let tv = TaggedValue::new_ptr(ptr);
unsafe_data: TaggedValue::new_ptr(ptr), // Register in the intern table so that future RcStr::from() calls for
} // the same content return this static version instead of allocating.
intern_static(tv, string.hash);
RcStr { unsafe_data: tv }
} }
/// Attempts to construct an RcStr but only if it can be constructed inline. /// Attempts to construct an RcStr but only if it can be constructed inline.
......
...@@ -23,6 +23,7 @@ use shrink_to_fit::ShrinkToFit; ...@@ -23,6 +23,7 @@ use shrink_to_fit::ShrinkToFit;
use triomphe::Arc; use triomphe::Arc;
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher}; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher};
pub use crate::dynamic::interned_count;
use crate::{ use crate::{
dynamic::{deref_from, hash_bytes, new_atom}, dynamic::{deref_from, hash_bytes, new_atom},
tagged_value::TaggedValue, tagged_value::TaggedValue,
...@@ -66,10 +67,13 @@ mod tagged_value; ...@@ -66,10 +67,13 @@ mod tagged_value;
/// Converting from an [`RcStr`] to a `&str` should be done with [`RcStr::as_str`]. Converting to a /// Converting from an [`RcStr`] to a `&str` should be done with [`RcStr::as_str`]. Converting to a
/// `String` should be done with [`RcStr::into_owned`]. /// `String` should be done with [`RcStr::into_owned`].
/// ///
/// ## Future Optimizations /// ## Interning
/// ///
/// This type is intentionally opaque to allow for optimizations to the underlying representation. /// Heap-allocated `RcStr` values are automatically interned: creating an `RcStr` from the same
/// Future implementations may use inline representations or interning. /// string content will reuse the existing allocation. This deduplicates memory and makes
/// equality checks via pointer identity fast. Static strings created with [`rcstr!`] are also
/// registered in the intern table, so `RcStr::from("...")` returns a zero-cost static copy
/// when a matching `rcstr!` constant exists.
// //
// If you want to change the underlying string type to `Arc<str>`, please ensure that you profile // If you want to change the underlying string type to `Arc<str>`, please ensure that you profile
// performance. The current implementation offers very cheap `String -> RcStr -> String`, meaning we // performance. The current implementation offers very cheap `String -> RcStr -> String`, meaning we
...@@ -143,8 +147,13 @@ impl RcStr { ...@@ -143,8 +147,13 @@ impl RcStr {
pub fn into_owned(self) -> String { pub fn into_owned(self) -> String {
match self.tag() { match self.tag() {
DYNAMIC_TAG => { DYNAMIC_TAG => {
// convert `self` into `arc` let md = ManuallyDrop::new(self);
let arc = unsafe { dynamic::restore_arc(ManuallyDrop::new(self).unsafe_data) }; let arc = unsafe { dynamic::restore_arc(md.unsafe_data) };
// Check if only this arc + intern table hold references.
// If so, evict from the table to enable try_unwrap to succeed.
if Arc::count(&arc) == 2 {
dynamic::intern_remove(md.unsafe_data);
}
match Arc::try_unwrap(arc) { match Arc::try_unwrap(arc) {
Ok(v) => v.value.into_string(), Ok(v) => v.value.into_string(),
Err(arc) => arc.value.as_str().to_string(), Err(arc) => arc.value.as_str().to_string(),
...@@ -399,7 +408,16 @@ impl_borrow_decode!(RcStr); ...@@ -399,7 +408,16 @@ impl_borrow_decode!(RcStr);
impl Drop for RcStr { impl Drop for RcStr {
fn drop(&mut self) { fn drop(&mut self) {
match self.tag() { match self.tag() {
DYNAMIC_TAG => unsafe { drop(dynamic::restore_arc(self.unsafe_data)) }, DYNAMIC_TAG => {
let arc = unsafe { dynamic::restore_arc(self.unsafe_data) };
// If the refcount is 2, only this arc and the intern table hold
// references. Remove from the intern table before dropping so
// the string is fully deallocated.
if Arc::count(&arc) == 2 {
dynamic::intern_remove(self.unsafe_data);
}
drop(arc);
}
STATIC_TAG => { STATIC_TAG => {
// do nothing, these are never deallocated // do nothing, these are never deallocated
} }
...@@ -519,19 +537,22 @@ mod tests { ...@@ -519,19 +537,22 @@ mod tests {
triomphe::Arc::count(&arc) triomphe::Arc::count(&arc)
} }
let str = RcStr::from("this is a long string that won't be inlined"); // Use a unique string to avoid interference from other tests' interned strings
let str = RcStr::from("test_refcount: a unique long string for this test");
assert_eq!(refcount(&str), 1); // Refcount is 2: one for `str`, one for the intern table entry
assert_eq!(refcount(&str), 1); // refcount should not modify the refcount itself assert_eq!(refcount(&str), 2);
assert_eq!(refcount(&str), 2); // refcount should not modify the refcount itself
let cloned_str = str.clone(); let cloned_str = str.clone();
assert_eq!(refcount(&str), 2); assert_eq!(refcount(&str), 3);
drop(cloned_str); drop(cloned_str);
assert_eq!(refcount(&str), 1); assert_eq!(refcount(&str), 2);
// into_owned evicts from intern table when refcount == 2, then unwraps
let _ = str.clone().into_owned(); let _ = str.clone().into_owned();
assert_eq!(refcount(&str), 1); assert_eq!(refcount(&str), 2);
} }
#[test] #[test]
...@@ -555,7 +576,10 @@ mod tests { ...@@ -555,7 +576,10 @@ mod tests {
const LONG: &str = "a very long string that lives forever"; const LONG: &str = "a very long string that lives forever";
let leaked = rcstr!(LONG); let leaked = rcstr!(LONG);
let not_leaked = RcStr::from(LONG); let not_leaked = RcStr::from(LONG);
assert_ne!(leaked.tag(), not_leaked.tag()); // With interning, RcStr::from() finds the static in the intern table
// and returns a STATIC-tagged copy — same tag, same pointer.
assert_eq!(leaked.tag(), not_leaked.tag());
assert_eq!(leaked.tag(), STATIC_TAG);
assert_eq!(leaked, not_leaked); assert_eq!(leaked, not_leaked);
} }
...@@ -624,4 +648,100 @@ mod tests { ...@@ -624,4 +648,100 @@ mod tests {
} }
} }
} }
#[test]
fn test_intern_pointer_identity() {
// Two RcStr::from() calls with the same long string should share the
// same underlying Arc (same TaggedValue bits).
let a = RcStr::from("intern_pointer_identity: same content");
let b = RcStr::from("intern_pointer_identity: same content");
assert_eq!(a.unsafe_data, b.unsafe_data);
assert_eq!(a.tag(), DYNAMIC_TAG);
}
fn refcount(str: &RcStr) -> usize {
assert!(str.tag() == DYNAMIC_TAG);
let arc = ManuallyDrop::new(unsafe { dynamic::restore_arc(str.unsafe_data) });
triomphe::Arc::count(&arc)
}
#[test]
fn test_intern_eviction() {
let s = RcStr::from("intern_eviction: unique string for eviction test");
assert_eq!(refcount(&s), 2); // s + intern table
let s2 = s.clone();
assert_eq!(refcount(&s), 3); // s + s2 + intern table
drop(s2);
assert_eq!(refcount(&s), 2);
// After dropping, re-creating the same string should get a fresh Arc
// (proving eviction happened)
let tv_before = s.unsafe_data;
drop(s);
let s_new = RcStr::from("intern_eviction: unique string for eviction test");
// New allocation — different pointer (eviction worked)
assert_ne!(s_new.unsafe_data, tv_before);
assert_eq!(refcount(&s_new), 2);
}
#[test]
fn test_intern_static_dedup() {
// After rcstr! registers a static, RcStr::from() returns STATIC
let static_str = rcstr!("intern_static_dedup: a long static string");
let dynamic_str = RcStr::from("intern_static_dedup: a long static string");
assert_eq!(static_str.tag(), STATIC_TAG);
assert_eq!(dynamic_str.tag(), STATIC_TAG);
assert_eq!(static_str.unsafe_data, dynamic_str.unsafe_data);
}
#[test]
fn test_intern_into_owned_evicts() {
let s = RcStr::from("intern_into_owned: unique string for into_owned test");
assert_eq!(refcount(&s), 2); // s + intern table
// into_owned with sole reference should succeed at try_unwrap
// (proving intern_remove was called first)
let owned = s.into_owned();
assert_eq!(
owned,
"intern_into_owned: unique string for into_owned test"
);
// After into_owned, re-creating gives a fresh interned string
let s_new = RcStr::from("intern_into_owned: unique string for into_owned test");
assert_eq!(refcount(&s_new), 2); // fresh: s_new + intern table
}
#[test]
fn test_intern_inline_not_interned() {
// Inline strings (≤6 bytes) should use INLINE_TAG, not be interned
let short = RcStr::from("hi");
assert_eq!(short.tag(), INLINE_TAG);
}
#[test]
fn test_intern_thread_safety() {
use std::thread;
const S: &str = "intern_thread_safety: shared across threads";
let handles: Vec<_> = (0..8)
.map(|_| {
thread::spawn(|| {
let strs: Vec<_> = (0..100).map(|_| RcStr::from(S)).collect();
// All should share the same Arc pointer
for s in &strs {
assert_eq!(s.unsafe_data, strs[0].unsafe_data);
}
drop(strs);
})
})
.collect();
for h in handles {
h.join().unwrap();
}
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment