Unverified Commit 2af85b58 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

make rcstrs on the heap/static slightly smaller (#93805)

### What?

Splits the previously unified `PrehashedString` (which held a `Payload`
enum of `String | &'static str`) into two separate types:
`StaticPrehashedString { value: &'static str, hash: u64 }` for atoms
produced by `rcstr!` / `make_const_prehashed_string`, and
`DynamicPrehashedString { value: Box<str>, hash: u64 }` for atoms held
in an `Arc`. The static and dynamic paths were already distinguished by
the `STATIC_TAG` / `DYNAMIC_TAG` bits in `RcStr`, so the runtime branch
on enum discriminant was redundant.

### Why?

- **16 bytes saved per heap-allocated `RcStr` value .** Dynamic atoms
drop the `String::capacity` field (which was always equal to `len` since
the contents are immutable) and because they are stored in a
`triomphe::Arc` this drops the Arc payload to 32 bytes instead of 40
which matches a mimalloc bucket (previously we were rounded up to a 48
byte bucket) so we save 16 bytes.
- **8 bytes saved per static `RcStr` value.** The linker will optimally
align our 24 byte struct.
- Removes a layer of dispatch on the hot path (`as_str`, `==`, `Hash`) —
typed deref to the correct variant instead of matching on `Payload`.
(i.o.w. one 'descreminent' traversal instead of 2)

### How?

- `dynamic.rs`: `Payload` enum removed; two structs replace
`PrehashedString`. `deref_from` split into `deref_static` and
`deref_dynamic`. `restore_arc` returns `Arc<DynamicPrehashedString>`.
- `lib.rs`: `as_str` and `into_owned` dispatch on `tag()` (STATIC vs
DYNAMIC vs INLINE) rather than `location()`. New `heap_hash_and_str`
helper for `PartialEq` and `Hash` to share the static/dynamic branch.
`into_owned`'s `try_unwrap` arm uses `String::from(Box<str>)` which
reuses the box allocation (still O(1)).
- `turbo-rcstr-macros`: emit `::turbo_rcstr::StaticPrehashedString`
instead of `::turbo_rcstr::PrehashedString`.
- Added a comment on `DynamicPrehashedString` noting the future move to
`triomphe::ThinArc` to fold the two heap allocations (Arc header + boxed
bytes) into one. Deferred because that change would make
`RcStr::from(String)` copy the bytes, invalidating the documented cheap
`String -> RcStr -> String` round-trip — wants a separate evaluation.


<!-- NEXT_JS_LLM_PR -->
parent b8c2854d
...@@ -40,7 +40,7 @@ pub fn rcstr(input: TokenStream) -> TokenStream { ...@@ -40,7 +40,7 @@ pub fn rcstr(input: TokenStream) -> TokenStream {
format!("::turbo_rcstr::inline_atom({lit}).unwrap()") format!("::turbo_rcstr::inline_atom({lit}).unwrap()")
} else { } else {
format!( format!(
"{{ static RCSTR_STORAGE: ::turbo_rcstr::PrehashedString = \ "{{ static RCSTR_STORAGE: ::turbo_rcstr::StaticPrehashedString = \
::turbo_rcstr::make_const_prehashed_string({lit}); const RCSTR: \ ::turbo_rcstr::make_const_prehashed_string({lit}); const RCSTR: \
::turbo_rcstr::RcStr = ::turbo_rcstr::from_static(&RCSTR_STORAGE); \ ::turbo_rcstr::RcStr = ::turbo_rcstr::from_static(&RCSTR_STORAGE); \
::turbo_rcstr::__rcstr_inventory_submit!( \ ::turbo_rcstr::__rcstr_inventory_submit!( \
...@@ -51,7 +51,7 @@ pub fn rcstr(input: TokenStream) -> TokenStream { ...@@ -51,7 +51,7 @@ pub fn rcstr(input: TokenStream) -> TokenStream {
format!( format!(
"{{ const TEXT: &str = {input}; if ::turbo_rcstr::is_atom_inlineable(TEXT) {{ \ "{{ const TEXT: &str = {input}; if ::turbo_rcstr::is_atom_inlineable(TEXT) {{ \
::turbo_rcstr::inline_atom(TEXT).unwrap() }} else {{ static RCSTR_STORAGE: \ ::turbo_rcstr::inline_atom(TEXT).unwrap() }} else {{ static RCSTR_STORAGE: \
::turbo_rcstr::PrehashedString = \ ::turbo_rcstr::StaticPrehashedString = \
::turbo_rcstr::make_const_prehashed_string(TEXT); const RCSTR: \ ::turbo_rcstr::make_const_prehashed_string(TEXT); const RCSTR: \
::turbo_rcstr::RcStr = ::turbo_rcstr::from_static(&RCSTR_STORAGE); \ ::turbo_rcstr::RcStr = ::turbo_rcstr::from_static(&RCSTR_STORAGE); \
::turbo_rcstr::__rcstr_inventory_submit!( \ ::turbo_rcstr::__rcstr_inventory_submit!( \
......
...@@ -7,49 +7,41 @@ use crate::{ ...@@ -7,49 +7,41 @@ use crate::{
is_atom_inlineable, tagged_value::TaggedValue, is_atom_inlineable, tagged_value::TaggedValue,
}; };
pub enum Payload { /// Read-only header for atoms allocated in static storage by the `rcstr!`
String(String), /// macro. The value lives for `'static`, so we store a `&'static str` rather
Ref(&'static str), /// than owning the bytes.
} pub struct StaticPrehashedString {
pub value: &'static str,
impl Payload {
pub(crate) fn as_str(&self) -> &str {
match self {
Payload::String(s) => s,
Payload::Ref(s) => s,
}
}
pub(crate) fn into_string(self) -> String {
match self {
Payload::String(s) => s,
Payload::Ref(r) => r.to_string(),
}
}
}
impl PartialEq for Payload {
fn eq(&self, other: &Self) -> bool {
self.as_str() == other.as_str()
}
}
pub struct PrehashedString {
pub value: Payload,
/// This is not the actual `fxhash`, but rather it's a value that passed to /// This is not the actual `fxhash`, but rather it's a value that passed to
/// `write_u64` of [rustc_hash::FxHasher]. /// `write_u64` of [rustc_hash::FxHasher].
pub hash: u64, pub hash: u64,
} }
pub unsafe fn cast(ptr: TaggedValue) -> *const PrehashedString { /// Heap-owned header for atoms held in an [`Arc`]. `Box<str>` instead of
ptr.get_ptr().cast() /// `String` because the contents are immutable — we save the `capacity` field
/// (and a tag/padding byte vs the old `Payload` enum) per atom.
///
/// TODO: collapse the two allocations (this header + the boxed bytes) into a
/// single [`triomphe::ThinArc`] so the hash and bytes share one allocation and
/// the inline pointer in [`crate::RcStr`] is one word. That change would make
/// `RcStr::from(String)` copy the bytes, which would invalidate the current
/// "cheap `String -> RcStr -> String`" property — worth a separate evaluation.
pub struct DynamicPrehashedString {
pub value: Box<str>,
pub hash: u64,
}
pub(crate) unsafe fn deref_static<'i>(ptr: TaggedValue) -> &'i StaticPrehashedString {
unsafe { &*(ptr.get_ptr() as *const StaticPrehashedString) }
} }
pub(crate) unsafe fn deref_from<'i>(ptr: TaggedValue) -> &'i PrehashedString { pub(crate) unsafe fn deref_dynamic<'i>(ptr: TaggedValue) -> &'i DynamicPrehashedString {
unsafe { &*cast(ptr) } unsafe { &*(ptr.get_ptr() as *const DynamicPrehashedString) }
} }
/// Caller should call `forget` (or `clone`) on the returned `Arc` /// Caller should call `forget` (or `clone`) on the returned `Arc`
pub unsafe fn restore_arc(v: TaggedValue) -> Arc<PrehashedString> { pub unsafe fn restore_arc(v: TaggedValue) -> Arc<DynamicPrehashedString> {
let ptr = v.get_ptr() as *const PrehashedString; let ptr = v.get_ptr() as *const DynamicPrehashedString;
unsafe { Arc::from_raw(ptr) } unsafe { Arc::from_raw(ptr) }
} }
...@@ -70,20 +62,22 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr { ...@@ -70,20 +62,22 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr {
let hash = hash_bytes(text.as_bytes()); let hash = hash_bytes(text.as_bytes());
let prehashed = PrehashedString { let prehashed = DynamicPrehashedString {
value: Payload::String(text.into()), // NOTE: This will capture as a Box<str> which will essentially
// `shrink_to_fit` the bytes.
value: text.into(),
hash, hash,
}; };
new_atom_from_prehashed(prehashed) new_atom_from_prehashed(prehashed)
} }
/// Construct a new dynamic RcStr from a PrehashedString /// Construct a new dynamic RcStr from a DynamicPrehashedString
pub(crate) fn new_atom_from_prehashed(prehashed: PrehashedString) -> RcStr { pub(crate) fn new_atom_from_prehashed(prehashed: DynamicPrehashedString) -> RcStr {
let entry: Arc<PrehashedString> = Arc::new(prehashed); let entry: Arc<DynamicPrehashedString> = Arc::new(prehashed);
let mut entry = Arc::into_raw(entry); let mut entry = Arc::into_raw(entry);
debug_assert!(0 == entry as u8 & TAG_MASK); debug_assert!(0 == entry as u8 & TAG_MASK);
entry = ((entry as usize) | DYNAMIC_TAG as usize) as *mut PrehashedString; entry = ((entry as usize) | DYNAMIC_TAG as usize) as *mut DynamicPrehashedString;
let ptr: NonNull<PrehashedString> = unsafe { let ptr: NonNull<DynamicPrehashedString> = unsafe {
// Safety: Arc::into_raw returns a non-null pointer // Safety: Arc::into_raw returns a non-null pointer
NonNull::new_unchecked(entry as *mut _) NonNull::new_unchecked(entry as *mut _)
}; };
...@@ -94,15 +88,15 @@ pub(crate) fn new_atom_from_prehashed(prehashed: PrehashedString) -> RcStr { ...@@ -94,15 +88,15 @@ pub(crate) fn new_atom_from_prehashed(prehashed: PrehashedString) -> RcStr {
} }
#[inline(always)] #[inline(always)]
pub(crate) const fn new_static_atom(string: &'static PrehashedString) -> RcStr { pub(crate) const fn new_static_atom(string: &'static StaticPrehashedString) -> RcStr {
let entry = string as *const PrehashedString; let entry = string as *const StaticPrehashedString;
const { const {
debug_assert!(align_of::<PrehashedString>() >= 4); debug_assert!(align_of::<StaticPrehashedString>() >= 4);
// This must be 00, so that we don't have to remove the tag, which we can't since it would // This must be 00, so that we don't have to remove the tag, which we can't since it would
// require pointer-integer casts in a const context. // require pointer-integer casts in a const context.
debug_assert!(STATIC_TAG == 0b_00); debug_assert!(STATIC_TAG == 0b_00);
} }
let ptr: NonNull<PrehashedString> = unsafe { let ptr: NonNull<StaticPrehashedString> = unsafe {
// Safety: references always return a non-null pointers // Safety: references always return a non-null pointers
NonNull::new_unchecked(entry as *mut _) NonNull::new_unchecked(entry as *mut _)
}; };
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment