Unverified Commit 2af85b58 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

make rcstrs on the heap/static slightly smaller (#93805)

### What?

Splits the previously unified `PrehashedString` (which held a `Payload`
enum of `String | &'static str`) into two separate types:
`StaticPrehashedString { value: &'static str, hash: u64 }` for atoms
produced by `rcstr!` / `make_const_prehashed_string`, and
`DynamicPrehashedString { value: Box<str>, hash: u64 }` for atoms held
in an `Arc`. The static and dynamic paths were already distinguished by
the `STATIC_TAG` / `DYNAMIC_TAG` bits in `RcStr`, so the runtime branch
on enum discriminant was redundant.

### Why?

- **16 bytes saved per heap-allocated `RcStr` value .** Dynamic atoms
drop the `String::capacity` field (which was always equal to `len` since
the contents are immutable) and because they are stored in a
`triomphe::Arc` this drops the Arc payload to 32 bytes instead of 40
which matches a mimalloc bucket (previously we were rounded up to a 48
byte bucket) so we save 16 bytes.
- **8 bytes saved per static `RcStr` value.** The linker will optimally
align our 24 byte struct.
- Removes a layer of dispatch on the hot path (`as_str`, `==`, `Hash`) —
typed deref to the correct variant instead of matching on `Payload`.
(i.o.w. one 'descreminent' traversal instead of 2)

### How?

- `dynamic.rs`: `Payload` enum removed; two structs replace
`PrehashedString`. `deref_from` split into `deref_static` and
`deref_dynamic`. `restore_arc` returns `Arc<DynamicPrehashedString>`.
- `lib.rs`: `as_str` and `into_owned` dispatch on `tag()` (STATIC vs
DYNAMIC vs INLINE) rather than `location()`. New `heap_hash_and_str`
helper for `PartialEq` and `Hash` to share the static/dynamic branch.
`into_owned`'s `try_unwrap` arm uses `String::from(Box<str>)` which
reuses the box allocation (still O(1)).
- `turbo-rcstr-macros`: emit `::turbo_rcstr::StaticPrehashedString`
instead of `::turbo_rcstr::PrehashedString`.
- Added a comment on `DynamicPrehashedString` noting the future move to
`triomphe::ThinArc` to fold the two heap allocations (Arc header + boxed
bytes) into one. Deferred because that change would make
`RcStr::from(String)` copy the bytes, invalidating the documented cheap
`String -> RcStr -> String` round-trip — wants a separate evaluation.


<!-- NEXT_JS_LLM_PR -->
parent b8c2854d
......@@ -40,7 +40,7 @@ pub fn rcstr(input: TokenStream) -> TokenStream {
format!("::turbo_rcstr::inline_atom({lit}).unwrap()")
} else {
format!(
"{{ static RCSTR_STORAGE: ::turbo_rcstr::PrehashedString = \
"{{ static RCSTR_STORAGE: ::turbo_rcstr::StaticPrehashedString = \
::turbo_rcstr::make_const_prehashed_string({lit}); const RCSTR: \
::turbo_rcstr::RcStr = ::turbo_rcstr::from_static(&RCSTR_STORAGE); \
::turbo_rcstr::__rcstr_inventory_submit!( \
......@@ -51,7 +51,7 @@ pub fn rcstr(input: TokenStream) -> TokenStream {
format!(
"{{ const TEXT: &str = {input}; if ::turbo_rcstr::is_atom_inlineable(TEXT) {{ \
::turbo_rcstr::inline_atom(TEXT).unwrap() }} else {{ static RCSTR_STORAGE: \
::turbo_rcstr::PrehashedString = \
::turbo_rcstr::StaticPrehashedString = \
::turbo_rcstr::make_const_prehashed_string(TEXT); const RCSTR: \
::turbo_rcstr::RcStr = ::turbo_rcstr::from_static(&RCSTR_STORAGE); \
::turbo_rcstr::__rcstr_inventory_submit!( \
......
......@@ -7,49 +7,41 @@ use crate::{
is_atom_inlineable, tagged_value::TaggedValue,
};
pub enum Payload {
String(String),
Ref(&'static str),
}
impl Payload {
pub(crate) fn as_str(&self) -> &str {
match self {
Payload::String(s) => s,
Payload::Ref(s) => s,
}
}
pub(crate) fn into_string(self) -> String {
match self {
Payload::String(s) => s,
Payload::Ref(r) => r.to_string(),
}
}
}
impl PartialEq for Payload {
fn eq(&self, other: &Self) -> bool {
self.as_str() == other.as_str()
}
}
pub struct PrehashedString {
pub value: Payload,
/// Read-only header for atoms allocated in static storage by the `rcstr!`
/// macro. The value lives for `'static`, so we store a `&'static str` rather
/// than owning the bytes.
pub struct StaticPrehashedString {
pub value: &'static str,
/// This is not the actual `fxhash`, but rather it's a value that passed to
/// `write_u64` of [rustc_hash::FxHasher].
pub hash: u64,
}
pub unsafe fn cast(ptr: TaggedValue) -> *const PrehashedString {
ptr.get_ptr().cast()
/// Heap-owned header for atoms held in an [`Arc`]. `Box<str>` instead of
/// `String` because the contents are immutable — we save the `capacity` field
/// (and a tag/padding byte vs the old `Payload` enum) per atom.
///
/// TODO: collapse the two allocations (this header + the boxed bytes) into a
/// single [`triomphe::ThinArc`] so the hash and bytes share one allocation and
/// the inline pointer in [`crate::RcStr`] is one word. That change would make
/// `RcStr::from(String)` copy the bytes, which would invalidate the current
/// "cheap `String -> RcStr -> String`" property — worth a separate evaluation.
pub struct DynamicPrehashedString {
pub value: Box<str>,
pub hash: u64,
}
pub(crate) unsafe fn deref_static<'i>(ptr: TaggedValue) -> &'i StaticPrehashedString {
unsafe { &*(ptr.get_ptr() as *const StaticPrehashedString) }
}
pub(crate) unsafe fn deref_from<'i>(ptr: TaggedValue) -> &'i PrehashedString {
unsafe { &*cast(ptr) }
pub(crate) unsafe fn deref_dynamic<'i>(ptr: TaggedValue) -> &'i DynamicPrehashedString {
unsafe { &*(ptr.get_ptr() as *const DynamicPrehashedString) }
}
/// Caller should call `forget` (or `clone`) on the returned `Arc`
pub unsafe fn restore_arc(v: TaggedValue) -> Arc<PrehashedString> {
let ptr = v.get_ptr() as *const PrehashedString;
pub unsafe fn restore_arc(v: TaggedValue) -> Arc<DynamicPrehashedString> {
let ptr = v.get_ptr() as *const DynamicPrehashedString;
unsafe { Arc::from_raw(ptr) }
}
......@@ -70,20 +62,22 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr {
let hash = hash_bytes(text.as_bytes());
let prehashed = PrehashedString {
value: Payload::String(text.into()),
let prehashed = DynamicPrehashedString {
// NOTE: This will capture as a Box<str> which will essentially
// `shrink_to_fit` the bytes.
value: text.into(),
hash,
};
new_atom_from_prehashed(prehashed)
}
/// Construct a new dynamic RcStr from a PrehashedString
pub(crate) fn new_atom_from_prehashed(prehashed: PrehashedString) -> RcStr {
let entry: Arc<PrehashedString> = Arc::new(prehashed);
/// Construct a new dynamic RcStr from a DynamicPrehashedString
pub(crate) fn new_atom_from_prehashed(prehashed: DynamicPrehashedString) -> RcStr {
let entry: Arc<DynamicPrehashedString> = Arc::new(prehashed);
let mut entry = Arc::into_raw(entry);
debug_assert!(0 == entry as u8 & TAG_MASK);
entry = ((entry as usize) | DYNAMIC_TAG as usize) as *mut PrehashedString;
let ptr: NonNull<PrehashedString> = unsafe {
entry = ((entry as usize) | DYNAMIC_TAG as usize) as *mut DynamicPrehashedString;
let ptr: NonNull<DynamicPrehashedString> = unsafe {
// Safety: Arc::into_raw returns a non-null pointer
NonNull::new_unchecked(entry as *mut _)
};
......@@ -94,15 +88,15 @@ pub(crate) fn new_atom_from_prehashed(prehashed: PrehashedString) -> RcStr {
}
#[inline(always)]
pub(crate) const fn new_static_atom(string: &'static PrehashedString) -> RcStr {
let entry = string as *const PrehashedString;
pub(crate) const fn new_static_atom(string: &'static StaticPrehashedString) -> RcStr {
let entry = string as *const StaticPrehashedString;
const {
debug_assert!(align_of::<PrehashedString>() >= 4);
debug_assert!(align_of::<StaticPrehashedString>() >= 4);
// This must be 00, so that we don't have to remove the tag, which we can't since it would
// require pointer-integer casts in a const context.
debug_assert!(STATIC_TAG == 0b_00);
}
let ptr: NonNull<PrehashedString> = unsafe {
let ptr: NonNull<StaticPrehashedString> = unsafe {
// Safety: references always return a non-null pointers
NonNull::new_unchecked(entry as *mut _)
};
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment