Unverified Commit e1bb911c authored by Matt Mastracci's avatar Matt Mastracci Committed by GitHub

turbo-tasks: precompute trait-impl vtables at compile time (#93092)

## What

Remove another lazy lock from trait vtables.

---------
Co-authored-by: 's avatarLuke Sandberg <lukesandberg@users.noreply.github.com>
parent 16150ccf
......@@ -2007,19 +2007,20 @@ dependencies = [
[[package]]
name = "ctor"
version = "0.8.0"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "352d39c2f7bef1d6ad73db6f5160efcaed66d94ef8c6c573a8410c00bf909a98"
checksum = "83cf0d42651b16c6dfe68685716d18480d18a9c39c62d76e8cf3eb6ed5d8bcbf"
dependencies = [
"ctor-proc-macro",
"dtor",
"link-section",
]
[[package]]
name = "ctor-proc-macro"
version = "0.0.7"
version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483"
[[package]]
name = "ctrlc"
......@@ -2398,18 +2399,18 @@ dependencies = [
[[package]]
name = "dtor"
version = "0.3.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1057d6c64987086ff8ed0fd3fbf377a6b7d205cc7715868cd401705f715cbe4"
checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b"
dependencies = [
"dtor-proc-macro",
]
[[package]]
name = "dtor-proc-macro"
version = "0.0.6"
version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5"
checksum = "2647271c92754afcb174e758003cfd1cbf1e43e5a7853d7b1813e63e19e39a73"
[[package]]
name = "dunce"
......@@ -4207,6 +4208,12 @@ dependencies = [
"smallvec",
]
[[package]]
name = "link-section"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b685d66585d646efe09fec763d796c291049c8b6bf84e04954bffc8748341f0d"
[[package]]
name = "linked-hash-map"
version = "0.5.6"
......@@ -9883,6 +9890,7 @@ dependencies = [
"bincode 2.0.1",
"codspeed-criterion-compat",
"concurrent-queue",
"ctor 0.10.1",
"dashmap 6.1.0",
"either",
"erased-serde",
......@@ -10105,7 +10113,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bincode 2.0.1",
"ctor 0.8.0",
"ctor 0.10.1",
"serde",
"tokio",
"trybuild",
......
......@@ -258,7 +258,7 @@ console-subscriber = "0.4.1"
const_format = "0.2.30"
crc32fast = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
ctor = "0.8"
ctor = "0.10"
crossbeam-channel = "0.5.8"
crossbeam-utils = "0.8"
dashmap = "6.1.0"
......
use quote::ToTokens;
use syn::{GenericArgument, Ident, Path, PathArguments, Type, TypeParamBound, spanned::Spanned};
pub fn get_cast_to_fat_pointer_ident(trait_ident: &Ident, struct_ident: &Ident) -> Ident {
Ident::new(
&format!("_cast_to_fat_pointer_{struct_ident}_{trait_ident}"),
trait_ident.span(),
)
}
pub fn get_native_function_ident(ident: &Ident) -> Ident {
Ident::new(
&format!("{}_FUNCTION", ident.to_string().to_uppercase()),
......@@ -22,6 +15,21 @@ pub fn get_trait_type_ident(ident: &Ident) -> Ident {
)
}
/// Name of the per-trait `VTableRegistry` static, populated by `#[ctor::ctor]` functions emitted
/// by each `value_impl` expansion.
pub fn get_trait_vtable_registry_ident(ident: &Ident) -> Ident {
Ident::new(&format!("__TurboTasksVTableRegistry_{ident}"), ident.span())
}
/// Name of the `#[ctor::ctor]` function that registers a per-impl vtable into its trait's
/// registry. Unique per (concrete-type, trait) pair.
pub fn get_vtable_register_fn_ident(struct_ident: &Ident, trait_ident: &Ident) -> Ident {
Ident::new(
&format!("__turbo_tasks_vtable_register_{struct_ident}_{trait_ident}"),
trait_ident.span(),
)
}
pub fn get_inherent_impl_function_ident(ty_ident: &Ident, fn_ident: &Ident) -> Ident {
Ident::new(
&format!(
......
......@@ -16,8 +16,8 @@ use crate::{
},
global_name::{global_name_for_method, global_name_for_trait_method_impl},
ident::{
get_cast_to_fat_pointer_ident, get_inherent_impl_function_ident, get_path_ident,
get_trait_impl_function_ident, get_type_ident,
get_inherent_impl_function_ident, get_path_ident, get_trait_impl_function_ident,
get_type_ident, get_vtable_register_fn_ident,
},
self_filter::is_self_used,
};
......@@ -173,12 +173,10 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
items: &[ImplItem],
) -> TokenStream2 {
let trait_ident = get_path_ident(trait_path);
let vtable_register_ident = get_vtable_register_fn_ident(ty_ident, &trait_ident);
let (impl_generics, _, where_clause) = generics.split_for_impl();
let cast_to_fat_pointer_ident: Ident =
get_cast_to_fat_pointer_ident(&trait_ident, ty_ident);
let mut trait_methods = Vec::new();
let mut trait_functions = Vec::with_capacity(items.len());
let mut trait_items = Vec::new();
......@@ -299,23 +297,34 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
value_type: <#ty as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::ValueType>>::DEF,
trait_type: <::std::boxed::Box<dyn #trait_path> as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::TraitType>>::DEF,
methods: &METHODS,
finalize_vtable_registry: || {
<::std::boxed::Box<dyn #trait_path> as turbo_tasks::VcValueTrait>::IMPL_VTABLES
.finalize();
},
}
}}
// These can execute later so they can reference trait_types during registration
turbo_tasks::macro_helpers::inventory_submit!{
turbo_tasks::macro_helpers::CollectableTraitCastFunctions(
<::std::boxed::Box<dyn #trait_path> as turbo_tasks::VcValueTrait>::get_trait_type_id,
<#ty as turbo_tasks::VcValueType>::get_value_type_id,
#cast_to_fat_pointer_ident as *const ()
)
}
// Queue this `impl Trait for Concrete` into the trait's `VTableRegistry` at program
// load. The ctor runs before `ValueType` ids are assigned, so it just appends to a
// `Vec`; the map keyed by `ValueTypeId` is built lazily inside the `VALUES`
// `LazyLock` initializer (via `CollectableTraitMethods::finalize_vtable_registry`).
// The vtable pointer is materialized at compile time via the null-fat-ptr trick, so
// there's no runtime `transmute` or indirect fn call.
#[turbo_tasks::macro_helpers::ctor::ctor(
crate_path = turbo_tasks::macro_helpers::ctor,
)]
#[allow(non_snake_case)]
fn #cast_to_fat_pointer_ident(raw: * const ()) -> *const dyn #trait_path {
let typed = raw as *const #ty;
typed as *const dyn #trait_path
fn #vtable_register_ident() {
<::std::boxed::Box<dyn #trait_path> as turbo_tasks::VcValueTrait>::IMPL_VTABLES
.register(
<#ty as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::ValueType>>::DEF,
{
let p: *const #ty = ::std::ptr::null();
// This attaches a fat pointer to the null pointer.
let fat: *const dyn #trait_path = p;
fat
},
);
}
// NOTE(alexkirsz) We can't have a general `turbo_tasks::Upcast<Box<dyn Trait>> for T where T: Trait` because
......
......@@ -11,7 +11,10 @@ use crate::{
get_receiver_style, split_function_attributes,
},
global_name::{global_name_for_trait_method, global_name_for_type},
ident::{get_trait_default_impl_function_ident, get_trait_type_ident},
ident::{
get_trait_default_impl_function_ident, get_trait_type_ident,
get_trait_vtable_registry_ident,
},
self_filter::is_self_used,
value_trait_arguments::ValueTraitArguments,
};
......@@ -67,6 +70,7 @@ pub fn value_trait(args: TokenStream, input: TokenStream) -> TokenStream {
let supertraits = supertraits.iter().collect::<Vec<_>>();
let trait_type_ident = get_trait_type_ident(trait_ident);
let trait_vtable_registry_ident = get_trait_vtable_registry_ident(trait_ident);
let mut dynamic_trait_fns = Vec::new();
let mut trait_methods: Vec<TokenStream2> = Vec::new();
let mut default_methods: Vec<TokenStream2> = Vec::new();
......@@ -335,21 +339,26 @@ pub fn value_trait(args: TokenStream, input: TokenStream) -> TokenStream {
const DEFAULTS: &[Option<&turbo_tasks::macro_helpers::NativeFunction>] = &[#(#default_methods),*];
}
// Per-trait `VTableRegistry`, populated during the ctor phase by `value_impl`
// expansions. `const`-constructible, so no `LazyLock` / first-access init. Kept
// private — the only reference is from the `VcValueTrait::IMPL_VTABLES` assoc const
// below.
#[doc(hidden)]
#[allow(non_upper_case_globals)]
static #trait_vtable_registry_ident:
turbo_tasks::macro_helpers::VTableRegistry<dyn #trait_ident> =
turbo_tasks::macro_helpers::VTableRegistry::new();
#[automatically_derived]
impl turbo_tasks::VcValueTrait for Box<dyn #trait_ident> {
type ValueTrait = dyn #trait_ident;
const IMPL_VTABLES: &'static turbo_tasks::macro_helpers::VTableRegistry<Self::ValueTrait> =
&#trait_vtable_registry_ident;
fn get_trait_type_id() -> turbo_tasks::TraitTypeId {
turbo_tasks::registry::get_trait_type_id(&#trait_type_ident)
}
// TODO: Remove this Lazy VTableRegistry once trait resolution is fully migrated
fn get_impl_vtables() -> &'static turbo_tasks::macro_helpers::VTableRegistry<Self::ValueTrait> {
static registry: ::std::sync::LazyLock<turbo_tasks::macro_helpers::VTableRegistry<dyn # trait_ident>> =
::std::sync::LazyLock::new(|| turbo_tasks::macro_helpers::VTableRegistry::new(turbo_tasks::registry::get_trait_type_id(&#trait_type_ident)));
&*registry
}
}
#[automatically_derived]
......
......@@ -28,6 +28,7 @@ async-trait = { workspace = true }
auto-hash-map = { workspace = true }
bincode = { workspace = true }
concurrent-queue = { workspace = true }
ctor = { workspace = true }
dashmap = { workspace = true }
either = { workspace = true }
erased-serde = { workspace = true }
......
......@@ -137,12 +137,15 @@ fn get_item<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, id: T::Id)
registry[*id as usize - 1]
}
/// Get the ID for a registered item. Forces registry init if needed, which
/// assigns IDs to all items as a side effect.
/// Read an item's assigned id directly, without touching `LazyLock`.
///
/// # Safety
///
/// Caller must guarantee that `init_registry` has already written the id for this `item`.
#[inline]
fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'static T) -> T::Id {
LazyLock::force(registry);
// SAFETY: The ID write happens-before this read thanks to the fence inside of LazyLock
unsafe fn get_id_unchecked<T: Registerable>(item: &'static T) -> T::Id {
// SAFETY: caller guarantees the id has been written. The write and this read are both
// inside the registry's single-threaded lazy init.
let n = unsafe { std::ptr::read(item.ty().id.get()) };
let Some(id) = NonZeroU16::new(n) else {
panic!(
......@@ -154,6 +157,15 @@ fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'stati
T::Id::from(id)
}
/// Get the ID for a registered item. Forces registry init if needed, which
/// assigns IDs to all items as a side effect.
#[inline]
fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'static T) -> T::Id {
LazyLock::force(registry);
// SAFETY: The ID write happens-before this read thanks to the fence inside of LazyLock
unsafe { get_id_unchecked(item) }
}
/// Validate that an ID is within the valid range
fn validate_id<T: Registerable>(
registry: &LazyLock<Box<[&'static T]>>,
......@@ -209,6 +221,20 @@ pub fn get_value_type_id(value: &'static ValueType) -> ValueTypeId {
get_id(&VALUES, value)
}
/// Read a `ValueType`'s assigned id directly, without touching `LazyLock`. See
/// [`get_id_unchecked`] for the safety contract.
///
/// # Safety
///
/// The only legitimate caller is `VTableRegistry::finalize`, which runs inside the `VALUES`
/// `LazyLock` initializer (via `register_all_trait_methods`), after `init_registry` has
/// assigned ids. Calling `get_value_type_id` from there would re-enter `LazyLock::force` and
/// deadlock.
#[inline]
pub(crate) unsafe fn get_value_type_id_unchecked(value: &'static ValueType) -> ValueTypeId {
unsafe { get_id_unchecked(value) }
}
#[inline]
pub fn get_value_type(id: ValueTypeId) -> &'static ValueType {
get_item(&VALUES, id)
......
......@@ -60,7 +60,7 @@ where
fn deref(&self) -> &Self::Target {
// This lookup will fail if the value type stored does not actually implement the trait,
// which implies a bug in either the registry code or the macro code.
let downcast_ptr = <Box<U> as VcValueTrait>::get_impl_vtables().cast(
let downcast_ptr = <Box<U> as VcValueTrait>::IMPL_VTABLES.cast(
self.shared_reference.type_id,
self.shared_reference.reference.0.as_ptr() as *const (),
);
......
......@@ -280,7 +280,9 @@ pub(crate) fn register_all_trait_methods(_: &[&'static ValueType]) {
for entry in inventory::iter::<CollectableTraitMethods> {
entry
.value_type
.register_trait(entry.trait_type, entry.methods)
.register_trait(entry.trait_type, entry.methods);
// Reigster all the rust vtables to support into_trait_ref calling stryles
(entry.finalize_vtable_registry)();
}
}
......
......@@ -123,14 +123,17 @@ pub unsafe trait VcValueType: ShrinkToFit + Sized + Send + Sync + 'static {
/// [`TraitRef`]: crate::TraitRef
pub trait VcValueTrait: NonLocalValue + Send + Sync + 'static {
// The concrete type of the value_trait implementing VcValueTrait
type ValueTrait: ?Sized;
type ValueTrait: ?Sized + std::ptr::Pointee<Metadata = std::ptr::DynMetadata<Self::ValueTrait>>;
/// The per-trait vtable registry, populated at program load by `#[ctor::ctor]`
/// functions emitted by each `#[turbo_tasks::value_impl]` expansion.
///
/// [`VTableRegistry::cast`] panics if the value type being cast doesn't implement this
/// trait.
const IMPL_VTABLES: &'static VTableRegistry<Self::ValueTrait>;
/// Returns the type id of the trait object.
fn get_trait_type_id() -> TraitTypeId;
/// Returns the vtable for an implementation of this trait.
/// Panics if ValueTypeId does not implement the trait.
fn get_impl_vtables() -> &'static VTableRegistry<Self::ValueTrait>;
}
/// Marker trait that indicates that a [`Vc<Self>`][crate::Vc] can be upcasted to a
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment