Unverified Commit e1bb911c authored by Matt Mastracci's avatar Matt Mastracci Committed by GitHub

turbo-tasks: precompute trait-impl vtables at compile time (#93092)

## What

Remove another lazy lock from trait vtables.

---------
Co-authored-by: 's avatarLuke Sandberg <lukesandberg@users.noreply.github.com>
parent 16150ccf
...@@ -2007,19 +2007,20 @@ dependencies = [ ...@@ -2007,19 +2007,20 @@ dependencies = [
[[package]] [[package]]
name = "ctor" name = "ctor"
version = "0.8.0" version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "352d39c2f7bef1d6ad73db6f5160efcaed66d94ef8c6c573a8410c00bf909a98" checksum = "83cf0d42651b16c6dfe68685716d18480d18a9c39c62d76e8cf3eb6ed5d8bcbf"
dependencies = [ dependencies = [
"ctor-proc-macro", "ctor-proc-macro",
"dtor", "dtor",
"link-section",
] ]
[[package]] [[package]]
name = "ctor-proc-macro" name = "ctor-proc-macro"
version = "0.0.7" version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483"
[[package]] [[package]]
name = "ctrlc" name = "ctrlc"
...@@ -2398,18 +2399,18 @@ dependencies = [ ...@@ -2398,18 +2399,18 @@ dependencies = [
[[package]] [[package]]
name = "dtor" name = "dtor"
version = "0.3.0" version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1057d6c64987086ff8ed0fd3fbf377a6b7d205cc7715868cd401705f715cbe4" checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b"
dependencies = [ dependencies = [
"dtor-proc-macro", "dtor-proc-macro",
] ]
[[package]] [[package]]
name = "dtor-proc-macro" name = "dtor-proc-macro"
version = "0.0.6" version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" checksum = "2647271c92754afcb174e758003cfd1cbf1e43e5a7853d7b1813e63e19e39a73"
[[package]] [[package]]
name = "dunce" name = "dunce"
...@@ -4207,6 +4208,12 @@ dependencies = [ ...@@ -4207,6 +4208,12 @@ dependencies = [
"smallvec", "smallvec",
] ]
[[package]]
name = "link-section"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b685d66585d646efe09fec763d796c291049c8b6bf84e04954bffc8748341f0d"
[[package]] [[package]]
name = "linked-hash-map" name = "linked-hash-map"
version = "0.5.6" version = "0.5.6"
...@@ -9883,6 +9890,7 @@ dependencies = [ ...@@ -9883,6 +9890,7 @@ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"concurrent-queue", "concurrent-queue",
"ctor 0.10.1",
"dashmap 6.1.0", "dashmap 6.1.0",
"either", "either",
"erased-serde", "erased-serde",
...@@ -10105,7 +10113,7 @@ version = "0.1.0" ...@@ -10105,7 +10113,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bincode 2.0.1", "bincode 2.0.1",
"ctor 0.8.0", "ctor 0.10.1",
"serde", "serde",
"tokio", "tokio",
"trybuild", "trybuild",
......
...@@ -258,7 +258,7 @@ console-subscriber = "0.4.1" ...@@ -258,7 +258,7 @@ console-subscriber = "0.4.1"
const_format = "0.2.30" const_format = "0.2.30"
crc32fast = "1.5.0" crc32fast = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3.0" } criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
ctor = "0.8" ctor = "0.10"
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5.8"
crossbeam-utils = "0.8" crossbeam-utils = "0.8"
dashmap = "6.1.0" dashmap = "6.1.0"
......
use quote::ToTokens; use quote::ToTokens;
use syn::{GenericArgument, Ident, Path, PathArguments, Type, TypeParamBound, spanned::Spanned}; use syn::{GenericArgument, Ident, Path, PathArguments, Type, TypeParamBound, spanned::Spanned};
pub fn get_cast_to_fat_pointer_ident(trait_ident: &Ident, struct_ident: &Ident) -> Ident {
Ident::new(
&format!("_cast_to_fat_pointer_{struct_ident}_{trait_ident}"),
trait_ident.span(),
)
}
pub fn get_native_function_ident(ident: &Ident) -> Ident { pub fn get_native_function_ident(ident: &Ident) -> Ident {
Ident::new( Ident::new(
&format!("{}_FUNCTION", ident.to_string().to_uppercase()), &format!("{}_FUNCTION", ident.to_string().to_uppercase()),
...@@ -22,6 +15,21 @@ pub fn get_trait_type_ident(ident: &Ident) -> Ident { ...@@ -22,6 +15,21 @@ pub fn get_trait_type_ident(ident: &Ident) -> Ident {
) )
} }
/// Name of the per-trait `VTableRegistry` static, populated by `#[ctor::ctor]` functions emitted
/// by each `value_impl` expansion.
pub fn get_trait_vtable_registry_ident(ident: &Ident) -> Ident {
Ident::new(&format!("__TurboTasksVTableRegistry_{ident}"), ident.span())
}
/// Name of the `#[ctor::ctor]` function that registers a per-impl vtable into its trait's
/// registry. Unique per (concrete-type, trait) pair.
pub fn get_vtable_register_fn_ident(struct_ident: &Ident, trait_ident: &Ident) -> Ident {
Ident::new(
&format!("__turbo_tasks_vtable_register_{struct_ident}_{trait_ident}"),
trait_ident.span(),
)
}
pub fn get_inherent_impl_function_ident(ty_ident: &Ident, fn_ident: &Ident) -> Ident { pub fn get_inherent_impl_function_ident(ty_ident: &Ident, fn_ident: &Ident) -> Ident {
Ident::new( Ident::new(
&format!( &format!(
......
...@@ -16,8 +16,8 @@ use crate::{ ...@@ -16,8 +16,8 @@ use crate::{
}, },
global_name::{global_name_for_method, global_name_for_trait_method_impl}, global_name::{global_name_for_method, global_name_for_trait_method_impl},
ident::{ ident::{
get_cast_to_fat_pointer_ident, get_inherent_impl_function_ident, get_path_ident, get_inherent_impl_function_ident, get_path_ident, get_trait_impl_function_ident,
get_trait_impl_function_ident, get_type_ident, get_type_ident, get_vtable_register_fn_ident,
}, },
self_filter::is_self_used, self_filter::is_self_used,
}; };
...@@ -173,12 +173,10 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream { ...@@ -173,12 +173,10 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
items: &[ImplItem], items: &[ImplItem],
) -> TokenStream2 { ) -> TokenStream2 {
let trait_ident = get_path_ident(trait_path); let trait_ident = get_path_ident(trait_path);
let vtable_register_ident = get_vtable_register_fn_ident(ty_ident, &trait_ident);
let (impl_generics, _, where_clause) = generics.split_for_impl(); let (impl_generics, _, where_clause) = generics.split_for_impl();
let cast_to_fat_pointer_ident: Ident =
get_cast_to_fat_pointer_ident(&trait_ident, ty_ident);
let mut trait_methods = Vec::new(); let mut trait_methods = Vec::new();
let mut trait_functions = Vec::with_capacity(items.len()); let mut trait_functions = Vec::with_capacity(items.len());
let mut trait_items = Vec::new(); let mut trait_items = Vec::new();
...@@ -299,23 +297,34 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream { ...@@ -299,23 +297,34 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
value_type: <#ty as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::ValueType>>::DEF, value_type: <#ty as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::ValueType>>::DEF,
trait_type: <::std::boxed::Box<dyn #trait_path> as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::TraitType>>::DEF, trait_type: <::std::boxed::Box<dyn #trait_path> as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::TraitType>>::DEF,
methods: &METHODS, methods: &METHODS,
finalize_vtable_registry: || {
<::std::boxed::Box<dyn #trait_path> as turbo_tasks::VcValueTrait>::IMPL_VTABLES
.finalize();
},
} }
}} }}
// These can execute later so they can reference trait_types during registration // Queue this `impl Trait for Concrete` into the trait's `VTableRegistry` at program
// load. The ctor runs before `ValueType` ids are assigned, so it just appends to a
turbo_tasks::macro_helpers::inventory_submit!{ // `Vec`; the map keyed by `ValueTypeId` is built lazily inside the `VALUES`
turbo_tasks::macro_helpers::CollectableTraitCastFunctions( // `LazyLock` initializer (via `CollectableTraitMethods::finalize_vtable_registry`).
<::std::boxed::Box<dyn #trait_path> as turbo_tasks::VcValueTrait>::get_trait_type_id, // The vtable pointer is materialized at compile time via the null-fat-ptr trick, so
<#ty as turbo_tasks::VcValueType>::get_value_type_id, // there's no runtime `transmute` or indirect fn call.
#cast_to_fat_pointer_ident as *const () #[turbo_tasks::macro_helpers::ctor::ctor(
) crate_path = turbo_tasks::macro_helpers::ctor,
} )]
#[allow(non_snake_case)] #[allow(non_snake_case)]
fn #cast_to_fat_pointer_ident(raw: * const ()) -> *const dyn #trait_path { fn #vtable_register_ident() {
let typed = raw as *const #ty; <::std::boxed::Box<dyn #trait_path> as turbo_tasks::VcValueTrait>::IMPL_VTABLES
typed as *const dyn #trait_path .register(
<#ty as turbo_tasks::macro_helpers::RegistryDef::<turbo_tasks::ValueType>>::DEF,
{
let p: *const #ty = ::std::ptr::null();
// This attaches a fat pointer to the null pointer.
let fat: *const dyn #trait_path = p;
fat
},
);
} }
// NOTE(alexkirsz) We can't have a general `turbo_tasks::Upcast<Box<dyn Trait>> for T where T: Trait` because // NOTE(alexkirsz) We can't have a general `turbo_tasks::Upcast<Box<dyn Trait>> for T where T: Trait` because
......
...@@ -11,7 +11,10 @@ use crate::{ ...@@ -11,7 +11,10 @@ use crate::{
get_receiver_style, split_function_attributes, get_receiver_style, split_function_attributes,
}, },
global_name::{global_name_for_trait_method, global_name_for_type}, global_name::{global_name_for_trait_method, global_name_for_type},
ident::{get_trait_default_impl_function_ident, get_trait_type_ident}, ident::{
get_trait_default_impl_function_ident, get_trait_type_ident,
get_trait_vtable_registry_ident,
},
self_filter::is_self_used, self_filter::is_self_used,
value_trait_arguments::ValueTraitArguments, value_trait_arguments::ValueTraitArguments,
}; };
...@@ -67,6 +70,7 @@ pub fn value_trait(args: TokenStream, input: TokenStream) -> TokenStream { ...@@ -67,6 +70,7 @@ pub fn value_trait(args: TokenStream, input: TokenStream) -> TokenStream {
let supertraits = supertraits.iter().collect::<Vec<_>>(); let supertraits = supertraits.iter().collect::<Vec<_>>();
let trait_type_ident = get_trait_type_ident(trait_ident); let trait_type_ident = get_trait_type_ident(trait_ident);
let trait_vtable_registry_ident = get_trait_vtable_registry_ident(trait_ident);
let mut dynamic_trait_fns = Vec::new(); let mut dynamic_trait_fns = Vec::new();
let mut trait_methods: Vec<TokenStream2> = Vec::new(); let mut trait_methods: Vec<TokenStream2> = Vec::new();
let mut default_methods: Vec<TokenStream2> = Vec::new(); let mut default_methods: Vec<TokenStream2> = Vec::new();
...@@ -335,21 +339,26 @@ pub fn value_trait(args: TokenStream, input: TokenStream) -> TokenStream { ...@@ -335,21 +339,26 @@ pub fn value_trait(args: TokenStream, input: TokenStream) -> TokenStream {
const DEFAULTS: &[Option<&turbo_tasks::macro_helpers::NativeFunction>] = &[#(#default_methods),*]; const DEFAULTS: &[Option<&turbo_tasks::macro_helpers::NativeFunction>] = &[#(#default_methods),*];
} }
// Per-trait `VTableRegistry`, populated during the ctor phase by `value_impl`
// expansions. `const`-constructible, so no `LazyLock` / first-access init. Kept
// private — the only reference is from the `VcValueTrait::IMPL_VTABLES` assoc const
// below.
#[doc(hidden)]
#[allow(non_upper_case_globals)]
static #trait_vtable_registry_ident:
turbo_tasks::macro_helpers::VTableRegistry<dyn #trait_ident> =
turbo_tasks::macro_helpers::VTableRegistry::new();
#[automatically_derived] #[automatically_derived]
impl turbo_tasks::VcValueTrait for Box<dyn #trait_ident> { impl turbo_tasks::VcValueTrait for Box<dyn #trait_ident> {
type ValueTrait = dyn #trait_ident; type ValueTrait = dyn #trait_ident;
const IMPL_VTABLES: &'static turbo_tasks::macro_helpers::VTableRegistry<Self::ValueTrait> =
&#trait_vtable_registry_ident;
fn get_trait_type_id() -> turbo_tasks::TraitTypeId { fn get_trait_type_id() -> turbo_tasks::TraitTypeId {
turbo_tasks::registry::get_trait_type_id(&#trait_type_ident) turbo_tasks::registry::get_trait_type_id(&#trait_type_ident)
} }
// TODO: Remove this Lazy VTableRegistry once trait resolution is fully migrated
fn get_impl_vtables() -> &'static turbo_tasks::macro_helpers::VTableRegistry<Self::ValueTrait> {
static registry: ::std::sync::LazyLock<turbo_tasks::macro_helpers::VTableRegistry<dyn # trait_ident>> =
::std::sync::LazyLock::new(|| turbo_tasks::macro_helpers::VTableRegistry::new(turbo_tasks::registry::get_trait_type_id(&#trait_type_ident)));
&*registry
}
} }
#[automatically_derived] #[automatically_derived]
......
...@@ -28,6 +28,7 @@ async-trait = { workspace = true } ...@@ -28,6 +28,7 @@ async-trait = { workspace = true }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
bincode = { workspace = true } bincode = { workspace = true }
concurrent-queue = { workspace = true } concurrent-queue = { workspace = true }
ctor = { workspace = true }
dashmap = { workspace = true } dashmap = { workspace = true }
either = { workspace = true } either = { workspace = true }
erased-serde = { workspace = true } erased-serde = { workspace = true }
......
...@@ -137,12 +137,15 @@ fn get_item<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, id: T::Id) ...@@ -137,12 +137,15 @@ fn get_item<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, id: T::Id)
registry[*id as usize - 1] registry[*id as usize - 1]
} }
/// Get the ID for a registered item. Forces registry init if needed, which /// Read an item's assigned id directly, without touching `LazyLock`.
/// assigns IDs to all items as a side effect. ///
/// # Safety
///
/// Caller must guarantee that `init_registry` has already written the id for this `item`.
#[inline] #[inline]
fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'static T) -> T::Id { unsafe fn get_id_unchecked<T: Registerable>(item: &'static T) -> T::Id {
LazyLock::force(registry); // SAFETY: caller guarantees the id has been written. The write and this read are both
// SAFETY: The ID write happens-before this read thanks to the fence inside of LazyLock // inside the registry's single-threaded lazy init.
let n = unsafe { std::ptr::read(item.ty().id.get()) }; let n = unsafe { std::ptr::read(item.ty().id.get()) };
let Some(id) = NonZeroU16::new(n) else { let Some(id) = NonZeroU16::new(n) else {
panic!( panic!(
...@@ -154,6 +157,15 @@ fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'stati ...@@ -154,6 +157,15 @@ fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'stati
T::Id::from(id) T::Id::from(id)
} }
/// Get the ID for a registered item. Forces registry init if needed, which
/// assigns IDs to all items as a side effect.
#[inline]
fn get_id<T: Registerable>(registry: &LazyLock<Box<[&'static T]>>, item: &'static T) -> T::Id {
LazyLock::force(registry);
// SAFETY: The ID write happens-before this read thanks to the fence inside of LazyLock
unsafe { get_id_unchecked(item) }
}
/// Validate that an ID is within the valid range /// Validate that an ID is within the valid range
fn validate_id<T: Registerable>( fn validate_id<T: Registerable>(
registry: &LazyLock<Box<[&'static T]>>, registry: &LazyLock<Box<[&'static T]>>,
...@@ -209,6 +221,20 @@ pub fn get_value_type_id(value: &'static ValueType) -> ValueTypeId { ...@@ -209,6 +221,20 @@ pub fn get_value_type_id(value: &'static ValueType) -> ValueTypeId {
get_id(&VALUES, value) get_id(&VALUES, value)
} }
/// Read a `ValueType`'s assigned id directly, without touching `LazyLock`. See
/// [`get_id_unchecked`] for the safety contract.
///
/// # Safety
///
/// The only legitimate caller is `VTableRegistry::finalize`, which runs inside the `VALUES`
/// `LazyLock` initializer (via `register_all_trait_methods`), after `init_registry` has
/// assigned ids. Calling `get_value_type_id` from there would re-enter `LazyLock::force` and
/// deadlock.
#[inline]
pub(crate) unsafe fn get_value_type_id_unchecked(value: &'static ValueType) -> ValueTypeId {
unsafe { get_id_unchecked(value) }
}
#[inline] #[inline]
pub fn get_value_type(id: ValueTypeId) -> &'static ValueType { pub fn get_value_type(id: ValueTypeId) -> &'static ValueType {
get_item(&VALUES, id) get_item(&VALUES, id)
......
...@@ -60,7 +60,7 @@ where ...@@ -60,7 +60,7 @@ where
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
// This lookup will fail if the value type stored does not actually implement the trait, // This lookup will fail if the value type stored does not actually implement the trait,
// which implies a bug in either the registry code or the macro code. // which implies a bug in either the registry code or the macro code.
let downcast_ptr = <Box<U> as VcValueTrait>::get_impl_vtables().cast( let downcast_ptr = <Box<U> as VcValueTrait>::IMPL_VTABLES.cast(
self.shared_reference.type_id, self.shared_reference.type_id,
self.shared_reference.reference.0.as_ptr() as *const (), self.shared_reference.reference.0.as_ptr() as *const (),
); );
......
...@@ -280,7 +280,9 @@ pub(crate) fn register_all_trait_methods(_: &[&'static ValueType]) { ...@@ -280,7 +280,9 @@ pub(crate) fn register_all_trait_methods(_: &[&'static ValueType]) {
for entry in inventory::iter::<CollectableTraitMethods> { for entry in inventory::iter::<CollectableTraitMethods> {
entry entry
.value_type .value_type
.register_trait(entry.trait_type, entry.methods) .register_trait(entry.trait_type, entry.methods);
// Reigster all the rust vtables to support into_trait_ref calling stryles
(entry.finalize_vtable_registry)();
} }
} }
......
...@@ -123,14 +123,17 @@ pub unsafe trait VcValueType: ShrinkToFit + Sized + Send + Sync + 'static { ...@@ -123,14 +123,17 @@ pub unsafe trait VcValueType: ShrinkToFit + Sized + Send + Sync + 'static {
/// [`TraitRef`]: crate::TraitRef /// [`TraitRef`]: crate::TraitRef
pub trait VcValueTrait: NonLocalValue + Send + Sync + 'static { pub trait VcValueTrait: NonLocalValue + Send + Sync + 'static {
// The concrete type of the value_trait implementing VcValueTrait // The concrete type of the value_trait implementing VcValueTrait
type ValueTrait: ?Sized; type ValueTrait: ?Sized + std::ptr::Pointee<Metadata = std::ptr::DynMetadata<Self::ValueTrait>>;
/// The per-trait vtable registry, populated at program load by `#[ctor::ctor]`
/// functions emitted by each `#[turbo_tasks::value_impl]` expansion.
///
/// [`VTableRegistry::cast`] panics if the value type being cast doesn't implement this
/// trait.
const IMPL_VTABLES: &'static VTableRegistry<Self::ValueTrait>;
/// Returns the type id of the trait object. /// Returns the type id of the trait object.
fn get_trait_type_id() -> TraitTypeId; fn get_trait_type_id() -> TraitTypeId;
/// Returns the vtable for an implementation of this trait.
/// Panics if ValueTypeId does not implement the trait.
fn get_impl_vtables() -> &'static VTableRegistry<Self::ValueTrait>;
} }
/// Marker trait that indicates that a [`Vc<Self>`][crate::Vc] can be upcasted to a /// Marker trait that indicates that a [`Vc<Self>`][crate::Vc] can be upcasted to a
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment