diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 3c9d9c7364e2..ff194d27565e 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -468,6 +468,7 @@ fn generate_the_pin_data( impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics #whr { + #[inline] fn clone(&self) -> Self { *self } } @@ -499,6 +500,7 @@ unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name # { type PinData = __ThePinData #ty_generics; + #[inline] unsafe fn __pin_data() -> Self::PinData { __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index ae9a0e68cd75..8e9fd18b993f 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -105,6 +105,7 @@ pub unsafe trait HasInitData { pub struct AllData(PhantomInvariant); impl Clone for AllData { + #[inline] fn clone(&self) -> Self { *self } @@ -127,6 +128,7 @@ pub fn __make_closure(self, f: F) -> F unsafe impl HasInitData for T { type InitData = AllData; + #[inline] unsafe fn __init_data() -> Self::InitData { AllData(PhantomInvariant::new()) } @@ -385,12 +387,14 @@ pub struct AlwaysFail { impl AlwaysFail { /// Creates a new initializer that always fails. + #[inline] pub fn new() -> Self { Self { _t: PhantomData } } } impl Default for AlwaysFail { + #[inline] fn default() -> Self { Self::new() } @@ -398,6 +402,7 @@ fn default() -> Self { // SAFETY: `__init` always fails, which is always okay. unsafe impl PinInit for AlwaysFail { + #[inline] unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { Err(()) } diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs index 641f4c7ce890..471652e8663a 100644 --- a/rust/pin-init/src/alloc.rs +++ b/rust/pin-init/src/alloc.rs @@ -35,6 +35,7 @@ fn try_pin_init(init: impl PinInit) -> Result, E> /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init(init: impl PinInit) -> Result, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { @@ -52,6 +53,7 @@ fn try_init(init: impl Init) -> Result E: From; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init(init: impl Init) -> Result { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { @@ -136,6 +138,7 @@ fn try_init(init: impl Init) -> Result impl InPlaceWrite for Box> { type Initialized = Box; + #[inline] fn write_init(mut self, init: impl Init) -> Result { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -145,6 +148,7 @@ fn write_init(mut self, init: impl Init) -> Result(mut self, init: impl PinInit) -> Result, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 6e9eb90db52c..7600cdbbbf98 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -955,6 +955,7 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { /// Ok(()) /// }); /// ``` + #[inline] fn pin_chain(self, f: F) -> ChainPinInit where F: FnOnce(Pin<&mut T>) -> Result<(), E>, @@ -1003,6 +1004,7 @@ unsafe impl PinInit for ChainPinInit I: PinInit, F: FnOnce(Pin<&mut T>) -> Result<(), E>, { + #[inline] unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: All requirements fulfilled since this function is `__init`. let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; @@ -1068,6 +1070,7 @@ pub unsafe trait Init: PinInit { /// Ok(()) /// }); /// ``` + #[inline] fn chain(self, f: F) -> ChainInit where F: FnOnce(&mut T) -> Result<(), E>, @@ -1095,6 +1098,7 @@ unsafe impl PinInit for ChainInit I: Init, F: FnOnce(&mut T) -> Result<(), E>, { + #[inline] unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: All requirements fulfilled since this function is `__init`. let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; @@ -1175,6 +1179,7 @@ unsafe fn __init(self, slot: *mut T) -> Result<(), E> { /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl PinInit { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. @@ -1187,6 +1192,7 @@ unsafe fn __init(self, slot: *mut T) -> Result<(), E> { /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_init(init: impl Init) -> impl Init { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. @@ -1283,6 +1289,7 @@ fn drop(&mut self) { /// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn init_array_from_fn( make_init: impl FnMut(usize) -> I, ) -> impl Init<[T; N], E> @@ -1307,6 +1314,7 @@ pub fn init_array_from_fn( /// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn pin_init_array_from_fn( make_init: impl FnMut(usize) -> I, ) -> impl PinInit<[T; N], E> @@ -1342,6 +1350,7 @@ pub fn pin_init_array_from_fn( /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`pin_init!`] invocation. +#[inline] pub fn pin_init_scope(make_init: F) -> impl PinInit where F: FnOnce() -> Result, @@ -1385,6 +1394,7 @@ pub fn pin_init_scope(make_init: F) -> impl PinInit /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`init!`] invocation. +#[inline] pub fn init_scope(make_init: F) -> impl Init where F: FnOnce() -> Result, @@ -1409,6 +1419,7 @@ unsafe impl Init for T {} // SAFETY: the `__init` function always returns `Ok(())` and initializes every field of // `slot`. Additionally, all pinning invariants of `T` are upheld. unsafe impl PinInit for T { + #[inline] unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self) }; @@ -1423,6 +1434,7 @@ unsafe impl Init for Result {} // - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. // - `Err(err)`, slot was not written to. unsafe impl PinInit for Result { + #[inline] unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self?) }; @@ -1449,6 +1461,7 @@ pub trait InPlaceWrite { impl InPlaceWrite for &'static mut MaybeUninit { type Initialized = &'static mut T; + #[inline] fn write_init(self, init: impl Init) -> Result { let slot = self.as_mut_ptr(); @@ -1459,6 +1472,7 @@ fn write_init(self, init: impl Init) -> Result { unsafe { Ok(self.assume_init_mut()) } } + #[inline] fn write_pin_init(self, init: impl PinInit) -> Result, E> { let slot = self.as_mut_ptr(); @@ -1764,6 +1778,7 @@ pub trait Wrapper { } impl Wrapper for UnsafeCell { + #[inline] fn pin_init(value_init: impl PinInit) -> impl PinInit { // SAFETY: `UnsafeCell` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1771,6 +1786,7 @@ fn pin_init(value_init: impl PinInit) -> impl PinInit { } impl Wrapper for MaybeUninit { + #[inline] fn pin_init(value_init: impl PinInit) -> impl PinInit { // SAFETY: `MaybeUninit` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1779,6 +1795,7 @@ fn pin_init(value_init: impl PinInit) -> impl PinInit { #[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] impl Wrapper for core::pin::UnsafePinned { + #[inline] fn pin_init(init: impl PinInit) -> impl PinInit { // SAFETY: `UnsafePinned` has a compatible layout to `T`. unsafe { cast_pin_init(init) }