mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-30 10:00:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'pin-init-v7.3' of https://github.com/Rust-for-Linux/linux into rust-next

Browse Source 
Pull pin-init updates from Gary Guo:
 "User-visible changes:

   - Merge the '__pinned_init' and '__init' methods and make 'Init'
     a marker trait.

   - Introduce public APIs 'raw_init' and 'raw_try_init' to prevent users
     from needing to invoke the internal '__pinned_init'/'__init' methods.

   - Emit errors for duplicate '#[pin]' attributes.

   - Link 'Zeroable::zeroed' and 'pin_init::zeroed' in documentation.

  Other changes:

   - Fix unwind safety issues.

   - Clean up lint 'allow' and 'expect's.

   - Overhaul '#[cfg]' handling to pave the way for tuple structs and
     self-referential structs.

   - Mark many functions as '#[inline]' for better codegen with '-C
     opt-level=s' ('CC_OPTIMIZE_FOR_SIZE')."

* tag 'pin-init-v7.3' of https://github.com/Rust-for-Linux/linux:
  rust: pin-init: add `#[inline]` to small functions
  rust: pin-init: remove `__pinned_init` method for `cfg(kernel)`
  rust: treewide: replace `__pinned_init` with `raw_[try_]init`
  rust: pin-init: add `raw_init` and `raw_try_init` and recommend over `__init`
  rust: pin-init: merge `__pinned_init` and `__init`
  rust: pin-init: examples: use `Wrapper::pin_init` instead of manual reimplementation
  rust: pin-init: mark `pin_init::zeroed` and `Zeroable::zeroed` as `#[inline]`
  rust: pin-init: docs: link `Zeroable::zeroed` and `pin_init::zeroed` in documentation
  rust: pin-init: internal: rework how `#[pin_data]` handles cfg
  rust: pin-init: make `[pin_]chain` unwind safe
  rust: pin-init: make `[pin_]init_array_from_fn` unwind safe
  rust: pin-init: internal: generate brace in macro for init code blocks
  rust: pin-init: internal: remove `allow` and `expect`s that don't fire
  rust: pin-init: remove redundant clippy expects in doc tests
  rust: pin-init: examples: fix incorrect drop
  rust: pin-init: internal: error on duplicate `#[pin]` attribute
This commit is contained in:
Miguel Ojeda
2026-08-09 18:54:08 +02:00
parent dc01dfb37b 1e26aea035
commit 5086dffd4b
18 changed files with 320 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/gpu/nova-core/gsp/cmdq.rs
View File

@@ -645,8 +645,8 @@ fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
// SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer
// fails.
unsafe {
msg_element.__init(core::ptr::from_mut(dst.header))?;
command.init().__init(core::ptr::from_mut(cmd))?;
pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?;
pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?;
}
// Fill the variable-length payload, which may be empty.

8
rust/kernel/alloc/kbox.rs
View File

@@ -372,13 +372,13 @@ pub fn pin_slice<Func, Item, E>(
// - `ptr` is a valid pointer to uninitialized memory.
// - `ptr` is not used if an error is returned.
// - `ptr` won't be moved until it is dropped, i.e. it is pinned.
unsafe { init(i).__pinned_init(ptr)? };
unsafe { pin_init::raw_try_init(ptr, init(i))? };
// SAFETY:
// - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to
// `with_capacity()` above.
// - The new value at index buffer.len() + 1 is the only element being added here, and
// it has been initialized above by `init(i).__pinned_init(ptr)`.
// it has been initialized above by `raw_try_init(ptr, i)`.
unsafe { buffer.inc_len(1) };
}
@@ -463,7 +463,7 @@ fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid.
unsafe { init.__init(slot)? };
unsafe { pin_init::raw_try_init(slot, init)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { Box::assume_init(self) })
}
@@ -473,7 +473,7 @@ fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Ini
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
unsafe { init.__pinned_init(slot)? };
unsafe { pin_init::raw_try_init(slot, init)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { Box::assume_init(self) }.into())
}

10
rust/kernel/dma.rs
View File

@@ -449,7 +449,7 @@ pub fn init_at<E>(&mut self, i: usize, init: impl Init<T, E>) -> Result
// - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on
// error cannot leave the element in an invalid state.
// - The DMA address has not been exposed yet, so there is no concurrent device access.
unsafe { init.__init(ptr)? };
unsafe { pin_init::raw_try_init(ptr, init)? };
Ok(())
}
@@ -791,10 +791,10 @@ pub fn init_with_attrs<E>(
// SAFETY:
// - `ptr` is valid, properly aligned, and points to exclusively owned memory.
// - If `__init` fails, `self` is dropped, which safely frees the underlying `Coherent`'s
// DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` requirements
// we are bypassing.
unsafe { init.__init(ptr)? };
// - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying
// `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop`
// requirements we are bypassing.
unsafe { pin_init::raw_try_init(ptr, init)? };
Ok(dmem)
}

2
rust/kernel/drm/device.rs
View File

@@ -244,7 +244,7 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<S
// SAFETY:
// - `raw_data` is a valid pointer to uninitialized memory.
// - `raw_data` will not move until it is dropped.
unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| {
unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| {
// SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the
// refcount must be non-zero.
unsafe { bindings::drm_dev_put(drm_dev) };

2
rust/kernel/drm/gpuvm/va.rs
View File

@@ -116,7 +116,7 @@ pub fn new(flags: AllocFlags) -> Result<GpuVaAlloc<T>, AllocError> {
pub(super) fn prepare(mut self, va_data: impl PinInit<T::VaData>) -> *mut bindings::drm_gpuva {
let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0);
// SAFETY: The `data` field is pinned.
let Ok(()) = unsafe { va_data.__pinned_init(&raw mut (*va_ptr).data) };
unsafe { pin_init::raw_init(&raw mut (*va_ptr).data, va_data) };
KBox::into_raw(self.0).cast()
}
}

2
rust/kernel/drm/gpuvm/vm_bo.rs
View File

@@ -181,7 +181,7 @@ pub(super) fn new(
};
let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
// SAFETY: `ptr->data` is a valid pinned location.
let Ok(()) = unsafe { value.__pinned_init(&raw mut (*raw_ptr).data) };
unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) };
// INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid
// as we just initialized it.
Ok(GpuVmBoAlloc(ptr))

6
rust/kernel/init.rs
View File

@@ -158,7 +158,9 @@ fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::Pi
{
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
pin_init_from_closure(|slot| {
pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e))
})
};
Self::try_pin_init(init, flags)
}
@@ -176,7 +178,7 @@ fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
{
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
init_from_closure(|slot| pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e)))
};
Self::try_init(init, flags)
}

2
rust/kernel/pwm.rs
View File

@@ -600,7 +600,7 @@ pub fn new<'a>(
let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) };
// SAFETY: We construct the `T` object in-place in the allocated private memory.
unsafe { data.__pinned_init(drvdata_ptr.cast()) }.inspect_err(|_| {
unsafe { pin_init::raw_try_init(drvdata_ptr.cast(), data) }.inspect_err(|_| {
// SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained
// from `pwmchip_alloc()`. We will not use pointer after this.
unsafe { bindings::pwmchip_put(c_chip_ptr) }

8
rust/kernel/sync/arc.rs
View File

@@ -717,7 +717,7 @@ fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid.
unsafe { init.__init(slot)? };
unsafe { pin_init::raw_try_init(slot, init)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() })
}
@@ -727,7 +727,7 @@ fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Ini
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
unsafe { init.__pinned_init(slot)? };
unsafe { pin_init::raw_try_init(slot, init)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() }.into())
}
@@ -795,7 +795,7 @@ pub unsafe fn assume_init(self) -> UniqueArc<T> {
#[inline]
pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
// SAFETY: The supplied pointer is valid for initialization.
match unsafe { init.__init(self.as_mut_ptr()) } {
match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } {
// SAFETY: Initialization completed successfully.
Ok(()) => Ok(unsafe { self.assume_init() }),
Err(err) => Err(err),
@@ -810,7 +810,7 @@ pub fn pin_init_with<E>(
) -> core::result::Result<Pin<UniqueArc<T>>, E> {
// SAFETY: The supplied pointer is valid for initialization and we will later pin the value
// to ensure it does not move.
match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } {
// SAFETY: Initialization completed successfully.
Ok(()) => Ok(unsafe { self.assume_init() }.into()),
Err(err) => Err(err),

8
rust/kernel/types.rs
View File

@@ -417,13 +417,13 @@ pub const fn cast_from(this: *const T) -> *const Self {
impl<T> Wrapper<T> for Opaque<T> {
/// Create an opaque pin-initializer from the given pin-initializer.
fn pin_init<E>(slot: impl PinInit<T, E>) -> impl PinInit<Self, E> {
Self::try_ffi_init(|ptr: *mut T| {
fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
Self::try_ffi_init(|slot: *mut T| {
// SAFETY:
// - `ptr` is a valid pointer to uninitialized memory,
// - `slot` is a valid pointer to uninitialized memory,
// - `slot` is not accessed on error,
// - `slot` is pinned in memory.
unsafe { PinInit::<T, E>::__pinned_init(slot, ptr) }
unsafe { pin_init::raw_try_init(slot, init) }
})
}
}

2
rust/macros/module.rs
View File

@@ -621,7 +621,7 @@ unsafe fn __init() -> ::kernel::ffi::c_int {
// SAFETY: No data race, since `__MOD` can only be accessed by this module
// and there only `__init` and `__exit` access it. These functions are only
// called once and `__exit` cannot be called before or during `__init`.
match unsafe { initer.__pinned_init(__MOD.as_mut_ptr()) } {
match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } {
Ok(m) => 0,
Err(e) => e.to_errno(),
}

11
rust/pin-init/examples/mutex.rs
View File

@@ -79,11 +79,7 @@ pub fn new(val: impl PinInit<T>) -> impl PinInit<Self> {
wait_list <- ListHead::new(),
spin_lock: SpinLock::new(),
locked: Cell::new(false),
data <- unsafe {
pin_init_from_closure(|slot: *mut UnsafeCell<T>| {
val.__pinned_init(slot.cast::<T>())
})
},
data <- UnsafeCell::pin_init(val),
})
}
@@ -91,7 +87,7 @@ pub fn new(val: impl PinInit<T>) -> impl PinInit<Self> {
pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> {
let mut sguard = self.spin_lock.acquire();
if self.locked.get() {
stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list));
stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list));
// println!("wait list length: {}", self.wait_list.size());
while self.locked.get() {
drop(sguard);
@@ -99,9 +95,6 @@ pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> {
thread::park();
sguard = self.spin_lock.acquire();
}
// This does have an effect, as the ListHead inside wait_entry implements Drop!
#[expect(clippy::drop_non_drop)]
drop(wait_entry);
}
self.locked.set(true);
unsafe {

10
rust/pin-init/examples/static_init.rs
View File

@@ -59,7 +59,7 @@ fn deref(&self) -> &Self::Target {
println!("doing init");
let ptr = self.cell.get().cast::<T>();
match self.init.take() {
Some(f) => unsafe { f.__pinned_init(ptr).unwrap() },
Some(f) => unsafe { pin_init::raw_init(ptr, f) },
None => unsafe { core::hint::unreachable_unchecked() },
}
self.present.set(true);
@@ -71,13 +71,11 @@ fn deref(&self) -> &Self::Target {
pub struct CountInit;
unsafe impl PinInit<CMutex<usize>> for CountInit {
unsafe fn __pinned_init(
self,
slot: *mut CMutex<usize>,
) -> Result<(), core::convert::Infallible> {
unsafe fn __init(self, slot: *mut CMutex<usize>) -> Result<(), core::convert::Infallible> {
let init = CMutex::new(0);
std::thread::sleep(std::time::Duration::from_millis(1000));
unsafe { init.__pinned_init(slot) }
unsafe { pin_init::raw_init(slot, init) };
Ok(())
}
}

8
rust/pin-init/internal/src/init.rs
View File

@@ -233,10 +233,12 @@ fn init_fields(
InitializerKind::Value { ident, .. } => ident,
InitializerKind::Init { ident, .. } => ident,
InitializerKind::Code { block, .. } => {
let stmt = &block.stmts;
res.extend(quote! {
#(#attrs)*
#[allow(unused_braces)]
#block
{
#(#stmt)*
}
});
continue;
}
@@ -334,7 +336,7 @@ fn make_field_check(
}),
};
quote! {
#[allow(unreachable_code, clippy::diverging_sub_expression)]
#[allow(unreachable_code)]
// We use unreachable code to perform field checks. They're still checked by the compiler.
// SAFETY: this code is never executed.
let _ = || unsafe {

94
rust/pin-init/internal/src/pin_data.rs
View File

@@ -1,13 +1,13 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
use proc_macro2::TokenStream;
use quote::{format_ident, quote};
use quote::{format_ident, quote, ToTokens};
use syn::{
parse::{End, Nothing, Parse},
parse_quote, parse_quote_spanned,
spanned::Spanned,
visit_mut::VisitMut,
Attribute, Field, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause,
Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause,
};
use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
@@ -35,10 +35,18 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
}
}
impl ToTokens for Args {
fn to_tokens(&self, tokens: &mut TokenStream) {
match self {
Self::Nothing(_) => (),
Self::PinnedDrop(kw) => kw.to_tokens(tokens),
}
}
}
struct FieldInfo<'a> {
field: &'a Field,
pinned: bool,
cfg_attrs: Vec<&'a Attribute>,
}
pub(crate) fn pin_data(
@@ -68,6 +76,55 @@ pub(crate) fn pin_data(
}
};
// Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all
// field cfgs first before continuing.
//
// We need to perform this after parsing so we can reliably detect field cfgs.
for (field_idx, field) in struct_.fields.iter_mut().enumerate() {
let cfg: Vec<_> = field
.attrs
.iter()
.filter(|a| a.path().is_ident("cfg"))
.map(|a| {
a.parse_args::<TokenStream>()
.expect("parse as token stream cannot fail")
})
.collect();
if cfg.is_empty() {
continue;
}
field.attrs.retain(|a| !a.path().is_ident("cfg"));
let cfg_true_struct = quote!(#struct_);
let punctuated = match &mut struct_.fields {
Fields::Named(fields) => &mut fields.named,
Fields::Unnamed(fields) => &mut fields.unnamed,
Fields::Unit => unreachable!(),
};
*punctuated = std::mem::take(punctuated)
.into_pairs()
.enumerate()
.filter(|&(i, _)| i != field_idx)
.map(|(_, p)| p)
.collect();
let cfg_false_struct = quote!(#struct_);
// Resolve one field at a time until we've got no more field cfgs.
//
// This is linear time because macro invocations with false cfg will not be expanded.
return Ok(quote!(
#[cfg(all(#(#cfg,)*))]
#[::pin_init::pin_data(#args)]
#cfg_true_struct
#[cfg(not(all(#(#cfg,)*)))]
#[::pin_init::pin_data(#args)]
#cfg_false_struct
));
}
// The generics might contain the `Self` type. Since this macro will define a new type with the
// same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed
// to this struct definition. Therefore we have to replace `Self` with the concrete name.
@@ -85,18 +142,19 @@ pub(crate) fn pin_data(
.map(|field| {
let len = field.attrs.len();
field.attrs.retain(|a| !a.path().is_ident("pin"));
let pinned = len != field.attrs.len();
let pinned_count = len - field.attrs.len();
if pinned_count > 1 {
dcx.error(&field, "#[pin] attribute specified more than once");
}
let cfg_attrs = field
.attrs
.iter()
.filter(|a| a.path().is_ident("cfg"))
.collect();
assert!(
!field.attrs.iter().any(|a| a.path().is_ident("cfg")),
"cfgs should be all resolved at this point"
);
FieldInfo {
field: &*field,
pinned,
cfg_attrs,
pinned: pinned_count != 0,
}
})
.collect();
@@ -182,9 +240,7 @@ fn generate_unpin_impl(
let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| {
let ident = f.field.ident.as_ref().unwrap();
let ty = &f.field.ty;
let cfg_attrs = &f.cfg_attrs;
quote!(
#(#cfg_attrs)*
#ident: #ty
)
});
@@ -242,7 +298,6 @@ fn drop(&mut self) {
// `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
// if it also implements `Drop`
trait MustNotImplDrop {}
#[expect(drop_bounds)]
impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {}
impl #impl_generics MustNotImplDrop for #ident #ty_generics
#whr
@@ -250,7 +305,6 @@ impl #impl_generics MustNotImplDrop for #ident #ty_generics
// We also take care to prevent users from writing a useless `PinnedDrop` implementation.
// They might implement `PinnedDrop` correctly for the struct, but forget to give
// `PinnedDrop` as the parameter to `#[pin_data]`.
#[expect(non_camel_case_types)]
trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized>
UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
@@ -279,7 +333,6 @@ fn generate_projections(
.iter()
.map(|field| {
let Field { vis, ident, ty, .. } = &field.field;
let cfg_attrs = &field.cfg_attrs;
let ident = ident
.as_ref()
@@ -287,11 +340,9 @@ fn generate_projections(
if field.pinned {
(
quote!(
#(#cfg_attrs)*
#vis #ident: ::core::pin::Pin<&'__pin mut #ty>,
),
quote!(
#(#cfg_attrs)*
// SAFETY: this field is structurally pinned.
#ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) },
),
@@ -299,11 +350,9 @@ fn generate_projections(
} else {
(
quote!(
#(#cfg_attrs)*
#vis #ident: &'__pin mut #ty,
),
quote!(
#(#cfg_attrs)*
#ident: &mut #this.#ident,
),
)
@@ -373,7 +422,6 @@ fn generate_the_pin_data(
.iter()
.map(|f| {
let Field { vis, ident, ty, .. } = f.field;
let cfg_attrs = &f.cfg_attrs;
let field_name = ident
.as_ref()
@@ -390,7 +438,6 @@ fn generate_the_pin_data(
/// - `(*slot).#field_name` is properly aligned.
/// - `(*slot).#field_name` points to uninitialized and exclusively accessed
/// memory.
#(#cfg_attrs)*
// Allow `non_snake_case` since the same warning will be emitted on
// the struct definition.
#[allow(non_snake_case)]
@@ -421,6 +468,7 @@ fn generate_the_pin_data(
impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics
#whr
{
#[inline]
fn clone(&self) -> Self { *self }
}
@@ -429,7 +477,6 @@ impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics
{}
#[allow(dead_code)] // Some functions might never be used and private.
#[expect(clippy::missing_safety_doc)]
impl #impl_generics __ThePinData #ty_generics
#whr
{
@@ -453,6 +500,7 @@ unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #
{
type PinData = __ThePinData #ty_generics;
#[inline]
unsafe fn __pin_data() -> Self::PinData {
__ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() }
}

13
rust/pin-init/src/__internal.rs
View File

@@ -105,6 +105,7 @@ pub unsafe trait HasInitData {
pub struct AllData<T: ?Sized>(PhantomInvariant<T>);
impl<T: ?Sized> Clone for AllData<T> {
#[inline]
fn clone(&self) -> Self {
*self
}
@@ -127,6 +128,7 @@ pub fn __make_closure<F, E>(self, f: F) -> F
unsafe impl<T: ?Sized> HasInitData for T {
type InitData = AllData<T>;
#[inline]
unsafe fn __init_data() -> Self::InitData {
AllData(PhantomInvariant::new())
}
@@ -181,7 +183,7 @@ pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mu
unsafe { this.value.assume_init_drop() };
}
// SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
unsafe { init.__init(this.value.as_mut_ptr())? };
// INVARIANT: `this.value` is initialized above.
this.is_init = true;
// SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
@@ -289,7 +291,7 @@ pub fn init<E>(self, init: impl PinInit<T, E>) -> Result<DropGuard<Pinned, T>, E
// - when `Err` is returned, we also propagate the error without touching `ptr`;
// also `self` is consumed so it cannot be touched further.
// - the drop guard will not hand out `&mut` (only `Pin<&mut T>`).
unsafe { init.__pinned_init(self.ptr)? };
unsafe { init.__init(self.ptr)? };
// SAFETY:
// - `self.ptr` is valid, properly aligned and pinned per type invariant.
@@ -385,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> {
impl<T: ?Sized> AlwaysFail<T> {
/// Creates a new initializer that always fails.
#[inline]
pub fn new() -> Self {
Self { _t: PhantomData }
}
}
impl<T: ?Sized> Default for AlwaysFail<T> {
#[inline]
fn default() -> Self {
Self::new()
}
}
// SAFETY: `__pinned_init` always fails, which is always okay.
// SAFETY: `__init` always fails, which is always okay.
unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> {
#[inline]
unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> {
Err(())
}
}

10
rust/pin-init/src/alloc.rs
View File

@@ -35,10 +35,11 @@ fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
/// type.
///
/// If `T: !Unpin` it will not be able to move afterwards.
#[inline]
fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> {
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
pin_init_from_closure(|slot| match init.__pinned_init(slot) {
pin_init_from_closure(|slot| match init.__init(slot) {
Ok(()) => Ok(()),
Err(i) => match i {},
})
@@ -52,6 +53,7 @@ fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
E: From<AllocError>;
/// Use the given initializer to in-place initialize a `T`.
#[inline]
fn init(init: impl Init<T>) -> Result<Self, AllocError> {
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
@@ -109,7 +111,7 @@ fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
let slot = slot.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
unsafe { init.__pinned_init(slot)? };
unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized and this is the only `Arc` to that data.
Ok(unsafe { Pin::new_unchecked(this.assume_init()) })
}
@@ -136,6 +138,7 @@ fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> {
type Initialized = Box<T>;
#[inline]
fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
@@ -145,11 +148,12 @@ fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E
Ok(unsafe { self.assume_init() })
}
#[inline]
fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
unsafe { init.__pinned_init(slot)? };
unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() }.into())
}

335
rust/pin-init/src/lib.rs
View File

@@ -70,7 +70,6 @@
//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
//!
//! ```rust
//! # #![expect(clippy::disallowed_names)]
//! # #![feature(allocator_api)]
//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
//! # use core::pin::Pin;
@@ -94,7 +93,6 @@
//! (or just the stack) to actually initialize a `Foo`:
//!
//! ```rust
//! # #![expect(clippy::disallowed_names)]
//! # #![feature(allocator_api)]
//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
//! # use core::{alloc::AllocError, pin::Pin};
@@ -456,7 +454,6 @@
/// # Examples
///
/// ```rust
/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
/// # use pin_init::*;
@@ -508,7 +505,6 @@ macro_rules! stack_pin_init {
/// # Examples
///
/// ```rust
/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/error.rs"] mod error; use error::Error;
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -535,7 +531,6 @@ macro_rules! stack_pin_init {
/// ```
///
/// ```rust
/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/error.rs"] mod error; use error::Error;
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -658,7 +653,6 @@ macro_rules! stack_try_pin_init {
/// Users of `Foo` can now create it like this:
///
/// ```rust
/// # #![expect(clippy::disallowed_names)]
/// # use pin_init::*;
/// # use core::pin::Pin;
/// # #[pin_data]
@@ -895,7 +889,7 @@ macro_rules! assert_pinned {
/// When implementing this trait you will need to take great care. Also there are probably very few
/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
///
/// The [`PinInit::__pinned_init`] function:
/// The [`PinInit::__init`] function:
/// - returns `Ok(())` if it initialized every field of `slot`,
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
/// - `slot` can be deallocated without UB occurring,
@@ -915,15 +909,33 @@ macro_rules! assert_pinned {
#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
#[must_use = "An initializer must be used in order to create its value."]
pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
/// Alias of [`PinInit::__init`].
///
/// New code should use `__init` instead.
///
/// # Safety
///
/// Same as `__init`.
#[inline(always)]
#[cfg(not(kernel))]
#[deprecated = "use `raw_try_init` instead"]
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: Per safety requirement.
unsafe { self.__init(slot) }
}
/// Initializes `slot`.
///
/// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`].
///
/// # Safety
///
/// - `slot` is a valid pointer to uninitialized memory.
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
/// deallocate.
/// - `slot` will not move until it is dropped, i.e. it will be pinned.
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
/// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved.
unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
/// First initializes the value using `self` then calls the function `f` with the initialized
/// value.
@@ -943,6 +955,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
/// Ok(())
/// });
/// ```
#[inline]
fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
where
F: FnOnce(Pin<&mut T>) -> Result<(), E>,
@@ -951,10 +964,38 @@ fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
}
}
/// Initializes `slot` with an initializer.
///
/// # Safety
///
/// - `slot` is a valid pointer to uninitialized memory.
/// - `slot` will not move until it is dropped, i.e. it will be pinned.
/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved.
#[inline(always)]
pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) {
// SAFETY: Per safety requirement.
unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) }
}
/// Fallibly initializes `slot` with an initializer.
///
/// # Safety
///
/// - `slot` is a valid pointer to uninitialized memory.
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
/// deallocate.
/// - `slot` will not move until it is dropped, i.e. it will be pinned.
/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved.
#[inline(always)]
pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> {
// SAFETY: Per safety requirement.
unsafe { init.__init(slot) }
}
/// An initializer returned by [`PinInit::pin_chain`].
pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
// SAFETY: The `__pinned_init` function is implemented such that it
// SAFETY: The `__init` function is implemented such that it
// - returns `Ok(())` on successful initialization,
// - returns `Err(err)` on error and in this case `slot` will be dropped.
// - considers `slot` pinned.
@@ -963,15 +1004,14 @@ unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
I: PinInit<T, E>,
F: FnOnce(Pin<&mut T>) -> Result<(), E>,
{
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: All requirements fulfilled since this function is `__pinned_init`.
unsafe { self.0.__pinned_init(slot)? };
// SAFETY: The above call initialized `slot` and we still have unique access.
let val = unsafe { &mut *slot };
// SAFETY: `slot` is considered pinned.
let val = unsafe { Pin::new_unchecked(val) };
// SAFETY: `slot` was initialized above.
(self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
#[inline]
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: All requirements fulfilled since this function is `__init`.
let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) };
let mut guard = slot.init(self.0)?;
(self.1)(guard.let_binding())?;
core::mem::forget(guard);
Ok(())
}
}
@@ -988,19 +1028,8 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
/// When implementing this trait you will need to take great care. Also there are probably very few
/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
///
/// The [`Init::__init`] function:
/// - returns `Ok(())` if it initialized every field of `slot`,
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
/// - `slot` can be deallocated without UB occurring,
/// - `slot` does not need to be dropped,
/// - `slot` is not partially initialized.
/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
///
/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
/// code as `__init`.
///
/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
/// move the pointee after initialization.
/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is
/// allowed to move the pointee after initialization.
///
#[cfg_attr(
kernel,
@@ -1014,15 +1043,6 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
#[must_use = "An initializer must be used in order to create its value."]
pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// Initializes `slot`.
///
/// # Safety
///
/// - `slot` is a valid pointer to uninitialized memory.
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
/// deallocate.
unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
/// First initializes the value using `self` then calls the function `f` with the initialized
/// value.
///
@@ -1031,7 +1051,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// # Examples
///
/// ```rust
/// # #![expect(clippy::disallowed_names)]
/// use pin_init::{init, init_zeroed, Init};
///
/// struct Foo {
@@ -1051,6 +1070,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// Ok(())
/// });
/// ```
#[inline]
fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
where
F: FnOnce(&mut T) -> Result<(), E>,
@@ -1062,62 +1082,55 @@ fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
/// An initializer returned by [`Init::chain`].
pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
// SAFETY: The `__init` function is implemented such that it
// - returns `Ok(())` on successful initialization,
// - returns `Err(err)` on error and in this case `slot` will be dropped.
// SAFETY: The `__init` function does not rely on the pinning requirement.
unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
where
I: Init<T, E>,
F: FnOnce(&mut T) -> Result<(), E>,
{
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: All requirements fulfilled since this function is `__init`.
unsafe { self.0.__pinned_init(slot)? };
// SAFETY: The above call initialized `slot` and we still have unique access.
(self.1)(unsafe { &mut *slot }).inspect_err(|_|
// SAFETY: `slot` was initialized above.
unsafe { core::ptr::drop_in_place(slot) })
}
}
// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
// SAFETY: The `__init` function is implemented such that it
// - returns `Ok(())` on successful initialization,
// - returns `Err(err)` on error and in this case `slot` will be dropped.
unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
where
I: Init<T, E>,
F: FnOnce(&mut T) -> Result<(), E>,
{
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
unsafe { self.__init(slot) }
#[inline]
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: All requirements fulfilled since this function is `__init`.
let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) };
let mut guard = slot.init(self.0)?;
(self.1)(guard.let_binding())?;
core::mem::forget(guard);
Ok(())
}
}
/// Implement `PinInit` and `Init` for closures.
///
/// It is unsafe to create this type, since the closure needs to fulfill the same safety
/// requirement as the `__pinned_init`/`__init` functions.
/// requirement as the `__init` functions.
struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>);
// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
// `__init` invariants.
unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T>
where
F: FnOnce(*mut T) -> Result<(), E>,
// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the
// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this
// implementation from being visible.
unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where
F: FnOnce(*mut T) -> Result<(), E>
{
#[inline]
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
(self.0)(slot)
}
}
// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
// `__pinned_init` invariants.
// `__init` invariants.
unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T>
where
F: FnOnce(*mut T) -> Result<(), E>,
{
#[inline]
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
(self.0)(slot)
}
}
@@ -1166,10 +1179,11 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
///
/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
/// pointer must result in a valid `U`.
#[inline]
pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> {
// SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
// requirements.
unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) }
unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) }
}
/// Changes the to be initialized type.
@@ -1178,6 +1192,7 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
///
/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
/// pointer must result in a valid `U`.
#[inline]
pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> {
// SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
// requirements.
@@ -1193,6 +1208,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
unsafe { init_from_closure(|_| Ok(())) }
}
/// Array initializer from element initializer.
struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>);
// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the
// elements that have been initialized so far are dropped, thus leaving the array uninitialized and
// ready to deallocate.
unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F>
where
F: FnMut(usize) -> I,
I: PinInit<T, E>,
{
unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> {
/// # Invariants
///
/// - `ptr[..num_init]` contains initialized elements of type `T`
/// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory
struct ArrayInitGuard<T> {
/// A pointer to the first element of the array.
ptr: *mut T,
/// The number of initialized elements in the array.
num_init: usize,
}
impl<T> Drop for ArrayInitGuard<T> {
#[inline]
fn drop(&mut self) {
// SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized.
unsafe {
core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut(
self.ptr,
self.num_init,
))
};
}
}
// INVARIANT: nothing is initialized yet.
let mut guard = ArrayInitGuard {
ptr: slot.cast::<T>(),
num_init: 0,
};
for i in 0..N {
// INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized
// thus far. This holds true for every `self.num_init = i`.
guard.num_init = i;
let init = (self.0)(i);
// SAFETY:
// - The subslot is derived from `slot` with a valid offset.
// - If `Err` is touched, the subslot is not touched further, the guard will drop
// previously initialized elements only.
// - `slot` is pinned so is the subslot.
unsafe { init.__init(&raw mut (*slot)[i]) }?;
}
// Dismiss the drop guard now that all elements are initialized.
core::mem::forget(guard);
Ok(())
}
}
// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the
// `__init` function that relies on `slot` being pinned.
unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F>
where
F: FnMut(usize) -> I,
I: Init<T, E>,
{
}
/// Initializes an array by initializing each element via the provided initializer.
///
/// # Examples
@@ -1203,32 +1289,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap();
/// assert_eq!(array.len(), 1_000);
/// ```
#[inline]
pub fn init_array_from_fn<I, const N: usize, T, E>(
mut make_init: impl FnMut(usize) -> I,
make_init: impl FnMut(usize) -> I,
) -> impl Init<[T; N], E>
where
I: Init<T, E>,
{
let init = move |slot: *mut [T; N]| {
let slot = slot.cast::<T>();
for i in 0..N {
let init = make_init(i);
// SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
let ptr = unsafe { slot.add(i) };
// SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
// requirements.
if let Err(e) = unsafe { init.__init(ptr) } {
// SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
// `Err` below, `slot` will be considered uninitialized memory.
unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
return Err(e);
}
}
Ok(())
};
// SAFETY: The initializer above initializes every element of the array. On failure it drops
// any initialized elements and returns `Err`.
unsafe { init_from_closure(init) }
ArrayInit(make_init, __internal::PhantomInvariant::new())
}
/// Initializes an array by initializing each element via the provided initializer.
@@ -1246,32 +1314,14 @@ pub fn init_array_from_fn<I, const N: usize, T, E>(
/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap();
/// assert_eq!(array.len(), 1_000);
/// ```
#[inline]
pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
mut make_init: impl FnMut(usize) -> I,
make_init: impl FnMut(usize) -> I,
) -> impl PinInit<[T; N], E>
where
I: PinInit<T, E>,
{
let init = move |slot: *mut [T; N]| {
let slot = slot.cast::<T>();
for i in 0..N {
let init = make_init(i);
// SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
let ptr = unsafe { slot.add(i) };
// SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
// requirements.
if let Err(e) = unsafe { init.__pinned_init(ptr) } {
// SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
// `Err` below, `slot` will be considered uninitialized memory.
unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
return Err(e);
}
}
Ok(())
};
// SAFETY: The initializer above initializes every element of the array. On failure it drops
// any initialized elements and returns `Err`.
unsafe { pin_init_from_closure(init) }
ArrayInit(make_init, __internal::PhantomInvariant::new())
}
/// Construct an initializer in a closure and run it.
@@ -1300,6 +1350,7 @@ pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the
/// initializer itself will fail with that error. If it returned `Ok`, then it will run the
/// initializer returned by the [`pin_init!`] invocation.
#[inline]
pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E>
where
F: FnOnce() -> Result<I, E>,
@@ -1307,13 +1358,13 @@ pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E>
{
// SAFETY:
// - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized,
// - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`.
// - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called
// from an initializer.
// - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`.
// - The safety requirements of `init.__init` are fulfilled, since it's being called from an
// initializer.
unsafe {
pin_init_from_closure(move |slot: *mut T| -> Result<(), E> {
let init = make_init()?;
init.__pinned_init(slot)
init.__init(slot)
})
}
}
@@ -1343,6 +1394,7 @@ pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E>
/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the
/// initializer itself will fail with that error. If it returned `Ok`, then it will run the
/// initializer returned by the [`init!`] invocation.
#[inline]
pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E>
where
F: FnOnce() -> Result<I, E>,
@@ -1361,8 +1413,13 @@ pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E>
}
}
// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`.
unsafe impl<T> Init<T> for T {
// SAFETY: The `__init` function does not rely on slot being pinned after it returns.
unsafe impl<T> Init<T> for T {}
// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of
// `slot`. Additionally, all pinning invariants of `T` are upheld.
unsafe impl<T> PinInit<T> for T {
#[inline]
unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self) };
@@ -1370,32 +1427,15 @@ unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
}
}
// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of
// `slot`. Additionally, all pinning invariants of `T` are upheld.
unsafe impl<T> PinInit<T> for T {
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self) };
Ok(())
}
}
// SAFETY: The `__init` function does not rely on slot being pinned after it returns.
unsafe impl<T, E> Init<T, E> for Result<T, E> {}
// SAFETY: when the `__init` function returns with
// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
// - `Err(err)`, slot was not written to.
unsafe impl<T, E> Init<T, E> for Result<T, E> {
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self?) };
Ok(())
}
}
// SAFETY: when the `__pinned_init` function returns with
// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
// - `Err(err)`, slot was not written to.
unsafe impl<T, E> PinInit<T, E> for Result<T, E> {
unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
#[inline]
unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self?) };
Ok(())
@@ -1421,6 +1461,7 @@ pub trait InPlaceWrite<T> {
impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
type Initialized = &'static mut T;
#[inline]
fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
@@ -1431,6 +1472,7 @@ fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
unsafe { Ok(self.assume_init_mut()) }
}
#[inline]
fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
@@ -1438,7 +1480,7 @@ fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initial
//
// The `'static` borrow guarantees the data will not be
// moved/invalidated until it gets dropped (which is never).
unsafe { init.__pinned_init(slot)? };
unsafe { init.__init(slot)? };
// SAFETY: The above call initialized the memory.
Ok(Pin::static_mut(unsafe { self.assume_init_mut() }))
@@ -1510,10 +1552,13 @@ fn init_zeroed() -> impl Init<Self>
/// Whenever a type implements [`Zeroable`], this function should be preferred over
/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
///
/// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead
/// when initialization is required in a `const` context.
///
/// # Examples
///
/// ```
/// use pin_init::{Zeroable, zeroed};
/// use pin_init::Zeroable;
///
/// #[derive(Zeroable)]
/// struct Point {
@@ -1521,10 +1566,11 @@ fn init_zeroed() -> impl Init<Self>
/// y: u32,
/// }
///
/// let point: Point = zeroed();
/// let point: Point = Zeroable::zeroed();
/// assert_eq!(point.x, 0);
/// assert_eq!(point.y, 0);
/// ```
#[inline]
fn zeroed() -> Self
where
Self: Sized,
@@ -1553,6 +1599,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
/// Whenever a type implements [`Zeroable`], this function should be preferred over
/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
///
/// While const traits remain unstable, this function serves as the `const` version of
/// [`Zeroable::zeroed()`].
///
/// # Examples
///
/// ```
@@ -1568,6 +1617,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
/// assert_eq!(point.x, 0);
/// assert_eq!(point.y, 0);
/// ```
#[inline]
pub const fn zeroed<T: Zeroable>() -> T {
// SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`.
unsafe { core::mem::zeroed() }
@@ -1728,6 +1778,7 @@ pub trait Wrapper<T> {
}
impl<T> Wrapper<T> for UnsafeCell<T> {
#[inline]
fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `UnsafeCell<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(value_init) }
@@ -1735,6 +1786,7 @@ fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
}
impl<T> Wrapper<T> for MaybeUninit<T> {
#[inline]
fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `MaybeUninit<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(value_init) }
@@ -1743,6 +1795,7 @@ fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
#[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))]
impl<T> Wrapper<T> for core::pin::UnsafePinned<T> {
#[inline]
fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `UnsafePinned<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(init) }