diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c8..dbc7a3017c78 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,6 +2,36 @@ #include +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + __rust_helper void rust_helper_smp_mb(void) { smp_mb(); diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 0f4ae673256d..0e55e2a0fb37 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -7,12 +7,21 @@ use pin_init::Wrapper; -use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use crate::{ + prelude::*, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, + types::Opaque, // +}; use core::{ marker::PhantomData, ops::Deref, - ptr::drop_in_place, - sync::atomic::{AtomicBool, Ordering}, + ptr::drop_in_place, // }; /// An object that can become inaccessible at runtime. @@ -65,7 +74,7 @@ /// ``` #[pin_data(PinnedDrop)] pub struct Revocable { - is_available: AtomicBool, + is_available: AtomicFlag, #[pin] data: Opaque, } @@ -84,7 +93,7 @@ impl Revocable { /// Creates a new revocable instance of the given data. pub fn new(data: impl PinInit) -> impl PinInit { try_pin_init!(Self { - is_available: AtomicBool::new(true), + is_available: AtomicFlag::new(true), data <- Opaque::pin_init(data), }? E) } @@ -98,7 +107,7 @@ pub fn new(data: impl PinInit) -> impl PinInit { /// because another CPU may be waiting to complete the revocation of this object. pub fn try_access(&self) -> Option> { let guard = rcu::read_lock(); - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // Since `self.is_available` is true, data is initialised and has to remain valid // because the RCU read side lock prevents it from being dropped. Some(RevocableGuard::new(self.data.get(), guard)) @@ -116,7 +125,7 @@ pub fn try_access(&self) -> Option> { /// allowed to sleep because another CPU may be waiting to complete the revocation of this /// object. pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // SAFETY: Since `self.is_available` is true, data is initialised and has to remain // valid because the RCU read side lock prevents it from being dropped. Some(unsafe { &*self.data.get() }) @@ -157,12 +166,11 @@ pub unsafe fn access(&self) -> &T { /// /// Callers must ensure that there are no more concurrent users of the revocable object. unsafe fn revoke_internal(&self) -> bool { - let revoke = self.is_available.swap(false, Ordering::Relaxed); + let revoke = self.is_available.xchg(false, Relaxed); if revoke { if SYNC { - // SAFETY: Just an FFI call, there are no further requirements. - unsafe { bindings::synchronize_rcu() }; + rcu::synchronize_rcu(); } // SAFETY: We know `self.data` is valid because only one CPU can succeed the diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the following memory accesses. //! - It provides ordering between all the preceding memory accesses and all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..1180695d533a 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,38 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses across the barrier. @@ -19,43 +51,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } -/// A full memory barrier. +trait MemoryBarrier { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. -#[inline(always)] -pub fn smp_mb() { +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb(_: T) { + T::run() +} + +/// Memory barrier between CPUs. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; + T::run() } else { - barrier(); + barrier() } } -/// A write-write memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory write accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; - } else { - barrier(); - } -} - -/// A read-read memory barrier. +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory read accesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb>(_: T) { + T::run() } diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..5aa0ce9ba01b 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -8,7 +8,11 @@ bindings, fs::File, prelude::*, - sync::{CondVar, LockClassKey}, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // }; use core::{marker::PhantomData, ops::Deref}; @@ -99,8 +103,6 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; // Wait for epoll items to be properly removed. - // - // SAFETY: Just an FFI call. - unsafe { bindings::synchronize_rcu() }; + synchronize_rcu(); } } diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..d867240be736 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,19 @@ fn drop(&mut self) { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +}