From 1fb92e0625596985b3941925afe7906fef41214b Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 24 Jun 2026 17:07:02 +0200 Subject: [PATCH 1/6] rust: sync: Add abstraction for synchronize_rcu() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit synchronize_rcu() is a frequently used C function which is always safe to be called. Add a safe abstraction for synchronize_rcu(). Signed-off-by: Philipp Stanner Reviewed-by: Onur Özkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo [boqun: Fix rustdoc reported by kernel test robot ] Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-3-phasta@kernel.org --- rust/kernel/sync/rcu.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..d867240be736 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,19 @@ fn drop(&mut self) { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +} From 042278f5fa9e681420ab486d6cbf464e26667461 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 24 Jun 2026 17:07:03 +0200 Subject: [PATCH 2/6] rust: revocable: Use safe synchronize_rcu() abstraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now have a safe wrapper for the foreign function synchronize_rcu(). Use it in revocable.rs. Signed-off-by: Philipp Stanner Reviewed-by: Onur Özkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-4-phasta@kernel.org --- rust/kernel/revocable.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 0f4ae673256d..f539603349f1 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -7,7 +7,11 @@ use pin_init::Wrapper; -use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use crate::{ + prelude::*, + sync::rcu, + types::Opaque, // +}; use core::{ marker::PhantomData, ops::Deref, @@ -161,8 +165,7 @@ unsafe fn revoke_internal(&self) -> bool { if revoke { if SYNC { - // SAFETY: Just an FFI call, there are no further requirements. - unsafe { bindings::synchronize_rcu() }; + rcu::synchronize_rcu(); } // SAFETY: We know `self.data` is valid because only one CPU can succeed the From 47c3367ff096e66f614ed3cdadece26ffc04b5e8 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 24 Jun 2026 17:07:04 +0200 Subject: [PATCH 3/6] rust: sync: Use safe synchronize_rcu() abstraction in poll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now have a safe wrapper for the foreign function synchronize_rcu(). Use it in poll.rs. Signed-off-by: Philipp Stanner Reviewed-by: Alice Ryhl Reviewed-by: Onur Özkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-5-phasta@kernel.org --- rust/kernel/sync/poll.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..5aa0ce9ba01b 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -8,7 +8,11 @@ bindings, fs::File, prelude::*, - sync::{CondVar, LockClassKey}, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // }; use core::{marker::PhantomData, ops::Deref}; @@ -99,8 +103,6 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; // Wait for epoll items to be properly removed. - // - // SAFETY: Just an FFI call. - unsafe { bindings::synchronize_rcu() }; + synchronize_rcu(); } } From 79d3d667af401eaa452f046595209aae6933c485 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 9 Jun 2026 16:38:37 +0100 Subject: [PATCH 4/6] rust: sync: Add helpers for mb, dma_mb and friends They supplement the existing smp_mb, smp_rmb and smp_wmb. Reviewed-by: Eliot Courtney Signed-off-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260609-rust-barrier-v2-1-30fcc48e1cd0@garyguo.net --- rust/helpers/barrier.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c8..dbc7a3017c78 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,6 +2,36 @@ #include +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + __rust_helper void rust_helper_smp_mb(void) { smp_mb(); From 72856afd33f2fa166c06f1536003e300e51ecf09 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 9 Jun 2026 16:38:38 +0100 Subject: [PATCH 5/6] rust: sync: Add generic memory barriers Implement a generic interface for memory barriers (full system/DMA/SMP). The interface uses a parameter to force user to specify their intent with barriers. Provide `Read`, `Write`, `Full` orderings which map to the existing `rmb()`, `wmb()` and `mb()`. Generic is used here instead of providing individual standalone functions to reduce code duplication; for example, the `CONFIG_SMP` check in `smp_mb` is uniformly implemented for all SMP barriers. This could extend to `virt_mb`'s if they're introduced in the future. It would also make it easier if new ordering types are introduced in the future (e.g. `Acquire`, `Release`). Signed-off-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260609-rust-barrier-v2-2-30fcc48e1cd0@garyguo.net --- rust/kernel/sync/atomic/ordering.rs | 2 +- rust/kernel/sync/barrier.rs | 131 +++++++++++++++++++++------- 2 files changed, 102 insertions(+), 31 deletions(-) diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the following memory accesses. //! - It provides ordering between all the preceding memory accesses and all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..1180695d533a 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,38 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses across the barrier. @@ -19,43 +51,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } -/// A full memory barrier. +trait MemoryBarrier { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. -#[inline(always)] -pub fn smp_mb() { +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb(_: T) { + T::run() +} + +/// Memory barrier between CPUs. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; + T::run() } else { - barrier(); + barrier() } } -/// A write-write memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory write accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; - } else { - barrier(); - } -} - -/// A read-read memory barrier. +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory read accesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb>(_: T) { + T::run() } From 3f90c16d413b2f68407f1c5bb112a1b16ea35dc4 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 16 Jul 2026 15:55:35 +0100 Subject: [PATCH 6/6] rust: revocable: Use LKMM atomics instead of Rust atomics Kernel code should use LKMM atomics. The existing code is `AtomicBool` with the need to use `xchg`, so convert it to `AtomicFlag`. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260716145536.3681630-1-gary@kernel.org --- rust/kernel/revocable.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index f539603349f1..0e55e2a0fb37 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -9,14 +9,19 @@ use crate::{ prelude::*, - sync::rcu, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, types::Opaque, // }; use core::{ marker::PhantomData, ops::Deref, - ptr::drop_in_place, - sync::atomic::{AtomicBool, Ordering}, + ptr::drop_in_place, // }; /// An object that can become inaccessible at runtime. @@ -69,7 +74,7 @@ /// ``` #[pin_data(PinnedDrop)] pub struct Revocable { - is_available: AtomicBool, + is_available: AtomicFlag, #[pin] data: Opaque, } @@ -88,7 +93,7 @@ impl Revocable { /// Creates a new revocable instance of the given data. pub fn new(data: impl PinInit) -> impl PinInit { try_pin_init!(Self { - is_available: AtomicBool::new(true), + is_available: AtomicFlag::new(true), data <- Opaque::pin_init(data), }? E) } @@ -102,7 +107,7 @@ pub fn new(data: impl PinInit) -> impl PinInit { /// because another CPU may be waiting to complete the revocation of this object. pub fn try_access(&self) -> Option> { let guard = rcu::read_lock(); - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // Since `self.is_available` is true, data is initialised and has to remain valid // because the RCU read side lock prevents it from being dropped. Some(RevocableGuard::new(self.data.get(), guard)) @@ -120,7 +125,7 @@ pub fn try_access(&self) -> Option> { /// allowed to sleep because another CPU may be waiting to complete the revocation of this /// object. pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // SAFETY: Since `self.is_available` is true, data is initialised and has to remain // valid because the RCU read side lock prevents it from being dropped. Some(unsafe { &*self.data.get() }) @@ -161,7 +166,7 @@ pub unsafe fn access(&self) -> &T { /// /// Callers must ensure that there are no more concurrent users of the revocable object. unsafe fn revoke_internal(&self) -> bool { - let revoke = self.is_available.swap(false, Ordering::Relaxed); + let revoke = self.is_available.xchg(false, Relaxed); if revoke { if SYNC {