diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h index ae45289e8ec7..2ca94e41823b 100644 --- a/include/asm-generic/qspinlock.h +++ b/include/asm-generic/qspinlock.h @@ -41,6 +41,7 @@ #include #include +#include #ifndef queued_spin_is_locked /** @@ -130,12 +131,32 @@ static __always_inline void queued_spin_release(struct qspinlock *lock) #endif #ifndef queued_spin_unlock + +DECLARE_TRACEPOINT(contended_release); + +extern void queued_spin_release_traced(struct qspinlock *lock); + /** * queued_spin_unlock - unlock a queued spinlock * @lock : Pointer to queued spinlock structure + * + * Generic tracing wrapper around the arch-overridable + * queued_spin_release(). */ static __always_inline void queued_spin_unlock(struct qspinlock *lock) { + /* + * Trace and release are combined in queued_spin_release_traced() so + * the compiler does not need to preserve the lock pointer across the + * function call, avoiding callee-saved register save/restore on the + * hot path. queued_spin_release() is therefore called both here and in + * queued_spin_release_traced(). Keep the two in sync. + */ + if (IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) && + tracepoint_enabled(contended_release)) { + queued_spin_release_traced(lock); + return; + } queued_spin_release(lock); } #endif diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks index 4198f0273ecd..1c6423aafcd4 100644 --- a/kernel/Kconfig.locks +++ b/kernel/Kconfig.locks @@ -243,6 +243,26 @@ config QUEUED_SPINLOCKS def_bool y if ARCH_USE_QUEUED_SPINLOCKS depends on SMP +config QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE + bool "Trace contended_release on queued spinlocks" + depends on QUEUED_SPINLOCKS && TRACEPOINTS + help + Fire the lock:contended_release tracepoint when a contended queued + spinlock is released, so it is possible to attribute a contended + spinlock to its holder. + + Architectures that can patch the unlock site do this at no cost and + do not need this option. + + Everywhere else the check is compiled into queued_spin_unlock() and + a small cost is paid on every unlock even when the tracepoint is + disabled: a static-branch NOP and possibly a few more instructions + to manage a stack frame. + + Sleeping locks fire lock:contended_release regardless of this option. + + If unsure, say N. + config BPF_ARCH_SPINLOCK bool diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index af8d122bb649..33fe6d437c8f 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -104,6 +104,28 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, #define queued_spin_lock_slowpath native_queued_spin_lock_slowpath #endif +#if !defined(queued_spin_unlock) && \ + IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) +/* + * Out-of-line trace-and-release path for queued_spin_unlock(), used when + * the contended_release tracepoint is enabled. + * + * queued_spin_release() is duplicated here on purpose: doing the release + * in this function (rather than tracing here and releasing in the caller) + * lets queued_spin_unlock() return right after the call, so the + * tracepoint-disabled hot path never has to keep lock live across a call + * in a callee-saved register. Keep this release in sync with the one in + * queued_spin_unlock(). + */ +void __lockfunc queued_spin_release_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + queued_spin_release(lock); +} +EXPORT_SYMBOL(queued_spin_release_traced); +#endif + #endif /* _GEN_PV_LOCK_SLOWPATH */ /**