mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-30 11:03:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

uprobes: Switch uretprobes_srcu to SRCU-fast-updown

Browse Source 
uretprobes_srcu currently uses normal SRCU, which issues
two smp_mb() per read lock/unlock pair. This overhead is
paid on every uretprobe hit.

Switch to SRCU-fast-updown, which eliminates the per-reader
memory barriers by moving the ordering cost to the
grace-period side (synchronize_rcu() instead of smp_mb()).
This is acceptable because grace periods (uprobe
unregistration) are infrequent compared to reader-side
uretprobe hits.

The updown flavor is required because the SRCU read lock is
taken in prepare_uretprobe() when a return instance is
created and is held until that return instance is finalized.
The traced thread returns to user space in between, so the
lock is inherently released in a different context from
where it was acquired: on the normal return path via
uprobe_handle_trampoline() -> hprobe_finalize(), or from
ri_timer() (expiry) or dup_utask() (fork) via
hprobe_expire(). srcu_down_read_fast() / srcu_up_read_fast()
are designed for this acquire-here / release-elsewhere
pattern and, unlike the same-context srcu_read_lock_fast()
variant, do not carry the lockdep read-side tracking that
would warn on it.

The short, same-context SRCU sections in ri_timer() and
dup_utask() (which guard the uprobe against reuse across the
hprobe_expire() cmpxchg) instead use guard(srcu_fast_updown)
for proper lockdep coverage.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://patch.msgid.link/20260706172744.3920417-3-puranjay@kernel.org
This commit is contained in:
Puranjay Mohan
2026-07-06 10:27:42 -07:00
committed by Peter Zijlstra
parent fa8a2d5513
commit 36c8b02c3f
2 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
include/linux/uprobes.h
View File

@@ -25,6 +25,7 @@ struct mm_struct;
struct inode;
struct notifier_block;
struct page;
struct srcu_ctr;
/*
* Allowed return values from uprobe consumer's handler callback
@@ -106,7 +107,7 @@ enum hprobe_state {
* underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an
* internal marker and is handled transparently by hprobe_fetch() helper.
*
* When uprobe is SRCU-protected, we also record srcu_idx value, necessary for
* When uprobe is SRCU-protected, we also record srcu_scp value, necessary for
* SRCU unlocking.
*
* See hprobe_expire() and hprobe_fetch() for details of race-free uprobe
@@ -115,7 +116,7 @@ enum hprobe_state {
*/
struct hprobe {
enum hprobe_state state;
int srcu_idx;
struct srcu_ctr __percpu *srcu_scp;
struct uprobe *uprobe;
};

29
kernel/events/uprobes.c
View File

@@ -54,7 +54,7 @@ static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
/* Covers return_instance's uprobe lifetime. */
DEFINE_STATIC_SRCU(uretprobes_srcu);
DEFINE_STATIC_SRCU_FAST_UPDOWN(uretprobes_srcu);
/* Have a copy of original instruction */
#define UPROBE_COPY_INSN 0
@@ -707,12 +707,13 @@ static void put_uprobe(struct uprobe *uprobe)
}
/* Initialize hprobe as SRCU-protected "leased" uprobe */
static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx)
static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe,
struct srcu_ctr __percpu *srcu_scp)
{
WARN_ON(!uprobe);
hprobe->state = HPROBE_LEASED;
hprobe->uprobe = uprobe;
hprobe->srcu_idx = srcu_idx;
hprobe->srcu_scp = srcu_scp;
}
/* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */
@@ -720,7 +721,7 @@ static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe)
{
hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE;
hprobe->uprobe = uprobe;
hprobe->srcu_idx = -1;
hprobe->srcu_scp = NULL;
}
/*
@@ -757,7 +758,7 @@ static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate)
{
switch (hstate) {
case HPROBE_LEASED:
__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp);
break;
case HPROBE_STABLE:
put_uprobe(hprobe->uprobe);
@@ -829,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get)
*/
if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) {
/* We won the race, we are the ones to unlock SRCU */
__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp);
return get && uprobe ? get_uprobe(uprobe) : uprobe;
}
@@ -2035,7 +2036,7 @@ static void ri_timer(struct timer_list *timer)
struct return_instance *ri;
/* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */
guard(srcu)(&uretprobes_srcu);
guard(srcu_fast_updown)(&uretprobes_srcu);
/* RCU protects return_instance from freeing. */
guard(rcu)();
@@ -2132,7 +2133,7 @@ static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
t->utask = n_utask;
/* protect uprobes from freeing, we'll need try_get_uprobe() them */
guard(srcu)(&uretprobes_srcu);
guard(srcu_fast_updown)(&uretprobes_srcu);
p = &n_utask->return_instances;
for (o = o_utask->return_instances; o; o = o->next) {
@@ -2244,8 +2245,8 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
{
struct uprobe_task *utask = current->utask;
unsigned long orig_ret_vaddr, trampoline_vaddr;
struct srcu_ctr __percpu *srcu_scp;
bool chained;
int srcu_idx;
if (!get_xol_area())
goto free;
@@ -2283,8 +2284,12 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
}
/* __srcu_read_lock() because SRCU lock survives switch to user space */
srcu_idx = __srcu_read_lock(&uretprobes_srcu);
/*
* Use srcu_down_read_fast() because the SRCU lock survives a switch to
* user space and can be unlocked from a different context by ri_timer()
* or dup_utask().
*/
srcu_scp = srcu_down_read_fast(&uretprobes_srcu);
ri->func = instruction_pointer(regs);
ri->stack = user_stack_pointer(regs);
@@ -2293,7 +2298,7 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
utask->depth++;
hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx);
hprobe_init_leased(&ri->hprobe, uprobe, srcu_scp);
ri->next = utask->return_instances;
rcu_assign_pointer(utask->return_instances, ri);