mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-27 15:17:04 -04:00
Code Issues Packages Projects Releases Wiki Activity

bpf: Fix mmap_lock leak in irq_work path

Browse Source 
stack_map_get_build_id_offset() introduced a per-CPU irq_work to defer
mmap_read_unlock() from NMI context, and bpf_find_vma() later reused the
same mmap_unlock_work. Both callers only check whether the work is busy
before taking mmap_lock, so a nested caller can reuse the slot before the
first caller queues it. Two read locks may then be acquired while only one
deferred unlock runs, leaking a read lock and blocking exit_mmap().

Reserve the per-CPU slot before mmap_read_trylock(). Use the same wrapper
in stackmap and bpf_find_vma() so both callers release the reservation on
trylock failure. Keep rejecting the slot while the irq_work remains busy.
Release it after the irq_work callback unlocks the mm.

Fixes: eac9153f2b ("bpf/stackmap: Fix deadlock with rq_lock in bpf_get_stack()")
Reported-by: syzbot+cdd6c0925e12b0af60cc@syzkaller.appspotmail.com
Reported-by: sashiko-bot@kernel.org
Signed-off-by: Sanghyun Park <sanghyun.park.cnu@gmail.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Closes: https://syzkaller.appspot.com/bug?extid=cdd6c0925e12b0af60cc
Closes: https://lore.kernel.org/r/20260630033745.B80201F000E9@smtp.kernel.org
Link: https://lore.kernel.org/bpf/20260805031425.2157475-2-sanghyun.park.cnu@gmail.com
This commit is contained in:
Sanghyun Park
2026-08-05 12:14:25 +09:00
committed by Daniel Borkmann
parent 3f562c537e
commit fa9dcacdcd
3 changed files with 56 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
kernel/bpf/mmap_unlock_work.h
View File

@@ -4,12 +4,15 @@
#ifndef __MMAP_UNLOCK_WORK_H__
#define __MMAP_UNLOCK_WORK_H__
#include <linux/atomic.h>
#include <linux/err.h>
#include <linux/irq_work.h>
/* irq_work to run mmap_read_unlock() in irq_work */
struct mmap_unlock_irq_work {
struct irq_work irq_work;
struct mm_struct *mm;
atomic_t active;
};
DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
@@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
* We cannot do mmap_read_unlock() when the irq is disabled, because of
* risk to deadlock with rq_lock. To look up vma when the irqs are
* disabled, we need to run mmap_read_unlock() in irq_work. We use a
* percpu variable to do the irq_work. If the irq_work is already used
* by another lookup, we fall over.
* percpu variable to do the irq_work. The active flag reserves the slot
* before mmap_read_trylock() and until the irq_work callback consumes mm.
*/
static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr)
static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void)
{
struct mmap_unlock_irq_work *work = NULL;
bool irq_work_busy = false;
struct mmap_unlock_irq_work *work;
if (irqs_disabled()) {
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
work = this_cpu_ptr(&mmap_unlock_work);
if (irq_work_is_busy(&work->irq_work)) {
/* cannot queue more up_read, fallback */
irq_work_busy = true;
}
} else {
/*
* PREEMPT_RT does not allow to trylock mmap sem in
* interrupt disabled context. Force the fallback code.
*/
irq_work_busy = true;
}
}
if (!irqs_disabled())
return NULL;
*work_ptr = work;
return irq_work_busy;
/*
* PREEMPT_RT does not allow to trylock mmap sem in interrupt
* disabled context. Force the fallback code.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT))
return ERR_PTR(-EBUSY);
work = this_cpu_ptr(&mmap_unlock_work);
if (irq_work_is_busy(&work->irq_work) ||
atomic_cmpxchg_acquire(&work->active, 0, 1))
return ERR_PTR(-EBUSY);
return work;
}
static inline void
bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work)
{
if (work)
atomic_set_release(&work->active, 0);
}
static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)

28
kernel/bpf/stackmap.c
View File

@@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i
static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
u32 trace_nr, bool user, bool may_fault)
{
struct mmap_unlock_irq_work *work = NULL;
bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
struct mmap_unlock_irq_work *work;
bool has_user_ctx = user && current && current->mm;
struct stack_map_build_id_cache cache = {};
struct vm_area_struct *vma;
@@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
return;
}
/* If the irq_work is in use, fall back to report ips. Same
* fallback is used for kernel stack (!user) on a stackmap with
* build_id.
*/
if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) {
/* cannot access current->mm, fall back to ips */
for (i = 0; i < trace_nr; i++)
stack_map_build_id_set_ip(&id_offs[i]);
return;
if (!has_user_ctx)
goto fallback;
work = bpf_mmap_unlock_guard_get();
if (IS_ERR(work))
goto fallback;
if (!mmap_read_trylock(current->mm)) {
bpf_mmap_unlock_guard_put(work);
goto fallback;
}
for (i = 0; i < trace_nr; i++) {
@@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
vma->vm_pgoff);
}
bpf_mmap_unlock_mm(work, current->mm);
return;
fallback:
/* cannot access current->mm, fall back to ips */
for (i = 0; i < trace_nr; i++)
stack_map_build_id_set_ip(&id_offs[i]);
}
static struct perf_callchain_entry *

14
kernel/bpf/task_iter.c
View File

@@ -753,9 +753,8 @@ static struct bpf_iter_reg task_vma_reg_info = {
BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags)
{
struct mmap_unlock_irq_work *work = NULL;
struct mmap_unlock_irq_work *work;
struct vm_area_struct *vma;
bool irq_work_busy = false;
bool __maybe_unused mmput_needed = false;
struct mm_struct *mm;
int ret = -ENOENT;
@@ -792,9 +791,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
if (!mm)
return -ENOENT;
irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
work = bpf_mmap_unlock_guard_get();
if (IS_ERR(work)) {
ret = PTR_ERR(work);
goto out;
}
if (irq_work_busy || !mmap_read_trylock(mm)) {
if (!mmap_read_trylock(mm)) {
bpf_mmap_unlock_guard_put(work);
ret = -EBUSY;
goto out;
}
@@ -1191,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry)
work = container_of(entry, struct mmap_unlock_irq_work, irq_work);
mmap_read_unlock_non_owner(work->mm);
work->mm = NULL;
bpf_mmap_unlock_guard_put(work);
}
static int __init task_iter_init(void)