diff --git a/include/linux/futex.h b/include/linux/futex.h index cb2a182547dd..51f4ccdc9092 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -105,7 +105,41 @@ static inline int futex_hash_free(struct mm_struct *mm) { return 0; } #endif /* !CONFIG_FUTEX */ #ifdef CONFIG_FUTEX_ROBUST_UNLOCK +#include + void futex_reset_cs_ranges(struct futex_mm_data *fd); +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr); + +static inline bool futex_within_robust_unlock(struct pt_regs *regs, + struct futex_unlock_cs_range *csr) +{ + unsigned long ip = instruction_pointer(regs); + + return ip >= csr->start_ip && ip < csr->start_ip + csr->len; +} + +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) +{ + struct futex_unlock_cs_range *csr; + + /* + * Avoid dereferencing current->mm if not returning from interrupt. + * current->rseq.event is going to be used subsequently, so bringing the + * cache line in is not a big deal. + */ + if (!current->rseq.event.user_irq) + return; + + csr = current->mm->futex.unlock.cs_ranges; + + /* The loop is optimized out for !COMPAT */ + for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) { + if (unlikely(futex_within_robust_unlock(regs, csr))) { + __futex_fixup_robust_unlock(regs, csr); + return; + } + } +} static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, unsigned long start, unsigned long end, bool sz32) @@ -114,7 +148,10 @@ static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned in fd->unlock.cs_ranges[idx].len = end - start; fd->unlock.cs_ranges[idx].pop_size32 = sz32; } -#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + #if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) void futex_mm_init(struct mm_struct *mm); diff --git a/include/vdso/futex.h b/include/vdso/futex.h new file mode 100644 index 000000000000..3cd175eefe64 --- /dev/null +++ b/include/vdso/futex.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _VDSO_FUTEX_H +#define _VDSO_FUTEX_H + +#include + +/** + * __vdso_futex_robust_list64_try_unlock - Try to unlock an uncontended robust futex + * with a 64-bit pending op pointer + * @lock: Pointer to the futex lock object + * @tid: The TID of the calling task + * @pop: Pointer to the task's robust_list_head::list_pending_op + * + * Return: The content of *@lock. On success this is the same as @tid. + * + * The function implements: + * if (atomic_try_cmpxchg(lock, &tid, 0)) + * *op = NULL; + * return tid; + * + * There is a race between a successful unlock and clearing the pending op + * pointer in the robust list head. If the calling task is interrupted in the + * race window and has to handle a (fatal) signal on return to user space then + * the kernel handles the clearing of @pending_op before attempting to deliver + * the signal. That ensures that a task cannot exit with a potentially invalid + * pending op pointer. + * + * User space uses it in the following way: + * + * if (__vdso_futex_robust_list64_try_unlock(lock, tid, &pending_op) != tid) + * err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....); + * + * If the unlock attempt fails due to the FUTEX_WAITERS bit set in the lock, + * then the syscall does the unlock, clears the pending op pointer and wakes the + * requested number of waiters. + */ +__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop); + +/** + * __vdso_futex_robust_list32_try_unlock - Try to unlock an uncontended robust futex + * with a 32-bit pending op pointer + * @lock: Pointer to the futex lock object + * @tid: The TID of the calling task + * @pop: Pointer to the task's robust_list_head::list_pending_op + * + * Return: The content of *@lock. On success this is the same as @tid. + * + * Same as __vdso_futex_robust_list64_try_unlock() just with a 32-bit @pop pointer. + */ +__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop); + +#endif diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 19d2244a9fef..e3d381fd3d25 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include +#include #include +#include #include #include #include +#include #include /* Workaround to allow gradual conversion of architecture code */ @@ -60,8 +61,10 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re if (ti_work & _TIF_PATCH_PENDING) klp_update_patch_state(current); - if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + futex_fixup_robust_unlock(regs); arch_do_signal_or_restart(regs); + } if (ti_work & _TIF_NOTIFY_RESUME) resume_user_mode_work(regs); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index aad6e501fabd..6ea4a97796a1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -46,6 +46,8 @@ #include #include +#include + #include "futex.h" #include "../locking/rtmutex_common.h" @@ -1446,6 +1448,22 @@ bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags) return robust_list_clear_pending(pop); } +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr) +{ + /* + * arch_futex_robust_unlock_get_pop() returns the list pending op pointer from + * @regs if the try_cmpxchg() succeeded. + */ + void __user *pop = arch_futex_robust_unlock_get_pop(regs); + + if (!pop) + return; + + futex_robust_list_clear_pending(pop, csr->pop_size32 ? FLAGS_ROBUST_LIST32 : 0); +} +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ + static void futex_cleanup(struct task_struct *tsk) { if (unlikely(tsk->futex.robust_list)) {