mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-05-19 08:11:11 -04:00
arm64: futex: Refactor futex atomic operation
Refactor the futex atomic operations using ll/sc instructions in preparation for FEAT_LSUI support. In addition, use named operands for the inline asm. No functional change. Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com> [catalin.marinas@arm.com: remove unnecessary stringify.h include] Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
This commit is contained in:
Yeoreum Yun
committed by
Catalin Marinas
Catalin Marinas
parent
42550d7d8a
commit
eaa3babcce
@@ -12,68 +12,133 @@
|
||||
|
||||
#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
|
||||
|
||||
#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
|
||||
do { \
|
||||
#define LLSC_FUTEX_ATOMIC_OP(op, insn) \
|
||||
static __always_inline int \
|
||||
__llsc_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
unsigned int loops = FUTEX_MAX_LOOPS; \
|
||||
int ret, oldval, newval; \
|
||||
\
|
||||
uaccess_enable_privileged(); \
|
||||
asm volatile( \
|
||||
" prfm pstl1strm, %2\n" \
|
||||
"1: ldxr %w1, %2\n" \
|
||||
asm volatile("// __llsc_futex_atomic_" #op "\n" \
|
||||
" prfm pstl1strm, %[uaddr]\n" \
|
||||
"1: ldxr %w[oldval], %[uaddr]\n" \
|
||||
insn "\n" \
|
||||
"2: stlxr %w0, %w3, %2\n" \
|
||||
" cbz %w0, 3f\n" \
|
||||
" sub %w4, %w4, %w0\n" \
|
||||
" cbnz %w4, 1b\n" \
|
||||
" mov %w0, %w6\n" \
|
||||
"2: stlxr %w[ret], %w[newval], %[uaddr]\n" \
|
||||
" cbz %w[ret], 3f\n" \
|
||||
" sub %w[loops], %w[loops], %w[ret]\n" \
|
||||
" cbnz %w[loops], 1b\n" \
|
||||
" mov %w[ret], %w[err]\n" \
|
||||
"3:\n" \
|
||||
" dmb ish\n" \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \
|
||||
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
|
||||
"+r" (loops) \
|
||||
: "r" (oparg), "Ir" (-EAGAIN) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret]) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret]) \
|
||||
: [ret] "=&r" (ret), [oldval] "=&r" (oldval), \
|
||||
[uaddr] "+Q" (*uaddr), [newval] "=&r" (newval), \
|
||||
[loops] "+r" (loops) \
|
||||
: [oparg] "r" (oparg), [err] "Ir" (-EAGAIN) \
|
||||
: "memory"); \
|
||||
uaccess_disable_privileged(); \
|
||||
} while (0)
|
||||
\
|
||||
if (!ret) \
|
||||
*oval = oldval; \
|
||||
\
|
||||
return ret; \
|
||||
}
|
||||
|
||||
LLSC_FUTEX_ATOMIC_OP(add, "add %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(or, "orr %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(and, "and %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(eor, "eor %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(set, "mov %w[newval], %w[oparg]")
|
||||
|
||||
static __always_inline int
|
||||
__llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
|
||||
uaccess_enable_privileged();
|
||||
asm volatile("//__llsc_futex_cmpxchg\n"
|
||||
" prfm pstl1strm, %[uaddr]\n"
|
||||
"1: ldxr %w[curval], %[uaddr]\n"
|
||||
" eor %w[tmp], %w[curval], %w[oldval]\n"
|
||||
" cbnz %w[tmp], 4f\n"
|
||||
"2: stlxr %w[tmp], %w[newval], %[uaddr]\n"
|
||||
" cbz %w[tmp], 3f\n"
|
||||
" sub %w[loops], %w[loops], %w[tmp]\n"
|
||||
" cbnz %w[loops], 1b\n"
|
||||
" mov %w[ret], %w[err]\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w[ret])
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w[ret])
|
||||
: [ret] "+r" (ret), [curval] "=&r" (val),
|
||||
[uaddr] "+Q" (*uaddr), [tmp] "=&r" (tmp),
|
||||
[loops] "+r" (loops)
|
||||
: [oldval] "r" (oldval), [newval] "r" (newval),
|
||||
[err] "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable_privileged();
|
||||
|
||||
if (!ret)
|
||||
*oval = val;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define FUTEX_ATOMIC_OP(op) \
|
||||
static __always_inline int \
|
||||
__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
return __llsc_futex_atomic_##op(oparg, uaddr, oval); \
|
||||
}
|
||||
|
||||
FUTEX_ATOMIC_OP(add)
|
||||
FUTEX_ATOMIC_OP(or)
|
||||
FUTEX_ATOMIC_OP(and)
|
||||
FUTEX_ATOMIC_OP(eor)
|
||||
FUTEX_ATOMIC_OP(set)
|
||||
|
||||
static __always_inline int
|
||||
__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval);
|
||||
}
|
||||
|
||||
static inline int
|
||||
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
|
||||
{
|
||||
int oldval = 0, ret, tmp;
|
||||
u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
int ret;
|
||||
u32 __user *uaddr;
|
||||
|
||||
if (!access_ok(_uaddr, sizeof(u32)))
|
||||
return -EFAULT;
|
||||
|
||||
uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
|
||||
switch (op) {
|
||||
case FUTEX_OP_SET:
|
||||
__futex_atomic_op("mov %w3, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_set(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_ADD:
|
||||
__futex_atomic_op("add %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_add(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_OR:
|
||||
__futex_atomic_op("orr %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_or(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_ANDN:
|
||||
__futex_atomic_op("and %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, ~oparg);
|
||||
ret = __futex_atomic_and(~oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_XOR:
|
||||
__futex_atomic_op("eor %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_eor(oparg, uaddr, oval);
|
||||
break;
|
||||
default:
|
||||
ret = -ENOSYS;
|
||||
}
|
||||
|
||||
if (!ret)
|
||||
*oval = oldval;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -81,40 +146,14 @@ static inline int
|
||||
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
|
||||
u32 oldval, u32 newval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
u32 __user *uaddr;
|
||||
|
||||
if (!access_ok(_uaddr, sizeof(u32)))
|
||||
return -EFAULT;
|
||||
|
||||
uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
uaccess_enable_privileged();
|
||||
asm volatile("// futex_atomic_cmpxchg_inatomic\n"
|
||||
" prfm pstl1strm, %2\n"
|
||||
"1: ldxr %w1, %2\n"
|
||||
" sub %w3, %w1, %w5\n"
|
||||
" cbnz %w3, 4f\n"
|
||||
"2: stlxr %w3, %w6, %2\n"
|
||||
" cbz %w3, 3f\n"
|
||||
" sub %w4, %w4, %w3\n"
|
||||
" cbnz %w4, 1b\n"
|
||||
" mov %w0, %w7\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
|
||||
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
|
||||
: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable_privileged();
|
||||
|
||||
if (!ret)
|
||||
*uval = val;
|
||||
|
||||
return ret;
|
||||
return __futex_cmpxchg(uaddr, oldval, newval, uval);
|
||||
}
|
||||
|
||||
#endif /* __ASM_FUTEX_H */
|
||||
|
||||
Reference in New Issue
Block a user