mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-12-28 05:34:13 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'bpf-riscv64-support-load-acquire-and-store-release-instructions'

Browse Source 
Peilin Ye says:

====================
bpf, riscv64: Support load-acquire and store-release instructions

Hi all!

Patchset [1] introduced BPF load-acquire (BPF_LOAD_ACQ) and
store-release (BPF_STORE_REL) instructions, and added x86-64 and arm64
JIT compiler support.  As a follow-up, this v2 patchset supports
load-acquire and store-release instructions for the riscv64 JIT
compiler, and introduces some related selftests/ changes.

Specifically:

 * PATCH 1 makes insn_def_regno() handle load-acquires properly for
   bpf_jit_needs_zext() (true for riscv64) architectures
 * PATCH 2, 3 from Andrea Parri add the actual support to the riscv64
   JIT compiler
 * PATCH 4 optimizes code emission by skipping redundant zext
   instructions inserted by the verifier
 * PATCH 5, 6 and 7 are minor selftest/ improvements
 * PATCH 8 enables (non-arena) load-acquire/store-release selftests for
   riscv64

v1: https://lore.kernel.org/bpf/cover.1745970908.git.yepeilin@google.com/
Changes since v1:

 * add Acked-by:, Reviewed-by: and Tested-by: tags from Lehui and Björn
 * simplify code logic in PATCH 1 (Lehui)
 * in PATCH 3, avoid changing 'return 0;' to 'return ret;' at the end of
   bpf_jit_emit_insn() (Lehui)

Please refer to individual patches for details.  Thanks!

[1] https://lore.kernel.org/all/cover.1741049567.git.yepeilin@google.com/
====================

Link: https://patch.msgid.link/cover.1746588351.git.yepeilin@google.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov
2025-05-09 10:05:28 -07:00
parent b69d4413aa d3131466b4
commit 32c563d109
8 changed files with 313 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
arch/riscv/net/bpf_jit.h
View File

@@ -608,6 +608,21 @@ static inline u32 rv_fence(u8 pred, u8 succ)
return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
}
static inline void emit_fence_r_rw(struct rv_jit_context *ctx)
{
emit(rv_fence(0x2, 0x3), ctx);
}
static inline void emit_fence_rw_w(struct rv_jit_context *ctx)
{
emit(rv_fence(0x3, 0x1), ctx);
}
static inline void emit_fence_rw_rw(struct rv_jit_context *ctx)
{
emit(rv_fence(0x3, 0x3), ctx);
}
static inline u32 rv_nop(void)
{
return rv_i_insn(0, 0, 0, 0, 0x13);

332
arch/riscv/net/bpf_jit_comp64.c
View File

@@ -473,11 +473,212 @@ static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx)
emit(hash, ctx);
}
static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
struct rv_jit_context *ctx)
static int emit_load_8(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
u8 r0;
int insns_start;
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lb(rd, off, rs), ctx);
else
emit(rv_lbu(rd, off, rs), ctx);
return ctx->ninsns - insns_start;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lb(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
return ctx->ninsns - insns_start;
}
static int emit_load_16(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
int insns_start;
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lh(rd, off, rs), ctx);
else
emit(rv_lhu(rd, off, rs), ctx);
return ctx->ninsns - insns_start;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lh(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
return ctx->ninsns - insns_start;
}
static int emit_load_32(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
int insns_start;
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lw(rd, off, rs), ctx);
else
emit(rv_lwu(rd, off, rs), ctx);
return ctx->ninsns - insns_start;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lw(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
return ctx->ninsns - insns_start;
}
static int emit_load_64(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
int insns_start;
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
emit_ld(rd, off, rs, ctx);
return ctx->ninsns - insns_start;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
emit_ld(rd, 0, RV_REG_T1, ctx);
return ctx->ninsns - insns_start;
}
static void emit_store_8(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
if (is_12b_int(off)) {
emit(rv_sb(rd, off, rs), ctx);
return;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit(rv_sb(RV_REG_T1, 0, rs), ctx);
}
static void emit_store_16(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
if (is_12b_int(off)) {
emit(rv_sh(rd, off, rs), ctx);
return;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit(rv_sh(RV_REG_T1, 0, rs), ctx);
}
static void emit_store_32(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
if (is_12b_int(off)) {
emit_sw(rd, off, rs, ctx);
return;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit_sw(RV_REG_T1, 0, rs, ctx);
}
static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
{
if (is_12b_int(off)) {
emit_sd(rd, off, rs, ctx);
return;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit_sd(RV_REG_T1, 0, rs, ctx);
}
static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
struct rv_jit_context *ctx)
{
u8 code = insn->code;
s32 imm = insn->imm;
s16 off = insn->off;
switch (imm) {
/* dst_reg = load_acquire(src_reg + off16) */
case BPF_LOAD_ACQ:
switch (BPF_SIZE(code)) {
case BPF_B:
emit_load_8(false, rd, off, rs, ctx);
break;
case BPF_H:
emit_load_16(false, rd, off, rs, ctx);
break;
case BPF_W:
emit_load_32(false, rd, off, rs, ctx);
break;
case BPF_DW:
emit_load_64(false, rd, off, rs, ctx);
break;
}
emit_fence_r_rw(ctx);
/* If our next insn is a redundant zext, return 1 to tell
* build_body() to skip it.
*/
if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))
return 1;
break;
/* store_release(dst_reg + off16, src_reg) */
case BPF_STORE_REL:
emit_fence_rw_w(ctx);
switch (BPF_SIZE(code)) {
case BPF_B:
emit_store_8(rd, off, rs, ctx);
break;
case BPF_H:
emit_store_16(rd, off, rs, ctx);
break;
case BPF_W:
emit_store_32(rd, off, rs, ctx);
break;
case BPF_DW:
emit_store_64(rd, off, rs, ctx);
break;
}
break;
default:
pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm);
return -EINVAL;
}
return 0;
}
static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
struct rv_jit_context *ctx)
{
u8 r0, code = insn->code;
s16 off = insn->off;
s32 imm = insn->imm;
int jmp_offset;
bool is64;
if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n");
return -EINVAL;
}
is64 = BPF_SIZE(code) == BPF_DW;
if (off) {
if (is_12b_int(off)) {
@@ -554,9 +755,14 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
jmp_offset = ninsns_rvoff(-6);
emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
emit(rv_fence(0x3, 0x3), ctx);
emit_fence_rw_rw(ctx);
break;
default:
pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);
return -EINVAL;
}
return 0;
}
#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
@@ -1650,8 +1856,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
{
int insn_len, insns_start;
bool sign_ext;
int insn_len;
sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
@@ -1663,78 +1869,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
switch (BPF_SIZE(code)) {
case BPF_B:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lb(rd, off, rs), ctx);
else
emit(rv_lbu(rd, off, rs), ctx);
insn_len = ctx->ninsns - insns_start;
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lb(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
insn_len = ctx->ninsns - insns_start;
insn_len = emit_load_8(sign_ext, rd, off, rs, ctx);
break;
case BPF_H:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lh(rd, off, rs), ctx);
else
emit(rv_lhu(rd, off, rs), ctx);
insn_len = ctx->ninsns - insns_start;
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lh(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
insn_len = ctx->ninsns - insns_start;
insn_len = emit_load_16(sign_ext, rd, off, rs, ctx);
break;
case BPF_W:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lw(rd, off, rs), ctx);
else
emit(rv_lwu(rd, off, rs), ctx);
insn_len = ctx->ninsns - insns_start;
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
if (sign_ext)
emit(rv_lw(rd, 0, RV_REG_T1), ctx);
else
emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
insn_len = ctx->ninsns - insns_start;
insn_len = emit_load_32(sign_ext, rd, off, rs, ctx);
break;
case BPF_DW:
if (is_12b_int(off)) {
insns_start = ctx->ninsns;
emit_ld(rd, off, rs, ctx);
insn_len = ctx->ninsns - insns_start;
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
insns_start = ctx->ninsns;
emit_ld(rd, 0, RV_REG_T1, ctx);
insn_len = ctx->ninsns - insns_start;
insn_len = emit_load_64(sign_ext, rd, off, rs, ctx);
break;
}
@@ -1879,49 +2023,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
/* STX: *(size *)(dst + off) = src */
case BPF_STX | BPF_MEM | BPF_B:
if (is_12b_int(off)) {
emit(rv_sb(rd, off, rs), ctx);
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit(rv_sb(RV_REG_T1, 0, rs), ctx);
emit_store_8(rd, off, rs, ctx);
break;
case BPF_STX | BPF_MEM | BPF_H:
if (is_12b_int(off)) {
emit(rv_sh(rd, off, rs), ctx);
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit(rv_sh(RV_REG_T1, 0, rs), ctx);
emit_store_16(rd, off, rs, ctx);
break;
case BPF_STX | BPF_MEM | BPF_W:
if (is_12b_int(off)) {
emit_sw(rd, off, rs, ctx);
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit_sw(RV_REG_T1, 0, rs, ctx);
emit_store_32(rd, off, rs, ctx);
break;
case BPF_STX | BPF_MEM | BPF_DW:
if (is_12b_int(off)) {
emit_sd(rd, off, rs, ctx);
break;
}
emit_imm(RV_REG_T1, off, ctx);
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
emit_sd(RV_REG_T1, 0, rs, ctx);
emit_store_64(rd, off, rs, ctx);
break;
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
emit_atomic(rd, rs, off, imm,
BPF_SIZE(code) == BPF_DW, ctx);
if (bpf_atomic_is_load_store(insn))
ret = emit_atomic_ld_st(rd, rs, insn, ctx);
else
ret = emit_atomic_rmw(rd, rs, insn, ctx);
if (ret)
return ret;
break;
case BPF_STX | BPF_PROBE_MEM32 | BPF_B:

3
arch/riscv/net/bpf_jit_core.c
View File

@@ -26,9 +26,8 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
int ret;
ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
/* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
if (ret > 0)
i++;
i++; /* skip the next instruction */
if (offset)
offset[i] = ctx->ninsns;
if (ret < 0)

12
kernel/bpf/verifier.c
View File

@@ -3649,16 +3649,16 @@ static int insn_def_regno(const struct bpf_insn *insn)
case BPF_ST:
return -1;
case BPF_STX:
if ((BPF_MODE(insn->code) == BPF_ATOMIC ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) &&
(insn->imm & BPF_FETCH)) {
if (BPF_MODE(insn->code) == BPF_ATOMIC ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
if (insn->imm == BPF_CMPXCHG)
return BPF_REG_0;
else
else if (insn->imm == BPF_LOAD_ACQ)
return insn->dst_reg;
else if (insn->imm & BPF_FETCH)
return insn->src_reg;
} else {
return -1;
}
return -1;
default:
return insn->dst_reg;
}

5
tools/testing/selftests/bpf/progs/bpf_misc.h
View File

@@ -225,8 +225,9 @@
#define CAN_USE_BPF_ST
#endif
#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86))
#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
#define CAN_USE_LOAD_ACQ_STORE_REL
#endif

48
tools/testing/selftests/bpf/progs/verifier_load_acquire.c
View File

@@ -10,65 +10,81 @@
SEC("socket")
__description("load-acquire, 8-bit")
__success __success_unpriv __retval(0x12)
__success __success_unpriv __retval(0)
__naked void load_acquire_8(void)
{
asm volatile (
"w1 = 0x12;"
"r0 = 0;"
"w1 = 0xfe;"
"*(u8 *)(r10 - 1) = w1;"
".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1));
".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1));
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(load_acquire_insn,
BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1))
BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1))
: __clobber_all);
}
SEC("socket")
__description("load-acquire, 16-bit")
__success __success_unpriv __retval(0x1234)
__success __success_unpriv __retval(0)
__naked void load_acquire_16(void)
{
asm volatile (
"w1 = 0x1234;"
"r0 = 0;"
"w1 = 0xfedc;"
"*(u16 *)(r10 - 2) = w1;"
".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2));
".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2));
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(load_acquire_insn,
BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2))
BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2))
: __clobber_all);
}
SEC("socket")
__description("load-acquire, 32-bit")
__success __success_unpriv __retval(0x12345678)
__success __success_unpriv __retval(0)
__naked void load_acquire_32(void)
{
asm volatile (
"w1 = 0x12345678;"
"r0 = 0;"
"w1 = 0xfedcba09;"
"*(u32 *)(r10 - 4) = w1;"
".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4));
".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4));
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(load_acquire_insn,
BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4))
BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4))
: __clobber_all);
}
SEC("socket")
__description("load-acquire, 64-bit")
__success __success_unpriv __retval(0x1234567890abcdef)
__success __success_unpriv __retval(0)
__naked void load_acquire_64(void)
{
asm volatile (
"r1 = 0x1234567890abcdef ll;"
"r0 = 0;"
"r1 = 0xfedcba0987654321 ll;"
"*(u64 *)(r10 - 8) = r1;"
".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8));
".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8));
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(load_acquire_insn,
BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8))
BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
: __clobber_all);
}

5
tools/testing/selftests/bpf/progs/verifier_precision.c
View File

@@ -91,8 +91,7 @@ __naked int bpf_end_bswap(void)
::: __clobber_all);
}
#if defined(ENABLE_ATOMICS_TESTS) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86))
#ifdef CAN_USE_LOAD_ACQ_STORE_REL
SEC("?raw_tp")
__success __log_level(2)
@@ -138,7 +137,7 @@ __naked int bpf_store_release(void)
: __clobber_all);
}
#endif /* load-acquire, store-release */
#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
#endif /* v4 instruction */
SEC("?raw_tp")

39
tools/testing/selftests/bpf/progs/verifier_store_release.c
View File

@@ -6,18 +6,21 @@
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86))
#ifdef CAN_USE_LOAD_ACQ_STORE_REL
SEC("socket")
__description("store-release, 8-bit")
__success __success_unpriv __retval(0x12)
__success __success_unpriv __retval(0)
__naked void store_release_8(void)
{
asm volatile (
"r0 = 0;"
"w1 = 0x12;"
".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1);
"w0 = *(u8 *)(r10 - 1);"
"w2 = *(u8 *)(r10 - 1);"
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(store_release_insn,
@@ -27,13 +30,17 @@ __naked void store_release_8(void)
SEC("socket")
__description("store-release, 16-bit")
__success __success_unpriv __retval(0x1234)
__success __success_unpriv __retval(0)
__naked void store_release_16(void)
{
asm volatile (
"r0 = 0;"
"w1 = 0x1234;"
".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1);
"w0 = *(u16 *)(r10 - 2);"
"w2 = *(u16 *)(r10 - 2);"
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(store_release_insn,
@@ -43,13 +50,17 @@ __naked void store_release_16(void)
SEC("socket")
__description("store-release, 32-bit")
__success __success_unpriv __retval(0x12345678)
__success __success_unpriv __retval(0)
__naked void store_release_32(void)
{
asm volatile (
"r0 = 0;"
"w1 = 0x12345678;"
".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1);
"w0 = *(u32 *)(r10 - 4);"
"w2 = *(u32 *)(r10 - 4);"
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(store_release_insn,
@@ -59,13 +70,17 @@ __naked void store_release_32(void)
SEC("socket")
__description("store-release, 64-bit")
__success __success_unpriv __retval(0x1234567890abcdef)
__success __success_unpriv __retval(0)
__naked void store_release_64(void)
{
asm volatile (
"r0 = 0;"
"r1 = 0x1234567890abcdef ll;"
".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1);
"r0 = *(u64 *)(r10 - 8);"
"r2 = *(u64 *)(r10 - 8);"
"if r2 == r1 goto 1f;"
"r0 = 1;"
"1:"
"exit;"
:
: __imm_insn(store_release_insn,
@@ -271,7 +286,7 @@ __naked void store_release_with_invalid_reg(void)
: __clobber_all);
}
#else
#else /* CAN_USE_LOAD_ACQ_STORE_REL */
SEC("socket")
__description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test")
@@ -281,6 +296,6 @@ int dummy_test(void)
return 0;
}
#endif
#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
char _license[] SEC("license") = "GPL";