mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-05-09 04:21:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 's390-bpf-use-kernel-s-expoline-thunks'

Browse Source 
Ilya Leoshkevich says:

====================
This series simplifies the s390 JIT by replacing the generation of
expolines (Spectre mitigation) with using the ones from the kernel
text. This is possible thanks to the V!=R s390 kernel rework.
Patch 1 is a small prerequisite for arch/s390 that I would like to
get in via the BPF tree. It has Heiko's Acked-by.
Patches 2 and 3 are the implementation.
====================

Link: https://patch.msgid.link/20250519223646.66382-1-iii@linux.ibm.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov
2025-05-22 08:40:57 -07:00
parent 4e2e6841ff 7f332f9fe9
commit d90f0bce57
2 changed files with 60 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
arch/s390/include/asm/nospec-branch.h
View File

@@ -26,8 +26,6 @@ static inline bool nospec_uses_trampoline(void)
return __is_defined(CC_USING_EXPOLINE) && !nospec_disable;
}
#ifdef CONFIG_EXPOLINE_EXTERN
void __s390_indirect_jump_r1(void);
void __s390_indirect_jump_r2(void);
void __s390_indirect_jump_r3(void);
@@ -44,8 +42,6 @@ void __s390_indirect_jump_r13(void);
void __s390_indirect_jump_r14(void);
void __s390_indirect_jump_r15(void);
#endif
#endif /* __ASSEMBLY__ */
#endif /* _ASM_S390_EXPOLINE_H */

123
arch/s390/net/bpf_jit_comp.c
View File

@@ -48,8 +48,6 @@ struct bpf_jit {
int lit64; /* Current position in 64-bit literal pool */
int base_ip; /* Base address for literal pool */
int exit_ip; /* Address of exit */
int r1_thunk_ip; /* Address of expoline thunk for 'br %r1' */
int r14_thunk_ip; /* Address of expoline thunk for 'br %r14' */
int tail_call_start; /* Tail call start offset */
int excnt; /* Number of exception table entries */
int prologue_plt_ret; /* Return address for prologue hotpatch PLT */
@@ -127,6 +125,18 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
jit->seen_regs |= (1 << r1);
}
static s32 off_to_pcrel(struct bpf_jit *jit, u32 off)
{
return off - jit->prg;
}
static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr)
{
if (jit->prg_buf)
return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg);
return 0;
}
#define REG_SET_SEEN(b1) \
({ \
reg_set_seen(jit, b1); \
@@ -201,7 +211,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
#define EMIT4_PCREL_RIC(op, mask, target) \
({ \
int __rel = ((target) - jit->prg) / 2; \
int __rel = off_to_pcrel(jit, target) / 2; \
_EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \
})
@@ -239,7 +249,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
#define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \
({ \
unsigned int rel = (int)((target) - jit->prg) / 2; \
unsigned int rel = off_to_pcrel(jit, target) / 2; \
_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \
(op2) | (mask) << 12); \
REG_SET_SEEN(b1); \
@@ -248,7 +258,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
#define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \
({ \
unsigned int rel = (int)((target) - jit->prg) / 2; \
unsigned int rel = off_to_pcrel(jit, target) / 2; \
_EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \
(rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \
REG_SET_SEEN(b1); \
@@ -257,29 +267,41 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \
({ \
int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\
_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\
REG_SET_SEEN(b1); \
REG_SET_SEEN(b2); \
})
#define EMIT6_PCREL_RILB(op, b, target) \
({ \
unsigned int rel = (int)((target) - jit->prg) / 2; \
_EMIT6((op) | reg_high(b) << 16 | rel >> 16, rel & 0xffff);\
REG_SET_SEEN(b); \
})
static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel)
{
u32 pc32dbl = (s32)(pcrel / 2);
#define EMIT6_PCREL_RIL(op, target) \
({ \
unsigned int rel = (int)((target) - jit->prg) / 2; \
_EMIT6((op) | rel >> 16, rel & 0xffff); \
})
_EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff);
}
static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel)
{
emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel);
REG_SET_SEEN(b);
}
#define EMIT6_PCREL_RILB(op, b, target) \
emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target))
#define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \
emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr))
static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel)
{
emit6_pcrel_ril(jit, op | mask << 20, pcrel);
}
#define EMIT6_PCREL_RILC(op, mask, target) \
({ \
EMIT6_PCREL_RIL((op) | (mask) << 20, (target)); \
})
emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target))
#define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \
emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr))
#define _EMIT6_IMM(op, imm) \
({ \
@@ -503,7 +525,7 @@ static void bpf_skip(struct bpf_jit *jit, int size)
{
if (size >= 6 && !is_valid_rel(size)) {
/* brcl 0xf,size */
EMIT6_PCREL_RIL(0xc0f4000000, size);
EMIT6_PCREL_RILC(0xc0040000, 0xf, size);
size -= 6;
} else if (size >= 4 && is_valid_rel(size)) {
/* brc 0xf,size */
@@ -618,28 +640,17 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
}
/*
* Emit an expoline for a jump that follows
* Jump using a register either directly or via an expoline thunk
*/
static void emit_expoline(struct bpf_jit *jit)
{
/* exrl %r0,.+10 */
EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
/* j . */
EMIT4_PCREL(0xa7f40000, 0);
}
/*
* Emit __s390_indirect_jump_r1 thunk if necessary
*/
static void emit_r1_thunk(struct bpf_jit *jit)
{
if (nospec_uses_trampoline()) {
jit->r1_thunk_ip = jit->prg;
emit_expoline(jit);
/* br %r1 */
_EMIT2(0x07f1);
}
}
#define EMIT_JUMP_REG(reg) do { \
if (nospec_uses_trampoline()) \
/* brcl 0xf,__s390_indirect_jump_rN */ \
EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \
__s390_indirect_jump_r ## reg); \
else \
/* br %rN */ \
_EMIT2(0x07f0 | reg); \
} while (0)
/*
* Call r1 either directly or via __s390_indirect_jump_r1 thunk
@@ -648,7 +659,8 @@ static void call_r1(struct bpf_jit *jit)
{
if (nospec_uses_trampoline())
/* brasl %r14,__s390_indirect_jump_r1 */
EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14,
__s390_indirect_jump_r1);
else
/* basr %r14,%r1 */
EMIT2(0x0d00, REG_14, REG_1);
@@ -664,16 +676,7 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
EMIT4(0xb9040000, REG_2, BPF_REG_0);
/* Restore registers */
save_restore_regs(jit, REGS_RESTORE, stack_depth, 0);
if (nospec_uses_trampoline()) {
jit->r14_thunk_ip = jit->prg;
/* Generate __s390_indirect_jump_r14 thunk */
emit_expoline(jit);
}
/* br %r14 */
_EMIT2(0x07fe);
if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
emit_r1_thunk(jit);
EMIT_JUMP_REG(14);
jit->prg = ALIGN(jit->prg, 8);
jit->prologue_plt = jit->prg;
@@ -1875,7 +1878,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
/* aghi %r1,tail_call_start */
EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start);
/* brcl 0xf,__s390_indirect_jump_r1 */
EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip);
EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf,
__s390_indirect_jump_r1);
} else {
/* bc 0xf,tail_call_start(%r1) */
_EMIT4(0x47f01000 + jit->tail_call_start);
@@ -2844,17 +2848,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
0xf000 | tjit->tccnt_off);
/* aghi %r15,stack_size */
EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size);
/* Emit an expoline for the following indirect jump. */
if (nospec_uses_trampoline())
emit_expoline(jit);
if (flags & BPF_TRAMP_F_SKIP_FRAME)
/* br %r14 */
_EMIT2(0x07fe);
EMIT_JUMP_REG(14);
else
/* br %r1 */
_EMIT2(0x07f1);
emit_r1_thunk(jit);
EMIT_JUMP_REG(1);
return 0;
}