mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 10:31:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

bpf: Derive the atomic load register in one place

Browse Source 
check_atomic_rmw() open codes the mapping from a BPF_ATOMIC to the register
it reads the old value into, the BPF_STX case of insn_def_regno() open codes
the very same mapping a second time, the const folding and the liveness
transfer functions a third and a fourth time, and BPF JITs need it as well
to know which register a faulting BPF_PROBE_ATOMIC has to clear.

Add a small helper so that all of them can share it. No functional change.
The BPF_LOAD_ACQ case is there for the JITs, which do walk all instruction
classes. const_reg_xfer() loses its explicit BPF_ATOMIC mode test since the
helper checks class and mode itself; the BPF_PROBE_ATOMIC it additionally
accepts cannot be seen there as it is only set from bpf_do_misc_fixups(),
that is, after const folding has run. arg_track_xfer() keeps its mode test
since that also guards the stack clearing next to it.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://patch.msgid.link/20260811131600.506721-1-daniel@iogearbox.net
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
This commit is contained in:
Daniel Borkmann
2026-08-11 15:15:55 +02:00
committed by Eduard Zingerman
parent 07cb86aa50
commit 41c5dbb4be
5 changed files with 33 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
include/linux/filter.h
View File

@@ -414,6 +414,30 @@ static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn)
insn->imm == BPF_LOAD_ACQ;
}
/*
* Given an instruction @insn, return the number of the BPF register that a
* BPF_ATOMIC reads the value at its memory operand into, or -1 if there is
* no such register. That is the register a BPF_PROBE_ATOMIC has to clear when
* the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to
* be a BPF_ATOMIC here.
*/
static inline int bpf_atomic_load_reg(const struct bpf_insn *insn)
{
if (BPF_CLASS(insn->code) != BPF_STX ||
(BPF_MODE(insn->code) != BPF_ATOMIC &&
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC))
return -1;
switch (insn->imm) {
case BPF_LOAD_ACQ:
return insn->dst_reg;
case BPF_CMPXCHG:
return BPF_REG_0;
default:
return (insn->imm & BPF_FETCH) ? insn->src_reg : -1;
}
}
/* Memory store, *(uint *) (dst_reg + off16) = imm32 */
#define BPF_ST_MEM(SIZE, DST, OFF, IMM) \

11
kernel/bpf/const_fold.c
View File

@@ -199,14 +199,9 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
ci_out[r] = unknown;
break;
case BPF_STX:
if (mode != BPF_ATOMIC)
break;
if (insn->imm == BPF_CMPXCHG)
ci_out[BPF_REG_0] = unknown;
else if (insn->imm == BPF_LOAD_ACQ)
*dst = unknown;
else if (insn->imm & BPF_FETCH)
*src = unknown;
r = bpf_atomic_load_reg(insn);
if (r >= 0)
ci_out[r] = unknown;
break;
}
}

11
kernel/bpf/fixups.c
View File

@@ -49,16 +49,7 @@ static int insn_def_regno(const struct bpf_insn *insn)
case BPF_ST:
return -1;
case BPF_STX:
if (BPF_MODE(insn->code) == BPF_ATOMIC ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
if (insn->imm == BPF_CMPXCHG)
return BPF_REG_0;
else if (insn->imm == BPF_LOAD_ACQ)
return insn->dst_reg;
else if (insn->imm & BPF_FETCH)
return insn->src_reg;
}
return -1;
return bpf_atomic_load_reg(insn);
default:
return insn->dst_reg;
}

9
kernel/bpf/liveness.c
View File

@@ -1209,12 +1209,9 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
clear_stack_for_all_offs(insn, at_out, insn->dst_reg,
at_stack_out, sz);
if (insn->imm == BPF_CMPXCHG)
at_out[BPF_REG_0] = none;
else if (insn->imm == BPF_LOAD_ACQ)
*dst = none;
else if (insn->imm & BPF_FETCH)
*src = none;
r = bpf_atomic_load_reg(insn);
if (r >= 0)
at_out[r] = none;
}
} else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) {
u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));

13
kernel/bpf/verifier.c
View File

@@ -6485,21 +6485,12 @@ static int check_atomic_rmw(struct bpf_verifier_env *env,
return -EACCES;
}
if (insn->imm & BPF_FETCH) {
if (insn->imm == BPF_CMPXCHG)
load_reg = BPF_REG_0;
else
load_reg = insn->src_reg;
load_reg = bpf_atomic_load_reg(insn);
if (load_reg >= 0) {
/* check and record load of old value */
err = check_reg_arg(env, load_reg, DST_OP);
if (err)
return err;
} else {
/* This instruction accesses a memory location but doesn't
* actually load it into a register.
*/
load_reg = -1;
}
dst_reg = cur_regs(env) + insn->dst_reg;