From 22a0cc10dacbafe1c28b6f513cc449cdd86d1cb1 Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Sat, 20 Jun 2026 02:44:16 +0000 Subject: [PATCH 01/16] selftests/bpf: don't modify the skb in the strparser parser prog sockmap_parse_prog.c is attached as an SK_SKB stream parser and modifies the skb: it calls bpf_skb_pull_data() and writes a byte into the packet. A stream parser runs on strparser's message head and must not modify it. A resize frees the frag_list segments strparser still tracks, leading to a use-after-free. Make the parser read-only. It only needs to return the message length, which keeps it attaching once packet-modifying parsers are rejected. Reviewed-by: Jiayuan Chen Signed-off-by: Sechang Lim Link: https://lore.kernel.org/r/20260620024423.4141004-2-rhkrqnwk98@gmail.com Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/sockmap_parse_prog.c | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c b/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c index c9abfe3a11af..56e9aebf05f2 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c @@ -5,28 +5,6 @@ SEC("sk_skb1") int bpf_prog1(struct __sk_buff *skb) { - void *data_end = (void *)(long) skb->data_end; - void *data = (void *)(long) skb->data; - __u8 *d = data; - int err; - - if (data + 10 > data_end) { - err = bpf_skb_pull_data(skb, 10); - if (err) - return SK_DROP; - - data_end = (void *)(long)skb->data_end; - data = (void *)(long)skb->data; - if (data + 10 > data_end) - return SK_DROP; - } - - /* This write/read is a bit pointless but tests the verifier and - * strparser handler for read/write pkt data and access into sk - * fields. - */ - d = data; - d[7] = 1; return skb->len; } From 31e2f36d3821811c03bddf5fd99ed8fc884fd222 Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Sat, 20 Jun 2026 02:44:17 +0000 Subject: [PATCH 02/16] bpf, sockmap: reject a packet-modifying SK_SKB stream parser sk_psock_strp_parse() runs the BPF_PROG_TYPE_SK_SKB stream-parser program to find the length of the next message. strparser assembles a message out of several received skbs by chaining them onto the head's frag_list and recording where to append the next one in strp->skb_nextp: *strp->skb_nextp = skb; strp->skb_nextp = &skb->next; and then calls the parser on the head: len = (*strp->cb.parse_msg)(strp, head); The parser is only meant to inspect the skb, but the program may call bpf_skb_change_tail() -- or the sibling bpf_skb_pull_data(), bpf_skb_change_head(), bpf_skb_adjust_room(), all allowed for SK_SKB. Once the head carries a frag_list these go ... -> skb_ensure_writable -> pskb_may_pull -> __pskb_pull_tail and __pskb_pull_tail() frees the frag_list skbs that strparser still tracks through skb_nextp: while ((list = skb_shinfo(skb)->frag_list) != insp) { skb_shinfo(skb)->frag_list = list->next; consume_skb(list); } strp->skb_nextp now points into a freed sk_buff. The next segment of the same message arrives in __strp_recv(), which links it with *strp->skb_nextp = skb, an 8-byte write into the freed skb. The free and the write happen in different __strp_recv() calls, so the message has to span at least three segments before it triggers. BUG: KASAN: slab-use-after-free in __strp_recv+0x447/0xda0 Write of size 8 at addr ffff88810db86140 by task repro/349 Call Trace: __strp_recv+0x447/0xda0 __tcp_read_sock+0x13d/0x590 tcp_bpf_strp_read_sock+0x195/0x320 strp_data_ready+0x267/0x340 sk_psock_strp_data_ready+0x1ce/0x350 tcp_data_queue+0x1364/0x2fd0 tcp_rcv_established+0xe07/0x1640 [...] Allocated by task 349: skb_clone+0x17b/0x210 __strp_recv+0x2c3/0xda0 __tcp_read_sock+0x13d/0x590 [...] Freed by task 349: kmem_cache_free+0x150/0x570 __pskb_pull_tail+0x57b/0xc20 skb_ensure_writable+0x236/0x260 __bpf_skb_change_tail+0x1d4/0x590 sk_skb_change_tail+0x2a/0x40 bpf_prog_1b285dcd6c41373e+0x27/0x30 bpf_prog_run_pin_on_cpu+0xf3/0x260 sk_psock_strp_parse+0x118/0x1e0 __strp_recv+0x4f6/0xda0 [...] The same resize also leaves the head's length inconsistent with its frags, so a later __pskb_pull_tail() can instead hit the BUG_ON(skb_copy_bits(...)) in net/core/skbuff.c. A stream parser is only meant to measure the next message, not to modify the packet. Reject a parser whose program can change packet data (prog->aux->changes_pkt_data) at attach time. The check is shared by sock_map_prog_update() and sock_map_link_update_prog(), which between them cover prog attach, link create and link update. Verdict programs are unaffected and may still modify the skb. Reviewed-by: Jiayuan Chen Signed-off-by: Sechang Lim Link: https://lore.kernel.org/r/20260620024423.4141004-3-rhkrqnwk98@gmail.com Signed-off-by: Alexei Starovoitov --- net/core/sock_map.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 99e3789492a0..c60ba6d292f9 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1515,6 +1515,17 @@ static int sock_map_prog_link_lookup(struct bpf_map *map, struct bpf_prog ***ppr return 0; } +static int sock_map_prog_attach_check(enum bpf_attach_type attach_type, + struct bpf_prog *prog) +{ + /* A stream parser must not modify the skb, only measure it. */ + if (prog && attach_type == BPF_SK_SKB_STREAM_PARSER && + prog->aux->changes_pkt_data) + return -EINVAL; + + return 0; +} + /* Handle the following four cases: * prog_attach: prog != NULL, old == NULL, link == NULL * prog_detach: prog == NULL, old != NULL, link == NULL @@ -1533,6 +1544,10 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, if (ret) return ret; + ret = sock_map_prog_attach_check(which, prog); + if (ret) + return ret; + /* for prog_attach/prog_detach/link_attach, return error if a bpf_link * exists for that prog. */ @@ -1776,6 +1791,11 @@ static int sock_map_link_update_prog(struct bpf_link *link, ret = -EINVAL; goto out; } + + ret = sock_map_prog_attach_check(link->attach_type, prog); + if (ret) + goto out; + if (!sockmap_link->map) { ret = -ENOLINK; goto out; From 05fb34384d20c49d596de34a47429e73ffb14959 Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Sat, 20 Jun 2026 02:44:18 +0000 Subject: [PATCH 03/16] selftests/bpf: test rejection of a packet-modifying SK_SKB stream parser Verify that attaching an SK_SKB stream parser that can modify the packet is rejected, while a read-only parser still attaches. Reviewed-by: Jiayuan Chen Signed-off-by: Sechang Lim Link: https://lore.kernel.org/r/20260620024423.4141004-4-rhkrqnwk98@gmail.com Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/sockmap_strp.c | 31 +++++++++++++++++++ .../selftests/bpf/progs/test_sockmap_strp.c | 7 +++++ 2 files changed, 38 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c index 621b3b71888e..1d7231728eaf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c @@ -431,6 +431,35 @@ static void test_sockmap_strp_verdict(int family, int sotype) test_sockmap_strp__destroy(strp); } +static void test_sockmap_strp_parser_reject(void) +{ + struct test_sockmap_strp *strp = NULL; + int parser_mod, parser_ro, link; + int err, map; + + strp = test_sockmap_strp__open_and_load(); + if (!ASSERT_OK_PTR(strp, "test_sockmap_strp__open_and_load")) + return; + + map = bpf_map__fd(strp->maps.sock_map); + parser_mod = bpf_program__fd(strp->progs.prog_skb_parser_resize); + parser_ro = bpf_program__fd(strp->progs.prog_skb_parser); + + err = bpf_prog_attach(parser_mod, map, BPF_SK_SKB_STREAM_PARSER, 0); + ASSERT_ERR(err, "bpf_prog_attach parser_mod"); + + link = bpf_link_create(parser_ro, map, BPF_SK_SKB_STREAM_PARSER, NULL); + if (!ASSERT_GE(link, 0, "bpf_link_create parser_ro")) + goto out; + + err = bpf_link_update(link, parser_mod, NULL); + ASSERT_ERR(err, "bpf_link_update parser_mod"); +out: + if (link >= 0) + close(link); + test_sockmap_strp__destroy(strp); +} + void test_sockmap_strp(void) { if (test__start_subtest("sockmap strp tcp pass")) @@ -451,4 +480,6 @@ void test_sockmap_strp(void) test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM); if (test__start_subtest("sockmap strp tcp dispatch")) test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM); + if (test__start_subtest("sockmap strp parser reject pkt mod")) + test_sockmap_strp_parser_reject(); } diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_strp.c b/tools/testing/selftests/bpf/progs/test_sockmap_strp.c index dde3d5bec515..fe88fa6d40bc 100644 --- a/tools/testing/selftests/bpf/progs/test_sockmap_strp.c +++ b/tools/testing/selftests/bpf/progs/test_sockmap_strp.c @@ -50,4 +50,11 @@ int prog_skb_parser_partial(struct __sk_buff *skb) return 10; } +SEC("sk_skb/stream_parser") +int prog_skb_parser_resize(struct __sk_buff *skb) +{ + bpf_skb_change_tail(skb, skb->len, 0); + return skb->len; +} + char _license[] SEC("license") = "GPL"; From 931a577fc79ea6a169a33f5538f4c1433235c358 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 23 Jun 2026 06:11:09 +0000 Subject: [PATCH 04/16] bpf: Reject offset refcount acquire arguments bpf_refcount_acquire() increments the refcount at the caller-supplied pointer plus the refcount field offset, then returns the caller-supplied pointer unchanged. The verifier records the return value as a base pointer to the refcounted object. bpf_list_pop_front() and bpf_rbtree_remove() can return embedded graph-node pointers as PTR_TO_BTF_ID | MEM_ALLOC with a fixed offset equal to the node field offset. Passing such a pointer directly to bpf_refcount_acquire() currently passes the refcounted-kptr type check. That makes the runtime operation start from base + node_off while the verifier models the returned pointer as the object base. Require refcount-acquire arguments to have zero fixed offset by carrying the requirement through check_func_arg_reg_off() to __check_ptr_off_reg(). Programs can still acquire a refcount from a graph-node-derived pointer after normalizing it with container_of(). Fixes: 7c50b1cb76aca ("bpf: Add bpf_refcount_acquire kfunc") Signed-off-by: Yiyang Chen Acked-by: Eduard Zingerman Acked-by: Yonghong Song Link: https://lore.kernel.org/r/2f894647f56f71838fdddeb97a3e057ed35ea92e.1782192383.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 21a365d436a5..3cdc2e90f643 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7996,9 +7996,10 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) +static int __check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type, + bool btf_id_fixed_off_ok) { u32 type = reg->type; @@ -8055,12 +8056,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero. This was already checked above. So pass - * fixed_off_ok as true to allow fixed offset for all other - * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we - * still need to do checks instead of returning. + * can be non-zero unless the caller requires otherwise. + * var_off always must be 0 for PTR_TO_BTF_ID, hence we still + * need to do checks instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, true); + return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8076,6 +8076,13 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, } } +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) +{ + return __check_func_arg_reg_off(env, reg, argno, arg_type, true); +} + static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -11947,6 +11954,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ enum bpf_arg_type arg_type = ARG_DONTCARE; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); + bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; @@ -12120,7 +12128,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_MEM: case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: case KF_ARG_PTR_TO_TIMER: @@ -12134,6 +12141,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_CTX: arg_type = ARG_PTR_TO_CTX; break; + case KF_ARG_PTR_TO_REFCOUNTED_KPTR: + arg_type = ARG_PTR_TO_BTF_ID; + btf_id_fixed_off_ok = false; + break; default: verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); return -EFAULT; @@ -12141,7 +12152,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; - ret = check_func_arg_reg_off(env, reg, argno, arg_type); + ret = __check_func_arg_reg_off(env, reg, argno, arg_type, + btf_id_fixed_off_ok); if (ret < 0) return ret; From 0371fb57a0c941592a5ad5ad5ac597d3b653ee73 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 23 Jun 2026 06:11:10 +0000 Subject: [PATCH 05/16] selftests/bpf: Cover refcount acquire node offsets Add regression coverage for bpf_refcount_acquire() on graph-node-derived pointers. The rejected case passes a popped list node pointer directly to bpf_refcount_acquire(), which must fail because the pointer carries a non-zero fixed offset. Signed-off-by: Yiyang Chen Reviewed-by: Emil Tsalapatis Acked-by: Yonghong Song Link: https://lore.kernel.org/r/bf2a2033ced272106292de4465b8ef3fb991c912.1782192383.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- .../bpf/progs/refcounted_kptr_fail.c | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 7247a20c0a3b..024ef2aae200 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -13,12 +13,20 @@ struct node_acquire { struct bpf_refcount refcount; }; +struct node_refcounted { + long key; + struct bpf_list_node list; + struct bpf_refcount refcount; +}; + extern void bpf_rcu_read_lock(void) __ksym; extern void bpf_rcu_read_unlock(void) __ksym; #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) private(A) struct bpf_spin_lock glock; private(A) struct bpf_rb_root groot __contains(node_acquire, node); +private(B) struct bpf_spin_lock lock; +private(B) struct bpf_list_head head __contains(node_refcounted, list); static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { @@ -93,6 +101,32 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) return 0; } +SEC("?tc") +__failure __msg("dereference of modified ptr_ ptr R1") +long refcount_acquire_list_node_offset(void *ctx) +{ + struct node_refcounted *node, *base, *ref; + struct bpf_list_node *list_node; + + node = bpf_obj_new(typeof(*node)); + if (!node) + return 1; + + bpf_spin_lock(&lock); + bpf_list_push_front(&head, &node->list); + list_node = bpf_list_pop_front(&head); + bpf_spin_unlock(&lock); + if (!list_node) + return 2; + + base = container_of(list_node, struct node_refcounted, list); + ref = bpf_refcount_acquire(list_node); + if (ref) + bpf_obj_drop(ref); + bpf_obj_drop(base); + return 0; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("function calls are not allowed while holding a lock") int BPF_PROG(rbtree_fail_sleepable_lock_across_rcu, From 72a85e9464a5332fb2cd7efd26d9295275ceda2d Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Tue, 23 Jun 2026 18:43:38 +0800 Subject: [PATCH 06/16] bpf: Mask pseudo pointer values in verifier logs print_bpf_insn() masks ldimm64 immediates for pointer-bearing pseudo sources when pointer leaks are not allowed, but the mask only covers BPF_PSEUDO_MAP_FD and BPF_PSEUDO_MAP_VALUE. BPF_PSEUDO_MAP_IDX, BPF_PSEUDO_MAP_IDX_VALUE, and BPF_PSEUDO_BTF_ID can also be resolved to kernel pointer values before the verifier log prints the instruction. Include them in the existing pointer classification so the log prints 0x0 instead of the rewritten address. Fixes: 4976b718c355 ("bpf: Introduce pseudo_btf_id") Fixes: 387544bfa291 ("bpf: Introduce fd_idx") Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/r/20260623-f01-13-pseudo-btf-id-cap-bpf-v2-1-a190ebb8f3e2@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman --- kernel/bpf/disasm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index f8a3c7eb451e..0391b3bc0073 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, */ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || - insn->src_reg == BPF_PSEUDO_MAP_VALUE; + insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE || + insn->src_reg == BPF_PSEUDO_BTF_ID; char tmp[64]; if (is_ptr && !allow_ptr_leaks) From 8a870967ca612933974808e6f4725613fea0cece Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Tue, 23 Jun 2026 18:43:39 +0800 Subject: [PATCH 07/16] selftests/bpf: Cover pseudo-BTF ksym log masking Add verifier_unpriv coverage for a raw socket-filter load of the bpf_prog_active typed ksym. The test verifies that the unprivileged load remains accepted and that the verbose verifier log prints the ldimm64 immediate as 0x0 instead of exposing a nonzero kernel address. Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/r/20260623-f01-13-pseudo-btf-id-cap-bpf-v2-2-a190ebb8f3e2@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman --- .../selftests/bpf/progs/verifier_unpriv.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 49f7bd05edad..42de5cff7e52 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -6,6 +6,8 @@ #include "../../../include/linux/filter.h" #include "bpf_misc.h" +extern const int bpf_prog_active __ksym; + #define BPF_SK_LOOKUP(func) \ /* struct bpf_sock_tuple tuple = {} */ \ "r2 = 0;" \ @@ -77,6 +79,23 @@ __naked void dummy_prog_loop1_socket(void) : __clobber_all); } +SEC("socket") +__description("unpriv: pseudo btf id log masks address") +__success_unpriv +__msg_unpriv("0: (18) r1 = 0x0") +__not_msg_unpriv("0: (18) r1 = 0x{{[1-9a-f][0-9a-f]*}}") +__retval_unpriv(0) +__log_level(2) +__naked void pseudo_btf_id_log_masks_address(void) +{ + asm volatile ("r1 = %[bpf_prog_active] ll;" + "r0 = 0;" + "exit;" + : + : __imm_addr(bpf_prog_active) + : __clobber_all); +} + SEC("socket") __description("unpriv: return pointer") __success __failure_unpriv __msg_unpriv("R0 leaks addr") From 26490a375cb9be9bac96b5171610fd85ca6c2305 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Wed, 24 Jun 2026 20:35:35 +0800 Subject: [PATCH 08/16] bpf: Fix insn_aux_data leak on verifier err_free_env path When bpf_check() allocates env->insn_aux_data successfully but later fails to allocate env->succ, it jumps directly to err_free_env. The existing vfree(env->insn_aux_data) sits before the err_free_env label, so that direct jump bypasses it and leaks insn_aux_data. Move vfree(env->insn_aux_data) into err_free_env so all early and late exit paths release it consistently. Fixes: 2f69c5685427 ("bpf: make bpf_insn_successors to return a pointer") Signed-off-by: KaFai Wan Reviewed-by: Anton Protopopov Link: https://lore.kernel.org/r/20260624123536.114757-1-kafai.wan@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3cdc2e90f643..6515d4d3c003 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20006,13 +20006,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (!is_priv) mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len); - vfree(env->insn_aux_data); err_free_env: bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + vfree(env->insn_aux_data); kvfree(env); return ret; } From 9b51a6155d14389876916726430da30eabb1d4ed Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Fri, 26 Jun 2026 17:52:52 +0200 Subject: [PATCH 09/16] bpf,fork: wipe ->bpf_storage before bailouts that access it Currently, copy_process() can bail out to free_task() before p->bpf_storage has been initialized, with this call graph (shown here for the !CONFIG_MEMCG case): copy_process dup_task_struct arch_dup_task_struct [copies the entire task_struct, including ->bpf_storage member] [RLIMIT_NPROC check fails] delayed_free_task free_task bpf_task_storage_free rcu_dereference(task->bpf_storage) bpf_local_storage_destroy In this case, the nascent task's ->bpf_storage member that bpf_local_storage_destroy() operates on is a plain copy of the parent's ->bpf_storage pointer, not a real initialized pointer. This leads to badness (kernel hangs, UAF). This is reachable as long as the process calling fork() has been inserted into a task storage map. Cc: stable@kernel.org Fixes: a10787e6d58c ("bpf: Enable task local storage for tracing programs") Signed-off-by: Jann Horn Signed-off-by: Andrii Nakryiko --- kernel/fork.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 13e38e89a1f3..f0e2e131a9a5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1009,6 +1009,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->mm_cid.active = 0; INIT_HLIST_NODE(&tsk->mm_cid.node); #endif + +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(tsk->bpf_storage, NULL); + tsk->bpf_ctx = NULL; +#endif return tsk; free_stack: @@ -2247,10 +2252,6 @@ __latent_entropy struct task_struct *copy_process( p->sequential_io = 0; p->sequential_io_avg = 0; #endif -#ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); - p->bpf_ctx = NULL; -#endif unwind_task_init(p); From a6f0643e4f63cfaa0d5d4a69de4f132eac4b8fe4 Mon Sep 17 00:00:00 2001 From: Matt Bobrowski Date: Sun, 28 Jun 2026 20:11:03 +0000 Subject: [PATCH 10/16] bpf: Reject BPF_MAP_TYPE_INODE_STORAGE creation if BPF LSM is uninitialized When CONFIG_BPF_LSM=y is set, BPF inode storage maps (BPF_MAP_TYPE_INODE_STORAGE) are compiled into the kernel. However, if the BPF LSM is not explicitly enabled at boot time (e.g. omitted from the "lsm=" boot parameter), lsm_prepare() is never executed for the BPF LSM. Consequently, the BPF inode security blob offset (bpf_lsm_blob_sizes.lbs_inode) is never initialized and remains at its default compiled size of 8 bytes instead of being updated to a valid offset past the reserved struct rcu_head (typically 16 bytes or more). When a privileged user creates and updates a BPF_MAP_TYPE_INODE_STORAGE map, bpf_inode() evaluates inode->i_security + 8. This erroneously aliases the struct rcu_head.func callback pointer at the beginning of the inode->i_security blob. During subsequent map element cleanup or inode destruction, writing NULL to owner_storage clears the queued RCU callback pointer. When rcu_do_batch() later executes the queued callback, it attempts an instruction fetch at address 0x0, triggering an immediate kernel panic. Fix this by introducing a global bpf_lsm_initialized boolean flag marked with __ro_after_init. Set this flag to true inside bpf_lsm_init() when the LSM framework successfully registers the BPF LSM. Gate map allocation in inode_storage_map_alloc() on this flag, returning -EOPNOTSUPP if the BPF LSM is in turn uninitialized. This fail-fast approach prevents userspace from allocating inode storage maps when the supporting BPF LSM infrastructure is absent, avoiding zombie map states. Fixes: 8ea636848aca ("bpf: Implement bpf_local_storage for inodes") Reported-by: oxsignal Signed-off-by: Matt Bobrowski Signed-off-by: Daniel Borkmann Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260628201103.3624525-1-mattbobrowski@google.com --- include/linux/bpf_lsm.h | 4 ++++ kernel/bpf/bpf_inode_storage.c | 9 +++++++++ security/bpf/hooks.c | 3 +++ 3 files changed, 16 insertions(+) diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 143775a27a2a..dda272d78f01 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -14,6 +14,8 @@ #ifdef CONFIG_BPF_LSM +extern bool bpf_lsm_initialized __ro_after_init; + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ RET bpf_lsm_##NAME(__VA_ARGS__); #include @@ -56,6 +58,8 @@ bool bpf_lsm_hook_returns_errno(u32 btf_id); #else /* !CONFIG_BPF_LSM */ +#define bpf_lsm_initialized false + static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id) { return false; diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c index 0da8d923e39d..f9e81060c1f4 100644 --- a/kernel/bpf/bpf_inode_storage.c +++ b/kernel/bpf/bpf_inode_storage.c @@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) { + /* + * Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM + * was not initialized by the LSM framework at boot. Without proper + * initialization, the BPF inode security blob offset remains unprepared, + * causing bpf_inode() to calculate an invalid memory offset and corrupt + * inode->i_security. + */ + if (!bpf_lsm_initialized) + return ERR_PTR(-EOPNOTSUPP); return bpf_local_storage_map_alloc(attr, &inode_cache); } diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c index 40efde233f3a..7b98f5d1e2be 100644 --- a/security/bpf/hooks.c +++ b/security/bpf/hooks.c @@ -7,6 +7,8 @@ #include #include +bool bpf_lsm_initialized __ro_after_init; + static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = { #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ LSM_HOOK_INIT(NAME, bpf_lsm_##NAME), @@ -24,6 +26,7 @@ static int __init bpf_lsm_init(void) { security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks), &bpf_lsmid); + bpf_lsm_initialized = true; pr_info("LSM support for eBPF active\n"); return 0; } From 96cce16e26dd02a8678f1e87f88a4b5cdb63b995 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:37:52 -0700 Subject: [PATCH 11/16] bpf: Support for hardening against JIT spraying The BPF JIT allocator packs many small programs into larger executable allocations and reuses space within those allocations as programs are loaded and freed. When fresh code is written into space that a previous program occupied, an indirect jump into the new program can reuse a branch prediction left behind by the old one. Flush the indirect branch predictors before reusing JIT memory so that indirect jumps into a newly written program don't reuse predictions from an old program that occupied the same space. Introduce bpf_arch_pred_flush_enabled static key and bpf_arch_pred_flush static call for flushing the branch predictors on JIT memory reuse. Architectures that need a flush, can update it to a predictor flush function. By default, its a NOP and does not emit any CALL. Allocations larger than a pack are not covered by this flush. That is safe because cBPF programs (the unprivileged attack surface) are bounded well below a pack size. Issue a warning if this assumption is ever violated while the flush is active. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- include/linux/filter.h | 10 ++++++++++ kernel/bpf/core.c | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/include/linux/filter.h b/include/linux/filter.h index 67d337ede91b..f68694f94ee7 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1314,6 +1315,15 @@ extern long bpf_jit_limit_max; typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); +/* + * Flush the indirect branch predictors before reusing JIT memory, so that + * indirect jumps into a newly written program don't reuse predictions left + * behind by an old program that occupied the same space. + */ +void bpf_arch_pred_flush(void); +DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush); +DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); struct bpf_binary_header * diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 649cce41e13f..7f0a17f128d4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -883,6 +884,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) memset(area, 0, size); } +DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); + +/* + * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the + * pack allocator test "is a predictor flush wired up at all" with a cheap + * static branch instead of repeatedly querying the static call target. + */ +DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -941,6 +951,14 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { + /* + * Allocations larger than a pack get their own pages, and + * predictors are not flushed for such allocation. This is only + * safe because cBPF programs (the unprivileged attack surface) + * are bounded well below a pack size. + */ + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); if (ptr) { @@ -971,6 +989,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); From a3af84b0fa00ead01fcd0e28b5d773ff25990a0d Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:07 -0700 Subject: [PATCH 12/16] x86/bugs: Enable IBPB flush on BPF JIT allocation Enable hardening against JIT spraying when Spectre-v2 mitigations are in use. Specifically, issue an IBPB flush on BPF JIT memory reuse. Skip enabling the IBPB flush if the BPF dispatcher is already using a retpoline sequence. This hardening applies only when BPF-JIT is in use. Guard the enabling under CONFIG_BPF_JIT so that bugs.c still builds with CONFIG_BPF_JIT=n. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Acked-by: Dave Hansen Signed-off-by: Daniel Borkmann --- arch/x86/include/asm/nospec-branch.h | 4 +++ arch/x86/kernel/cpu/bugs.c | 50 +++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 4f4b5e8a1574..b68892e6d58c 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -388,6 +388,10 @@ extern void srso_alias_return_thunk(void); extern void entry_untrain_ret(void); extern void write_ibpb(void); +#ifdef CONFIG_BPF_JIT +extern void bpf_arch_ibpb(void); +#endif + #ifdef CONFIG_X86_64 extern void clear_bhb_loop(void); #endif diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 83f51cab0b1e..d9af230c0512 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -1651,8 +1652,21 @@ static inline const char *spectre_v2_module_string(void) { return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; } + +/* + * The "retpoline sequence" is the "call;mov;ret" sequence that + * replaces normal indirect branch instructions. Differentiate + * *the* retpoline sequence from the LFENCE-prefixed indirect + * branches that simply use the retpoline infrastructure. + */ +static inline bool retpoline_seq_enabled(void) +{ + return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE); +} + #else static inline const char *spectre_v2_module_string(void) { return ""; } +static inline bool retpoline_seq_enabled(void) { return false; } #endif #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" @@ -2095,8 +2109,7 @@ static void __init bhi_apply_mitigation(void) return; /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ - if (boot_cpu_has(X86_FEATURE_RETPOLINE) && - !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { + if (retpoline_seq_enabled()) { spec_ctrl_disable_kernel_rrsba(); if (rrsba_disabled) return; @@ -2238,6 +2251,27 @@ static void __init spectre_v2_update_mitigation(void) pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); } +#ifdef CONFIG_BPF_JIT +static void __bpf_arch_ibpb(void *unused) +{ + write_ibpb(); +} + +void bpf_arch_ibpb(void) +{ + on_each_cpu(__bpf_arch_ibpb, NULL, 1); +} + +static bool __init cpu_wants_ibpb_bpf(void) +{ + /* A genuine retpoline already neutralizes ring0 indirect predictions */ + if (retpoline_seq_enabled()) + return false; + + return boot_cpu_has(X86_FEATURE_IBPB); +} +#endif + static void __init spectre_v2_apply_mitigation(void) { if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) @@ -2314,6 +2348,14 @@ static void __init spectre_v2_apply_mitigation(void) setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); pr_info("Enabling Restricted Speculation for firmware calls\n"); } + +#ifdef CONFIG_BPF_JIT + if (cpu_wants_ibpb_bpf()) { + static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb); + static_branch_enable(&bpf_pred_flush_enabled); + pr_info("Enabling IBPB for BPF\n"); + } +#endif } static void update_stibp_msr(void * __unused) @@ -3490,9 +3532,7 @@ static const char *spectre_bhi_state(void) return "; BHI: BHI_DIS_S"; else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) return "; BHI: SW loop, KVM: SW loop"; - else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && - !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) && - rrsba_disabled) + else if (retpoline_seq_enabled() && rrsba_disabled) return "; BHI: Retpoline"; else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) return "; BHI: Vulnerable, KVM: SW loop"; From 0bb99f2cfaae6822d734d69722de30af823efdf3 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:23 -0700 Subject: [PATCH 13/16] bpf: Restrict JIT predictor flush to cBPF Currently predictor flush on memory reuse is done for all BPF JIT allocations, but only cBPF programs can be loaded by an unprivileged user. eBPF is privileged by default, and flushing predictors for all CPUs on every eBPF reuse penalizes the common case for no security benefit. eBPF allocations can be frequent on busy systems, only flush predictors for cBPF programs. Trampoline and dispatcher allocations also skip the flush as they are eBPF-only. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- arch/arm64/net/bpf_jit_comp.c | 4 ++-- arch/loongarch/net/bpf_jit.c | 5 +++-- arch/powerpc/net/bpf_jit_comp.c | 4 ++-- arch/riscv/net/bpf_jit_comp64.c | 2 +- arch/riscv/net/bpf_jit_core.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 5 +++-- include/linux/filter.h | 5 +++-- kernel/bpf/core.c | 13 ++++++++----- kernel/bpf/dispatcher.c | 2 +- 9 files changed, 25 insertions(+), 18 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index f6bcc0e1a950..b0075ece4a6e 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2177,7 +2177,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr image_size = extable_offset + extable_size; ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u64), &header, &image_ptr, - jit_fill_hole); + jit_fill_hole, was_classic); if (!ro_header) goto out_off; @@ -2870,7 +2870,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 058ffbbaad85..3f3f0335d63c 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1762,7 +1762,7 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -2228,7 +2228,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr image_size = prog_size + extable_size; /* Now we know the size of the structure to make */ ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), - &header, &image_ptr, jit_fill_hole); + &header, &image_ptr, jit_fill_hole, + bpf_prog_was_classic(prog)); if (!ro_header) goto out_offset; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index d4a17e18c9fb..7b07b43575f1 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -295,7 +295,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, - bpf_jit_fill_ill_insns); + bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp)); if (!fhdr) goto out_err; @@ -588,7 +588,7 @@ bool bpf_jit_inlines_helper_call(s32 imm) void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index c03c1de16b79..f9d5347ba966 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1321,7 +1321,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 4365d07aaf54..ce3bd3762e08 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -109,7 +109,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr bpf_jit_binary_pack_alloc(prog_size + extable_size, &jit_data->ro_image, sizeof(u32), &jit_data->header, &jit_data->image, - bpf_fill_ill_insns); + bpf_fill_ill_insns, + bpf_prog_was_classic(prog)); if (!jit_data->ro_header) goto out_offset; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 054e043ffcd2..de7515ea1bea 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3653,7 +3653,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -3965,7 +3965,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* allocate module memory for x86 insns and extable */ header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size, &image, align, &rw_header, &rw_image, - jit_fill_hole); + jit_fill_hole, + bpf_prog_was_classic(prog)); if (!header) goto out_addrs; prog->aux->extable = (void *) image + roundup(proglen, align); diff --git a/include/linux/filter.h b/include/linux/filter.h index f68694f94ee7..14acb2455746 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1338,7 +1338,7 @@ void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic); void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) @@ -1352,7 +1352,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, unsigned int alignment, struct bpf_binary_header **rw_hdr, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns); + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic); int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, struct bpf_binary_header *rw_header); void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7f0a17f128d4..1614ccc3f111 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -942,7 +942,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins return NULL; } -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); struct bpf_prog_pack *pack; @@ -957,7 +957,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) * safe because cBPF programs (the unprivileged attack surface) * are bounded well below a pack size. */ - if (static_branch_unlikely(&bpf_pred_flush_enabled)) + if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); @@ -989,7 +989,9 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: - static_call_cond(bpf_arch_pred_flush)(); + /* Flush only for cBPF as it may contain a crafted gadget */ + if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1149,7 +1151,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, struct bpf_binary_header **rw_header, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns) + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic) { struct bpf_binary_header *ro_header; u32 size, hole, start; @@ -1162,7 +1165,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(size)) return NULL; - ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); if (!ro_header) { bpf_jit_uncharge_modmem(size); return NULL; diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index b77db7413f8c..ea2d60dc1fee 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); From a23c1c5396a91680703360d1ee28a44657c503c4 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:38 -0700 Subject: [PATCH 14/16] bpf: Skip redundant IBPB in pack allocator bpf_prog_pack_alloc() issues IBPB on all CPUs on every cBPF allocation, even when reusing chunks from an existing pack where no new memory was touched since the last IBPB. Since IBPB on all CPUs is heavy, Dave Hansen suggested to track allocation since last IBPB, and only issue IBPB at reuse for the chunks that have not seen an IBPB since they were last freed. Track per-pack whether an IBPB is needed via arch_flush_needed. Set it when allocating a chunk, reset on IBPB flush. On reuse, conditionally issue the flush. Since IBPB invalidates all BTB entries, clear the flag on all packs after flushing. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1614ccc3f111..50aba113ef9d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -876,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, struct bpf_prog_pack { struct list_head list; void *ptr; + bool arch_flush_needed; unsigned long bitmap[]; }; @@ -928,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; set_vm_flush_reset_perms(pack->ptr); err = set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); @@ -990,8 +993,15 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool found_free_area: /* Flush only for cBPF as it may contain a crafted gadget */ - if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + if (static_branch_unlikely(&bpf_pred_flush_enabled) && + pack->arch_flush_needed && + was_classic) { + struct bpf_prog_pack *p; + static_call_cond(bpf_arch_pred_flush)(); + list_for_each_entry(p, &pack_list, list) + p->arch_flush_needed = false; + } bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1029,6 +1039,9 @@ void bpf_prog_pack_free(void *ptr, u32 size) "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); + + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); From a9b1f19a6a673ba06820898d0f1ad02883ea1639 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:54 -0700 Subject: [PATCH 15/16] bpf: Prefer packs that won't trigger an IBPB flush on allocation Currently BPF pack allocator picks the chunks from the first available pack. While this is okay, it naturally leads to more frequent flushes when there are multiple packs in the system that weren't used since the last flush. As an optimization prefer allocating the new programs from packs that are unused since last flush. When all packs are dirty, allocation forces a flush and marks all packs clean. Below are some future optimizations ideas: 1. Currently, the "dirty" tracking is only done at the pack-level. Flush frequency can further be reduced with chunk-level tracking. This requires a new bitmap per-pack to track the dirty state. 2. IBPB flush is done on all CPUs, even if only a single CPU ran the BPF program. On a system with hundreds of CPUs this could be a major bottleneck forcing hundreds of IPIs to deliver the flush. The solution is to track the CPUs where a BPF program ran, and issue IBPB only on those CPUs. 3. Avoid IBPB when flush is already done at other sources (e.g. context switch). Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 50aba113ef9d..1b32b9f2491f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -948,8 +948,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); - struct bpf_prog_pack *pack; - unsigned long pos; + struct bpf_prog_pack *pack, *fallback_pack = NULL; + unsigned long pos, fallback_pos = 0; void *ptr = NULL; mutex_lock(&pack_mutex); @@ -981,8 +981,29 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool list_for_each_entry(pack, &pack_list, list) { pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, nbits, 0); - if (pos < BPF_PROG_CHUNK_COUNT) + if (pos >= BPF_PROG_CHUNK_COUNT) + continue; + /* Flush not enabled, use any pack */ + if (!static_branch_unlikely(&bpf_pred_flush_enabled)) goto found_free_area; + /* + * cBPF reuse of a dirty pack triggers a flush, so prefer a + * clean pack for cBPF. eBPF never flushes, so pick the first + * free pack, dirty or clean. + */ + if (!was_classic || !pack->arch_flush_needed) + goto found_free_area; + if (!fallback_pack) { + fallback_pack = pack; + fallback_pos = pos; + } + } + + /* No preferred pack found */ + if (fallback_pack) { + pack = fallback_pack; + pos = fallback_pos; + goto found_free_area; } pack = alloc_new_pack(bpf_fill_ill_insns); From b72e29e0f7ee329d89f86db8700c8ea99b4a370a Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:39:29 -0700 Subject: [PATCH 16/16] bpf: Prefer dirty packs for eBPF allocations The pack allocator only flushes predictors when reusing a dirty pack for cBPF, eBPF allocations never trigger a flush. Currently, eBPF picks the first free pack, which could be a clean pack. As an optimization, leaving a clean pack for cBPF can avoid flushes. Prefer dirty packs for eBPF and keep clean packs free for cBPF. This mirrors the existing cBPF preference for clean packs: each program kind prefers the pack that avoids an extra flush, and falls back to the other kind only when no preferred pack has room. eBPF reuse of a dirty pack is harmless since eBPF being privileged does not flush. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1b32b9f2491f..6e19a030da6f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -988,10 +988,10 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool goto found_free_area; /* * cBPF reuse of a dirty pack triggers a flush, so prefer a - * clean pack for cBPF. eBPF never flushes, so pick the first - * free pack, dirty or clean. + * clean pack for cBPF. eBPF never flushes, so steer it to a + * dirty pack and keep clean packs free for cBPF. */ - if (!was_classic || !pack->arch_flush_needed) + if (was_classic ^ pack->arch_flush_needed) goto found_free_area; if (!fallback_pack) { fallback_pack = pack;