From 964c0747686abda124cf718a91ac1c0c325ca52a Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Sat, 14 Feb 2026 20:40:38 +0800 Subject: [PATCH 1/5] bpf: allow calling bpf_kptr_xchg while holding a lock For the following scenario: struct tree_node { struct bpf_rb_node node; struct request __kptr *req; u64 key; }; struct bpf_rb_root tree_root __contains(tree_node, node); struct bpf_spin_lock tree_lock; If we need to traverse all nodes in the rbtree, retrieve the __kptr pointer from each node, and read kernel data from the referenced object, using bpf_kptr_xchg appears unavoidable. This patch skips the BPF verifier checks for bpf_kptr_xchg when called while holding a lock. Signed-off-by: Kaitao Cheng Link: https://lore.kernel.org/r/20260214124042.62229-2-pilgrimtao@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2ef00f9b94fe..ee63f27aa5e4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20998,7 +20998,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) if (env->cur_state->active_locks) { if ((insn->src_reg == BPF_REG_0 && - insn->imm != BPF_FUNC_spin_unlock) || + insn->imm != BPF_FUNC_spin_unlock && + insn->imm != BPF_FUNC_kptr_xchg) || (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { verbose(env, From ee9886c40aaeb0fa0817e28d995bb7c58d66ab25 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Sat, 14 Feb 2026 20:40:39 +0800 Subject: [PATCH 2/5] bpf: allow using bpf_kptr_xchg even if the NON_OWN_REF flag is set When traversing an rbtree using bpf_rbtree_left/right, if bpf_kptr_xchg is used to access the __kptr pointer contained in a node, it currently requires first removing the node with bpf_rbtree_remove and clearing the NON_OWN_REF flag, then re-adding the node to the original rbtree with bpf_rbtree_add after usage. This process significantly degrades rbtree traversal performance. The patch enables accessing __kptr pointers with the NON_OWN_REF flag set while holding the lock, eliminating the need for this remove-read-add sequence. Signed-off-by: Kaitao Cheng Signed-off-by: Feng Yang Link: https://lore.kernel.org/r/20260214124042.62229-3-pilgrimtao@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ee63f27aa5e4..7426dba5e5d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9309,7 +9309,8 @@ static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } static const struct bpf_reg_types kptr_xchg_dest_types = { .types = { PTR_TO_MAP_VALUE, - PTR_TO_BTF_ID | MEM_ALLOC + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, } }; static const struct bpf_reg_types dynptr_types = { @@ -9473,6 +9474,7 @@ found: } case PTR_TO_BTF_ID | MEM_ALLOC: case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: + case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && meta->func_id != BPF_FUNC_kptr_xchg) { verifier_bug(env, "unimplemented handling of MEM_ALLOC"); From 580fa3430b5d164fe8841e99825601d7a85af0d3 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Sat, 14 Feb 2026 20:40:40 +0800 Subject: [PATCH 3/5] selftests/bpf: Add supplementary tests for bpf_kptr_xchg 1. Allow using bpf_kptr_xchg while holding a lock. 2. When the rb_node contains a __kptr pointer, we do not need to perform a remove-read-add operation. This patch implements the following workflow: 1. Construct a rbtree with 16 elements. 2. Traverse the rbtree, locate the kptr pointer in the target node, and read the content pointed to by the pointer. 3. Remove all nodes from the rbtree. Signed-off-by: Kaitao Cheng Signed-off-by: Feng Yang Link: https://lore.kernel.org/r/20260214124042.62229-4-pilgrimtao@gmail.com Signed-off-by: Alexei Starovoitov --- .../testing/selftests/bpf/prog_tests/rbtree.c | 6 + tools/testing/selftests/bpf/progs/bpf_misc.h | 4 + .../selftests/bpf/progs/rbtree_search_kptr.c | 167 ++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/rbtree_search_kptr.c diff --git a/tools/testing/selftests/bpf/prog_tests/rbtree.c b/tools/testing/selftests/bpf/prog_tests/rbtree.c index d8f3d7a45fe9..a854fb38e418 100644 --- a/tools/testing/selftests/bpf/prog_tests/rbtree.c +++ b/tools/testing/selftests/bpf/prog_tests/rbtree.c @@ -9,6 +9,7 @@ #include "rbtree_btf_fail__wrong_node_type.skel.h" #include "rbtree_btf_fail__add_wrong_type.skel.h" #include "rbtree_search.skel.h" +#include "rbtree_search_kptr.skel.h" static void test_rbtree_add_nodes(void) { @@ -193,3 +194,8 @@ void test_rbtree_search(void) { RUN_TESTS(rbtree_search); } + +void test_rbtree_search_kptr(void) +{ + RUN_TESTS(rbtree_search_kptr); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index c9bfbe1bafc1..0904fe14ad1d 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -188,6 +188,10 @@ #define POINTER_VALUE 0xbadcafe #define TEST_DATA_LEN 64 +#ifndef __aligned +#define __aligned(x) __attribute__((aligned(x))) +#endif + #ifndef __used #define __used __attribute__((used)) #endif diff --git a/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c b/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c new file mode 100644 index 000000000000..069fc64b0167 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ + +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +#define NR_NODES 16 + +struct node_data { + int data; +}; + +struct tree_node { + struct bpf_rb_node node; + u64 key; + struct node_data __kptr * node_data; +}; + +#define private(name) SEC(".data." #name) __hidden __aligned(8) + +private(A) struct bpf_rb_root root __contains(tree_node, node); +private(A) struct bpf_spin_lock lock; + +static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct tree_node *node_a, *node_b; + + node_a = container_of(a, struct tree_node, node); + node_b = container_of(b, struct tree_node, node); + + return node_a->key < node_b->key; +} + +SEC("syscall") +__retval(0) +long rbtree_search_kptr(void *ctx) +{ + struct tree_node *tnode; + struct bpf_rb_node *rb_n; + struct node_data __kptr * node_data; + int lookup_key = NR_NODES / 2; + int lookup_data = NR_NODES / 2; + int i, data, ret = 0; + + for (i = 0; i < NR_NODES && can_loop; i++) { + tnode = bpf_obj_new(typeof(*tnode)); + if (!tnode) + return __LINE__; + + node_data = bpf_obj_new(typeof(*node_data)); + if (!node_data) { + bpf_obj_drop(tnode); + return __LINE__; + } + + tnode->key = i; + node_data->data = i; + + node_data = bpf_kptr_xchg(&tnode->node_data, node_data); + if (node_data) + bpf_obj_drop(node_data); + + bpf_spin_lock(&lock); + bpf_rbtree_add(&root, &tnode->node, less); + bpf_spin_unlock(&lock); + } + + bpf_spin_lock(&lock); + rb_n = bpf_rbtree_root(&root); + while (rb_n && can_loop) { + tnode = container_of(rb_n, struct tree_node, node); + node_data = bpf_kptr_xchg(&tnode->node_data, NULL); + if (!node_data) { + ret = __LINE__; + goto fail; + } + + data = node_data->data; + node_data = bpf_kptr_xchg(&tnode->node_data, node_data); + if (node_data) { + bpf_spin_unlock(&lock); + bpf_obj_drop(node_data); + return __LINE__; + } + + if (lookup_key == tnode->key) { + if (data == lookup_data) + break; + + ret = __LINE__; + goto fail; + } + + if (lookup_key < tnode->key) + rb_n = bpf_rbtree_left(&root, rb_n); + else + rb_n = bpf_rbtree_right(&root, rb_n); + } + bpf_spin_unlock(&lock); + + while (can_loop) { + bpf_spin_lock(&lock); + rb_n = bpf_rbtree_first(&root); + if (!rb_n) { + bpf_spin_unlock(&lock); + return 0; + } + + rb_n = bpf_rbtree_remove(&root, rb_n); + if (!rb_n) { + ret = __LINE__; + goto fail; + } + bpf_spin_unlock(&lock); + + tnode = container_of(rb_n, struct tree_node, node); + + node_data = bpf_kptr_xchg(&tnode->node_data, NULL); + if (node_data) + bpf_obj_drop(node_data); + + bpf_obj_drop(tnode); + } + + return 0; +fail: + bpf_spin_unlock(&lock); + return ret; +} + + +SEC("syscall") +__failure __msg("R1 type=scalar expected=map_value, ptr_, ptr_") +long non_own_ref_kptr_xchg_no_lock(void *ctx) +{ + struct tree_node *tnode; + struct bpf_rb_node *rb_n; + struct node_data __kptr * node_data; + int data; + + bpf_spin_lock(&lock); + rb_n = bpf_rbtree_first(&root); + if (!rb_n) { + bpf_spin_unlock(&lock); + return __LINE__; + } + bpf_spin_unlock(&lock); + + tnode = container_of(rb_n, struct tree_node, node); + node_data = bpf_kptr_xchg(&tnode->node_data, NULL); + if (!node_data) + return __LINE__; + + data = node_data->data; + if (data < 0) + return __LINE__; + + node_data = bpf_kptr_xchg(&tnode->node_data, node_data); + if (node_data) + return __LINE__; + + return 0; +} + +char _license[] SEC("license") = "GPL"; From fb1590448ff7e79e304f9bbe662fc20334038b50 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Sat, 14 Feb 2026 20:40:41 +0800 Subject: [PATCH 4/5] bpf: allow using bpf_kptr_xchg even if the MEM_RCU flag is set For the following scenario: struct tree_node { struct bpf_refcount ref; struct bpf_rb_node node; struct node_data __kptr * node_data; u64 key; }; This means node_data would have the type PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU. When traversing an rbtree using bpf_rbtree_left/right, if we need to use bpf_kptr_xchg to read the __kptr pointer, we still need to follow the remove-read-add sequence. This patch allows us to use bpf_kptr_xchg to directly read the __kptr pointer without any prior operations. Signed-off-by: Kaitao Cheng Signed-off-by: Feng Yang Link: https://lore.kernel.org/r/20260214124042.62229-5-pilgrimtao@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7426dba5e5d1..1153a828ce8d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9311,6 +9311,7 @@ static const struct bpf_reg_types kptr_xchg_dest_types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, } }; static const struct bpf_reg_types dynptr_types = { @@ -9475,6 +9476,7 @@ found: case PTR_TO_BTF_ID | MEM_ALLOC: case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: + case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && meta->func_id != BPF_FUNC_kptr_xchg) { verifier_bug(env, "unimplemented handling of MEM_ALLOC"); From ed8fa4b8894f7e80d3a6bf63e4cb880c7545c63a Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Sat, 14 Feb 2026 20:40:42 +0800 Subject: [PATCH 5/5] selftests/bpf: Add test case for rbtree nodes that contain both bpf_refcount and kptr fields. Allow bpf_kptr_xchg to directly operate on pointers marked with NON_OWN_REF | MEM_RCU. In the example demonstrated in this patch, as long as "struct bpf_refcount ref" exists, the __kptr pointer is guaranteed to carry the MEM_RCU flag. The ref member itself does not need to be explicitly used. Signed-off-by: Kaitao Cheng Link: https://lore.kernel.org/r/20260214124042.62229-6-pilgrimtao@gmail.com Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/rbtree_search_kptr.c | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c b/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c index 069fc64b0167..610aae45e2dc 100644 --- a/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c +++ b/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c @@ -18,11 +18,21 @@ struct tree_node { struct node_data __kptr * node_data; }; +struct tree_node_ref { + struct bpf_refcount ref; + struct bpf_rb_node node; + u64 key; + struct node_data __kptr * node_data; +}; + #define private(name) SEC(".data." #name) __hidden __aligned(8) private(A) struct bpf_rb_root root __contains(tree_node, node); private(A) struct bpf_spin_lock lock; +private(B) struct bpf_rb_root root_r __contains(tree_node_ref, node); +private(B) struct bpf_spin_lock lock_r; + static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { struct tree_node *node_a, *node_b; @@ -130,6 +140,119 @@ fail: return ret; } +static bool less_r(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct tree_node_ref *node_a, *node_b; + + node_a = container_of(a, struct tree_node_ref, node); + node_b = container_of(b, struct tree_node_ref, node); + + return node_a->key < node_b->key; +} + +SEC("syscall") +__retval(0) +long rbtree_search_kptr_ref(void *ctx) +{ + struct tree_node_ref *tnode_r, *tnode_m; + struct bpf_rb_node *rb_n; + struct node_data __kptr * node_data; + int lookup_key = NR_NODES / 2; + int lookup_data = NR_NODES / 2; + int i, data, ret = 0; + + for (i = 0; i < NR_NODES && can_loop; i++) { + tnode_r = bpf_obj_new(typeof(*tnode_r)); + if (!tnode_r) + return __LINE__; + + node_data = bpf_obj_new(typeof(*node_data)); + if (!node_data) { + bpf_obj_drop(tnode_r); + return __LINE__; + } + + tnode_r->key = i; + node_data->data = i; + + node_data = bpf_kptr_xchg(&tnode_r->node_data, node_data); + if (node_data) + bpf_obj_drop(node_data); + + /* Unused reference */ + tnode_m = bpf_refcount_acquire(tnode_r); + if (!tnode_m) + return __LINE__; + + bpf_spin_lock(&lock_r); + bpf_rbtree_add(&root_r, &tnode_r->node, less_r); + bpf_spin_unlock(&lock_r); + + bpf_obj_drop(tnode_m); + } + + bpf_spin_lock(&lock_r); + rb_n = bpf_rbtree_root(&root_r); + while (rb_n && can_loop) { + tnode_r = container_of(rb_n, struct tree_node_ref, node); + node_data = bpf_kptr_xchg(&tnode_r->node_data, NULL); + if (!node_data) { + ret = __LINE__; + goto fail; + } + + data = node_data->data; + node_data = bpf_kptr_xchg(&tnode_r->node_data, node_data); + if (node_data) { + bpf_spin_unlock(&lock_r); + bpf_obj_drop(node_data); + return __LINE__; + } + + if (lookup_key == tnode_r->key) { + if (data == lookup_data) + break; + + ret = __LINE__; + goto fail; + } + + if (lookup_key < tnode_r->key) + rb_n = bpf_rbtree_left(&root_r, rb_n); + else + rb_n = bpf_rbtree_right(&root_r, rb_n); + } + bpf_spin_unlock(&lock_r); + + while (can_loop) { + bpf_spin_lock(&lock_r); + rb_n = bpf_rbtree_first(&root_r); + if (!rb_n) { + bpf_spin_unlock(&lock_r); + return 0; + } + + rb_n = bpf_rbtree_remove(&root_r, rb_n); + if (!rb_n) { + ret = __LINE__; + goto fail; + } + bpf_spin_unlock(&lock_r); + + tnode_r = container_of(rb_n, struct tree_node_ref, node); + + node_data = bpf_kptr_xchg(&tnode_r->node_data, NULL); + if (node_data) + bpf_obj_drop(node_data); + + bpf_obj_drop(tnode_r); + } + + return 0; +fail: + bpf_spin_unlock(&lock_r); + return ret; +} SEC("syscall") __failure __msg("R1 type=scalar expected=map_value, ptr_, ptr_")