diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index f5eaeb2493d4..14a5fdfa0421 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -782,8 +782,8 @@ __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) return -EINVAL; } - /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ - if ((s64)end - (s64)start > BPF_MAX_LOOPS) { + /* start <= end here, so end - start fits in a u32 without overflow */ + if ((u32)(end - start) > BPF_MAX_LOOPS) { s->cur = s->end = 0; return -E2BIG; } @@ -802,12 +802,11 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) { struct bpf_iter_num_kern *s = (void *)it; - /* check failed initialization or if we are done (same behavior); - * need to be careful about overflow, so convert to s64 for checks, - * e.g., if s->cur == s->end == INT_MAX, we can't just do - * s->cur + 1 >= s->end + /* + * s->cur < s->end while iterating, else s->cur == s->end == 0; the signed + * s->cur + 1 >= s->end holds even when s->cur + 1 wraps (start == INT_MIN). */ - if ((s64)(s->cur + 1) >= s->end) { + if (s->cur + 1 >= s->end) { s->cur = s->end = 0; return NULL; } @@ -819,9 +818,7 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) { - struct bpf_iter_num_kern *s = (void *)it; - - s->cur = s->end = 0; + /* no-op */ } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 09588b7b08b0..4db151e24355 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20006,6 +20006,51 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1); insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0); *cnt = 6; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) { + /* inline bpf_iter_num_new(&it, start, end); R1=&it, R2=start, R3=end */ + int i = 0; + + /* if (start > end) goto einval; */ + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8); + /* r0 = (u32)end - (u32)start; if (r0 > BPF_MAX_LOOPS) goto e2big; */ + insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3); + insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2); + insn_buf[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, BPF_MAX_LOOPS, 8); + /* s->cur = start - 1; s->end = end; return 0; */ + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insn_buf[i++] = BPF_JMP_A(5); + /* einval: s->cur = s->end = 0; return -EINVAL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); + insn_buf[i++] = BPF_JMP_A(2); + /* e2big: s->cur = s->end = 0; return -E2BIG; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); + *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { + /* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */ + int i = 0; + + /* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */ + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); + /* s->cur = r0; return &s->cur; */ + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); + insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); + insn_buf[i++] = BPF_JMP_A(2); + /* done: s->cur = s->end = 0; return NULL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_destroy]) { + /* bpf_iter_num_destroy() is a no-op; emit a nop to drop the call */ + insn_buf[0] = BPF_JMP_A(0); + *cnt = 1; } if (env->insn_aux_data[insn_idx].arg_prog) { diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 2749b26fd4cd..d3655a706482 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -974,6 +974,7 @@ $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ $(OUTPUT)/perfbuf_bench.skel.h $(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h $(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h +$(OUTPUT)/bench_bpf_for.o: $(OUTPUT)/bpf_for_bench.skel.h $(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h $(OUTPUT)/bench_bpf_hashmap_full_update.o: $(OUTPUT)/bpf_hashmap_full_update_bench.skel.h $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h @@ -999,6 +1000,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_ringbufs.o \ $(OUTPUT)/bench_bloom_filter_map.o \ $(OUTPUT)/bench_bpf_loop.o \ + $(OUTPUT)/bench_bpf_for.o \ $(OUTPUT)/bench_strncmp.o \ $(OUTPUT)/bench_bpf_hashmap_full_update.o \ $(OUTPUT)/bench_local_storage.o \ diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 3d9d2cd7764b..b86b73456d3c 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -276,6 +276,7 @@ static const struct argp_option opts[] = { extern struct argp bench_ringbufs_argp; extern struct argp bench_bloom_map_argp; extern struct argp bench_bpf_loop_argp; +extern struct argp bench_bpf_for_argp; extern struct argp bench_local_storage_argp; extern struct argp bench_local_storage_rcu_tasks_trace_argp; extern struct argp bench_strncmp_argp; @@ -292,6 +293,7 @@ static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 }, { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 }, + { &bench_bpf_for_argp, 0, "bpf_for loop benchmark", 0 }, { &bench_local_storage_argp, 0, "local_storage benchmark", 0 }, { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 }, { &bench_local_storage_rcu_tasks_trace_argp, 0, @@ -557,6 +559,7 @@ extern const struct bench bench_bloom_false_positive; extern const struct bench bench_hashmap_without_bloom; extern const struct bench bench_hashmap_with_bloom; extern const struct bench bench_bpf_loop; +extern const struct bench bench_bpf_for; extern const struct bench bench_strncmp_no_helper; extern const struct bench bench_strncmp_helper; extern const struct bench bench_bpf_hashmap_full_update; @@ -640,6 +643,7 @@ static const struct bench *benchs[] = { &bench_hashmap_without_bloom, &bench_hashmap_with_bloom, &bench_bpf_loop, + &bench_bpf_for, &bench_strncmp_no_helper, &bench_strncmp_helper, &bench_bpf_hashmap_full_update, diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_for.c b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c new file mode 100644 index 000000000000..730c51ad2dec --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include "bench.h" +#include "bpf_for_bench.skel.h" + +/* BPF triggering benchmarks */ +static struct ctx { + struct bpf_for_bench *skel; +} ctx; + +static struct { + __u32 nr_loops; +} args = { + /* + * Default to a large loop count so the per-iteration bpf_iter_num_next() cost dominates + * the one-time bpf_iter_num_new()/destroy() setup and teardown. + */ + .nr_loops = 1000, +}; + +enum { + ARG_NR_LOOPS = 4000, +}; + +static const struct argp_option opts[] = { + { "nr_loops", ARG_NR_LOOPS, "nr_loops", 0, + "Set number of iterations for the bpf_for() loop"}, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_NR_LOOPS: + args.nr_loops = strtol(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +/* exported into benchmark runner */ +const struct argp bench_bpf_for_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumer!\n"); + exit(1); + } +} + +static void *producer(void *input) +{ + while (true) + /* trigger the bpf program */ + syscall(__NR_getpgid); + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->hits, 0); +} + +static void setup(void) +{ + struct bpf_link *link; + + setup_libbpf(); + + ctx.skel = bpf_for_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + link = bpf_program__attach(ctx.skel->progs.benchmark); + if (!link) { + fprintf(stderr, "failed to attach program!\n"); + exit(1); + } + + ctx.skel->bss->nr_loops = args.nr_loops; +} + +const struct bench bench_bpf_for = { + .name = "bpf-for", + .argp = &bench_bpf_for_argp, + .validate = validate, + .setup = setup, + .producer_thread = producer, + .measure = measure, + .report_progress = ops_report_progress, + .report_final = ops_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh new file mode 100755 index 000000000000..7da6453920da --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +for t in 1 4 8 12 16; do +for i in 10 100 500 1000 5000 10000 50000 100000 500000 1000000; do +subtitle "nr_loops: $i, nr_threads: $t" + summarize_ops "bpf_for: " \ + "$($RUN_BENCH -p $t --nr_loops $i bpf-for)" + printf "\n" +done +done diff --git a/tools/testing/selftests/bpf/progs/bpf_for_bench.c b/tools/testing/selftests/bpf/progs/bpf_for_bench.c new file mode 100644 index 000000000000..f9c723051fc7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_for_bench.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int nr_loops; +long hits; + +static int outer_loop(__u32 index, void *data) +{ + int i; + + /* + * Empty body: the work being measured is the open-coded numeric iterator itself + * (bpf_iter_num_new/next/destroy behind bpf_for()). + */ + bpf_for(i, 0, nr_loops) + ; + __sync_add_and_fetch(&hits, nr_loops); + return 0; +} + +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int benchmark(void *ctx) +{ + bpf_loop(1000, outer_loop, NULL, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index 0fa70b133d93..62d7df9e80be 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -88,6 +88,89 @@ int iter_err_unsafe_asm_loop(const void *ctx) return 0; } +/* + * Naked function, so there is no compiler-generated glue and the whole inlined program can be + * matched. Pinned to arches whose JITs zero-extend 32-bit writes implicitly + * (bpf_jit_needs_zext() == false); on arches that need explicit zero-extension the verifier + * interleaves "wN = wN" insns and the fixed shape below would not match. The inlining itself is + * arch independent, so checking it on these arches is sufficient. + * + * bpf_iter_num_new() emits the full range check (distance computation and both the -EINVAL and + * -E2BIG error paths); bpf_iter_num_next() and bpf_iter_num_destroy() are inlined too. + */ +SEC("raw_tp") +__arch_x86_64 +__arch_arm64 +__success +__xlated("r6 = r10") +__xlated("r6 += -8") +__xlated("call unknown") +__xlated("r3 = r0") +__xlated("r3 &= 65535") +__xlated("r1 = r6") +__xlated("r2 = 0") +/* bpf_iter_num_new(&it, 0, ) with the range check kept */ +__xlated("if w2 s> w3 goto pc+8") +__xlated("w0 = w3") +__xlated("w0 -= w2") +__xlated("if r0 > 0x800000 goto pc+8") +__xlated("w2 += -1") +__xlated("*(u32 *)(r1 +0) = r2") +__xlated("*(u32 *)(r1 +4) = r3") +__xlated("r0 = 0") +__xlated("goto pc+5") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -22") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -7") +__xlated("r1 = r6") +/* bpf_iter_num_next(&it) */ +__xlated("r0 = *(u32 *)(r1 +0)") +__xlated("w0 += 1") +__xlated("r2 = *(u32 *)(r1 +4)") +__xlated("if w0 s>= w2 goto pc+3") +__xlated("*(u32 *)(r1 +0) = r0") +__xlated("r0 = r1") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = 0") +__xlated("if r0 != 0x0 goto pc-11") +__xlated("r1 = r6") +/* bpf_iter_num_destroy(&it) is inlined to a nop */ +__xlated("goto pc+0") +__xlated("r0 = 0") +__xlated("exit") +int __naked iter_num_new_inlined(void) +{ + asm volatile ( + /* r6 points to struct bpf_iter_num on the stack */ + "r6 = r10;" + "r6 += -8;" + /* non-constant end so the range checks are kept */ + "call %[bpf_get_prandom_u32];" + "r3 = r0;" + "r3 &= 0xffff;" + "r1 = r6;" + "r2 = 0;" + "call %[bpf_iter_num_new];" + "1:" + "r1 = r6;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r6;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy) + : __clobber_common, "r6" + ); +} + SEC("raw_tp") __success int iter_while_loop(const void *ctx)