diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index cbde86d082cc..021be6d93dfb 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -472,6 +472,13 @@ type. An example is shown below:: } late_initcall(init_subsystem); +At kernel build time the ``resolve_btfids`` tool finds all kfuncs declared with +``BTF_KFUNCS_START()`` and emits their BTF annotations into the kernel's BTF. +For each kfunc it emits a ``bpf_kfunc`` BTF decl tag, a ``bpf_fastcall`` decl +tag when the kfunc is flagged ``KF_FASTCALL``, and the ``address_space(1)`` type +attribute on the return value and/or arguments flagged ``KF_ARENA_RET``, +``KF_ARENA_ARG1`` or ``KF_ARENA_ARG2`` (see section 2.8). + 2.7 Specifying no-cast aliases with ___init -------------------------------------------- diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst index 1ca8c5f73ad0..0aa232b117b5 100644 --- a/Documentation/process/changes.rst +++ b/Documentation/process/changes.rst @@ -147,11 +147,6 @@ Since Linux 5.2, if CONFIG_DEBUG_INFO_BTF is selected, the build system generates BTF (BPF Type Format) from DWARF in vmlinux, a bit later from kernel modules as well. This requires pahole v1.22 or later. -Since Linux 7.0, kfuncs annotated with KF_IMPLICIT_ARGS require pahole v1.26 -or later. Without it, such kfuncs will have incorrect BTF prototypes in -vmlinux, causing BPF programs to fail to load with a "func_proto incompatible -with vmlinux" error. Many sched_ext kfuncs are affected. - It is found in the 'dwarves' or 'pahole' distro packages or from https://fedorapeople.org/~acme/dwarves/. diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf index e66e13e79653..a1812985a61a 100644 --- a/scripts/Makefile.btf +++ b/scripts/Makefile.btf @@ -14,9 +14,7 @@ pahole-flags-$(call test-ge, $(pahole-ver), 125) += --skip_encoding_btf_inconsis else # Switch to using --btf_features for v1.26 and later. -pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs - -pahole-flags-$(call test-ge, $(pahole-ver), 130) += --btf_features=attributes +pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func pahole-flags-$(call test-ge, $(pahole-ver), 131) += --btf_features=layout diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 85488935909d..d2e4176339da 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -58,6 +58,17 @@ * __BTF_ID__func__vfs_fallocate__5: * .zero 4 * .word (1 << 3) | (1 << 1) | (1 << 2) + * + * In addition to resolving BTF IDs, resolve_btfids performs kernel-specific + * BTF-to-BTF transformations for kfuncs found in BTF_SET8_KFUNCS sets. For + * each such kfunc it: + * + * - emits a "bpf_kfunc" decl tag, and "bpf_fastcall" when KF_FASTCALL is set; + * - wraps the return value and/or arguments flagged KF_ARENA_RET, + * KF_ARENA_ARG1 or KF_ARENA_ARG2 with the "address_space(1)" type attribute; + * - rewrites the prototype of KF_IMPLICIT_ARGS kfuncs. + * + * These kfunc annotations were historically produced by pahole. */ #define _GNU_SOURCE @@ -161,8 +172,16 @@ struct object { u32 addr_syms_cap; }; +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" + +#define KF_FASTCALL (1 << 12) +#define KF_ARENA_RET (1 << 13) +#define KF_ARENA_ARG1 (1 << 14) +#define KF_ARENA_ARG2 (1 << 15) #define KF_IMPLICIT_ARGS (1 << 16) #define KF_IMPL_SUFFIX "_impl" +#define TYPE_ATTR_ARENA "address_space(1)" struct kfunc { struct rb_node rb_node; @@ -1229,7 +1248,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct continue; tag_name = btf__name_by_offset(btf, t->name_off); - if (strcmp(tag_name, "bpf_kfunc") == 0) + if (strcmp(tag_name, DECL_TAG_KFUNC) == 0) continue; idx = btf_decl_tag(t)->component_idx; @@ -1280,6 +1299,141 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct return 0; } +static bool is_arena_arg(struct kfunc *kfunc, u32 idx) +{ + switch (idx) { + case 0: + return kfunc->flags & KF_ARENA_ARG1; + case 1: + return kfunc->flags & KF_ARENA_ARG2; + default: + return false; + } +} + +static s32 arena_tag_ptr(struct btf *btf, u32 ptr_id, struct kfunc *kfunc) +{ + const struct btf_type *ptr = btf__type_by_id(btf, ptr_id); + s32 tag_id, new_ptr_id; + + if (!btf_is_ptr(ptr)) { + pr_err("ERROR: resolve_btfids: kfunc %s: arena type is not a pointer\n", + kfunc->name); + return -EINVAL; + } + + tag_id = btf__add_type_attr(btf, TYPE_ATTR_ARENA, ptr->type); + if (tag_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a type attr to BTF: %d\n", + kfunc->name, tag_id); + return tag_id; + } + + new_ptr_id = btf__add_ptr(btf, tag_id); + if (new_ptr_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a pointer to BTF: %d\n", + kfunc->name, new_ptr_id); + } + + return new_ptr_id; +} + +/* + * Add a FUNC_PROTO for @kfunc with each relevant pointer tagged with + * an "address_space(1)" attribute. The original proto may be shared + * with other FUNCs, so it is never modified in place. + */ +static s32 add_arena_tagged_proto(struct btf *btf, struct kfunc *kfunc) +{ + const struct btf_type *func = btf__type_by_id(btf, kfunc->btf_id); + u32 proto_id = func->type; + const struct btf_type *proto = btf__type_by_id(btf, proto_id); + u32 nr_params = btf_vlen(proto); + s32 ret_type_id = proto->type; + const struct btf_type *t; + struct btf_param *params; + s32 new_proto_id, id; + const char *name; + int err, i; + + if (kfunc->flags & KF_ARENA_RET) { + ret_type_id = arena_tag_ptr(btf, ret_type_id, kfunc); + if (ret_type_id < 0) + return ret_type_id; + } + + new_proto_id = btf__add_func_proto(btf, ret_type_id); + if (new_proto_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a func proto to BTF: %d\n", + kfunc->name, new_proto_id); + return new_proto_id; + } + + for (i = 0; i < nr_params; i++) { + /* btf__add_func_param() below may move the proto, re-fetch */ + proto = btf__type_by_id(btf, proto_id); + name = btf__name_by_offset(btf, btf_params(proto)[i].name_off); + + err = btf__add_func_param(btf, name ?: "", btf_params(proto)[i].type); + if (err < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a proto param to BTF: %d\n", + kfunc->name, err); + return err; + } + } + + for (i = 0; i < nr_params; i++) { + if (!is_arena_arg(kfunc, i)) + continue; + + t = btf__type_by_id(btf, new_proto_id); + params = btf_params(t); + + id = arena_tag_ptr(btf, params[i].type, kfunc); + if (id < 0) + return id; + + t = btf__type_by_id(btf, new_proto_id); + params = btf_params(t); + params[i].type = id; + } + + pr_debug("added arena-tagged proto for kfunc %s: %d\n", kfunc->name, new_proto_id); + + return new_proto_id; +} + +static int process_kfunc_with_arena_flags(struct btf2btf_context *ctx, + struct kfunc *kfunc) +{ + struct btf_type *t; + s32 proto_id; + + proto_id = add_arena_tagged_proto(ctx->btf, kfunc); + if (proto_id < 0) + return proto_id; + + t = (struct btf_type *)btf__type_by_id(ctx->btf, kfunc->btf_id); + t->type = proto_id; + + return 0; +} + +static int add_decl_tag(struct btf2btf_context *ctx, const char *tag_name, + u32 target_btf_id, int component_idx) +{ + s32 new_id; + + new_id = btf__add_decl_tag(ctx->btf, tag_name, target_btf_id, component_idx); + if (new_id < 0) { + pr_err("ERROR: resolve_btfids: failed to add '%s' decl tag for BTF id %u: %d\n", + tag_name, target_btf_id, new_id); + return new_id; + } + + return push_decl_tag_id(ctx, new_id); +} + static int btf2btf(struct object *obj) { struct btf2btf_context ctx = {}; @@ -1293,12 +1447,27 @@ static int btf2btf(struct object *obj) for (next = rb_first(&ctx.kfuncs); next; next = rb_next(next)) { struct kfunc *kfunc = rb_entry(next, struct kfunc, rb_node); - if (!(kfunc->flags & KF_IMPLICIT_ARGS)) - continue; - - err = process_kfunc_with_implicit_args(&ctx, kfunc); + err = add_decl_tag(&ctx, DECL_TAG_KFUNC, kfunc->btf_id, -1); if (err) goto out; + + if (kfunc->flags & KF_FASTCALL) { + err = add_decl_tag(&ctx, DECL_TAG_FASTCALL, kfunc->btf_id, -1); + if (err) + goto out; + } + + if (kfunc->flags & KF_IMPLICIT_ARGS) { + err = process_kfunc_with_implicit_args(&ctx, kfunc); + if (err) + goto out; + } + + if (kfunc->flags & (KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2)) { + err = process_kfunc_with_arena_flags(&ctx, kfunc); + if (err) + goto out; + } } err = 0; @@ -1379,6 +1548,12 @@ static int finalize_btf(struct object *obj) struct btf *base_btf = obj->base_btf, *btf = obj->btf; int err; + err = btf__dedup(obj->btf, NULL); + if (err) { + pr_err("FAILED to dedup BTF: %s\n", strerror(errno)); + goto out_err; + } + if (obj->base_btf && obj->distill_base) { err = btf__distill_base(obj->btf, &base_btf, &btf); if (err) { diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index ac51fd454821..732cfed35e1c 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -12,9 +12,22 @@ #define BTF_DATA_FILE "resolve_btfids.test.o.BTF" +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" +#define TYPE_ATTR_ARENA "address_space(1)" + #ifndef KF_FASTCALL #define KF_FASTCALL (1 << 12) #endif +#ifndef KF_ARENA_RET +#define KF_ARENA_RET (1 << 13) +#endif +#ifndef KF_ARENA_ARG1 +#define KF_ARENA_ARG1 (1 << 14) +#endif +#ifndef KF_ARENA_ARG2 +#define KF_ARENA_ARG2 (1 << 15) +#endif struct symbol { const char *name; @@ -41,6 +54,8 @@ struct kfunc_symbol { static struct kfunc_symbol kfunc_symbols[] = { { "kfunc_a", -1, 0 }, { "kfunc_b", -1, KF_FASTCALL }, + { "kfunc_c", -1, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2 }, + { "kfunc_d", -1, KF_ARENA_ARG2 }, }; /* Align the .BTF_ids section to 4 bytes */ @@ -88,6 +103,8 @@ BTF_SET_END(test_set) BTF_KFUNCS_START(test_kfunc_set) BTF_ID_FLAGS(func, kfunc_a) BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) BTF_KFUNCS_END(test_kfunc_set) /* @@ -95,6 +112,8 @@ BTF_KFUNCS_END(test_kfunc_set) * actually sort at least one of the two sets. */ BTF_KFUNCS_START(test_kfunc_set_rev) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) BTF_ID_FLAGS(func, kfunc_a) BTF_KFUNCS_END(test_kfunc_set_rev) @@ -159,6 +178,28 @@ static int resolve_symbols(struct btf *btf) return 0; } +static bool btf_has_decl_tag(struct btf *btf, const char *tag_name, s32 target_id) +{ + const struct btf_type *t; + const char *name; + int nr, id; + + nr = btf__type_cnt(btf); + for (id = 1; id < nr; id++) { + t = btf__type_by_id(btf, id); + if (!btf_is_decl_tag(t)) + continue; + if (t->type != (__u32)target_id) + continue; + if (btf_decl_tag(t)->component_idx != -1) + continue; + name = btf__name_by_offset(btf, t->name_off); + if (strcmp(name, tag_name) == 0) + return true; + } + return false; +} + static void check_kfunc_set(struct btf_id_set8 *set) { unsigned int i, j; @@ -184,6 +225,22 @@ static void check_kfunc_set(struct btf_id_set8 *set) } } +/* True if @id is PTR -> TYPE_TAG(kflag=1, "address_space(1)") -> pointee */ +static bool is_arena_tagged_ptr(struct btf *btf, __u32 id) +{ + const struct btf_type *ptr, *tag; + const char *name; + + ptr = btf__type_by_id(btf, id); + if (!btf_is_ptr(ptr)) + return false; + tag = btf__type_by_id(btf, ptr->type); + if (!btf_is_type_tag(tag) || !btf_kflag(tag)) + return false; + name = btf__name_by_offset(btf, tag->name_off); + return strcmp(name, TYPE_ATTR_ARENA) == 0; +} + void test_resolve_btfids(void) { __u32 *test_list, *test_lists[] = { test_list_local, test_list_global }; @@ -227,6 +284,55 @@ void test_resolve_btfids(void) check_kfunc_set(&test_kfunc_set); check_kfunc_set(&test_kfunc_set_rev); + /* Check resolve_btfids emitted a bpf_kfunc decl_tag for each kfunc */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_KFUNC, + kfunc_symbols[i].id), + kfunc_symbols[i].name); + } + + /* Check resolve_btfids emitted bpf_fastcall for KF_FASTCALL kfuncs */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].flags & KF_FASTCALL) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_FASTCALL, + kfunc_symbols[i].id), + kfunc_symbols[i].name); + } + } + + /* + * Check resolve_btfids wrapped exactly the arena-flagged return/args + * with the address_space(1) type attribute, and left other + * pointers/returns untouched. + */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + const struct btf_type *fn, *proto; + const struct btf_param *params; + const char *name = kfunc_symbols[i].name; + u32 fl = kfunc_symbols[i].flags; + __u32 nr; + + fn = btf__type_by_id(btf, kfunc_symbols[i].id); + if (!ASSERT_TRUE(btf_is_func(fn), name)) + continue; + proto = btf__type_by_id(btf, fn->type); + if (!ASSERT_TRUE(btf_is_func_proto(proto), name)) + continue; + params = btf_params(proto); + nr = btf_vlen(proto); + + ASSERT_EQ(is_arena_tagged_ptr(btf, proto->type), + !!(fl & KF_ARENA_RET), name); + if (nr > 0) { + ASSERT_EQ(is_arena_tagged_ptr(btf, params[0].type), + !!(fl & KF_ARENA_ARG1), name); + } + if (nr > 1) { + ASSERT_EQ(is_arena_tagged_ptr(btf, params[1].type), + !!(fl & KF_ARENA_ARG2), name); + } + } + out: btf__free(btf); } diff --git a/tools/testing/selftests/bpf/progs/btf_data.c b/tools/testing/selftests/bpf/progs/btf_data.c index 8587658012c3..ec34f7a6e038 100644 --- a/tools/testing/selftests/bpf/progs/btf_data.c +++ b/tools/testing/selftests/bpf/progs/btf_data.c @@ -58,3 +58,13 @@ int kfunc_b(struct root_struct *root) { return 0; } + +struct root_struct *kfunc_c(struct root_struct *a, struct root_struct *b) +{ + return a; +} + +int kfunc_d(struct root_struct *a, struct root_struct *b) +{ + return 0; +}