diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c91b5a4bda03..2f1d24fef857 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6842,6 +6842,15 @@ struct bpf_link_info { __u32 flags; __u32 pid; } uprobe_multi; + struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 75495a5c3507..76ab51deaa6b 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3700,6 +3700,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) kvfree(tr_link); } +static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies); + u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs); + u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids); + struct bpf_tracing_multi_link *tr_link; + u32 ucount = info->tracing_multi.count; + bool has_cookies, show_addrs; + int err = 0; + + if ((uids || ucookies || uaddrs) && !ucount) + return -EINVAL; + + tr_link = container_of(link, struct bpf_tracing_multi_link, link); + + info->tracing_multi.attach_type = tr_link->link.attach_type; + info->tracing_multi.count = tr_link->nodes_cnt; + info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf); + + if (!uids && !ucookies && !uaddrs) + return 0; + + if (ucount < tr_link->nodes_cnt) + err = -ENOSPC; + else + ucount = tr_link->nodes_cnt; + + has_cookies = !!tr_link->cookies; + show_addrs = kallsyms_show_value(current_cred()); + + for (int i = 0; i < ucount; i++) { + struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u64 addr, cookie; + u32 id; + + bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id); + + addr = show_addrs ? mnode->trampoline->ip : 0; + cookie = has_cookies ? tr_link->cookies[i] : 0; + + if (uids && put_user(id, uids + i)) + return -EFAULT; + if (uaddrs && put_user(addr, uaddrs + i)) + return -EFAULT; + if (ucookies && put_user(cookie, ucookies + i)) + return -EFAULT; + + cond_resched(); + } + + return err; +} + #ifdef CONFIG_PROC_FS static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) @@ -3730,6 +3784,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, static const struct bpf_link_ops bpf_tracing_multi_link_lops = { .release = bpf_tracing_multi_link_release, .dealloc_deferred = bpf_tracing_multi_link_dealloc, + .fill_link_info = bpf_tracing_multi_link_fill_link_info, #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_tracing_multi_show_fdinfo, #endif diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index bdcd717b0348..088d1d206065 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val) return (__u64 *) u64_to_ptr(val); } +static __u32 *u64_to_u32_arr(__u64 val) +{ + return (__u32 *)u64_to_ptr(val); +} + +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled) +{ + struct kernel_sym *sym; + + if (!addr) + return NULL; + + sym = kernel_syms_search(&dd, addr); + if (!sym && is_ibt_enabled && addr >= 4) + sym = kernel_syms_search(&dd, addr - 4); + + return sym; +} + static void show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) { @@ -403,6 +422,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) jsonw_end_array(json_wtr); } +static void +show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + show_link_attach_type_json(info->tracing_multi.attach_type, wtr); + jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count); + jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id); + jsonw_name(wtr, "funcs"); + + jsonw_start_array(wtr); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + struct kernel_sym *sym; + __u64 addr = addrs[i]; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + jsonw_start_object(wtr); + jsonw_uint_field(wtr, "id", ids[i]); + jsonw_uint_field(wtr, "addr", addr); + if (sym) { + jsonw_string_field(wtr, "func", sym->name); + if (sym->module[0] == '\0') { + jsonw_name(wtr, "module"); + jsonw_null(wtr); + } else { + jsonw_string_field(wtr, "module", sym->module); + } + } + jsonw_uint_field(wtr, "cookie", cookies[i]); + jsonw_end_object(wtr); + } + jsonw_end_array(wtr); +} + static void show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr) { @@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_json(info, json_wtr); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_json(info, json_wtr); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) } } +static void show_tracing_multi_plain(struct bpf_link_info *info) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!info->tracing_multi.count) + return; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + printf("\n\t"); + show_link_attach_type_plain(info->tracing_multi.attach_type); + printf("btf_obj_id %u ", info->tracing_multi.btf_obj_id); + printf("count %u ", info->tracing_multi.count); + + printf("\n\t%-16s %-16s %-16s %s", + "btf_id", "addr", "cookie", "func [module]"); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + __u64 addr = addrs[i]; + struct kernel_sym *sym; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]); + if (sym) { + printf(" %s", sym->name); + if (sym->module[0] != '\0') + printf(" [%s]", sym->module); + } + } +} + static void show_perf_event_kprobe_plain(struct bpf_link_info *info) { const char *buf; @@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_plain(info); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_plain(info); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) static int do_show_link(int fd) { __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL; + __u32 *ids = NULL; struct bpf_link_info info; __u32 len = sizeof(info); char path_buf[PATH_MAX]; @@ -1114,6 +1226,26 @@ static int do_show_link(int fd) goto again; } } + if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) { + count = info.tracing_multi.count; + if (count) { + ids = calloc(count, sizeof(__u32)); + addrs = calloc(count, sizeof(__u64)); + cookies = calloc(count, sizeof(__u64)); + if (!ids || !addrs || !cookies) { + p_err("mem alloc failed"); + close(fd); + free(cookies); + free(addrs); + free(ids); + return -ENOMEM; + } + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = ptr_to_u64(cookies); + goto again; + } + } if (info.type == BPF_LINK_TYPE_PERF_EVENT) { switch (info.perf_event.type) { case BPF_PERF_EVENT_TRACEPOINT: @@ -1153,6 +1285,7 @@ static int do_show_link(int fd) free(cookies); free(offsets); free(addrs); + free(ids); close(fd); return 0; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index c91b5a4bda03..2f1d24fef857 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -6842,6 +6842,15 @@ struct bpf_link_info { __u32 flags; __u32 pid; } uprobe_multi; + struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index f589eefbf9fb..0918321c8e63 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "trace_helpers.h" #include "test_fill_link_info.skel.h" @@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 }; #define KPROBE_FUNC "bpf_fentry_test1" static __u64 kprobe_addr; +static const char * const tmulti_syms[] = { + "bpf_fentry_test2", + "bpf_fentry_test1", + "bpf_fentry_test3", +}; + +static __u64 tmulti_cookies[] = { 30, 10, 20 }; +#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms) + +struct tmulti_target { + const char *name; + __u64 addr; + __u64 cookie; + __u32 id; +}; + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel, bpf_link__destroy(link); } +static int tmulti_target_cmp(const void *a, const void *b) +{ + const struct tmulti_target *ta = a; + const struct tmulti_target *tb = b; + + return (ta->id > tb->id) - (ta->id < tb->id); +} + +static int setup_tmulti_targets(const struct bpf_program *prog, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info; + __u32 len = sizeof(prog_info); + struct btf *btf; + int err, i; + __s32 id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return -1; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC); + if (!ASSERT_GT(id, 0, "btf__find_by_name_kind")) + goto error; + + targets[i].name = tmulti_syms[i]; + targets[i].addr = ksym_get_addr(tmulti_syms[i]); + targets[i].cookie = tmulti_cookies[i]; + targets[i].id = id; + } + + memset(&prog_info, 0, len); + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + goto error; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + goto error; + *btf_obj_id = prog_info.attach_btf_obj_id; + + /* + * The kernel tracing multi attach sorts ids. We sort as well, + * so we can easily compare ids and cookies later. + */ + qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp); + btf__free(btf); + return 0; + +error: + btf__free(btf); + return -1; +} + +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, + const struct tmulti_target *targets, + __u32 btf_obj_id, bool has_cookies) +{ + enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); + __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT]; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + memset(&info, 0, sizeof(info)); + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + + memset(ids, 0, sizeof(ids)); + memset(cookies, 0, sizeof(cookies)); + memset(addrs, 0, sizeof(addrs)); + + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; + info.tracing_multi.count = TRACING_MULTI_CNT; + + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); + + if (targets[i].addr) { + struct ksym *ksym; + + if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) + return -1; + ksym = ksym_search(addrs[i]); + if (!ASSERT_OK_PTR(ksym, "ksym_search")) + return -1; + ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name"); + } else { + ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs"); + } + } + + return 0; +} + +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets) +{ + __u32 ids[TRACING_MULTI_CNT] = {}; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + /* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count"); + + /* Smaller than actual count provided -> ENOSPC */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT - 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count"); + for (i = 0; i < TRACING_MULTI_CNT - 1; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids"); + /* check that the last entry is not populated */ + ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids"); + + /* Bigger than actual count provided -> OK */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT + 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_OK(err, "tracing_multi.big_count"); + for (i = 0; i < TRACING_MULTI_CNT; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + + /* Invalid ids pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids"); + + /* Invalid cookies pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.cookies = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies"); + + /* Invalid addrs pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.addrs = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs"); +} + +static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, + bool has_cookies, bool invalid) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tmulti_target targets[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT], btf_obj_id; + __u64 cookies[TRACING_MULTI_CNT]; + struct bpf_link *link; + int link_fd, err, i; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ids[i] = targets[i].id; + cookies[i] = targets[i].cookie; + } + + opts.ids = ids; + opts.cnt = TRACING_MULTI_CNT; + if (has_cookies) + opts.cookies = cookies; + + link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + link_fd = bpf_link__fd(link); + if (invalid) { + verify_tracing_multi_invalid_user_buffer(link_fd, targets); + } else { + err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, + targets, btf_obj_id, has_cookies); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + } + + bpf_link__destroy(link); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -640,6 +875,13 @@ void test_fill_link_info(void) if (test__start_subtest("kprobe_multi_invalid_ubuff")) test_kprobe_multi_fill_link_info(skel, true, true, true); + if (test__start_subtest("tracing_multi_link_info")) { + test_tracing_multi_fill_link_info(skel, false, false); + test_tracing_multi_fill_link_info(skel, true, false); + } + if (test__start_subtest("tracing_multi_invalid_ubuff")) + test_tracing_multi_fill_link_info(skel, true, true); + if (test__start_subtest("uprobe_multi_link_info")) test_uprobe_multi_fill_link_info(skel, false, false); if (test__start_subtest("uretprobe_multi_link_info")) diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c index 137bd6292163..c85081538e93 100644 --- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c +++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c @@ -58,4 +58,10 @@ int BPF_PROG(umulti_run) return 0; } +SEC("fentry.multi") +int BPF_PROG(tmulti_run) +{ + return 0; +} + char _license[] SEC("license") = "GPL";