mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-27 23:25:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

bpf: Factor stackid_fastpath function from __bpf_get_stackid

Browse Source 
The new stackid_fastpath does the fast stack hash and trace check, that
does not need new bucket allocation. It covers both just-ip and buildid
code paths.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20260803210149.296496-3-jolsa@kernel.org
This commit is contained in:
Jiri Olsa
2026-08-03 23:01:39 +02:00
committed by Andrii Nakryiko
parent 15b837759a
commit 0ca56befcf
1 changed files with 28 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
kernel/bpf/stackmap.c
View File

@@ -511,6 +511,7 @@ struct stackid {
u32 len;
u32 hash;
u32 id;
bool hash_matches;
};
static int stackid_init(struct stackid *stackid, struct bpf_map *map,
@@ -531,28 +532,46 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
stackid->id = stackid->hash & (smap->n_buckets - 1);
stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
return 0;
}
static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
struct perf_callchain_entry *trace, u64 flags)
{
int err;
err = stackid_init(stackid, map, trace, flags);
if (err)
return err;
/* fast cmp */
if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP)
return stackid->id;
if (stack_map_use_build_id(map))
return -ENOENT;
if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
return stackid->id;
if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
return -EEXIST;
return -ENOENT;
}
static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
struct perf_callchain_entry *trace, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
struct stack_map_bucket *new_bucket, *old_bucket;
bool user = flags & BPF_F_USER_STACK;
bool hash_matches;
u32 trace_len, i;
int err;
err = stackid_init(stackid, map, trace, flags);
if (err)
err = stackid_fastpath(stackid, map, trace, flags);
if (err != -ENOENT)
return err;
hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
/* fast cmp */
if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
return stackid->id;
if (stack_map_use_build_id(map)) {
struct bpf_stack_build_id *id_offs;
@@ -567,7 +586,7 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
id_offs[i].ip = stackid->ips[i];
stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
if (hash_matches && stackid->bucket->nr == stackid->nr &&
if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) {
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
return stackid->id;
@@ -577,12 +596,6 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
return -EEXIST;
}
} else {
if (hash_matches && stackid->bucket->nr == stackid->nr &&
memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
return stackid->id;
if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
return -EEXIST;
new_bucket = (struct stack_map_bucket *)
pcpu_freelist_pop(&smap->freelist);
if (unlikely(!new_bucket))