diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index c801a330aece..cbde86d082cc 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -250,6 +250,34 @@ Or:: ... } +2.3.7 __const_map and __map Annotations +--------------------------------------- + +These annotations are used for ``struct bpf_map *`` arguments and distinguish a +verifier-known map from an opaque one. + +``__const_map`` indicates a map must be known at the verification time, i.e. a +concrete map fd the BPF program references directly. + +An example is given below:: + + __bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, + unsigned int flags) + { + ... + } + +``__map`` indicates an opaque ``struct bpf_map *`` that may be resolved +at run time. The argument may take either a map fd or a ``PTR_TO_BTF_ID`` +``struct bpf_map`` pointer. + +An example is given below:: + + __bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, ...) + { + ... + } + .. _BPF_kfunc_nodef: 2.4 Using an existing kernel function @@ -411,7 +439,7 @@ Example declaration: .. code-block:: c __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { ... } Example usage in BPF program: diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7bfc28673124..356884587ae1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -888,8 +888,8 @@ enum bpf_arg_type { ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ARG_PTR_TO_ARENA, - ARG_CONST_SIZE, /* number of bytes accessed from memory */ - ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ + ARG_MEM_SIZE, /* number of bytes accessed from memory */ + ARG_MEM_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ @@ -960,6 +960,21 @@ enum bpf_return_type { }; static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); +/* The longest tracepoint has 12 args. + * See include/trace/bpf_probe.h + * + * Also reuse this macro for maximum number of arguments a BPF function + * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via + * stack arg slots. The JIT may map some stack arg slots to registers based + * on the native calling convention (e.g., arg 6 to R9 on x86-64). + */ +#define MAX_BPF_FUNC_ARGS 12 + +/* The maximum number of arguments passed through registers + * a single function may have. + */ +#define MAX_BPF_FUNC_REG_ARGS 5 + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -984,7 +999,7 @@ struct bpf_func_proto { enum bpf_arg_type arg4_type; enum bpf_arg_type arg5_type; }; - enum bpf_arg_type arg_type[5]; + enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS]; }; union { struct { @@ -994,7 +1009,7 @@ struct bpf_func_proto { u32 *arg4_btf_id; u32 *arg5_btf_id; }; - u32 *arg_btf_id[5]; + u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1002,7 +1017,7 @@ struct bpf_func_proto { size_t arg4_size; size_t arg5_size; }; - size_t arg_size[5]; + size_t arg_size[MAX_BPF_FUNC_ARGS]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); @@ -1192,21 +1207,6 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The longest tracepoint has 12 args. - * See include/trace/bpf_probe.h - * - * Also reuse this macro for maximum number of arguments a BPF function - * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via - * stack arg slots. The JIT may map some stack arg slots to registers based - * on the native calling convention (e.g., arg 6 to R9 on x86-64). - */ -#define MAX_BPF_FUNC_ARGS 12 - -/* The maximum number of arguments passed through registers - * a single function may have. - */ -#define MAX_BPF_FUNC_REG_ARGS 5 - /* The argument is a structure or a union. */ #define BTF_FMODEL_STRUCT_ARG BIT(0) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 682c2cd3b844..a2a40caca0a0 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -15,7 +15,7 @@ * ensures that umax_value + (int)off + (int)size cannot overflow a u64. */ #define BPF_MAX_VAR_OFF (1 << 29) -/* Maximum variable size permitted for ARG_CONST_SIZE[_OR_ZERO]. This ensures +/* Maximum variable size permitted for ARG_MEM_SIZE[_OR_ZERO]. This ensures * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) @@ -1302,7 +1302,6 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } -/* only use after check_attach_btf_id() */ static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog) { return (prog->type == BPF_PROG_TYPE_EXT && prog->aux->saved_dst_prog_type) ? @@ -1479,10 +1478,17 @@ struct ret_mem_desc { bool found; }; +/* A constant scalar argument; Populated by process_const_arg() */ +struct arg_constant_desc { + u64 value; + bool found; +}; + struct bpf_call_arg_meta { /* Common */ struct btf *btf; u32 func_id; + const struct bpf_func_proto *fn; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1496,10 +1502,7 @@ struct bpf_call_arg_meta { u32 kfunc_flags; const struct btf_type *func_proto; const char *func_name; - struct { - u64 value; - bool found; - } arg_constant; + struct arg_constant_desc arg_constant; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later @@ -1614,6 +1617,7 @@ enum bpf_reg_arg_type { struct bpf_kfunc_desc { struct btf_func_model func_model; + struct bpf_func_proto proto; u32 func_id; s32 imm; u16 offset; @@ -1621,13 +1625,15 @@ struct bpf_kfunc_desc { }; struct bpf_kfunc_desc_tab { + u32 nr_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during * verification. JITs do lookups by bpf_insn, where func_id may not be * available, therefore at the end of verification do_misc_fixups() * sorts this by imm and offset. + * + * Grown one entry at a time by bpf_add_kfunc_call(). */ - struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; - u32 nr_descs; + struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2e4ae0ef0860..2f473ad4fd7c 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -636,7 +636,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * r5 += 1 * ... * call bpf_perf_event_output#25 - * where .arg5_type = ARG_CONST_SIZE_OR_ZERO + * where .arg5_type = ARG_MEM_SIZE_OR_ZERO * * and this case: * r6 = 1 diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 3983b4ce73c8..82c5988417a0 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -186,7 +186,7 @@ static const struct bpf_func_proto bpf_ima_inode_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_inode_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -205,7 +205,7 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_file_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4eeeaeb69790..5e8ac45ce56a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8709,7 +8709,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c..fb9357b64cad 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2305,7 +2305,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2347,7 +2347,7 @@ static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, @@ -2367,7 +2367,7 @@ static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, @@ -2393,7 +2393,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 88b38db47de9..4709a5ad0474 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -278,7 +278,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) @@ -539,7 +539,7 @@ const struct bpf_func_proto bpf_strtol_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(s64), @@ -567,7 +567,7 @@ const struct bpf_func_proto bpf_strtoul_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), @@ -583,7 +583,7 @@ static const struct bpf_func_proto bpf_strncmp_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_CONST_STR, }; @@ -627,7 +627,7 @@ const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = { .arg1_type = ARG_ANYTHING, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { @@ -653,7 +653,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size, @@ -675,7 +675,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -706,7 +706,7 @@ const struct bpf_func_proto bpf_copy_from_user_task_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_BTF_ID, .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], @@ -1093,10 +1093,10 @@ const struct bpf_func_proto bpf_snprintf_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx) @@ -1888,7 +1888,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, }; @@ -1943,7 +1943,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_DYNPTR, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -2004,7 +2004,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = { .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -3404,10 +3404,10 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN(1, "A call to BPF exception callback should never return\n"); } -__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) +__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags) { struct bpf_async_kern *async = (struct bpf_async_kern *)wq; - struct bpf_map *map = p__map; + struct bpf_map *map = p__const_map; BUILD_BUG_ON(sizeof(struct bpf_async_kern) > sizeof(struct bpf_wq)); BUILD_BUG_ON(__alignof__(struct bpf_async_kern) != __alignof__(struct bpf_wq)); @@ -4643,17 +4643,17 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_SIGNAL); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_SIGNAL); } /** @@ -4661,17 +4661,17 @@ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct b * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_RESUME); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b..c1bf7a197a96 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -634,7 +634,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 41fe87d7302f..463f94ba1cc4 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -781,7 +781,7 @@ const struct bpf_func_proto bpf_get_stack_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -797,7 +797,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -831,7 +831,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -848,7 +848,7 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -911,7 +911,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 94091130bcc5..8d111da88655 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3043,6 +3043,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id; prog->aux->dst_prog = dst_prog; + if (dst_prog) { + prog->aux->saved_dst_prog_type = dst_prog->type; + prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type; + } prog->aux->dev_bound = !!attr->prog_ifindex; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; @@ -6559,7 +6563,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; const struct bpf_func_proto * __weak @@ -6606,7 +6610,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8d0635ee48c7..b274004fccfd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1854,32 +1854,34 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty reg->dynptr.first_slot = first_slot; } +/* + * Refine the return type of the bpf_map_lookup_elem() for special map types: + * map-in-map, xskmap, sockmap and sockhash. + */ +static void refine_map_lookup_value(struct bpf_reg_state *reg) +{ + enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL; + const struct bpf_map *map = reg->map_ptr; + + if (map->inner_map_meta) { + reg->type = CONST_PTR_TO_MAP | maybe_null; + reg->map_ptr = map->inner_map_meta; + /* transfer reg's id which is unique for every map_lookup_elem + * as UID of the inner map. + */ + if (btf_record_has_field(map->inner_map_meta->record, + BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) + reg->map_uid = reg->id; + } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { + reg->type = PTR_TO_XDP_SOCK | maybe_null; + } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || + map->map_type == BPF_MAP_TYPE_SOCKHASH) { + reg->type = PTR_TO_SOCKET | maybe_null; + } +} + static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) { - if (base_type(reg->type) == PTR_TO_MAP_VALUE) { - const struct bpf_map *map = reg->map_ptr; - - if (map->inner_map_meta) { - reg->type = CONST_PTR_TO_MAP; - reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem - * as UID of the inner map. - */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) { - reg->map_uid = reg->id; - } - } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { - reg->type = PTR_TO_XDP_SOCK; - } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || - map->map_type == BPF_MAP_TYPE_SOCKHASH) { - reg->type = PTR_TO_SOCKET; - } else { - reg->type = PTR_TO_MAP_VALUE; - } - return; - } - reg->type &= ~PTR_MAYBE_NULL; } @@ -2719,8 +2721,12 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto); + int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { + struct bpf_call_arg_meta meta; struct bpf_kfunc_btf_tab *btf_tab; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; @@ -2806,11 +2812,30 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (err) return err; - desc = &tab->descs[tab->nr_descs++]; + memset(&meta, 0, sizeof(meta)); + meta.btf = kfunc.btf; + meta.func_id = kfunc.id; + meta.func_proto = kfunc.proto; + meta.func_name = kfunc.name; + meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0; + + tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT); + if (!tab) + return -ENOMEM; + prog_aux->kfunc_tab = tab; + + desc = &tab->descs[tab->nr_descs]; + memset(desc, 0, sizeof(*desc)); + + err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + if (err) + return err; + desc->func_id = func_id; desc->offset = offset; desc->addr = addr; desc->func_model = func_model; + tab->nr_descs++; sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; @@ -4924,6 +4949,18 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; +static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) +{ + enum bpf_reg_type type; + + for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { + if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) + return type; + } + + return NOT_INIT; +} + static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ @@ -6667,7 +6704,7 @@ static int check_stack_range_initialized( */ bool clobber = type == BPF_WRITE; /* - * Negative access_size signals global subprog/kfunc arg check where + * Negative access_size signals global subprog arg check where * STACK_POISON slots are acceptable. static stack liveness * might have determined that subprog doesn't read them, * but BTF based global subprog validation isn't accurate enough. @@ -6869,11 +6906,11 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, - argno_t size_argno, enum bpf_access_type access_type, + argno_t size_argno, u32 access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) { - int err; + int err = 0; /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. @@ -6910,8 +6947,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, reg_arg_name(env, size_argno)); return -EACCES; } - err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - access_type, zero_size_allowed, meta); + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_READ, zero_size_allowed, meta); + if (!err && access_type & BPF_WRITE) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_WRITE, zero_size_allowed, meta); + if (!err) { int regno = reg_from_argno(size_argno); @@ -6920,15 +6963,15 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, else err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); } + return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, u32 mem_size) + argno_t argno, u32 mem_size, enum bpf_access_type access_type, + struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(reg->type); - struct bpf_reg_state saved_reg; - int err; + int size, err = 0; if (bpf_register_is_null(reg)) return 0; @@ -6939,22 +6982,16 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return -EACCES; } - /* Assuming that the register contains a value check if the memory - * access is safe. Temporarily save and restore the register's state as - * the conversion shouldn't be visible to a caller. + /* + * Only a global subprog (meta == NULL) may read poisoned stack slots: + * its static stack liveness proved the callee body skips them. */ - if (may_be_null) { - saved_reg = *reg; - mark_ptr_not_null_reg(reg); - } + size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; - int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; - - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL); - - if (may_be_null) - *reg = saved_reg; + if (access_type & BPF_READ) + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); + if (!err && (access_type & BPF_WRITE)) + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); return err; } @@ -6993,28 +7030,33 @@ static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf return 0; } -static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) +static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(mem_reg->type); - struct bpf_reg_state saved_reg; - struct bpf_call_arg_meta meta; + int regno = reg_from_argno(argno); int err; - memset(&meta, 0, sizeof(meta)); - - if (may_be_null) { - saved_reg = *mem_reg; - mark_ptr_not_null_reg(mem_reg); + if (meta->arg_constant.found) { + verifier_bug(env, "only one constant argument permitted"); + return -EFAULT; } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta); + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); + return -EINVAL; + } - if (may_be_null) - *mem_reg = saved_reg; + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err < 0) + return err; - return err; + meta->arg_constant.found = true; + meta->arg_constant.value = reg->var_off.value; + + return 0; } enum { @@ -7224,18 +7266,6 @@ static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - -static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - static int process_kptr_func(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta) { @@ -7738,8 +7768,7 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, static bool arg_type_is_mem_size(enum bpf_arg_type type) { - return type == ARG_CONST_SIZE || - type == ARG_CONST_SIZE_OR_ZERO; + return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO; } static bool arg_type_is_raw_mem(enum bpf_arg_type type) @@ -7885,8 +7914,8 @@ static const struct bpf_reg_types dynptr_types = { static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &mem_types, [ARG_PTR_TO_MAP_VALUE] = &mem_types, - [ARG_CONST_SIZE] = &scalar_types, - [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, + [ARG_MEM_SIZE] = &scalar_types, + [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, @@ -8286,11 +8315,49 @@ static int get_constant_map_key(struct bpf_verifier_env *env, static bool can_elide_value_nullness(const struct bpf_map *map); +static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) +{ + /* Use map_uid (which is unique id of inner map) to reject: + * inner_map1 = bpf_map_lookup_elem(outer_map, key1) + * inner_map2 = bpf_map_lookup_elem(outer_map, key2) + * if (inner_map1 && inner_map2) { + * timer = bpf_map_lookup_elem(inner_map1); + * if (timer) + * // mismatch would have been allowed + * bpf_timer_init(timer, inner_map2); + * } + * + * Comparing map_ptr is enough to distinguish normal and outer maps. + */ + if (meta->map.ptr && + (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) { + argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1); + struct btf_record *rec = meta->map.ptr->record; + const char *obj_name = "workqueue"; + + if (rec->timer_off >= 0) + obj_name = "timer"; + else if (rec->task_work_off >= 0) + obj_name = "bpf_task_work"; + + verbose(env, "%s pointer in %s map_uid=%d ", + obj_name, reg_arg_name(env, obj_argno), meta->map.uid); + verbose(env, "doesn't match map pointer in %s map_uid=%d\n", + reg_arg_name(env, argno), reg->map_uid); + return -EINVAL; + } + + meta->map.ptr = reg->map_ptr; + meta->map.uid = reg->map_uid; + return 0; +} + static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, - const struct bpf_func_proto *fn, int insn_idx) { + const struct bpf_func_proto *fn = meta->fn; u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; @@ -8361,29 +8428,9 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ - if (meta->map.ptr) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * timer = bpf_map_lookup_elem(inner_map1); - * if (timer) - * // mismatch would have been allowed - * bpf_timer_init(timer, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - verbose(env, - "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; + err = process_map_ptr_arg(env, reg, argno, meta); + if (err) + return err; break; case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: @@ -8466,7 +8513,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, argno, meta); + err = process_timer_func(env, reg, argno, &meta->map); if (err) return err; break; @@ -8478,22 +8525,21 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, * next is_mem_size argument below. */ if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, argno, fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta); if (err) return err; if (arg_type & MEM_ALIGNED) err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; - case ARG_CONST_SIZE: + case ARG_MEM_SIZE: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; - case ARG_CONST_SIZE_OR_ZERO: + case ARG_MEM_SIZE_OR_ZERO: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, @@ -8824,6 +8870,8 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_a int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (!arg_type_is_raw_mem(fn->arg_type[i])) continue; if (meta->arg_raw_mem.regno) @@ -8872,6 +8920,8 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) @@ -8893,6 +8943,8 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) @@ -8909,6 +8961,8 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) return false; @@ -9252,7 +9306,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -10298,9 +10352,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, fn, insn_idx); + err = check_func_arg(env, i, &meta, insn_idx); if (err) return err; } @@ -10518,10 +10573,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].map_ptr = meta.map.ptr; regs[BPF_REG_0].map_uid = meta.map.uid; regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; - if (!type_may_be_null(ret_flag) && + if (type_may_be_null(ret_flag) || btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { regs[BPF_REG_0].id = ++env->id_gen; } + /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */ + refine_map_lookup_value(®s[BPF_REG_0]); break; case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); @@ -10619,7 +10676,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (type_may_be_null(regs[BPF_REG_0].type)) + if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id) regs[BPF_REG_0].id = ++env->id_gen; if (is_ptr_cast_function(func_id) && @@ -10772,26 +10829,24 @@ static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) } static bool is_kfunc_arg_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__sz"); } static bool is_kfunc_arg_const_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__szk"); @@ -10812,6 +10867,11 @@ static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) return btf_param_match_suffix(btf, arg, "__map"); } +static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__const_map"); +} + static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__alloc"); @@ -11043,6 +11103,11 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, } enum kfunc_ptr_arg_type { + KF_ARG_CONST_MEM_SIZE, + KF_ARG_MEM_SIZE, + KF_ARG_CONST, + KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, + KF_ARG_ANYTHING, KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ @@ -11052,13 +11117,11 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_LIST_NODE, KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, - KF_ARG_PTR_TO_MAP, + KF_ARG_CONST_MAP_PTR, KF_ARG_PTR_TO_TIMER, KF_ARG_PTR_TO_WORKQUEUE, KF_ARG_PTR_TO_IRQ_FLAG, @@ -11319,108 +11382,154 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } -static enum kfunc_ptr_arg_type -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, - struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta, - const struct btf_type *t, const struct btf_type *ref_t, - const char *ref_tname, const struct btf_param *args, - int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) +static int +get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + const struct btf_param *args, int arg, int nargs) { - bool arg_mem_size = false; + const struct btf_type *t, *ref_t = NULL; + argno_t argno = argno_from_arg(arg + 1); + const char *ref_tname = NULL; + int arg_type; - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - return KF_ARG_PTR_TO_CTX; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); - if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) - arg_mem_size = true; + /* Scalar arguments are classified from their BTF suffix/name alone. */ + if (btf_type_is_scalar(t)) { + if (is_kfunc_arg_constant(meta->btf, &args[arg])) + return KF_ARG_CONST; + if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) + return KF_ARG_CONST_MEM_SIZE; + if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) + return KF_ARG_MEM_SIZE; + if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || + is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) + return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; + return KF_ARG_ANYTHING; + } + + if (!btf_type_is_ptr(t)) { + verbose(env, "Unrecognized %s type %s\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + + ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - return KF_ARG_PTR_TO_CTX; - - if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && - !arg_mem_size) - return KF_ARG_PTR_TO_NULL; - - if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_ALLOC_BTF_ID; - - if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_REFCOUNTED_KPTR; - - if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_iter(meta, arg, &args[arg])) - return KF_ARG_PTR_TO_ITER; - - if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_HEAD; - - if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_NODE; - - if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_ROOT; - - if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_NODE; - - if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CONST_STR; - - if (is_kfunc_arg_map(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_MAP; - - if (is_kfunc_arg_wq(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_WORKQUEUE; - - if (is_kfunc_arg_timer(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TIMER; - - if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TASK_WORK; - - if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_IRQ_FLAG; - - if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RES_SPIN_LOCK; - - if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { - if (!btf_type_is_struct(ref_t)) { - verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", - meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname); + if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || + meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) + arg_type = KF_ARG_PTR_TO_CTX; + else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) + arg_type = KF_ARG_PTR_TO_CTX; + else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_DYNPTR; + else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = KF_ARG_PTR_TO_ITER; + else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_HEAD; + else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_NODE; + else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_ROOT; + else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_NODE; + else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CONST_STR; + else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) + arg_type = KF_ARG_CONST_MAP_PTR; + else if (is_kfunc_arg_map(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_BTF_ID; + else if (is_kfunc_arg_wq(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_WORKQUEUE; + else if (is_kfunc_arg_timer(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TIMER; + else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CALLBACK; + else if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { + if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - return KF_ARG_PTR_TO_BTF_ID; + arg_type = KF_ARG_PTR_TO_MEM; + } else if (btf_type_is_struct(ref_t)) + /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ + arg_type = KF_ARG_PTR_TO_BTF_ID; + else { + /* + * Otherwise this is a fixed-size memory buffer supported by + * check_helper_mem_access(): a pointer to a scalar or a struct of + * scalars. The access size is derived from the pointed-to BTF type. + */ + if (!btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } - if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CALLBACK; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) + arg_type |= PTR_MAYBE_NULL; - /* This is the catch all argument type of register types supported by - * check_helper_mem_access. However, we only allow when argument type is - * pointer to scalar, or struct composed (recursively) of scalars. When - * arg_mem_size is true, the pointer can be void *. - */ - if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && - (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { - verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", - reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); + return arg_type; +} + +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto) +{ + const struct btf *btf = meta->btf; + const struct btf_param *args; + u32 i, nargs; + int arg_type; + + args = (const struct btf_param *)(meta->func_proto + 1); + nargs = btf_type_vlen(meta->func_proto); + if (nargs > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s has %d > %d args\n", meta->func_name, + nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } - return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; + if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d args\n", + meta->func_name, nargs); + return -ENOTSUPP; + } + + for (i = 0; i < nargs; i++) { + if (is_kfunc_arg_prog_aux(btf, &args[i]) || + is_kfunc_arg_ignore(btf, &args[i]) || + is_kfunc_arg_implicit(meta, i)) + continue; + + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + if (arg_type < 0) + return arg_type; + + proto->arg_type[i] = arg_type; + } + + return 0; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, @@ -12006,16 +12115,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_ARGS) { - verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_ARGS); - return -EINVAL; - } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - func_name, nargs); - return -ENOTSUPP; - } ret = check_outgoing_stack_args(env, caller, nargs); if (ret) @@ -12032,8 +12131,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; - bool is_ret_buf_sz = false; - int kf_arg_type; + int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ @@ -12056,52 +12154,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_scalar(t)) { - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } - - if (is_kfunc_arg_constant(meta->btf, &args[i])) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret < 0) - return ret; - meta->arg_constant.found = true; - meta->arg_constant.value = reg->var_off.value; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { - meta->r0_rdonly = true; - is_ret_buf_sz = true; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { - is_ret_buf_sz = true; - } - - if (is_ret_buf_sz) { - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) - return ret; - } - continue; - } - - if (!btf_type_is_ptr(t)) { - verbose(env, "Unrecognized %s type %s\n", - reg_arg_name(env, argno), btf_type_str(t)); - return -EINVAL; - } - - if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && !is_kfunc_arg_nullable(meta->btf, &args[i])) { verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); @@ -12118,76 +12171,36 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); + if (btf_type_is_ptr(t)) { + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } - kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, - args, i, nargs, argno, reg); - if (kf_arg_type < 0) - return kf_arg_type; - switch (kf_arg_type) { - case KF_ARG_PTR_TO_NULL: + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) continue; - case KF_ARG_PTR_TO_MAP: - if (!reg->map_ptr) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || - reg->map_ptr->record->task_work_off >= 0)) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * wq = bpf_map_lookup_elem(inner_map1); - * if (wq) - * // mismatch would have been allowed - * bpf_wq_init(wq, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - if (reg->map_ptr->record->task_work_off >= 0) { - verbose(env, - "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - verbose(env, - "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; - fallthrough; + + if (is_kfunc_arg_map(btf, &args[i])) { + ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; + ref_t = btf_type_by_id(btf_vmlinux, ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } + + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + case KF_ARG_CONST_MEM_SIZE: + case KF_ARG_MEM_SIZE: + case KF_ARG_ANYTHING: + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(env, reg)) { - if (!is_kfunc_rcu(meta)) { - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } - fallthrough; + case KF_ARG_CONST_MAP_PTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: case KF_ARG_PTR_TO_RB_ROOT: case KF_ARG_PTR_TO_RB_NODE: case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: @@ -12218,7 +12231,35 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; - switch (kf_arg_type) { + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + break; + case KF_ARG_ANYTHING: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + break; + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) + meta->r0_rdonly = true; + ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (ret < 0) + return ret; + break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", @@ -12387,75 +12428,100 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MAP: - /* If argument has '__map' suffix expect 'struct bpf_map *' */ - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - fallthrough; + case KF_ARG_CONST_MAP_PTR: + if (base_type(reg->type) != CONST_PTR_TO_MAP || + type_may_be_null(reg->type)) { + verbose(env, "pointer in %s isn't map pointer\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + ret = process_map_ptr_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + break; case KF_ARG_PTR_TO_BTF_ID: /* Only base_type is checked, further checks are done here */ - if ((base_type(reg->type) != PTR_TO_BTF_ID || - (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && - !reg2btf_ids[base_type(reg->type)]) { - verbose(env, "%s is %s ", reg_arg_name(env, argno), - reg_type_str(env, reg->type)); - verbose(env, "expected %s or socket\n", - reg_type_str(env, base_type(reg->type) | - (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_MEM: - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; - } - ret = check_mem_reg(env, reg, argno, type_size); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = reg; - const struct btf_param *buff_arg = &args[i]; - struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); - const struct btf_param *size_arg = &args[i + 1]; - argno_t next_argno = argno_from_arg(i + 2); + if (base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) { + if (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type)) { + if (!is_kfunc_rcu(meta)) { + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } - if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, - argno, next_argno); - if (ret < 0) { - verbose(env, "%s and ", reg_arg_name(env, argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, next_argno)); + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + if (ret < 0) return ret; - } + break; } - if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(size_reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, next_argno)); + if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + + verbose(env, "%s is %s expected %s %s", + reg_arg_name(env, argno), reg_type_str(env, reg->type), + btf_type_str(ref_t), ref_tname); + if (reg2btf_type != NOT_INIT) + verbose(env, " or %s", reg_type_str(env, reg2btf_type)); + verbose(env, "\n"); + return -EINVAL; + } + + /* + * If the register does not contain btf id but the argument type is a pointer to + * scalar-only struct, allow verifying it as a fixed size memory. + */ + kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + fallthrough; + case KF_ARG_PTR_TO_MEM: + if (kf_arg_type & MEM_FIXED_SIZE) { + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } - meta->arg_constant.found = true; - meta->arg_constant.value = size_reg->var_off.value; + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); + if (ret < 0) + return ret; + } + break; + case KF_ARG_CONST_MEM_SIZE: + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + fallthrough; + case KF_ARG_MEM_SIZE: + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); + struct bpf_reg_state *size_reg = reg; + argno_t buff_argno = argno_from_arg(i); + + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; } - /* Skip next '__sz' or '__szk' argument */ - i++; + if (bpf_register_is_null(buff_reg)) + break; + + ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, + BPF_READ | BPF_WRITE, true, meta); + if (ret < 0) { + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + return ret; + } break; } case KF_ARG_PTR_TO_CALLBACK: @@ -12515,7 +12581,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me reg_arg_name(env, argno)); return -EINVAL; } - ret = process_timer_kfunc(env, reg, argno, meta); + ret = process_timer_func(env, reg, argno, &meta->map); if (ret < 0) return ret; break; @@ -12976,6 +13042,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int err, insn_idx = *insn_idx_p; const struct btf_param *args; u32 i, nargs, ptr_type_id; + struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -12992,6 +13059,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, func_name = meta.func_name; insn_aux = &env->insn_aux_data[insn_idx]; + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) { + verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm); + return -EFAULT; + } + meta.fn = &desc->proto; + insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta); if (!insn->off && diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 76ab51deaa6b..891897f8a1b3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -221,7 +221,7 @@ const struct bpf_func_proto bpf_probe_read_user_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -258,7 +258,7 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -273,7 +273,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -308,7 +308,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -328,7 +328,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -347,7 +347,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ @@ -383,7 +383,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; #define MAX_TRACE_PRINTK_VARARGS 3 @@ -418,7 +418,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; static void __set_printk_clr_event(struct work_struct *work) @@ -474,9 +474,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, }; const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void) @@ -518,9 +518,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) @@ -535,7 +535,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, @@ -559,7 +559,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -633,7 +633,7 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void) @@ -730,7 +730,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static DEFINE_PER_CPU(int, bpf_event_output_nest_level); @@ -996,7 +996,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1053,9 +1053,9 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1218,7 +1218,7 @@ const struct bpf_func_proto bpf_get_branch_snapshot_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) @@ -1421,7 +1421,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, @@ -1462,7 +1462,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1524,12 +1524,12 @@ BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx, } static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { - .func = bpf_perf_prog_read_value, - .gpl_only = true, - .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .func = bpf_perf_prog_read_value, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, @@ -1566,7 +1566,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1646,7 +1646,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; extern const struct bpf_func_proto bpf_skb_output_proto; @@ -1701,7 +1701,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/net/core/filter.c b/net/core/filter.c index c21c1daecf9d..eb4d299b1fec 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1747,7 +1747,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1784,7 +1784,7 @@ static const struct bpf_func_proto bpf_skb_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len) @@ -1823,7 +1823,7 @@ static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, @@ -1867,7 +1867,7 @@ static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -2063,9 +2063,9 @@ static const struct bpf_func_proto bpf_csum_diff_proto = { .pkt_access = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -2626,7 +2626,7 @@ static const struct bpf_func_proto bpf_redirect_neigh_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -4199,7 +4199,7 @@ static const struct bpf_func_proto bpf_xdp_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len) @@ -4231,7 +4231,7 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len) @@ -4794,7 +4794,7 @@ static const struct bpf_func_proto bpf_skb_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BTF_ID_LIST_SINGLE(bpf_skb_output_btf_ids, struct, sk_buff) @@ -4808,7 +4808,7 @@ const struct bpf_func_proto bpf_skb_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static unsigned short bpf_tunnel_key_af(u64 flags) @@ -4891,7 +4891,7 @@ static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -4926,7 +4926,7 @@ static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static struct metadata_dst __percpu *md_dst; @@ -5008,7 +5008,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -5036,7 +5036,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * @@ -5208,7 +5208,7 @@ static const struct bpf_func_proto bpf_xdp_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BTF_ID_LIST_SINGLE(bpf_xdp_output_btf_ids, struct, xdp_buff) @@ -5222,7 +5222,7 @@ const struct bpf_func_proto bpf_xdp_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb) @@ -5769,7 +5769,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sk_getsockopt, struct sock *, sk, int, level, @@ -5786,7 +5786,7 @@ const struct bpf_func_proto bpf_sk_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sk_setsockopt_nodelay, struct sock *, sk, int, level, @@ -5810,7 +5810,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level, @@ -5827,7 +5827,7 @@ const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_unlocked_sk_getsockopt, struct sock *, sk, int, level, @@ -5844,7 +5844,7 @@ const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx, @@ -5861,7 +5861,7 @@ static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_addr_getsockopt, struct bpf_sock_addr_kern *, ctx, @@ -5878,7 +5878,7 @@ static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; static int sk_bpf_set_get_bypass_prot_mem(struct sock *sk, @@ -5923,7 +5923,7 @@ static const struct bpf_func_proto bpf_sock_create_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_create_getsockopt, struct sock *, sk, int, level, @@ -5949,7 +5949,7 @@ static const struct bpf_func_proto bpf_sock_create_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock, @@ -5975,7 +5975,7 @@ static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock, @@ -6085,7 +6085,7 @@ static const struct bpf_func_proto bpf_sock_ops_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock, @@ -6152,7 +6152,7 @@ static const struct bpf_func_proto bpf_bind_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; #ifdef CONFIG_XFRM @@ -6205,7 +6205,7 @@ static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; #endif @@ -6612,7 +6612,7 @@ static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6672,7 +6672,7 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6870,7 +6870,7 @@ static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { @@ -6880,7 +6880,7 @@ static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF) @@ -6924,7 +6924,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; static void bpf_update_srh_state(struct sk_buff *skb) @@ -7013,7 +7013,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_action_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset, @@ -7254,7 +7254,7 @@ static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7273,7 +7273,7 @@ static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7292,7 +7292,7 @@ static const struct bpf_func_proto bpf_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7316,7 +7316,7 @@ static const struct bpf_func_proto bpf_tc_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7340,7 +7340,7 @@ static const struct bpf_func_proto bpf_tc_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7364,7 +7364,7 @@ static const struct bpf_func_proto bpf_tc_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7402,7 +7402,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7426,7 +7426,7 @@ static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7450,7 +7450,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7470,7 +7470,7 @@ static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7489,7 +7489,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7508,7 +7508,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7828,9 +7828,9 @@ static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len, @@ -7897,9 +7897,9 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags) @@ -8053,7 +8053,7 @@ static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -8131,7 +8131,7 @@ static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -8226,7 +8226,7 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = { .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv6, struct ipv6hdr *, iph, @@ -8258,7 +8258,7 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph, @@ -11731,7 +11731,7 @@ static const struct bpf_func_proto sk_reuseport_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; BPF_CALL_5(sk_reuseport_load_bytes_relative, @@ -11749,7 +11749,7 @@ static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index be97f6887f0e..b79bafca68f7 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -61,12 +61,14 @@ #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" +#include "verifier_map_lookup_refine.skel.h" #include "verifier_map_ptr.skel.h" #include "verifier_map_ptr_mixing.skel.h" #include "verifier_map_ret_val.skel.h" #include "verifier_masking.skel.h" #include "verifier_may_goto_1.skel.h" #include "verifier_may_goto_2.skel.h" +#include "verifier_mem_size_reg.skel.h" #include "verifier_meta_access.skel.h" #include "verifier_movsx.skel.h" #include "verifier_mtu.skel.h" @@ -215,12 +217,14 @@ void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } +void test_verifier_map_lookup_refine(void) { RUN(verifier_map_lookup_refine); } void test_verifier_map_ptr(void) { RUN(verifier_map_ptr); } void test_verifier_map_ptr_mixing(void) { RUN(verifier_map_ptr_mixing); } void test_verifier_map_ret_val(void) { RUN(verifier_map_ret_val); } void test_verifier_masking(void) { RUN(verifier_masking); } void test_verifier_may_goto_1(void) { RUN(verifier_may_goto_1); } void test_verifier_may_goto_2(void) { RUN(verifier_may_goto_2); } +void test_verifier_mem_size_reg(void) { RUN(verifier_mem_size_reg); } void test_verifier_meta_access(void) { RUN(verifier_meta_access); } void test_verifier_movsx(void) { RUN(verifier_movsx); } void test_verifier_mul(void) { RUN(verifier_mul); } diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index d0d65d6d450c..efe7bcae70f8 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("R1 pointer type STRUCT cgroup must point") +__failure __msg("R1 is fp expected STRUCT cgroup") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 94489ac64da8..340bd7db79f0 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -1589,7 +1589,7 @@ int xdp_invalid_ctx(void *ctx) __u32 hdr_size = sizeof(struct ethhdr); /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") -__failure __msg("unbounded memory access") +__failure __msg("must be a known constant") int dynptr_slice_var_len1(struct __sk_buff *skb) { struct bpf_dynptr ptr; diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index 5b4453747c23..0952d1ebf0f1 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -137,7 +137,7 @@ int helper_param_not_ok(void *ctx) p = bpf_rdonly_cast(0, 0); /* - * Any helper with ARG_CONST_SIZE_OR_ZERO constraint will do, + * Any helper with ARG_MEM_SIZE_OR_ZERO constraint will do, * the most permissive constraint */ bpf_copy_from_user(p, 0, (void *)42); diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 8942b5478129..5c99b1e6532b 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -50,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("R1 pointer type STRUCT task_struct must point") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index bc038ac2df98..1a273e416fed 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1195,7 +1195,7 @@ l0_%=: r1 = r6; \ r3 += -8; \ r5 = 0; \ /* The 4th argument of bpf_skb_store_bytes is defined as \ - * ARG_CONST_SIZE, so 0 is not allowed. The 'r4 != 0' \ + * ARG_MEM_SIZE, so 0 is not allowed. The 'r4 != 0' \ * is providing us this exclusion of zero from initial \ * [0, 7] range. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index f2c54e4d89eb..343fc08d9747 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -85,7 +85,7 @@ __naked void stack_bitwise_and_zero_included(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -278,7 +278,7 @@ __naked void stack_jmp_no_min_check(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -778,7 +778,7 @@ __naked void variable_memory_8_bytes_leak(void) r3 += 1; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory region [1, 64]\ * at &fp[-64] is not fully initialized. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c index 6d2a38597c34..c6603a118fdc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c @@ -91,7 +91,7 @@ l0_%=: exit; \ /* Call a function taking a pointer and a size which doesn't allow the size to * be zero (i.e. bpf_trace_printk() declares the second argument to be - * ARG_CONST_SIZE, not ARG_CONST_SIZE_OR_ZERO). We attempt to pass zero for the + * ARG_MEM_SIZE, not ARG_MEM_SIZE_OR_ZERO). We attempt to pass zero for the * size and expect to fail. */ SEC("tracepoint") diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index b606b5dca734..7918646e5bfc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -154,7 +154,7 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_value_or_null expected=map_ptr") +__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c new file mode 100644 index 000000000000..c01abf54923d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +struct inner_map { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} inner_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __array(values, struct inner_map); +} outer_map SEC(".maps") = { + .values = { [0] = &inner_map }, +}; + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_kfunc_mem_buf(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + __u32 key = 0; + void *inner; + char *p; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + bpf_dynptr_from_skb(skb, 0, &dptr); + /* arg3 is mem+size */ + p = bpf_dynptr_slice(&dptr, 0, inner, 4); + if (p) + return p[0]; + return 0; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) +{ + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is mem+size */ + return bpf_csum_diff(inner, 4, NULL, 0, 0) + skb->len; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) +{ + char th[sizeof(struct tcphdr)] = {}; + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is fixed-sized mem */ + return bpf_tcp_raw_check_syncookie_ipv4(inner, (void *)th); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c new file mode 100644 index 000000000000..7e24706a764e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The __szk size of a kfunc memory/size pair must be marked precise even when + * the nullable buffer is passed as NULL. + */ +SEC("?tc") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r4 stack= before") +int dynptr_slice_null_buf_size_precise(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + char *p; + + bpf_dynptr_from_skb(skb, 0, &dptr); + + p = bpf_dynptr_slice(&dptr, 0, NULL, 8); + if (p) + return p[0]; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 2870738d93f7..8f0c45421f89 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("R1 pointer type STRUCT task_struct must point to scalar, or struct with scalar") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; @@ -98,7 +98,7 @@ int BPF_PROG(path_d_path_kfunc_null) } SEC("lsm.s/task_alloc") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task) { struct path *root; @@ -112,7 +112,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task } SEC("lsm.s/file_open") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_current) { struct path *pwd; diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 302d712e0d7e..8cd626e04551 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 pointer type STRUCT prog_test_fail1 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail1", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\nR1 pointer type STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 pointer type STRUCT prog_test_fail3 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail3", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, },