Commit bc049387 authored by Kumar Kartikeya Dwivedi's avatar Kumar Kartikeya Dwivedi Committed by Alexei Starovoitov
Browse files

bpf: Add support for __prog argument suffix to pass in prog->aux 



Instead of hardcoding the list of kfuncs that need prog->aux passed to
them with a combination of fixup_kfunc_call adjustment + __ign suffix,
combine both in __prog suffix, which ignores the argument passed in, and
fixes it up to the prog->aux. This allows kfuncs to have the prog->aux
passed into them without having to touch the verifier.

Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: default avatarKumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20250513142812.1021591-1-memxor@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 3880cdbe

Documentation/bpf/kfuncs.rst

+17 −0
Original line number Diff line number Diff line
@@ -160,6 +160,23 @@ Or:: 
                ...

        }



2.2.6 __prog Annotation

---------------------------

This annotation is used to indicate that the argument needs to be fixed up to

the bpf_prog_aux of the caller BPF program. Any value passed into this argument

is ignored, and rewritten by the verifier.



An example is given below::



        __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,

                                                 int (callback_fn)(void *map, int *key, void *value),

                                                 unsigned int flags,

                                                 void *aux__prog)

         {

                struct bpf_prog_aux *aux = aux__prog;

                ...

         }



.. _BPF_kfunc_nodef:



2.3 Using an existing kernel function

include/linux/bpf_verifier.h

+1 −0
Original line number Diff line number Diff line
@@ -591,6 +591,7 @@ struct bpf_insn_aux_data { 
	 * bpf_fastcall pattern.

	 */

	u8 fastcall_spills_num:3;

	u8 arg_prog:4;



	/* below fields are initialized once */

	unsigned int orig_idx; /* original instruction index */

kernel/bpf/helpers.c

+2 −2
Original line number Diff line number Diff line
@@ -3002,9 +3002,9 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags) 
__bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,

					 int (callback_fn)(void *map, int *key, void *value),

					 unsigned int flags,

					 void *aux__ign)

					 void *aux__prog)

{

	struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__ign;

	struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__prog;

	struct bpf_async_kern *async = (struct bpf_async_kern *)wq;



	if (flags)

kernel/bpf/verifier.c

+27 −6
Original line number Diff line number Diff line
@@ -322,6 +322,7 @@ struct bpf_kfunc_call_arg_meta { 
	struct btf *arg_btf;

	u32 arg_btf_id;

	bool arg_owning_ref;

	bool arg_prog;









	struct {









		struct btf_field *field;








@@ -11897,6 +11898,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param







	return btf_param_match_suffix(btf, arg, "__irq_flag");









}

















static bool is_kfunc_arg_prog(const struct btf *btf, const struct btf_param *arg)









{









	return btf_param_match_suffix(btf, arg, "__prog");









}

















static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,









					  const struct btf_param *arg,









					  const char *name)








@@ -12938,6 +12944,17 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_







		if (is_kfunc_arg_ignore(btf, &args[i]))









			continue;

















		if (is_kfunc_arg_prog(btf, &args[i])) {









			/* Used to reject repeated use of __prog. */









			if (meta->arg_prog) {









				verbose(env, "Only 1 prog->aux argument supported per-kfunc\n");









				return -EFAULT;









			}









			meta->arg_prog = true;









			cur_aux(env)->arg_prog = regno;









			continue;









		}

















		if (btf_type_is_scalar(t)) {









			if (reg->type != SCALAR_VALUE) {









				verbose(env, "R%d is not a scalar\n", regno);








@@ -21517,13 +21534,17 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,







		   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {









		insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);









		*cnt = 1;









	} else if (is_bpf_wq_set_callback_impl_kfunc(desc->func_id)) {









		struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) };









	}

















		insn_buf[0] = ld_addrs[0];









		insn_buf[1] = ld_addrs[1];









		insn_buf[2] = *insn;









		*cnt = 3;









	if (env->insn_aux_data[insn_idx].arg_prog) {









		u32 regno = env->insn_aux_data[insn_idx].arg_prog;









		struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };









		int idx = *cnt;

















		insn_buf[idx++] = ld_addrs[0];









		insn_buf[idx++] = ld_addrs[1];









		insn_buf[idx++] = *insn;









		*cnt = idx;









	}









	return 0;









}