Merge branch 'enhance-bpf-global-subprogs-with-argument-tags' (85dd93ac) · Commits · git / linux-net

include/linux/bpf.h

+1 −6

Original line number	Diff line number	Diff line
		@@ -2466,12 +2466,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
		struct btf_func_model *m);

		struct bpf_reg_state;
		int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state *regs);
		int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state *regs);
		int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state reg, u32 nargs);
		int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
		int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog prog,
		struct btf btf, const struct btf_type t);
		const char btf_find_decl_tag_value(const struct btf btf, const struct btf_type *pt,

include/linux/bpf_verifier.h

+21 −8

Original line number	Diff line number	Diff line
		@@ -606,6 +606,13 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)

		#define BPF_MAX_SUBPROGS 256

		struct bpf_subprog_arg_info {
		enum bpf_arg_type arg_type;
		union {
		u32 mem_size;
		};
		};

		struct bpf_subprog_info {
		/* 'start' has to be the first field otherwise find_subprog() won't work */
		u32 start; /* insn idx of function entry point */
		@@ -617,6 +624,10 @@ struct bpf_subprog_info {
		bool is_cb: 1;
		bool is_async_cb: 1;
		bool is_exception_cb: 1;
		bool args_cached: 1;

		u8 arg_cnt;
		struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
		};

		struct bpf_verifier_env;
		@@ -727,6 +738,16 @@ struct bpf_verifier_env {
		char tmp_str_buf[TMP_STR_BUF_LEN];
		};

		static inline struct bpf_func_info_aux subprog_aux(struct bpf_verifier_env env, int subprog)
		{
		return &env->prog->aux->func_info_aux[subprog];
		}

		static inline struct bpf_subprog_info subprog_info(struct bpf_verifier_env env, int subprog)
		{
		return &env->subprog_info[subprog];
		}

		__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
		const char *fmt, va_list args);
		__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
		@@ -764,14 +785,6 @@ bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
		void
		bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);

		int check_ptr_off_reg(struct bpf_verifier_env *env,
		const struct bpf_reg_state *reg, int regno);
		int check_func_arg_reg_off(struct bpf_verifier_env *env,
		const struct bpf_reg_state *reg, int regno,
		enum bpf_arg_type arg_type);
		int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg,
		u32 regno, u32 mem_size);

		/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
		static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
		struct btf *btf, u32 btf_id)

kernel/bpf/btf.c

+78 −204

Original line number	Diff line number	Diff line
		@@ -6765,222 +6765,64 @@ int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog pr
		return btf_check_func_type_match(log, btf1, t1, btf2, t2);
		}

		static int btf_check_func_arg_match(struct bpf_verifier_env *env,
		const struct btf *btf, u32 func_id,
		struct bpf_reg_state *regs,
		bool ptr_to_mem_ok,
		bool processing_call)
		static bool btf_is_dynptr_ptr(const struct btf btf, const struct btf_type t)
		{
		enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
		struct bpf_verifier_log *log = &env->log;
		const char func_name, ref_tname;
		const struct btf_type t, ref_t;
		const struct btf_param *args;
		u32 i, nargs, ref_id;
		int ret;
		const char *name;

		t = btf_type_by_id(btf, func_id);
		if (!t \|\| !btf_type_is_func(t)) {
		/* These checks were already done by the verifier while loading
		* struct bpf_func_info or in add_kfunc_call().
		*/
		bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
		func_id);
		return -EFAULT;
		}
		func_name = btf_name_by_offset(btf, t->name_off);
		t = btf_type_by_id(btf, t->type); /* skip PTR */

		while (btf_type_is_modifier(t))
		t = btf_type_by_id(btf, t->type);
		if (!t \|\| !btf_type_is_func_proto(t)) {
		bpf_log(log, "Invalid BTF of func %s\n", func_name);
		return -EFAULT;
		}
		args = (const struct btf_param *)(t + 1);
		nargs = btf_type_vlen(t);
		if (nargs > MAX_BPF_FUNC_REG_ARGS) {
		bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
		MAX_BPF_FUNC_REG_ARGS);
		return -EINVAL;
		}

		/* check that BTF function arguments match actual types that the
		* verifier sees.
		*/
		for (i = 0; i < nargs; i++) {
		enum bpf_arg_type arg_type = ARG_DONTCARE;
		u32 regno = i + 1;
		struct bpf_reg_state *reg = &regs[regno];

		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
		if (btf_type_is_scalar(t)) {
		if (reg->type == SCALAR_VALUE)
		continue;
		bpf_log(log, "R%d is not a scalar\n", regno);
		return -EINVAL;
		}

		if (!btf_type_is_ptr(t)) {
		bpf_log(log, "Unrecognized arg#%d type %s\n",
		i, btf_type_str(t));
		return -EINVAL;
		}

		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
		ref_tname = btf_name_by_offset(btf, ref_t->name_off);

		ret = check_func_arg_reg_off(env, reg, regno, arg_type);
		if (ret < 0)
		return ret;

		if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
		/* If function expects ctx type in BTF check that caller
		* is passing PTR_TO_CTX.
		*/
		if (reg->type != PTR_TO_CTX) {
		bpf_log(log,
		"arg#%d expected pointer to ctx, but got %s\n",
		i, btf_type_str(t));
		return -EINVAL;
		}
		} else if (ptr_to_mem_ok && processing_call) {
		const struct btf_type *resolve_ret;
		u32 type_size;

		resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
		if (IS_ERR(resolve_ret)) {
		bpf_log(log,
		"arg#%d reference type('%s %s') size cannot be determined: %ld\n",
		i, btf_type_str(ref_t), ref_tname,
		PTR_ERR(resolve_ret));
		return -EINVAL;
		}

		if (check_mem_reg(env, reg, regno, type_size))
		return -EINVAL;
		} else {
		bpf_log(log, "reg type unsupported for arg#%d function %s#%d\n", i,
		func_name, func_id);
		return -EINVAL;
		}
		}

		return 0;
		/* allow either struct or struct forward declaration */
		if (btf_type_is_struct(t) \|\|
		(btf_type_is_fwd(t) && btf_type_kflag(t) == 0)) {
		name = btf_str_by_offset(btf, t->name_off);
		return name && strcmp(name, "bpf_dynptr") == 0;
		}

		/* Compare BTF of a function declaration with given bpf_reg_state.
		* Returns:
		* EFAULT - there is a verifier bug. Abort verification.
		* EINVAL - there is a type mismatch or BTF is not available.
		* 0 - BTF matches with what bpf_reg_state expects.
		* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
		*/
		int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state *regs)
		{
		struct bpf_prog *prog = env->prog;
		struct btf *btf = prog->aux->btf;
		bool is_global;
		u32 btf_id;
		int err;

		if (!prog->aux->func_info)
		return -EINVAL;

		btf_id = prog->aux->func_info[subprog].type_id;
		if (!btf_id)
		return -EFAULT;

		if (prog->aux->func_info_aux[subprog].unreliable)
		return -EINVAL;

		is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
		err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, false);

		/* Compiler optimizations can remove arguments from static functions
		* or mismatched type can be passed into a global function.
		* In such cases mark the function as unreliable from BTF point of view.
		*/
		if (err)
		prog->aux->func_info_aux[subprog].unreliable = true;
		return err;
		}

		/* Compare BTF of a function call with given bpf_reg_state.
		* Returns:
		* EFAULT - there is a verifier bug. Abort verification.
		* EINVAL - there is a type mismatch or BTF is not available.
		* 0 - BTF matches with what bpf_reg_state expects.
		* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
		*
		* NOTE: the code is duplicated from btf_check_subprog_arg_match()
		* because btf_check_func_arg_match() is still doing both. Once that
		* function is split in 2, we can call from here btf_check_subprog_arg_match()
		* first, and then treat the calling part in a new code path.
		*/
		int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state *regs)
		{
		struct bpf_prog *prog = env->prog;
		struct btf *btf = prog->aux->btf;
		bool is_global;
		u32 btf_id;
		int err;

		if (!prog->aux->func_info)
		return -EINVAL;

		btf_id = prog->aux->func_info[subprog].type_id;
		if (!btf_id)
		return -EFAULT;

		if (prog->aux->func_info_aux[subprog].unreliable)
		return -EINVAL;

		is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
		err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, true);

		/* Compiler optimizations can remove arguments from static functions
		* or mismatched type can be passed into a global function.
		* In such cases mark the function as unreliable from BTF point of view.
		*/
		if (err)
		prog->aux->func_info_aux[subprog].unreliable = true;
		return err;
		return false;
		}

		/* Convert BTF of a function into bpf_reg_state if possible
		/* Process BTF of a function to produce high-level expectation of function
		* arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information
		* is cached in subprog info for reuse.
		* Returns:
		* EFAULT - there is a verifier bug. Abort verification.
		* EINVAL - cannot convert BTF.
		* 0 - Successfully converted BTF into bpf_reg_state
		* (either PTR_TO_CTX or SCALAR_VALUE).
		* 0 - Successfully processed BTF and constructed argument expectations.
		*/
		int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state regs, u32 arg_cnt)
		int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
		{
		bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL;
		struct bpf_subprog_info *sub = subprog_info(env, subprog);
		struct bpf_verifier_log *log = &env->log;
		struct bpf_prog *prog = env->prog;
		enum bpf_prog_type prog_type = prog->type;
		struct btf *btf = prog->aux->btf;
		const struct btf_param *args;
		const struct btf_type t, ref_t;
		const struct btf_type t, ref_t, *fn_t;
		u32 i, nargs, btf_id;
		const char *tname;

		if (!prog->aux->func_info \|\|
		prog->aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL) {
		if (sub->args_cached)
		return 0;

		if (!prog->aux->func_info) {
		bpf_log(log, "Verifier bug\n");
		return -EFAULT;
		}

		btf_id = prog->aux->func_info[subprog].type_id;
		if (!btf_id) {
		if (!is_global) /* not fatal for static funcs */
		return -EINVAL;
		bpf_log(log, "Global functions need valid BTF\n");
		return -EFAULT;
		}

		t = btf_type_by_id(btf, btf_id);
		if (!t \|\| !btf_type_is_func(t)) {
		fn_t = btf_type_by_id(btf, btf_id);
		if (!fn_t \|\| !btf_type_is_func(fn_t)) {
		/* These checks were already done by the verifier while loading
		* struct bpf_func_info
		*/
		@@ -6988,11 +6830,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		subprog);
		return -EFAULT;
		}
		tname = btf_name_by_offset(btf, t->name_off);

		if (log->level & BPF_LOG_LEVEL)
		bpf_log(log, "Validating %s() func#%d...\n",
		tname, subprog);
		tname = btf_name_by_offset(btf, fn_t->name_off);

		if (prog->aux->func_info_aux[subprog].unreliable) {
		bpf_log(log, "Verifier bug in function %s()\n", tname);
		@@ -7001,7 +6839,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		if (prog_type == BPF_PROG_TYPE_EXT)
		prog_type = prog->aux->dst_prog->type;

		t = btf_type_by_id(btf, t->type);
		t = btf_type_by_id(btf, fn_t->type);
		if (!t \|\| !btf_type_is_func_proto(t)) {
		bpf_log(log, "Invalid type of function %s()\n", tname);
		return -EFAULT;
		@@ -7013,7 +6851,6 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		tname, nargs, MAX_BPF_FUNC_REG_ARGS);
		return -EINVAL;
		}
		*arg_cnt = nargs;
		/* check that function returns int, exception cb also requires this */
		t = btf_type_by_id(btf, t->type);
		while (btf_type_is_modifier(t))
		@@ -7028,24 +6865,54 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		* Only PTR_TO_CTX and SCALAR are supported atm.
		*/
		for (i = 0; i < nargs; i++) {
		struct bpf_reg_state *reg = &regs[i + 1];
		bool is_nonnull = false;
		const char *tag;

		t = btf_type_by_id(btf, args[i].type);

		tag = btf_find_decl_tag_value(btf, fn_t, i, "arg:");
		if (IS_ERR(tag) && PTR_ERR(tag) == -ENOENT) {
		tag = NULL;
		} else if (IS_ERR(tag)) {
		bpf_log(log, "arg#%d type's tag fetching failure: %ld\n", i, PTR_ERR(tag));
		return PTR_ERR(tag);
		}
		/* 'arg:<tag>' decl_tag takes precedence over derivation of
		* register type from BTF type itself
		*/
		if (tag) {
		/* disallow arg tags in static subprogs */
		if (!is_global) {
		bpf_log(log, "arg#%d type tag is not supported in static functions\n", i);
		return -EOPNOTSUPP;
		}
		if (strcmp(tag, "ctx") == 0) {
		sub->args[i].arg_type = ARG_PTR_TO_CTX;
		continue;
		}
		if (strcmp(tag, "nonnull") == 0)
		is_nonnull = true;
		}

		while (btf_type_is_modifier(t))
		t = btf_type_by_id(btf, t->type);
		if (btf_type_is_int(t) \|\| btf_is_any_enum(t)) {
		reg->type = SCALAR_VALUE;
		sub->args[i].arg_type = ARG_ANYTHING;
		continue;
		}
		if (btf_type_is_ptr(t) && btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
		sub->args[i].arg_type = ARG_PTR_TO_CTX;
		continue;
		}
		if (btf_type_is_ptr(t)) {
		if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
		reg->type = PTR_TO_CTX;
		if (btf_type_is_ptr(t) && btf_is_dynptr_ptr(btf, t)) {
		sub->args[i].arg_type = ARG_PTR_TO_DYNPTR \| MEM_RDONLY;
		continue;
		}
		if (is_global && btf_type_is_ptr(t)) {
		u32 mem_size;

		t = btf_type_skip_modifiers(btf, t->type, NULL);

		ref_t = btf_resolve_size(btf, t, &reg->mem_size);
		ref_t = btf_resolve_size(btf, t, &mem_size);
		if (IS_ERR(ref_t)) {
		bpf_log(log,
		"arg#%d reference type('%s %s') size cannot be determined: %ld\n",
		@@ -7054,15 +6921,22 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
		return -EINVAL;
		}

		reg->type = PTR_TO_MEM \| PTR_MAYBE_NULL;
		reg->id = ++env->id_gen;

		sub->args[i].arg_type = is_nonnull ? ARG_PTR_TO_MEM : ARG_PTR_TO_MEM_OR_NULL;
		sub->args[i].mem_size = mem_size;
		continue;
		}
		if (is_nonnull) {
		bpf_log(log, "arg#%d marked as non-null, but is not a pointer type\n", i);
		return -EINVAL;
		}
		bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
		i, btf_type_str(t), tname);
		return -EINVAL;
		}

		sub->arg_cnt = nargs;
		sub->args_cached = true;

		return 0;
		}

kernel/bpf/verifier.c

+144 −40

Original line number	Diff line number	Diff line
		@@ -437,16 +437,6 @@ static const char subprog_name(const struct bpf_verifier_env env, int subprog)
		return btf_type_name(env->prog->aux->btf, info->type_id);
		}

		static struct bpf_func_info_aux subprog_aux(const struct bpf_verifier_env env, int subprog)
		{
		return &env->prog->aux->func_info_aux[subprog];
		}

		static struct bpf_subprog_info subprog_info(struct bpf_verifier_env env, int subprog)
		{
		return &env->subprog_info[subprog];
		}

		static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)
		{
		struct bpf_subprog_info *info = subprog_info(env, subprog);
		@@ -5137,7 +5127,7 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env,
		return 0;
		}

		int check_ptr_off_reg(struct bpf_verifier_env *env,
		static int check_ptr_off_reg(struct bpf_verifier_env *env,
		const struct bpf_reg_state *reg, int regno)
		{
		return __check_ptr_off_reg(env, reg, regno, false);
		@@ -7310,7 +7300,7 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
		return err;
		}

		int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg,
		static int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg,
		u32 regno, u32 mem_size)
		{
		bool may_be_null = type_may_be_null(reg->type);
		@@ -8296,7 +8286,7 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
		return field;
		}

		int check_func_arg_reg_off(struct bpf_verifier_env *env,
		static int check_func_arg_reg_off(struct bpf_verifier_env *env,
		const struct bpf_reg_state *reg, int regno,
		enum bpf_arg_type arg_type)
		{
		@@ -9259,6 +9249,102 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls
		return err;
		}

		static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
		const struct btf *btf,
		struct bpf_reg_state *regs)
		{
		struct bpf_subprog_info *sub = subprog_info(env, subprog);
		struct bpf_verifier_log *log = &env->log;
		u32 i;
		int ret;

		ret = btf_prepare_func_args(env, subprog);
		if (ret)
		return ret;

		/* check that BTF function arguments match actual types that the
		* verifier sees.
		*/
		for (i = 0; i < sub->arg_cnt; i++) {
		u32 regno = i + 1;
		struct bpf_reg_state *reg = &regs[regno];
		struct bpf_subprog_arg_info *arg = &sub->args[i];

		if (arg->arg_type == ARG_ANYTHING) {
		if (reg->type != SCALAR_VALUE) {
		bpf_log(log, "R%d is not a scalar\n", regno);
		return -EINVAL;
		}
		} else if (arg->arg_type == ARG_PTR_TO_CTX) {
		ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
		if (ret < 0)
		return ret;
		/* If function expects ctx type in BTF check that caller
		* is passing PTR_TO_CTX.
		*/
		if (reg->type != PTR_TO_CTX) {
		bpf_log(log, "arg#%d expects pointer to ctx\n", i);
		return -EINVAL;
		}
		} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
		ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
		if (ret < 0)
		return ret;
		if (check_mem_reg(env, reg, regno, arg->mem_size))
		return -EINVAL;
		if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) {
		bpf_log(log, "arg#%d is expected to be non-NULL\n", i);
		return -EINVAL;
		}
		} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR \| MEM_RDONLY)) {
		ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);
		if (ret)
		return ret;
		} else {
		bpf_log(log, "verifier bug: unrecognized arg#%d type %d\n",
		i, arg->arg_type);
		return -EFAULT;
		}
		}

		return 0;
		}

		/* Compare BTF of a function call with given bpf_reg_state.
		* Returns:
		* EFAULT - there is a verifier bug. Abort verification.
		* EINVAL - there is a type mismatch or BTF is not available.
		* 0 - BTF matches with what bpf_reg_state expects.
		* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
		*/
		static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
		struct bpf_reg_state *regs)
		{
		struct bpf_prog *prog = env->prog;
		struct btf *btf = prog->aux->btf;
		u32 btf_id;
		int err;

		if (!prog->aux->func_info)
		return -EINVAL;

		btf_id = prog->aux->func_info[subprog].type_id;
		if (!btf_id)
		return -EFAULT;

		if (prog->aux->func_info_aux[subprog].unreliable)
		return -EINVAL;

		err = btf_check_func_arg_match(env, subprog, btf, regs);
		/* Compiler optimizations can remove arguments from static functions
		* or mismatched type can be passed into a global function.
		* In such cases mark the function as unreliable from BTF point of view.
		*/
		if (err)
		prog->aux->func_info_aux[subprog].unreliable = true;
		return err;
		}

		static int push_callback_call(struct bpf_verifier_env env, struct bpf_insn insn,
		int insn_idx, int subprog,
		set_callee_state_fn set_callee_state_cb)
		@@ -19909,6 +19995,7 @@ static void free_states(struct bpf_verifier_env *env)
		static int do_check_common(struct bpf_verifier_env *env, int subprog)
		{
		bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
		struct bpf_subprog_info *sub = subprog_info(env, subprog);
		struct bpf_verifier_state *state;
		struct bpf_reg_state *regs;
		int ret, i;
		@@ -19935,54 +20022,71 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
		state->first_insn_idx = env->subprog_info[subprog].start;
		state->last_insn_idx = -1;


		regs = state->frame[state->curframe]->regs;
		if (subprog \|\| env->prog->type == BPF_PROG_TYPE_EXT) {
		u32 nargs;
		const char *sub_name = subprog_name(env, subprog);
		struct bpf_subprog_arg_info *arg;
		struct bpf_reg_state *reg;

		ret = btf_prepare_func_args(env, subprog, regs, &nargs);
		verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);
		ret = btf_prepare_func_args(env, subprog);
		if (ret)
		goto out;

		if (subprog_is_exc_cb(env, subprog)) {
		state->frame[0]->in_exception_callback_fn = true;
		/* We have already ensured that the callback returns an integer, just
		* like all global subprogs. We need to determine it only has a single
		* scalar argument.
		*/
		if (nargs != 1 \|\| regs[BPF_REG_1].type != SCALAR_VALUE) {
		if (sub->arg_cnt != 1 \|\| sub->args[0].arg_type != ARG_ANYTHING) {
		verbose(env, "exception cb only supports single integer argument\n");
		ret = -EINVAL;
		goto out;
		}
		}
		for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
		if (regs[i].type == PTR_TO_CTX)
		for (i = BPF_REG_1; i <= sub->arg_cnt; i++) {
		arg = &sub->args[i - BPF_REG_1];
		reg = &regs[i];

		if (arg->arg_type == ARG_PTR_TO_CTX) {
		reg->type = PTR_TO_CTX;
		mark_reg_known_zero(env, regs, i);
		else if (regs[i].type == SCALAR_VALUE)
		} else if (arg->arg_type == ARG_ANYTHING) {
		reg->type = SCALAR_VALUE;
		mark_reg_unknown(env, regs, i);
		else if (base_type(regs[i].type) == PTR_TO_MEM) {
		const u32 mem_size = regs[i].mem_size;

		} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR \| MEM_RDONLY)) {
		/* assume unspecial LOCAL dynptr type */
		__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);
		} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
		reg->type = PTR_TO_MEM;
		if (arg->arg_type & PTR_MAYBE_NULL)
		reg->type \|= PTR_MAYBE_NULL;
		mark_reg_known_zero(env, regs, i);
		regs[i].mem_size = mem_size;
		regs[i].id = ++env->id_gen;
		reg->mem_size = arg->mem_size;
		reg->id = ++env->id_gen;
		} else {
		WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n",
		i - BPF_REG_1, arg->arg_type);
		ret = -EFAULT;
		goto out;
		}
		}
		} else {
		/* if main BPF program has associated BTF info, validate that
		* it's matching expected signature, and otherwise mark BTF
		* info for main program as unreliable
		*/
		if (env->prog->aux->func_info_aux) {
		ret = btf_prepare_func_args(env, 0);
		if (ret \|\| sub->arg_cnt != 1 \|\| sub->args[0].arg_type != ARG_PTR_TO_CTX)
		env->prog->aux->func_info_aux[0].unreliable = true;
		}

		/* 1st arg to a function */
		regs[BPF_REG_1].type = PTR_TO_CTX;
		mark_reg_known_zero(env, regs, BPF_REG_1);
		ret = btf_check_subprog_arg_match(env, subprog, regs);
		if (ret == -EFAULT)
		/* unlikely verifier bug. abort.
		* ret == 0 and ret < 0 are sadly acceptable for
		* main() function due to backward compatibility.
		* Like socket filter program may be written as:
		* int bpf_prog(struct pt_regs *ctx)
		* and never dereference that ctx in the program.
		* 'struct pt_regs' is a type mismatch for socket
		* filter that should be using 'struct __sk_buff'.
		*/
		goto out;
		}

		ret = do_check(env);

tools/lib/bpf/bpf_helpers.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -188,6 +188,9 @@ enum libbpf_tristate {
		!!sym; \
		})

		#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx")))
		#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull")))

		#ifndef ___bpf_concat
		#define ___bpf_concat(a, b) a ## b
		#endif