Commit 46ffc1f7 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-split-verifier-c' 



Alexei Starovoitov says:

====================
v3->v4: Restore few minor comments and undo few function moves
v2->v3: Actually restore comments lost in patch 3
(instead of adding them to patch 4)
v1->v2: Restore comments lost in patch 3

verifier.c is huge. Split it into logically independent pieces.
No functional changes.
The diff is impossible to review over email.
'git show' shows minimal actual changes. Only plenty of moved lines.
Such split may cause backport headaches.
We should have split it long ago.
Even after split verifier.c is still 20k lines,
but further split is harder.
====================

Acked-by: default avatarKumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Link: https://patch.msgid.link/20260412152936.54262-1-alexei.starovoitov@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 2ec74a05 99a832a2

include/linux/bpf_verifier.h

+284 −1
Original line number Diff line number Diff line
@@ -279,6 +279,8 @@ static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) 
			  (1 << BPF_REG_3) | (1 << BPF_REG_4) | \

			  (1 << BPF_REG_5))



#define BPF_MAIN_FUNC (-1)



#define BPF_DYNPTR_SIZE		sizeof(struct bpf_dynptr_kern)

#define BPF_DYNPTR_NR_SLOTS		(BPF_DYNPTR_SIZE / BPF_REG_SIZE)
@@ -983,6 +985,41 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, 
		bpf_log(&env->log, "verifier bug: " fmt "\n", ##args);				\

	})



static inline void mark_prune_point(struct bpf_verifier_env *env, int idx)

{

	env->insn_aux_data[idx].prune_point = true;

}



static inline bool bpf_is_prune_point(struct bpf_verifier_env *env, int insn_idx)

{

	return env->insn_aux_data[insn_idx].prune_point;

}



static inline void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)

{

	env->insn_aux_data[idx].force_checkpoint = true;

}



static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)

{

	return env->insn_aux_data[insn_idx].force_checkpoint;

}



static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx)

{

	env->insn_aux_data[idx].calls_callback = true;

}



static inline bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx)

{

	return env->insn_aux_data[insn_idx].calls_callback;

}



static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx)

{

	env->insn_aux_data[idx].jmp_point = true;

}



static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)

{

	struct bpf_verifier_state *cur = env->cur_state;
@@ -1024,6 +1061,11 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) 
		*btf_id = key & 0x7FFFFFFF;

}



int bpf_check_btf_info_early(struct bpf_verifier_env *env,

			     const union bpf_attr *attr, bpfptr_t uattr);

int bpf_check_btf_info(struct bpf_verifier_env *env,

		       const union bpf_attr *attr, bpfptr_t uattr);



int bpf_check_attach_target(struct bpf_verifier_log *log,

			    const struct bpf_prog *prog,

			    const struct bpf_prog *tgt_prog,
@@ -1033,6 +1075,89 @@ void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); 


int mark_chain_precision(struct bpf_verifier_env *env, int regno);



int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx);

int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st);



void bpf_clear_jmp_history(struct bpf_verifier_state *state);

int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,

			    const struct bpf_verifier_state *src);

struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx);

void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);

void bpf_free_backedges(struct bpf_scc_visit *visit);

int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,

			 int insn_flags, u64 linked_regs);

void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist);

void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,

			   struct bpf_reg_state *reg);

void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg);

void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,

				  struct bpf_verifier_state *st);

void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);

int bpf_mark_chain_precision(struct bpf_verifier_env *env,

			     struct bpf_verifier_state *starting_state,

			     int regno, bool *changed);



static inline int bpf_get_spi(s32 off)

{

	return (-off - 1) / BPF_REG_SIZE;

}



static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env,

					      const struct bpf_reg_state *reg)

{

	struct bpf_verifier_state *cur = env->cur_state;



	return cur->frame[reg->frameno];

}



/* Return IP for a given frame in a call stack */

static inline u32 bpf_frame_insn_idx(struct bpf_verifier_state *st, u32 frame)

{

	return frame == st->curframe

	       ? st->insn_idx

	       : st->frame[frame + 1]->callsite;

}



static inline bool bpf_is_jmp_point(struct bpf_verifier_env *env, int insn_idx)

{

	return env->insn_aux_data[insn_idx].jmp_point;

}



static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack)

{

	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;

}



static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack)

{

	return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE;

}



static inline bool bpf_register_is_null(struct bpf_reg_state *reg)

{

	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);

}



static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)

{

	bt->reg_masks[frame] |= 1 << reg;

}



static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)

{

	bt->stack_masks[frame] |= 1ull << slot;

}



static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)

{

	return bt->reg_masks[frame] & (1 << reg);

}



static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)

{

	return bt->stack_masks[frame] & (1ull << slot);

}



bool bpf_map_is_rdonly(const struct bpf_map *map);

int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,

			bool is_ldsx);
@@ -1179,13 +1304,91 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en 
int bpf_jmp_offset(struct bpf_insn *insn);

struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);

void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);

bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);

bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);



int bpf_find_subprog(struct bpf_verifier_env *env, int off);

int bpf_compute_const_regs(struct bpf_verifier_env *env);

int bpf_prune_dead_branches(struct bpf_verifier_env *env);

int bpf_check_cfg(struct bpf_verifier_env *env);

int bpf_compute_postorder(struct bpf_verifier_env *env);

int bpf_compute_scc(struct bpf_verifier_env *env);



struct bpf_map_desc {

	struct bpf_map *ptr;

	int uid;

};



struct bpf_kfunc_call_arg_meta {

	/* In parameters */

	struct btf *btf;

	u32 func_id;

	u32 kfunc_flags;

	const struct btf_type *func_proto;

	const char *func_name;

	/* Out parameters */

	u32 ref_obj_id;

	u8 release_regno;

	bool r0_rdonly;

	u32 ret_btf_id;

	u64 r0_size;

	u32 subprogno;

	struct {

		u64 value;

		bool found;

	} arg_constant;



	/* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,

	 * generally to pass info about user-defined local kptr types to later

	 * verification logic

	 *   bpf_obj_drop/bpf_percpu_obj_drop

	 *     Record the local kptr type to be drop'd

	 *   bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)

	 *     Record the local kptr type to be refcount_incr'd and use

	 *     arg_owning_ref to determine whether refcount_acquire should be

	 *     fallible

	 */

	struct btf *arg_btf;

	u32 arg_btf_id;

	bool arg_owning_ref;

	bool arg_prog;



	struct {

		struct btf_field *field;

	} arg_list_head;

	struct {

		struct btf_field *field;

	} arg_rbtree_root;

	struct {

		enum bpf_dynptr_type type;

		u32 id;

		u32 ref_obj_id;

	} initialized_dynptr;

	struct {

		u8 spi;

		u8 frameno;

	} iter;

	struct bpf_map_desc map;

	u64 mem_size;

};



int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,

			 const struct bpf_func_proto **ptr);

int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id,

			     s16 offset, struct bpf_kfunc_call_arg_meta *meta);

bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn);

bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn);

static inline bool bpf_is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)

{

	return meta->kfunc_flags & KF_ITER_NEXT;

}



static inline bool bpf_is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)

{

	return meta->kfunc_flags & KF_SLEEPABLE;

}

bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta);

struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem);

int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off);

bool bpf_insn_is_cond_jump(u8 code);

bool bpf_is_may_goto_insn(struct bpf_insn *insn);
@@ -1204,5 +1407,85 @@ int bpf_stack_liveness_init(struct bpf_verifier_env *env); 
void bpf_stack_liveness_free(struct bpf_verifier_env *env);

int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);

bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);

int bpf_compute_live_registers(struct bpf_verifier_env *env);



#define BPF_MAP_KEY_POISON	(1ULL << 63)

#define BPF_MAP_KEY_SEEN	(1ULL << 62)



static inline bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)

{

	return aux->map_ptr_state.poison;

}



static inline bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)

{

	return aux->map_ptr_state.unpriv;

}



static inline bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)

{

	return aux->map_key_state & BPF_MAP_KEY_POISON;

}



static inline bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)

{

	return !(aux->map_key_state & BPF_MAP_KEY_SEEN);

}



static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)

{

	return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);

}



#define MAX_PACKET_OFF 0xffff

#define CALLER_SAVED_REGS 6



enum bpf_reg_arg_type {

	SRC_OP,		/* register is used as source operand */

	DST_OP,		/* register is used as destination operand */

	DST_OP_NO_MARK	/* same as above, check only, don't mark */

};



#define MAX_KFUNC_DESCS 256



struct bpf_kfunc_desc {

	struct btf_func_model func_model;

	u32 func_id;

	s32 imm;

	u16 offset;

	unsigned long addr;

};



struct bpf_kfunc_desc_tab {

	/* Sorted by func_id (BTF ID) and offset (fd_array offset) during

	 * verification. JITs do lookups by bpf_insn, where func_id may not be

	 * available, therefore at the end of verification do_misc_fixups()

	 * sorts this by imm and offset.

	 */

	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];

	u32 nr_descs;

};



/* Functions exported from verifier.c, used by fixups.c */

bool bpf_is_reg64(struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t);

void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);

void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog);

bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env);

bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm);

int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset);

int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

			 struct bpf_insn *insn_buf, int insn_idx, int *cnt);



/* Functions in fixups.c, called from bpf_check() */

int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);

int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);

void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env);

int bpf_opt_remove_dead_code(struct bpf_verifier_env *env);

int bpf_opt_remove_nops(struct bpf_verifier_env *env);

int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr);

int bpf_convert_ctx_accesses(struct bpf_verifier_env *env);

int bpf_jit_subprogs(struct bpf_verifier_env *env);

int bpf_fixup_call_args(struct bpf_verifier_env *env);

int bpf_do_misc_fixups(struct bpf_verifier_env *env);



#endif /* _LINUX_BPF_VERIFIER_H */

kernel/bpf/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_ 
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o

obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o

obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o

obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o

obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o

obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o

obj-$(CONFIG_BPF_JIT) += trampoline.o

kernel/bpf/backtrack.c

0 → 100644
+934 −0

File added.

Preview size limit exceeded, changes collapsed.

kernel/bpf/cfg.c

0 → 100644
+872 −0

File added.

Preview size limit exceeded, changes collapsed.

kernel/bpf/check_btf.c

0 → 100644
+463 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */

#include <linux/bpf.h>

#include <linux/bpf_verifier.h>

#include <linux/filter.h>

#include <linux/btf.h>



#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)



static int check_abnormal_return(struct bpf_verifier_env *env)

{

	int i;



	for (i = 1; i < env->subprog_cnt; i++) {

		if (env->subprog_info[i].has_ld_abs) {

			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");

			return -EINVAL;

		}

		if (env->subprog_info[i].has_tail_call) {

			verbose(env, "tail_call is not allowed in subprogs without BTF\n");

			return -EINVAL;

		}

	}

	return 0;

}



/* The minimum supported BTF func info size */

#define MIN_BPF_FUNCINFO_SIZE	8

#define MAX_FUNCINFO_REC_SIZE	252



static int check_btf_func_early(struct bpf_verifier_env *env,

				const union bpf_attr *attr,

				bpfptr_t uattr)

{

	u32 krec_size = sizeof(struct bpf_func_info);

	const struct btf_type *type, *func_proto;

	u32 i, nfuncs, urec_size, min_size;

	struct bpf_func_info *krecord;

	struct bpf_prog *prog;

	const struct btf *btf;

	u32 prev_offset = 0;

	bpfptr_t urecord;

	int ret = -ENOMEM;



	nfuncs = attr->func_info_cnt;

	if (!nfuncs) {

		if (check_abnormal_return(env))

			return -EINVAL;

		return 0;

	}



	urec_size = attr->func_info_rec_size;

	if (urec_size < MIN_BPF_FUNCINFO_SIZE ||

	    urec_size > MAX_FUNCINFO_REC_SIZE ||

	    urec_size % sizeof(u32)) {

		verbose(env, "invalid func info rec size %u\n", urec_size);

		return -EINVAL;

	}



	prog = env->prog;

	btf = prog->aux->btf;



	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);

	min_size = min_t(u32, krec_size, urec_size);



	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);

	if (!krecord)

		return -ENOMEM;



	for (i = 0; i < nfuncs; i++) {

		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);

		if (ret) {

			if (ret == -E2BIG) {

				verbose(env, "nonzero tailing record in func info");

				/* set the size kernel expects so loader can zero

				 * out the rest of the record.

				 */

				if (copy_to_bpfptr_offset(uattr,

							  offsetof(union bpf_attr, func_info_rec_size),

							  &min_size, sizeof(min_size)))

					ret = -EFAULT;

			}

			goto err_free;

		}



		if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {

			ret = -EFAULT;

			goto err_free;

		}



		/* check insn_off */

		ret = -EINVAL;

		if (i == 0) {

			if (krecord[i].insn_off) {

				verbose(env,

					"nonzero insn_off %u for the first func info record",

					krecord[i].insn_off);

				goto err_free;

			}

		} else if (krecord[i].insn_off <= prev_offset) {

			verbose(env,

				"same or smaller insn offset (%u) than previous func info record (%u)",

				krecord[i].insn_off, prev_offset);

			goto err_free;

		}



		/* check type_id */

		type = btf_type_by_id(btf, krecord[i].type_id);

		if (!type || !btf_type_is_func(type)) {

			verbose(env, "invalid type id %d in func info",

				krecord[i].type_id);

			goto err_free;

		}



		func_proto = btf_type_by_id(btf, type->type);

		if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))

			/* btf_func_check() already verified it during BTF load */

			goto err_free;



		prev_offset = krecord[i].insn_off;

		bpfptr_add(&urecord, urec_size);

	}



	prog->aux->func_info = krecord;

	prog->aux->func_info_cnt = nfuncs;

	return 0;



err_free:

	kvfree(krecord);

	return ret;

}



static int check_btf_func(struct bpf_verifier_env *env,

			  const union bpf_attr *attr,

			  bpfptr_t uattr)

{

	const struct btf_type *type, *func_proto, *ret_type;

	u32 i, nfuncs, urec_size;

	struct bpf_func_info *krecord;

	struct bpf_func_info_aux *info_aux = NULL;

	struct bpf_prog *prog;

	const struct btf *btf;

	bpfptr_t urecord;

	bool scalar_return;

	int ret = -ENOMEM;



	nfuncs = attr->func_info_cnt;

	if (!nfuncs) {

		if (check_abnormal_return(env))

			return -EINVAL;

		return 0;

	}

	if (nfuncs != env->subprog_cnt) {

		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");

		return -EINVAL;

	}



	urec_size = attr->func_info_rec_size;



	prog = env->prog;

	btf = prog->aux->btf;



	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);



	krecord = prog->aux->func_info;

	info_aux = kzalloc_objs(*info_aux, nfuncs,

				GFP_KERNEL_ACCOUNT | __GFP_NOWARN);

	if (!info_aux)

		return -ENOMEM;



	for (i = 0; i < nfuncs; i++) {

		/* check insn_off */

		ret = -EINVAL;



		if (env->subprog_info[i].start != krecord[i].insn_off) {

			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");

			goto err_free;

		}



		/* Already checked type_id */

		type = btf_type_by_id(btf, krecord[i].type_id);

		info_aux[i].linkage = BTF_INFO_VLEN(type->info);

		/* Already checked func_proto */

		func_proto = btf_type_by_id(btf, type->type);



		ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);

		scalar_return =

			btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);

		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {

			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");

			goto err_free;

		}

		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {

			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");

			goto err_free;

		}



		env->subprog_info[i].name = btf_name_by_offset(btf, type->name_off);

		bpfptr_add(&urecord, urec_size);

	}



	prog->aux->func_info_aux = info_aux;

	return 0;



err_free:

	kfree(info_aux);

	return ret;

}



#define MIN_BPF_LINEINFO_SIZE	offsetofend(struct bpf_line_info, line_col)

#define MAX_LINEINFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE



static int check_btf_line(struct bpf_verifier_env *env,

			  const union bpf_attr *attr,

			  bpfptr_t uattr)

{

	u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;

	struct bpf_subprog_info *sub;

	struct bpf_line_info *linfo;

	struct bpf_prog *prog;

	const struct btf *btf;

	bpfptr_t ulinfo;

	int err;



	nr_linfo = attr->line_info_cnt;

	if (!nr_linfo)

		return 0;

	if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))

		return -EINVAL;



	rec_size = attr->line_info_rec_size;

	if (rec_size < MIN_BPF_LINEINFO_SIZE ||

	    rec_size > MAX_LINEINFO_REC_SIZE ||

	    rec_size & (sizeof(u32) - 1))

		return -EINVAL;



	/* Need to zero it in case the userspace may

	 * pass in a smaller bpf_line_info object.

	 */

	linfo = kvzalloc_objs(struct bpf_line_info, nr_linfo,

			      GFP_KERNEL_ACCOUNT | __GFP_NOWARN);

	if (!linfo)

		return -ENOMEM;



	prog = env->prog;

	btf = prog->aux->btf;



	s = 0;

	sub = env->subprog_info;

	ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);

	expected_size = sizeof(struct bpf_line_info);

	ncopy = min_t(u32, expected_size, rec_size);

	for (i = 0; i < nr_linfo; i++) {

		err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);

		if (err) {

			if (err == -E2BIG) {

				verbose(env, "nonzero tailing record in line_info");

				if (copy_to_bpfptr_offset(uattr,

							  offsetof(union bpf_attr, line_info_rec_size),

							  &expected_size, sizeof(expected_size)))

					err = -EFAULT;

			}

			goto err_free;

		}



		if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {

			err = -EFAULT;

			goto err_free;

		}



		/*

		 * Check insn_off to ensure

		 * 1) strictly increasing AND

		 * 2) bounded by prog->len

		 *

		 * The linfo[0].insn_off == 0 check logically falls into

		 * the later "missing bpf_line_info for func..." case

		 * because the first linfo[0].insn_off must be the

		 * first sub also and the first sub must have

		 * subprog_info[0].start == 0.

		 */

		if ((i && linfo[i].insn_off <= prev_offset) ||

		    linfo[i].insn_off >= prog->len) {

			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",

				i, linfo[i].insn_off, prev_offset,

				prog->len);

			err = -EINVAL;

			goto err_free;

		}



		if (!prog->insnsi[linfo[i].insn_off].code) {

			verbose(env,

				"Invalid insn code at line_info[%u].insn_off\n",

				i);

			err = -EINVAL;

			goto err_free;

		}



		if (!btf_name_by_offset(btf, linfo[i].line_off) ||

		    !btf_name_by_offset(btf, linfo[i].file_name_off)) {

			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);

			err = -EINVAL;

			goto err_free;

		}



		if (s != env->subprog_cnt) {

			if (linfo[i].insn_off == sub[s].start) {

				sub[s].linfo_idx = i;

				s++;

			} else if (sub[s].start < linfo[i].insn_off) {

				verbose(env, "missing bpf_line_info for func#%u\n", s);

				err = -EINVAL;

				goto err_free;

			}

		}



		prev_offset = linfo[i].insn_off;

		bpfptr_add(&ulinfo, rec_size);

	}



	if (s != env->subprog_cnt) {

		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",

			env->subprog_cnt - s, s);

		err = -EINVAL;

		goto err_free;

	}



	prog->aux->linfo = linfo;

	prog->aux->nr_linfo = nr_linfo;



	return 0;



err_free:

	kvfree(linfo);

	return err;

}



#define MIN_CORE_RELO_SIZE	sizeof(struct bpf_core_relo)

#define MAX_CORE_RELO_SIZE	MAX_FUNCINFO_REC_SIZE



static int check_core_relo(struct bpf_verifier_env *env,

			   const union bpf_attr *attr,

			   bpfptr_t uattr)

{

	u32 i, nr_core_relo, ncopy, expected_size, rec_size;

	struct bpf_core_relo core_relo = {};

	struct bpf_prog *prog = env->prog;

	const struct btf *btf = prog->aux->btf;

	struct bpf_core_ctx ctx = {

		.log = &env->log,

		.btf = btf,

	};

	bpfptr_t u_core_relo;

	int err;



	nr_core_relo = attr->core_relo_cnt;

	if (!nr_core_relo)

		return 0;

	if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))

		return -EINVAL;



	rec_size = attr->core_relo_rec_size;

	if (rec_size < MIN_CORE_RELO_SIZE ||

	    rec_size > MAX_CORE_RELO_SIZE ||

	    rec_size % sizeof(u32))

		return -EINVAL;



	u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);

	expected_size = sizeof(struct bpf_core_relo);

	ncopy = min_t(u32, expected_size, rec_size);



	/* Unlike func_info and line_info, copy and apply each CO-RE

	 * relocation record one at a time.

	 */

	for (i = 0; i < nr_core_relo; i++) {

		/* future proofing when sizeof(bpf_core_relo) changes */

		err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);

		if (err) {

			if (err == -E2BIG) {

				verbose(env, "nonzero tailing record in core_relo");

				if (copy_to_bpfptr_offset(uattr,

							  offsetof(union bpf_attr, core_relo_rec_size),

							  &expected_size, sizeof(expected_size)))

					err = -EFAULT;

			}

			break;

		}



		if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {

			err = -EFAULT;

			break;

		}



		if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {

			verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",

				i, core_relo.insn_off, prog->len);

			err = -EINVAL;

			break;

		}



		err = bpf_core_apply(&ctx, &core_relo, i,

				     &prog->insnsi[core_relo.insn_off / 8]);

		if (err)

			break;

		bpfptr_add(&u_core_relo, rec_size);

	}

	return err;

}



int bpf_check_btf_info_early(struct bpf_verifier_env *env,

			     const union bpf_attr *attr,

			     bpfptr_t uattr)

{

	struct btf *btf;

	int err;



	if (!attr->func_info_cnt && !attr->line_info_cnt) {

		if (check_abnormal_return(env))

			return -EINVAL;

		return 0;

	}



	btf = btf_get_by_fd(attr->prog_btf_fd);

	if (IS_ERR(btf))

		return PTR_ERR(btf);

	if (btf_is_kernel(btf)) {

		btf_put(btf);

		return -EACCES;

	}

	env->prog->aux->btf = btf;



	err = check_btf_func_early(env, attr, uattr);

	if (err)

		return err;

	return 0;

}



int bpf_check_btf_info(struct bpf_verifier_env *env,

		       const union bpf_attr *attr,

		       bpfptr_t uattr)

{

	int err;



	if (!attr->func_info_cnt && !attr->line_info_cnt) {

		if (check_abnormal_return(env))

			return -EINVAL;

		return 0;

	}



	err = check_btf_func(env, attr, uattr);

	if (err)

		return err;



	err = check_btf_line(env, attr, uattr);

	if (err)

		return err;



	err = check_core_relo(env, attr, uattr);

	if (err)

		return err;



	return 0;

}