Commit 2f69c568 authored by Anton Protopopov's avatar Anton Protopopov Committed by Alexei Starovoitov
Browse files

bpf: make bpf_insn_successors to return a pointer 



The bpf_insn_successors() function is used to return successors
to a BPF instruction. So far, an instruction could have 0, 1 or 2
successors. Prepare the verifier code to introduction of instructions
with more than 2 successors (namely, indirect jumps).

To do this, introduce a new struct, struct bpf_iarray, containing
an array of bpf instruction indexes and make bpf_insn_successors
to return a pointer of that type. The storage for all instructions
is allocated in the env->succ, which holds an array of size 2,
to be used for all instructions.

Signed-off-by: default avatarAnton Protopopov <a.s.protopopov@gmail.com>
Acked-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251019202145.3944697-10-a.s.protopopov@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 44481e49

include/linux/bpf_verifier.h

+11 −1
Original line number Diff line number Diff line
@@ -509,6 +509,15 @@ struct bpf_map_ptr_state { 
#define BPF_ALU_SANITIZE		(BPF_ALU_SANITIZE_SRC | \

					 BPF_ALU_SANITIZE_DST)



/*

 * An array of BPF instructions.

 * Primary usage: return value of bpf_insn_successors.

 */

struct bpf_iarray {

	int cnt;

	u32 items[];

};



struct bpf_insn_aux_data {

	union {

		enum bpf_reg_type ptr_type;	/* pointer type for load/store insns */
@@ -828,6 +837,7 @@ struct bpf_verifier_env { 
	/* array of pointers to bpf_scc_info indexed by SCC id */

	struct bpf_scc_info **scc_info;

	u32 scc_cnt;

	struct bpf_iarray *succ;

};



static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@@ -1050,7 +1060,7 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st 


struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);

int bpf_jmp_offset(struct bpf_insn *insn);

int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]);

struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);

void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);

bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);

kernel/bpf/liveness.c

+24 −12
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ 
 *   - read and write marks propagation.

 * - The propagation phase is a textbook live variable data flow analysis:

 *

 *     state[cc, i].live_after = U [state[cc, s].live_before for s in insn_successors(i)]

 *     state[cc, i].live_after = U [state[cc, s].live_before for s in bpf_insn_successors(i)]

 *     state[cc, i].live_before =

 *       (state[cc, i].live_after / state[cc, i].must_write) U state[i].may_read

 *
@@ -54,7 +54,7 @@ 
 *   The equation for "must_write_acc" propagation looks as follows:

 *

 *     state[cc, i].must_write_acc =

 *       ∩ [state[cc, s].must_write_acc for s in insn_successors(i)]

 *       ∩ [state[cc, s].must_write_acc for s in bpf_insn_successors(i)]

 *       U state[cc, i].must_write

 *

 *   (An intersection of all "must_write_acc" for instruction successors
@@ -447,7 +447,12 @@ int bpf_jmp_offset(struct bpf_insn *insn) 
__diag_push();

__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for opcode_info_tbl");



inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])

/*

 * Returns an array of instructions succ, with succ->items[0], ...,

 * succ->items[n-1] with successor instructions, where n=succ->cnt

 */

inline struct bpf_iarray *

bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)

{

	static const struct opcode_info {

		bool can_jump;
@@ -474,19 +479,25 @@ inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]) 
		_J(BPF_JSET,  {.can_jump = true,  .can_fallthrough = true}),

	#undef _J

	};

	struct bpf_prog *prog = env->prog;

	struct bpf_insn *insn = &prog->insnsi[idx];

	const struct opcode_info *opcode_info;

	int i = 0, insn_sz;

	struct bpf_iarray *succ;

	int insn_sz;



	/* pre-allocated array of size up to 2; reset cnt, as it may have been used already */

	succ = env->succ;

	succ->cnt = 0;



	opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)];

	insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;

	if (opcode_info->can_fallthrough)

		succ[i++] = idx + insn_sz;

		succ->items[succ->cnt++] = idx + insn_sz;



	if (opcode_info->can_jump)

		succ[i++] = idx + bpf_jmp_offset(insn) + 1;

		succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1;



	return i;

	return succ;

}



__diag_pop();
@@ -548,11 +559,12 @@ static inline bool update_insn(struct bpf_verifier_env *env, 
	struct bpf_insn_aux_data *aux = env->insn_aux_data;

	u64 new_before, new_after, must_write_acc;

	struct per_frame_masks *insn, *succ_insn;

	u32 succ_num, s, succ[2];

	struct bpf_iarray *succ;

	u32 s;

	bool changed;



	succ_num = bpf_insn_successors(env->prog, insn_idx, succ);

	if (unlikely(succ_num == 0))

	succ = bpf_insn_successors(env, insn_idx);

	if (succ->cnt == 0)

		return false;



	changed = false;
@@ -564,8 +576,8 @@ static inline bool update_insn(struct bpf_verifier_env *env, 
	 * of successors plus all "must_write" slots of instruction itself.

	 */

	must_write_acc = U64_MAX;

	for (s = 0; s < succ_num; ++s) {

		succ_insn = get_frame_masks(instance, frame, succ[s]);

	for (s = 0; s < succ->cnt; ++s) {

		succ_insn = get_frame_masks(instance, frame, succ->items[s]);

		new_after |= succ_insn->live_before;

		must_write_acc &= succ_insn->must_write_acc;

	}

kernel/bpf/verifier.c

+40 −20
Original line number Diff line number Diff line
@@ -17805,6 +17805,22 @@ static int mark_fastcall_patterns(struct bpf_verifier_env *env) 
	return 0;

}









static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)









{









	size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);









	struct bpf_iarray *new;

















	new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);









	if (!new) {









		/* this is what callers always want, so simplify the call site */









		kvfree(old);









		return NULL;









	}

















	new->cnt = n_elem;









	return new;









}

















/* Visits the instruction at index t and returns one of the following:









 *  < 0 - an error occurred









 *  DONE_EXPLORING - the instruction was fully explored








@@ -18025,8 +18041,9 @@ static int check_cfg(struct bpf_verifier_env *env)







 */









static int compute_postorder(struct bpf_verifier_env *env)









{









	u32 cur_postorder, i, top, stack_sz, s, succ_cnt, succ[2];









	u32 cur_postorder, i, top, stack_sz, s;









	int *stack = NULL, *postorder = NULL, *state = NULL;









	struct bpf_iarray *succ;

















	postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);









	state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);








@@ -18050,11 +18067,11 @@ static int compute_postorder(struct bpf_verifier_env *env)







				stack_sz--;









				continue;









			}









			succ_cnt = bpf_insn_successors(env->prog, top, succ);









			for (s = 0; s < succ_cnt; ++s) {









				if (!state[succ[s]]) {









					stack[stack_sz++] = succ[s];









					state[succ[s]] |= DISCOVERED;









			succ = bpf_insn_successors(env, top);









			for (s = 0; s < succ->cnt; ++s) {









				if (!state[succ->items[s]]) {









					stack[stack_sz++] = succ->items[s];









					state[succ->items[s]] |= DISCOVERED;









				}









			}









			state[top] |= EXPLORED;








@@ -24313,14 +24330,13 @@ static int compute_live_registers(struct bpf_verifier_env *env)







		for (i = 0; i < env->cfg.cur_postorder; ++i) {









			int insn_idx = env->cfg.insn_postorder[i];









			struct insn_live_regs *live = &state[insn_idx];









			int succ_num;









			u32 succ[2];









			struct bpf_iarray *succ;









			u16 new_out = 0;









			u16 new_in = 0;

















			succ_num = bpf_insn_successors(env->prog, insn_idx, succ);









			for (int s = 0; s < succ_num; ++s)









				new_out |= state[succ[s]].in;









			succ = bpf_insn_successors(env, insn_idx);









			for (int s = 0; s < succ->cnt; ++s)









				new_out |= state[succ->items[s]].in;









			new_in = (new_out & ~live->def) | live->use;









			if (new_out != live->out || new_in != live->in) {









				live->in = new_in;








@@ -24373,11 +24389,11 @@ static int compute_scc(struct bpf_verifier_env *env)







	const u32 insn_cnt = env->prog->len;









	int stack_sz, dfs_sz, err = 0;









	u32 *stack, *pre, *low, *dfs;









	u32 succ_cnt, i, j, t, w;









	u32 i, j, t, w;









	u32 next_preorder_num;









	u32 next_scc_id;









	bool assign_scc;









	u32 succ[2];









	struct bpf_iarray *succ;

















	next_preorder_num = 1;









	next_scc_id = 1;








@@ -24484,12 +24500,12 @@ static int compute_scc(struct bpf_verifier_env *env)







				stack[stack_sz++] = w;









			}









			/* Visit 'w' successors */









			succ_cnt = bpf_insn_successors(env->prog, w, succ);









			for (j = 0; j < succ_cnt; ++j) {









				if (pre[succ[j]]) {









					low[w] = min(low[w], low[succ[j]]);









			succ = bpf_insn_successors(env, w);









			for (j = 0; j < succ->cnt; ++j) {









				if (pre[succ->items[j]]) {









					low[w] = min(low[w], low[succ->items[j]]);









				} else {









					dfs[dfs_sz++] = succ[j];









					dfs[dfs_sz++] = succ->items[j];









					goto dfs_continue;









				}









			}








@@ -24506,8 +24522,8 @@ static int compute_scc(struct bpf_verifier_env *env)







			 * or if component has a self reference.









			 */









			assign_scc = stack[stack_sz - 1] != w;









			for (j = 0; j < succ_cnt; ++j) {









				if (succ[j] == w) {









			for (j = 0; j < succ->cnt; ++j) {









				if (succ->items[j] == w) {









					assign_scc = true;









					break;









				}








@@ -24569,6 +24585,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3







		goto err_free_env;









	for (i = 0; i < len; i++)









		env->insn_aux_data[i].orig_idx = i;









	env->succ = iarray_realloc(NULL, 2);









	if (!env->succ)









		goto err_free_env;









	env->prog = *prog;









	env->ops = bpf_verifier_ops[env->prog->type];
















@@ -24817,6 +24836,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3







	bpf_stack_liveness_free(env);









	kvfree(env->cfg.insn_postorder);









	kvfree(env->scc_info);









	kvfree(env->succ);









	kvfree(env);









	return ret;









}