Commit 0f79bb89 authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'bpf-introduce-may_goto-and-cond_break' 

Alexei Starovoitov says:

====================
bpf: Introduce may_goto and cond_break

From: Alexei Starovoitov <ast@kernel.org>

v5 -> v6:
- Rename BPF_JMA to BPF_JCOND
- Addressed Andrii's review comments

v4 -> v5:
- rewrote patch 1 to avoid fake may_goto_reg and use 'u32 may_goto_cnt' instead.
  This way may_goto handling is similar to bpf_loop() processing.
- fixed bug in patch 2 that RANGE_WITHIN should not use
  rold->type == NOT_INIT as a safe signal.
- patch 3 fixed negative offset computation in cond_break macro
- using bpf_arena and cond_break recompiled lib/glob.c as bpf prog
  and it works! It will be added as a selftest to arena series.

v3 -> v4:
- fix drained issue reported by John.
  may_goto insn could be implemented with sticky state (once
  reaches 0 it stays 0), but the verifier shouldn't assume that.
  It has to explore both branches.
  Arguably drained iterator state shouldn't be there at all.
  bpf_iter_css_next() is not sticky. Can be fixed, but auditing all
  iterators for stickiness. That's an orthogonal discussion.
- explained JMA name reasons in patch 1
- fixed test_progs-no_alu32 issue and added another test

v2 -> v3: Major change
- drop bpf_can_loop() kfunc and introduce may_goto instruction instead
  kfunc is a function call while may_goto doesn't consume any registers
  and LLVM can produce much better code due to less register pressure.
- instead of counting from zero to BPF_MAX_LOOPS start from it instead
  and break out of the loop when count reaches zero
- use may_goto instruction in cond_break macro
- recognize that 'exact' state comparison doesn't need to be truly exact.
  regsafe() should ignore precision and liveness marks, but range_within
  logic is safe to use while evaluating open coded iterators.
====================

Link: https://lore.kernel.org/r/20240306031929.42666-1-alexei.starovoitov@gmail.com


Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents 9a9d1d36 0c8bbf99

include/linux/bpf_verifier.h

+2 −0
Original line number Diff line number Diff line
@@ -449,6 +449,7 @@ struct bpf_verifier_state { 
	u32 jmp_history_cnt;

	u32 dfs_depth;

	u32 callback_unroll_depth;

	u32 may_goto_depth;

};



#define bpf_get_spilled_reg(slot, frame, mask)				\
@@ -619,6 +620,7 @@ struct bpf_subprog_info { 
	u32 start; /* insn idx of function entry point */

	u32 linfo_idx; /* The idx to the main_prog->aux->linfo */

	u16 stack_depth; /* max. stack depth used by this function */

	u16 stack_extra;

	bool has_tail_call: 1;

	bool tail_call_reachable: 1;

	bool has_ld_abs: 1;

include/uapi/linux/bpf.h

+5 −0
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ 
#define BPF_JSGE	0x70	/* SGE is signed '>=', GE in x86 */

#define BPF_JSLT	0xc0	/* SLT is signed, '<' */

#define BPF_JSLE	0xd0	/* SLE is signed, '<=' */

#define BPF_JCOND	0xe0	/* conditional pseudo jumps: may_goto, goto_or_nop */

#define BPF_CALL	0x80	/* function call */

#define BPF_EXIT	0x90	/* function return */
@@ -50,6 +51,10 @@ 
#define BPF_XCHG	(0xe0 | BPF_FETCH)	/* atomic exchange */

#define BPF_CMPXCHG	(0xf0 | BPF_FETCH)	/* atomic compare-and-write */



enum bpf_cond_pseudo_jmp {

	BPF_MAY_GOTO = 0,

};



/* Register numbers */

enum {

	BPF_REG_0 = 0,

kernel/bpf/core.c

+1 −0
Original line number Diff line number Diff line
@@ -1675,6 +1675,7 @@ bool bpf_opcode_in_insntable(u8 code) 
		[BPF_LD | BPF_IND | BPF_B] = true,

		[BPF_LD | BPF_IND | BPF_H] = true,

		[BPF_LD | BPF_IND | BPF_W] = true,

		[BPF_JMP | BPF_JCOND] = true,

	};

#undef BPF_INSN_3_TBL

#undef BPF_INSN_2_TBL

kernel/bpf/disasm.c

+4 −0
Original line number Diff line number Diff line
@@ -322,6 +322,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, 
		} else if (insn->code == (BPF_JMP | BPF_JA)) {

			verbose(cbs->private_data, "(%02x) goto pc%+d\n",

				insn->code, insn->off);

		} else if (insn->code == (BPF_JMP | BPF_JCOND) &&

			   insn->src_reg == BPF_MAY_GOTO) {

			verbose(cbs->private_data, "(%02x) may_goto pc%+d\n",

				insn->code, insn->off);

		} else if (insn->code == (BPF_JMP32 | BPF_JA)) {

			verbose(cbs->private_data, "(%02x) gotol pc%+d\n",

				insn->code, insn->imm);

kernel/bpf/verifier.c

+162 −50
Original line number Diff line number Diff line
@@ -533,6 +533,16 @@ static bool is_async_callback_calling_insn(struct bpf_insn *insn) 
	return bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm);

}









static bool is_may_goto_insn(struct bpf_insn *insn)









{









	return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;









}

















static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)









{









	return is_may_goto_insn(&env->prog->insnsi[insn_idx]);









}

















static bool is_storage_get_function(enum bpf_func_id func_id)









{









	return func_id == BPF_FUNC_sk_storage_get ||








@@ -1429,6 +1439,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,







	dst_state->dfs_depth = src->dfs_depth;









	dst_state->callback_unroll_depth = src->callback_unroll_depth;









	dst_state->used_as_loop_entry = src->used_as_loop_entry;









	dst_state->may_goto_depth = src->may_goto_depth;









	for (i = 0; i <= src->curframe; i++) {









		dst = dst_state->frame[i];









		if (!dst) {








@@ -14871,11 +14882,36 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,







	int err;

















	/* Only conditional jumps are expected to reach here. */









	if (opcode == BPF_JA || opcode > BPF_JSLE) {









	if (opcode == BPF_JA || opcode > BPF_JCOND) {









		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);









		return -EINVAL;









	}

















	if (opcode == BPF_JCOND) {









		struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;









		int idx = *insn_idx;

















		if (insn->code != (BPF_JMP | BPF_JCOND) ||









		    insn->src_reg != BPF_MAY_GOTO ||









		    insn->dst_reg || insn->imm || insn->off == 0) {









			verbose(env, "invalid may_goto off %d imm %d\n",









				insn->off, insn->imm);









			return -EINVAL;









		}









		prev_st = find_prev_entry(env, cur_st->parent, idx);

















		/* branch out 'fallthrough' insn as a new state to explore */









		queued_st = push_stack(env, idx + 1, idx, false);









		if (!queued_st)









			return -ENOMEM;

















		queued_st->may_goto_depth++;









		if (prev_st)









			widen_imprecise_scalars(env, prev_st, queued_st);









		*insn_idx += insn->off;









		return 0;









	}

















	/* check src2 operand */









	err = check_reg_arg(env, insn->dst_reg, SRC_OP);









	if (err)








@@ -15659,6 +15695,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)







	default:









		/* conditional jump with two edges */









		mark_prune_point(env, t);









		if (is_may_goto_insn(insn))









			mark_force_checkpoint(env, t);

















		ret = push_insn(t, t + 1, FALLTHROUGH, env);









		if (ret)








@@ -16222,8 +16260,8 @@ static int check_btf_info(struct bpf_verifier_env *env,







}

















/* check %cur's range satisfies %old's */









static bool range_within(struct bpf_reg_state *old,









			 struct bpf_reg_state *cur)









static bool range_within(const struct bpf_reg_state *old,









			 const struct bpf_reg_state *cur)









{









	return old->umin_value <= cur->umin_value &&









	       old->umax_value >= cur->umax_value &&








@@ -16387,21 +16425,28 @@ static bool regs_exact(const struct bpf_reg_state *rold,







	       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);









}

















enum exact_level {









	NOT_EXACT,









	EXACT,









	RANGE_WITHIN









};

















/* Returns true if (rold safe implies rcur safe) */









static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,









		    struct bpf_reg_state *rcur, struct bpf_idmap *idmap, bool exact)









		    struct bpf_reg_state *rcur, struct bpf_idmap *idmap,









		    enum exact_level exact)









{









	if (exact)









	if (exact == EXACT)









		return regs_exact(rold, rcur, idmap);

















	if (!(rold->live & REG_LIVE_READ))









	if (!(rold->live & REG_LIVE_READ) && exact == NOT_EXACT)









		/* explored state didn't use this */









		return true;









	if (rold->type == NOT_INIT)









	if (rold->type == NOT_INIT) {









		if (exact == NOT_EXACT || rcur->type == NOT_INIT)









			/* explored state can't have used this */









			return true;









	if (rcur->type == NOT_INIT)









		return false;









	}

















	/* Enforce that register types have to match exactly, including their









	 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general








@@ -16436,7 +16481,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,







			return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&









			       check_scalar_ids(rold->id, rcur->id, idmap);









		}









		if (!rold->precise)









		if (!rold->precise && exact == NOT_EXACT)









			return true;









		/* Why check_ids() for scalar registers?









		 *








@@ -16547,7 +16592,8 @@ static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,







}

















static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,









		      struct bpf_func_state *cur, struct bpf_idmap *idmap, bool exact)









		      struct bpf_func_state *cur, struct bpf_idmap *idmap,









		      enum exact_level exact)









{









	int i, spi;
















@@ -16560,12 +16606,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,















		spi = i / BPF_REG_SIZE;

















		if (exact &&









		if (exact != NOT_EXACT &&









		    old->stack[spi].slot_type[i % BPF_REG_SIZE] !=









		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])









			return false;

















		if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ) && !exact) {









		if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)









		    && exact == NOT_EXACT) {









			i += BPF_REG_SIZE - 1;









			/* explored state didn't use this */









			continue;








@@ -16711,7 +16758,7 @@ static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur,







 * the current state will reach 'bpf_exit' instruction safely









 */









static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,









			      struct bpf_func_state *cur, bool exact)









			      struct bpf_func_state *cur, enum exact_level exact)









{









	int i;
















@@ -16738,7 +16785,7 @@ static void reset_idmap_scratch(struct bpf_verifier_env *env)







static bool states_equal(struct bpf_verifier_env *env,









			 struct bpf_verifier_state *old,









			 struct bpf_verifier_state *cur,









			 bool exact)









			 enum exact_level exact)









{









	int i;
















@@ -17112,7 +17159,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)







			 * => unsafe memory access at 11 would not be caught.









			 */









			if (is_iter_next_insn(env, insn_idx)) {









				if (states_equal(env, &sl->state, cur, true)) {









				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {









					struct bpf_func_state *cur_frame;









					struct bpf_reg_state *iter_state, *iter_reg;









					int spi;








@@ -17135,15 +17182,23 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)







				}









				goto skip_inf_loop_check;









			}









			if (is_may_goto_insn_at(env, insn_idx)) {









				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {









					update_loop_entry(cur, &sl->state);









					goto hit;









				}









				goto skip_inf_loop_check;









			}









			if (calls_callback(env, insn_idx)) {









				if (states_equal(env, &sl->state, cur, true))









				if (states_equal(env, &sl->state, cur, RANGE_WITHIN))









					goto hit;









				goto skip_inf_loop_check;









			}









			/* attempt to detect infinite loop to avoid unnecessary doomed work */









			if (states_maybe_looping(&sl->state, cur) &&









			    states_equal(env, &sl->state, cur, true) &&









			    states_equal(env, &sl->state, cur, EXACT) &&









			    !iter_active_depths_differ(&sl->state, cur) &&









			    sl->state.may_goto_depth == cur->may_goto_depth &&









			    sl->state.callback_unroll_depth == cur->callback_unroll_depth) {









				verbose_linfo(env, insn_idx, "; ");









				verbose(env, "infinite loop detected at insn %d\n", insn_idx);








@@ -17199,7 +17254,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)







		 */









		loop_entry = get_loop_entry(&sl->state);









		force_exact = loop_entry && loop_entry->branches > 0;









		if (states_equal(env, &sl->state, cur, force_exact)) {









		if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {









			if (force_exact)









				update_loop_entry(cur, loop_entry);









hit:








@@ -19408,7 +19463,10 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







	struct bpf_insn insn_buf[16];









	struct bpf_prog *new_prog;









	struct bpf_map *map_ptr;









	int i, ret, cnt, delta = 0;









	int i, ret, cnt, delta = 0, cur_subprog = 0;









	struct bpf_subprog_info *subprogs = env->subprog_info;









	u16 stack_depth = subprogs[cur_subprog].stack_depth;









	u16 stack_depth_extra = 0;

















	if (env->seen_exception && !env->exception_callback_subprog) {









		struct bpf_insn patch[] = {








@@ -19428,7 +19486,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







		mark_subprog_exc_cb(env, env->exception_callback_subprog);









	}

















	for (i = 0; i < insn_cnt; i++, insn++) {









	for (i = 0; i < insn_cnt;) {









		/* Make divide-by-zero exceptions impossible. */









		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||









		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||








@@ -19467,7 +19525,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */








@@ -19487,7 +19545,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Rewrite pointer arithmetic to mitigate speculation attacks. */








@@ -19502,7 +19560,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			aux = &env->insn_aux_data[i + delta];









			if (!aux->alu_state ||









			    aux->alu_state == BPF_ALU_NON_POINTER)









				continue;









				goto next_insn;

















			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;









			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==








@@ -19540,19 +19598,39 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		if (is_may_goto_insn(insn)) {









			int stack_off = -stack_depth - 8;

















			stack_depth_extra = 8;









			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);









			insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2);









			insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);









			insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off);









			cnt = 4;

















			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);









			if (!new_prog)









				return -ENOMEM;

















			delta += cnt - 1;









			env->prog = prog = new_prog;









			insn = new_prog->insnsi + i + delta;









			goto next_insn;









		}

















		if (insn->code != (BPF_JMP | BPF_CALL))









			continue;









			goto next_insn;









		if (insn->src_reg == BPF_PSEUDO_CALL)









			continue;









			goto next_insn;









		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {









			ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);









			if (ret)









				return ret;









			if (cnt == 0)









				continue;









				goto next_insn;

















			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);









			if (!new_prog)








@@ -19561,7 +19639,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta	 += cnt - 1;









			env->prog = prog = new_prog;









			insn	  = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		if (insn->imm == BPF_FUNC_get_route_realm)








@@ -19609,11 +19687,11 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







				}

















				insn->imm = ret + 1;









				continue;









				goto next_insn;









			}

















			if (!bpf_map_ptr_unpriv(aux))









				continue;









				goto next_insn;

















			/* instead of changing every JIT dealing with tail_call









			 * emit two extra insns:








@@ -19642,7 +19720,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		if (insn->imm == BPF_FUNC_timer_set_callback) {








@@ -19754,7 +19832,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







				delta    += cnt - 1;









				env->prog = prog = new_prog;









				insn      = new_prog->insnsi + i + delta;









				continue;









				goto next_insn;









			}

















			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,








@@ -19785,31 +19863,31 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			switch (insn->imm) {









			case BPF_FUNC_map_lookup_elem:









				insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);









				continue;









				goto next_insn;









			case BPF_FUNC_map_update_elem:









				insn->imm = BPF_CALL_IMM(ops->map_update_elem);









				continue;









				goto next_insn;









			case BPF_FUNC_map_delete_elem:









				insn->imm = BPF_CALL_IMM(ops->map_delete_elem);









				continue;









				goto next_insn;









			case BPF_FUNC_map_push_elem:









				insn->imm = BPF_CALL_IMM(ops->map_push_elem);









				continue;









				goto next_insn;









			case BPF_FUNC_map_pop_elem:









				insn->imm = BPF_CALL_IMM(ops->map_pop_elem);









				continue;









				goto next_insn;









			case BPF_FUNC_map_peek_elem:









				insn->imm = BPF_CALL_IMM(ops->map_peek_elem);









				continue;









				goto next_insn;









			case BPF_FUNC_redirect_map:









				insn->imm = BPF_CALL_IMM(ops->map_redirect);









				continue;









				goto next_insn;









			case BPF_FUNC_for_each_map_elem:









				insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);









				continue;









				goto next_insn;









			case BPF_FUNC_map_lookup_percpu_elem:









				insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);









				continue;









				goto next_insn;









			}

















			goto patch_call_imm;








@@ -19837,7 +19915,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Implement bpf_get_func_arg inline. */








@@ -19862,7 +19940,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Implement bpf_get_func_ret inline. */








@@ -19890,7 +19968,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Implement get_func_arg_cnt inline. */








@@ -19905,7 +19983,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)















			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Implement bpf_get_func_ip inline. */








@@ -19920,7 +19998,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)















			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}

















		/* Implement bpf_kptr_xchg inline */








@@ -19938,7 +20016,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			continue;









			goto next_insn;









		}









patch_call_imm:









		fn = env->ops->get_func_proto(insn->imm, env->prog);








@@ -19952,6 +20030,40 @@ static int do_misc_fixups(struct bpf_verifier_env *env)







			return -EFAULT;









		}









		insn->imm = fn->func - __bpf_call_base;









next_insn:









		if (subprogs[cur_subprog + 1].start == i + delta + 1) {









			subprogs[cur_subprog].stack_depth += stack_depth_extra;









			subprogs[cur_subprog].stack_extra = stack_depth_extra;









			cur_subprog++;









			stack_depth = subprogs[cur_subprog].stack_depth;









			stack_depth_extra = 0;









		}









		i++;









		insn++;









	}

















	env->prog->aux->stack_depth = subprogs[0].stack_depth;









	for (i = 0; i < env->subprog_cnt; i++) {









		int subprog_start = subprogs[i].start;









		int stack_slots = subprogs[i].stack_extra / 8;

















		if (!stack_slots)









			continue;









		if (stack_slots > 1) {









			verbose(env, "verifier bug: stack_slots supports may_goto only\n");









			return -EFAULT;









		}

















		/* Add ST insn to subprog prologue to init extra stack */









		insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP,









					 -subprogs[i].stack_depth, BPF_MAX_LOOPS);









		/* Copy first actual insn to preserve it */









		insn_buf[1] = env->prog->insnsi[subprog_start];

















		new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);









		if (!new_prog)









			return -ENOMEM;









		env->prog = prog = new_prog;









	}

















	/* Since poke tab is now finalized, publish aux to tracker. */