Commit 6fee7a7e authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-follow-up-on-gen_epilogue' 



Martin KaFai Lau says:

====================
bpf: Follow up on gen_epilogue

From: Martin KaFai Lau <martin.lau@kernel.org>

The set addresses some follow ups on the earlier gen_epilogue
patch set.
====================

Acked-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20240904180847.56947-1-martin.lau@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 46f4ea04 00750788

include/linux/bpf_verifier.h

+1 −1
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ 
 */

#define TMP_STR_BUF_LEN 320

/* Patch buffer size */

#define INSN_BUF_SIZE 16

#define INSN_BUF_SIZE 32



/* Liveness marks, used for registers and spilled-regs (in stack slots).

 * Read marks propagate upwards until they find a write mark; they record that

kernel/bpf/verifier.c

+43 −42
Original line number Diff line number Diff line
@@ -21232,7 +21232,7 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 
					int position,

					s32 stack_base,

					u32 callback_subprogno,

					u32 *cnt)

					u32 *total_cnt)

{

	s32 r6_offset = stack_base + 0 * BPF_REG_SIZE;

	s32 r7_offset = stack_base + 1 * BPF_REG_SIZE;
@@ -21241,55 +21241,56 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 
	int reg_loop_cnt = BPF_REG_7;

	int reg_loop_ctx = BPF_REG_8;









	struct bpf_insn *insn_buf = env->insn_buf;









	struct bpf_prog *new_prog;









	u32 callback_start;









	u32 call_insn_offset;









	s32 callback_offset;









	u32 cnt = 0;

















	/* This represents an inlined version of bpf_iter.c:bpf_loop,









	 * be careful to modify this code in sync.









	 */









	struct bpf_insn insn_buf[] = {

















	/* Return error and jump to the end of the patch if









	 * expected number of iterations is too big.









	 */









		BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2),









		BPF_MOV32_IMM(BPF_REG_0, -E2BIG),









		BPF_JMP_IMM(BPF_JA, 0, 0, 16),









	insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2);









	insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG);









	insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16);









	/* spill R6, R7, R8 to use these as loop vars */









		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset),









		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset),









		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset),









	insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset);









	insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset);









	insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset);









	/* initialize loop vars */









		BPF_MOV64_REG(reg_loop_max, BPF_REG_1),









		BPF_MOV32_IMM(reg_loop_cnt, 0),









		BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3),









	insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1);









	insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0);









	insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3);









	/* loop header,









	 * if reg_loop_cnt >= reg_loop_max skip the loop body









	 */









		BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5),









	insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5);









	/* callback call,









	 * correct callback offset would be set after patching









	 */









		BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt),









		BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx),









		BPF_CALL_REL(0),









	insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt);









	insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx);









	insn_buf[cnt++] = BPF_CALL_REL(0);









	/* increment loop counter */









		BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1),









	insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1);









	/* jump to loop header if callback returned 0 */









		BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6),









	insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6);









	/* return value of bpf_loop,









	 * set R0 to the number of iterations









	 */









		BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt),









	insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt);









	/* restore original values of R6, R7, R8 */









		BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset),









		BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset),









		BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset),









	};









	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset);









	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset);









	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset);

















	*cnt = ARRAY_SIZE(insn_buf);









	new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt);









	*total_cnt = cnt;









	new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);









	if (!new_prog)









		return new_prog;