Commit 47c7f3b7 authored by Saket Kumar Bhaskar's avatar Saket Kumar Bhaskar Committed by Madhavan Srinivasan
Browse files

powerpc64/bpf: Implement PROBE_MEM32 pseudo instructions 



Add support for [LDX | STX | ST], PROBE_MEM32, [B | H | W | DW]
instructions.  They are similar to PROBE_MEM instructions with the
following differences:
- PROBE_MEM32 supports store.
- PROBE_MEM32 relies on the verifier to clear upper 32-bit of the
src/dst register
- PROBE_MEM32 adds 64-bit kern_vm_start address (which is stored in _R26
in the prologue). Due to bpf_arena constructions such _R26 + reg +
off16 access is guaranteed to be within arena virtual range, so no
address check at run-time.
- PROBE_MEM32 allows STX and ST. If they fault the store is a nop. When
LDX faults the destination register is zeroed.

To support these on powerpc, we do tmp1 = _R26 + src/dst reg and then use
tmp1 as the new src/dst register. This allows us to reuse most of the
code for normal [LDX | STX | ST].

Additionally, bpf_jit_emit_probe_mem_store() is introduced to emit
instructions for storing memory values depending on the size (byte,
halfword, word, doubleword).

Stack layout is adjusted to introduce a new NVR (_R26) and to make
BPF_PPC_STACKFRAME quadword aligned (local_tmp_var is increased by
8 bytes).

Reviewed-by: default avatarHari Bathini <hbathini@linux.ibm.com>
Tested-by: default avatarVenkat Rao Bagalkote <venkat88@linux.ibm.com>
Signed-off-by: default avatarSaket Kumar Bhaskar <skb99@linux.ibm.com>
Signed-off-by: default avatarMadhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/20250904100835.1100423-2-skb99@linux.ibm.com
parent 46104a7d

arch/powerpc/net/bpf_jit.h

+3 −2
Original line number Diff line number Diff line
@@ -161,9 +161,10 @@ struct codegen_context { 
	unsigned int seen;

	unsigned int idx;

	unsigned int stack_size;

	int b2p[MAX_BPF_JIT_REG + 2];

	int b2p[MAX_BPF_JIT_REG + 3];

	unsigned int exentry_idx;

	unsigned int alt_exit_addr;

	u64 arena_vm_start;

};



#define bpf_to_ppc(r)	(ctx->b2p[r])
@@ -201,7 +202,7 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, 


int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,

			  struct codegen_context *ctx, int insn_idx,

			  int jmp_off, int dst_reg);

			  int jmp_off, int dst_reg, u32 code);



#endif

arch/powerpc/net/bpf_jit_comp.c

+7 −3
Original line number Diff line number Diff line
@@ -204,6 +204,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 


	/* Make sure that the stack is quadword aligned. */

	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);

	cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);



	/* Scouting faux-generate pass 0 */

	if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
@@ -326,7 +327,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 
 */

int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,

			  struct codegen_context *ctx, int insn_idx, int jmp_off,

			  int dst_reg)

			  int dst_reg, u32 code)

{

	off_t offset;

	unsigned long pc;
@@ -355,6 +356,9 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass 
		(ctx->exentry_idx * BPF_FIXUP_LEN * 4);



	fixup[0] = PPC_RAW_LI(dst_reg, 0);

	if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX)

		fixup[0] = PPC_RAW_NOP();



	if (IS_ENABLED(CONFIG_PPC32))

		fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
@@ -579,7 +583,7 @@ static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_contex 
{

	if (IS_ENABLED(CONFIG_PPC64)) {

		/* See bpf_jit_stack_tailcallcnt() */

		int tailcallcnt_offset = 6 * 8;

		int tailcallcnt_offset = 7 * 8;



		EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset));

		EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset));
@@ -594,7 +598,7 @@ static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_cont 
{

	if (IS_ENABLED(CONFIG_PPC64)) {

		/* See bpf_jit_stack_tailcallcnt() */

		int tailcallcnt_offset = 6 * 8;

		int tailcallcnt_offset = 7 * 8;



		EMIT(PPC_RAW_LL(_R3, _R1, -tailcallcnt_offset));

		EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset - tailcallcnt_offset));

arch/powerpc/net/bpf_jit_comp32.c

+1 −1
Original line number Diff line number Diff line
@@ -1087,7 +1087,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 
				}



				ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, insn_idx,

							    jmp_off, dst_reg);

							    jmp_off, dst_reg, code);

				if (ret)

					return ret;

			}

arch/powerpc/net/bpf_jit_comp64.c

+144 −18
Original line number Diff line number Diff line
@@ -25,18 +25,18 @@ 
 * with our redzone usage.

 *

 *		[	prev sp		] <-------------

 *		[   nv gpr save area	] 5*8		|

 *		[   nv gpr save area	] 6*8		|

 *		[    tail_call_cnt	] 8		|

 *		[    local_tmp_var	] 16		|

 *		[    local_tmp_var	] 24		|

 * fp (r31) -->	[   ebpf stack space	] upto 512	|

 *		[     frame header	] 32/112	|

 * sp (r1) --->	[    stack pointer	] --------------

 */



/* for gpr non volatile registers BPG_REG_6 to 10 */

#define BPF_PPC_STACK_SAVE	(5*8)

#define BPF_PPC_STACK_SAVE	(6*8)

/* for bpf JIT code internal usage */

#define BPF_PPC_STACK_LOCALS	24

#define BPF_PPC_STACK_LOCALS	32

/* stack frame excluding BPF stack, ensure this is quadword aligned */

#define BPF_PPC_STACKFRAME	(STACK_FRAME_MIN_SIZE + \

				 BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE)
@@ -44,6 +44,7 @@ 
/* BPF register usage */

#define TMP_REG_1	(MAX_BPF_JIT_REG + 0)

#define TMP_REG_2	(MAX_BPF_JIT_REG + 1)

#define ARENA_VM_START  (MAX_BPF_JIT_REG + 2)



/* BPF to ppc register mappings */

void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
@@ -67,10 +68,12 @@ void bpf_jit_init_reg_mapping(struct codegen_context *ctx) 
	ctx->b2p[BPF_REG_AX] = _R12;

	ctx->b2p[TMP_REG_1] = _R9;

	ctx->b2p[TMP_REG_2] = _R10;

	/* non volatile register for kern_vm_start address */

	ctx->b2p[ARENA_VM_START] = _R26;

}



/* PPC NVR range -- update this if we ever use NVRs below r27 */

#define BPF_PPC_NVR_MIN		_R27

/* PPC NVR range -- update this if we ever use NVRs below r26 */

#define BPF_PPC_NVR_MIN		_R26



static inline bool bpf_has_stack_frame(struct codegen_context *ctx)

{
@@ -89,9 +92,9 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 
 *		[	prev sp		] <-------------

 *		[	  ...       	] 		|

 * sp (r1) --->	[    stack pointer	] --------------

 *		[   nv gpr save area	] 5*8

 *		[   nv gpr save area	] 6*8

 *		[    tail_call_cnt	] 8

 *		[    local_tmp_var	] 16

 *		[    local_tmp_var	] 24

 *		[   unused red zone	] 224

 */

static int bpf_jit_stack_local(struct codegen_context *ctx)
@@ -99,12 +102,12 @@ static int bpf_jit_stack_local(struct codegen_context *ctx) 
	if (bpf_has_stack_frame(ctx))

		return STACK_FRAME_MIN_SIZE + ctx->stack_size;

	else

		return -(BPF_PPC_STACK_SAVE + 24);

		return -(BPF_PPC_STACK_SAVE + 32);

}



static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)

{

	return bpf_jit_stack_local(ctx) + 16;

	return bpf_jit_stack_local(ctx) + 24;

}



static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
@@ -170,10 +173,17 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 
		if (bpf_is_seen_register(ctx, bpf_to_ppc(i)))

			EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));



	if (ctx->arena_vm_start)

		EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1,

				 bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));



	/* Setup frame pointer to point to the bpf stack area */

	if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)))

		EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,

				STACK_FRAME_MIN_SIZE + ctx->stack_size));



	if (ctx->arena_vm_start)

		PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start);

}



static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
@@ -185,6 +195,10 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx 
		if (bpf_is_seen_register(ctx, bpf_to_ppc(i)))

			EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));



	if (ctx->arena_vm_start)

		EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1,

				bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));



	/* Tear down our stack frame */

	if (bpf_has_stack_frame(ctx)) {

		EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size));
@@ -396,11 +410,11 @@ void bpf_stf_barrier(void); 
asm (

"		.global bpf_stf_barrier		;"

"	bpf_stf_barrier:			;"

"		std	21,-64(1)		;"

"		std	22,-56(1)		;"

"		std	21,-80(1)		;"

"		std	22,-72(1)		;"

"		sync				;"

"		ld	21,-64(1)		;"

"		ld	22,-56(1)		;"

"		ld	21,-80(1)		;"

"		ld	22,-72(1)		;"

"		ori	31,31,0			;"

"		.rept 14			;"

"		b	1f			;"
@@ -409,6 +423,36 @@ asm ( 
"		blr				;"

);



static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off,

					u32 code, u32 *image)

{

	u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);

	u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);



	switch (BPF_SIZE(code)) {

	case BPF_B:

		EMIT(PPC_RAW_STB(src_reg, tmp1_reg, off));

		break;

	case BPF_H:

		EMIT(PPC_RAW_STH(src_reg, tmp1_reg, off));

		break;

	case BPF_W:

		EMIT(PPC_RAW_STW(src_reg, tmp1_reg, off));

		break;

	case BPF_DW:

		if (off % 4) {

			EMIT(PPC_RAW_LI(tmp2_reg, off));

			EMIT(PPC_RAW_STDX(src_reg, tmp1_reg, tmp2_reg));

		} else {

			EMIT(PPC_RAW_STD(src_reg, tmp1_reg, off));

		}

		break;

	default:

		return -EINVAL;

	}

	return 0;

}



static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image)

{

	u32 code = insn.code;
@@ -960,6 +1004,50 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 
			}

			break;



		case BPF_STX | BPF_PROBE_MEM32 | BPF_B:

		case BPF_STX | BPF_PROBE_MEM32 | BPF_H:

		case BPF_STX | BPF_PROBE_MEM32 | BPF_W:

		case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:



			EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START)));



			ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image);

			if (ret)

				return ret;



			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,

						    ctx->idx - 1, 4, -1, code);

			if (ret)

				return ret;



			break;



		case BPF_ST | BPF_PROBE_MEM32 | BPF_B:

		case BPF_ST | BPF_PROBE_MEM32 | BPF_H:

		case BPF_ST | BPF_PROBE_MEM32 | BPF_W:

		case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:



			EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START)));



			if (BPF_SIZE(code) == BPF_W || BPF_SIZE(code) == BPF_DW) {

				PPC_LI32(tmp2_reg, imm);

				src_reg = tmp2_reg;

			} else {

				EMIT(PPC_RAW_LI(tmp2_reg, imm));

				src_reg = tmp2_reg;

			}



			ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image);

			if (ret)

				return ret;



			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,

						    ctx->idx - 1, 4, -1, code);

			if (ret)

				return ret;



			break;



		/*

		 * BPF_STX ATOMIC (atomic ops)

		 */
@@ -1112,9 +1200,10 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 
				 * Check if 'off' is word aligned for BPF_DW, because

				 * we might generate two instructions.

				 */

				if ((BPF_SIZE(code) == BPF_DW ||

				    (BPF_SIZE(code) == BPF_B && BPF_MODE(code) == BPF_PROBE_MEMSX)) &&

						(off & 3))

				if ((BPF_SIZE(code) == BPF_DW && (off & 3)) ||

				    (BPF_SIZE(code) == BPF_B &&

				     BPF_MODE(code) == BPF_PROBE_MEMSX) ||

				    (BPF_SIZE(code) == BPF_B && BPF_MODE(code) == BPF_MEMSX))

					PPC_JMP((ctx->idx + 3) * 4);

				else

					PPC_JMP((ctx->idx + 2) * 4);
@@ -1160,12 +1249,49 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 


			if (BPF_MODE(code) == BPF_PROBE_MEM) {

				ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,

							    ctx->idx - 1, 4, dst_reg);

							    ctx->idx - 1, 4, dst_reg, code);

				if (ret)

					return ret;

			}

			break;



		/* dst = *(u64 *)(ul) (src + ARENA_VM_START + off) */

		case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:

		case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:

		case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:

		case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:



			EMIT(PPC_RAW_ADD(tmp1_reg, src_reg, bpf_to_ppc(ARENA_VM_START)));



			switch (size) {

			case BPF_B:

				EMIT(PPC_RAW_LBZ(dst_reg, tmp1_reg, off));

				break;

			case BPF_H:

				EMIT(PPC_RAW_LHZ(dst_reg, tmp1_reg, off));

				break;

			case BPF_W:

				EMIT(PPC_RAW_LWZ(dst_reg, tmp1_reg, off));

				break;

			case BPF_DW:

				if (off % 4) {

					EMIT(PPC_RAW_LI(tmp2_reg, off));

					EMIT(PPC_RAW_LDX(dst_reg, tmp1_reg, tmp2_reg));

				} else {

					EMIT(PPC_RAW_LD(dst_reg, tmp1_reg, off));

				}

				break;

			}



			if (size != BPF_DW && insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;



			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,

						    ctx->idx - 1, 4, dst_reg, code);

			if (ret)

				return ret;

			break;



		/*

		 * Doubleword load

		 * 16 byte instruction that uses two 'struct bpf_insn'