Commit 45ed2e8b authored by Saket Kumar Bhaskar's avatar Saket Kumar Bhaskar Committed by Madhavan Srinivasan
Browse files

powerpc64/bpf: Introduce bpf_jit_emit_atomic_ops() to emit atomic instructions 



The existing code for emitting bpf atomic instruction sequences for
atomic operations such as XCHG, CMPXCHG, ADD, AND, OR, and XOR has been
refactored into a reusable function, bpf_jit_emit_ppc_atomic_op().
It also computes the jump offset and tracks the instruction index for jited
LDARX/LWARX to be used in case it causes a fault.

Reviewed-by: default avatarHari Bathini <hbathini@linux.ibm.com>
Tested-by: default avatarVenkat Rao Bagalkote <venkat88@linux.ibm.com>
Signed-off-by: default avatarSaket Kumar Bhaskar <skb99@linux.ibm.com>
Signed-off-by: default avatarMadhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/20250904100835.1100423-4-skb99@linux.ibm.com
parent a2485d06

arch/powerpc/net/bpf_jit_comp64.c

+115 −88
Original line number Diff line number Diff line
@@ -423,6 +423,111 @@ asm ( 
"		blr				;"

);



static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx,

				   const struct bpf_insn *insn, u32 *jmp_off,

				   u32 *tmp_idx, u32 *addrp)

{

	u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);

	u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);

	u32 size = BPF_SIZE(insn->code);

	u32 src_reg = bpf_to_ppc(insn->src_reg);

	u32 dst_reg = bpf_to_ppc(insn->dst_reg);

	s32 imm = insn->imm;



	u32 save_reg = tmp2_reg;

	u32 ret_reg = src_reg;

	u32 fixup_idx;



	/* Get offset into TMP_REG_1 */

	EMIT(PPC_RAW_LI(tmp1_reg, insn->off));

       /*

	* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'

	* before and after the operation.

	*

	* This is a requirement in the Linux Kernel Memory Model.

	* See __cmpxchg_u64() in asm/cmpxchg.h as an example.

	*/

	if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))

		EMIT(PPC_RAW_SYNC());



	*tmp_idx = ctx->idx;



	/* load value from memory into TMP_REG_2 */

	if (size == BPF_DW)

		EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));

	else

		EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));

	/* Save old value in _R0 */

	if (imm & BPF_FETCH)

		EMIT(PPC_RAW_MR(_R0, tmp2_reg));



	switch (imm) {

	case BPF_ADD:

	case BPF_ADD | BPF_FETCH:

		EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));

		break;

	case BPF_AND:

	case BPF_AND | BPF_FETCH:

		EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));

		break;

	case BPF_OR:

	case BPF_OR | BPF_FETCH:

		EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));

		break;

	case BPF_XOR:

	case BPF_XOR | BPF_FETCH:

		EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));

		break;

	case BPF_CMPXCHG:

	       /*

		* Return old value in BPF_REG_0 for BPF_CMPXCHG &

		* in src_reg for other cases.

		*/

		ret_reg = bpf_to_ppc(BPF_REG_0);



		/* Compare with old value in BPF_R0 */

		if (size == BPF_DW)

			EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));

		else

			EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));

		/* Don't set if different from old value */

		PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);

		fallthrough;

	case BPF_XCHG:

		save_reg = src_reg;

		break;

	default:

		return -EOPNOTSUPP;

	}



	/* store new value */

	if (size == BPF_DW)

		EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));

	else

		EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));

	/* we're done if this succeeded */

	PPC_BCC_SHORT(COND_NE, *tmp_idx * 4);

	fixup_idx = ctx->idx;



	if (imm & BPF_FETCH) {

		/* Emit 'sync' to enforce full ordering */

		if (IS_ENABLED(CONFIG_SMP))

			EMIT(PPC_RAW_SYNC());

		EMIT(PPC_RAW_MR(ret_reg, _R0));

		/*

		 * Skip unnecessary zero-extension for 32-bit cmpxchg.

		 * For context, see commit 39491867ace5.

		 */

		if (size != BPF_DW && imm == BPF_CMPXCHG &&

		    insn_is_zext(insn + 1))

			*addrp = ctx->idx * 4;

	}



	*jmp_off = (fixup_idx - *tmp_idx) * 4;



	return 0;

}



static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off,

					u32 code, u32 *image)

{
@@ -538,7 +643,6 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 
		u32 size = BPF_SIZE(code);

		u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);

		u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);

		u32 save_reg, ret_reg;

		s16 off = insn[i].off;

		s32 imm = insn[i].imm;

		bool func_addr_fixed;
@@ -546,6 +650,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 
		u64 imm64;

		u32 true_cond;

		u32 tmp_idx;

		u32 jmp_off;



		/*

		 * addrs[] maps a BPF bytecode address into a real offset from
@@ -1080,93 +1185,15 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code 
				return -EOPNOTSUPP;

			}



			save_reg = tmp2_reg;

			ret_reg = src_reg;



			/* Get offset into TMP_REG_1 */

			EMIT(PPC_RAW_LI(tmp1_reg, off));

			/*

			 * Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'

			 * before and after the operation.

			 *

			 * This is a requirement in the Linux Kernel Memory Model.

			 * See __cmpxchg_u64() in asm/cmpxchg.h as an example.

			 */

			if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))

				EMIT(PPC_RAW_SYNC());

			tmp_idx = ctx->idx * 4;

			/* load value from memory into TMP_REG_2 */

			if (size == BPF_DW)

				EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));

			else

				EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));



			/* Save old value in _R0 */

			if (imm & BPF_FETCH)

				EMIT(PPC_RAW_MR(_R0, tmp2_reg));



			switch (imm) {

			case BPF_ADD:

			case BPF_ADD | BPF_FETCH:

				EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));

				break;

			case BPF_AND:

			case BPF_AND | BPF_FETCH:

				EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));

				break;

			case BPF_OR:

			case BPF_OR | BPF_FETCH:

				EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));

				break;

			case BPF_XOR:

			case BPF_XOR | BPF_FETCH:

				EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));

				break;

			case BPF_CMPXCHG:

				/*

				 * Return old value in BPF_REG_0 for BPF_CMPXCHG &

				 * in src_reg for other cases.

				 */

				ret_reg = bpf_to_ppc(BPF_REG_0);



				/* Compare with old value in BPF_R0 */

				if (size == BPF_DW)

					EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));

				else

					EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));

				/* Don't set if different from old value */

				PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);

				fallthrough;

			case BPF_XCHG:

				save_reg = src_reg;

				break;

			default:

			ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],

						      &jmp_off, &tmp_idx, &addrs[i + 1]);

			if (ret) {

				if (ret == -EOPNOTSUPP) {

					pr_err_ratelimited(

						"eBPF filter atomic op code %02x (@%d) unsupported\n",

						code, i);

				return -EOPNOTSUPP;

				}



			/* store new value */

			if (size == BPF_DW)

				EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));

			else

				EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));

			/* we're done if this succeeded */

			PPC_BCC_SHORT(COND_NE, tmp_idx);



			if (imm & BPF_FETCH) {

				/* Emit 'sync' to enforce full ordering */

				if (IS_ENABLED(CONFIG_SMP))

					EMIT(PPC_RAW_SYNC());

				EMIT(PPC_RAW_MR(ret_reg, _R0));

				/*

				 * Skip unnecessary zero-extension for 32-bit cmpxchg.

				 * For context, see commit 39491867ace5.

				 */

				if (size != BPF_DW && imm == BPF_CMPXCHG &&

				    insn_is_zext(&insn[i + 1]))

					addrs[++i] = ctx->idx * 4;

				return ret;

			}

			break;