Commit 8afd3170 authored by Andrea Parri's avatar Andrea Parri Committed by Alexei Starovoitov
Browse files

bpf, riscv64: Support load-acquire and store-release instructions 



Support BPF load-acquire (BPF_LOAD_ACQ) and store-release
(BPF_STORE_REL) instructions in the riscv64 JIT compiler.  For example,
consider the following 64-bit load-acquire (assuming little-endian):

  db 10 00 00 00 01 00 00  r1 = load_acquire((u64 *)(r1 + 0x0))
  95 00 00 00 00 00 00 00  exit

  opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX
  imm (0x00000100): BPF_LOAD_ACQ

The JIT compiler will emit an LD instruction followed by a FENCE R,RW
instruction for the above, e.g.:

  ld x7,0(x6)
  fence r,rw

Similarly, consider the following 16-bit store-release:

  cb 21 00 00 10 01 00 00  store_release((u16 *)(r1 + 0x0), w2)
  95 00 00 00 00 00 00 00  exit

  opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX
  imm (0x00000110): BPF_STORE_REL

A FENCE RW,W instruction followed by an SH instruction will be emitted,
e.g.:

  fence rw,w
  sh x2,0(x4)

8-bit and 16-bit load-acquires are zero-extending (cf., LBU, LHU).  The
verifier always rejects misaligned load-acquires/store-releases (even if
BPF_F_ANY_ALIGNMENT is set), so the emitted load and store instructions
are guaranteed to be single-copy atomic.

Introduce primitives to emit the relevant (and the most common/used in
the kernel) fences, i.e. fences with R -> RW, RW -> W and RW -> RW.

Rename emit_atomic() to emit_atomic_rmw() to make it clear that it only
handles RMW atomics, and replace its is64 parameter to allow to perform
the required checks on the opsize (BPF_SIZE(code)).

Acked-by: default avatarBjörn Töpel <bjorn@kernel.org>
Tested-by: Björn Töpel <bjorn@rivosinc.com> # QEMU/RVA23
Signed-off-by: default avatarAndrea Parri <parri.andrea@gmail.com>
Co-developed-by: default avatarPeilin Ye <yepeilin@google.com>
Signed-off-by: default avatarPeilin Ye <yepeilin@google.com>
Reviewed-by: default avatarPu Lehui <pulehui@huawei.com>
Link: https://lore.kernel.org/r/3059c560e537ad43ed19055d2ebbd970c698095a.1746588351.git.yepeilin@google.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 118ae46b

arch/riscv/net/bpf_jit.h

+15 −0
Original line number Diff line number Diff line
@@ -608,6 +608,21 @@ static inline u32 rv_fence(u8 pred, u8 succ) 
	return rv_i_insn(imm11_0, 0, 0, 0, 0xf);

}



static inline void emit_fence_r_rw(struct rv_jit_context *ctx)

{

	emit(rv_fence(0x2, 0x3), ctx);

}



static inline void emit_fence_rw_w(struct rv_jit_context *ctx)

{

	emit(rv_fence(0x3, 0x1), ctx);

}



static inline void emit_fence_rw_rw(struct rv_jit_context *ctx)

{

	emit(rv_fence(0x3, 0x3), ctx);

}



static inline u32 rv_nop(void)

{

	return rv_i_insn(0, 0, 0, 0, 0x13);

arch/riscv/net/bpf_jit_comp64.c

+70 −5
Original line number Diff line number Diff line
@@ -607,11 +607,65 @@ static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 
	emit_sd(RV_REG_T1, 0, rs, ctx);

}



static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,

static int emit_atomic_ld_st(u8 rd, u8 rs, s16 off, s32 imm, u8 code, struct rv_jit_context *ctx)

{

	switch (imm) {

	/* dst_reg = load_acquire(src_reg + off16) */

	case BPF_LOAD_ACQ:

		switch (BPF_SIZE(code)) {

		case BPF_B:

			emit_load_8(false, rd, off, rs, ctx);

			break;

		case BPF_H:

			emit_load_16(false, rd, off, rs, ctx);

			break;

		case BPF_W:

			emit_load_32(false, rd, off, rs, ctx);

			break;

		case BPF_DW:

			emit_load_64(false, rd, off, rs, ctx);

			break;

		}

		emit_fence_r_rw(ctx);

		break;

	/* store_release(dst_reg + off16, src_reg) */

	case BPF_STORE_REL:

		emit_fence_rw_w(ctx);

		switch (BPF_SIZE(code)) {

		case BPF_B:

			emit_store_8(rd, off, rs, ctx);

			break;

		case BPF_H:

			emit_store_16(rd, off, rs, ctx);

			break;

		case BPF_W:

			emit_store_32(rd, off, rs, ctx);

			break;

		case BPF_DW:

			emit_store_64(rd, off, rs, ctx);

			break;

		}

		break;

	default:

		pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm);

		return -EINVAL;

	}



	return 0;

}



static int emit_atomic_rmw(u8 rd, u8 rs, s16 off, s32 imm, u8 code,

			   struct rv_jit_context *ctx)

{

	u8 r0;

	int jmp_offset;

	bool is64;



	if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {

		pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n");

		return -EINVAL;

	}

	is64 = BPF_SIZE(code) == BPF_DW;



	if (off) {

		if (is_12b_int(off)) {
@@ -688,9 +742,14 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 
		     rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);

		jmp_offset = ninsns_rvoff(-6);

		emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);

		emit(rv_fence(0x3, 0x3), ctx);

		emit_fence_rw_rw(ctx);

		break;

	default:

		pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);

		return -EINVAL;

	}



	return 0;

}



#define BPF_FIXUP_OFFSET_MASK   GENMASK(26, 0)
@@ -1962,10 +2021,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_STX | BPF_MEM | BPF_DW:

		emit_store_64(rd, off, rs, ctx);

		break;

	case BPF_STX | BPF_ATOMIC | BPF_B:

	case BPF_STX | BPF_ATOMIC | BPF_H:

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		emit_atomic(rd, rs, off, imm,

			    BPF_SIZE(code) == BPF_DW, ctx);

		if (bpf_atomic_is_load_store(insn))

			ret = emit_atomic_ld_st(rd, rs, off, imm, code, ctx);

		else

			ret = emit_atomic_rmw(rd, rs, off, imm, code, ctx);

		if (ret)

			return ret;

		break;



	case BPF_STX | BPF_PROBE_MEM32 | BPF_B: