Commit 88044230 authored by Peilin Ye's avatar Peilin Ye Committed by Alexei Starovoitov
Browse files

bpf: Introduce load-acquire and store-release instructions 

Introduce BPF instructions with load-acquire and store-release
semantics, as discussed in [1].  Define 2 new flags:

  #define BPF_LOAD_ACQ    0x100
  #define BPF_STORE_REL   0x110

A "load-acquire" is a BPF_STX | BPF_ATOMIC instruction with the 'imm'
field set to BPF_LOAD_ACQ (0x100).

Similarly, a "store-release" is a BPF_STX | BPF_ATOMIC instruction with
the 'imm' field set to BPF_STORE_REL (0x110).

Unlike existing atomic read-modify-write operations that only support
BPF_W (32-bit) and BPF_DW (64-bit) size modifiers, load-acquires and
store-releases also support BPF_B (8-bit) and BPF_H (16-bit).  As an
exception, however, 64-bit load-acquires/store-releases are not
supported on 32-bit architectures (to fix a build error reported by the
kernel test robot).

An 8- or 16-bit load-acquire zero-extends the value before writing it to
a 32-bit register, just like ARM64 instruction LDARH and friends.

Similar to existing atomic read-modify-write operations, misaligned
load-acquires/store-releases are not allowed (even if
BPF_F_ANY_ALIGNMENT is set).

As an example, consider the following 64-bit load-acquire BPF
instruction (assuming little-endian):

  db 10 00 00 00 01 00 00  r0 = load_acquire((u64 *)(r1 + 0x0))

  opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX
  imm (0x00000100): BPF_LOAD_ACQ

Similarly, a 16-bit BPF store-release:

  cb 21 00 00 10 01 00 00  store_release((u16 *)(r1 + 0x0), w2)

  opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX
  imm (0x00000110): BPF_STORE_REL

In arch/{arm64,s390,x86}/net/bpf_jit_comp.c, have
bpf_jit_supports_insn(..., /*in_arena=*/true) return false for the new
instructions, until the corresponding JIT compiler supports them in
arena.

[1] https://lore.kernel.org/all/20240729183246.4110549-1-yepeilin@google.com/



Acked-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Acked-by: default avatarIlya Leoshkevich <iii@linux.ibm.com>
Cc: kernel test robot <lkp@intel.com>
Signed-off-by: default avatarPeilin Ye <yepeilin@google.com>
Link: https://lore.kernel.org/r/a217f46f0e445fbd573a1a024be5c6bf1d5fe716.1741049567.git.yepeilin@google.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 3a6fa573

arch/arm64/net/bpf_jit_comp.c

+4 −0
Original line number Diff line number Diff line
@@ -2667,8 +2667,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 
	if (!in_arena)

		return true;

	switch (insn->code) {

	case BPF_STX | BPF_ATOMIC | BPF_B:

	case BPF_STX | BPF_ATOMIC | BPF_H:

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		if (bpf_atomic_is_load_store(insn))

			return false;

		if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))

			return false;

	}

arch/s390/net/bpf_jit_comp.c

+10 −4
Original line number Diff line number Diff line
@@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void) 


bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)

{

	/*

	 * Currently the verifier uses this function only to check which

	 * atomic stores to arena are supported, and they all are.

	 */

	if (!in_arena)

		return true;

	switch (insn->code) {

	case BPF_STX | BPF_ATOMIC | BPF_B:

	case BPF_STX | BPF_ATOMIC | BPF_H:

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		if (bpf_atomic_is_load_store(insn))

			return false;

	}

	return true;

}

arch/x86/net/bpf_jit_comp.c

+4 −0
Original line number Diff line number Diff line
@@ -3771,8 +3771,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 
	if (!in_arena)

		return true;

	switch (insn->code) {

	case BPF_STX | BPF_ATOMIC | BPF_B:

	case BPF_STX | BPF_ATOMIC | BPF_H:

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		if (bpf_atomic_is_load_store(insn))

			return false;

		if (insn->imm == (BPF_AND | BPF_FETCH) ||

		    insn->imm == (BPF_OR | BPF_FETCH) ||

		    insn->imm == (BPF_XOR | BPF_FETCH))

include/linux/bpf.h

+15 −0
Original line number Diff line number Diff line
@@ -991,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) 
	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;

}



/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an

 * atomic load or store, and false if it is a read-modify-write instruction.

 */

static inline bool

bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)

{

	switch (atomic_insn->imm) {

	case BPF_LOAD_ACQ:

	case BPF_STORE_REL:

		return true;

	default:

		return false;

	}

}



struct bpf_prog_ops {

	int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,

			union bpf_attr __user *uattr);

include/linux/filter.h

+2 −0
Original line number Diff line number Diff line
@@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) 
 *   BPF_XOR | BPF_FETCH      src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);

 *   BPF_XCHG                 src_reg = atomic_xchg(dst_reg + off16, src_reg)

 *   BPF_CMPXCHG              r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)

 *   BPF_LOAD_ACQ             dst_reg = smp_load_acquire(src_reg + off16)

 *   BPF_STORE_REL            smp_store_release(dst_reg + off16, src_reg)

 */



#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF)			\