Commit d90f0bce authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 's390-bpf-use-kernel-s-expoline-thunks' 

Ilya Leoshkevich says:

====================
This series simplifies the s390 JIT by replacing the generation of
expolines (Spectre mitigation) with using the ones from the kernel
text. This is possible thanks to the V!=R s390 kernel rework.
Patch 1 is a small prerequisite for arch/s390 that I would like to
get in via the BPF tree. It has Heiko's Acked-by.
Patches 2 and 3 are the implementation.
====================

Link: https://patch.msgid.link/20250519223646.66382-1-iii@linux.ibm.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 4e2e6841 7f332f9f

arch/s390/include/asm/nospec-branch.h

+0 −4
Original line number Diff line number Diff line
@@ -26,8 +26,6 @@ static inline bool nospec_uses_trampoline(void) 
	return __is_defined(CC_USING_EXPOLINE) && !nospec_disable;

}



#ifdef CONFIG_EXPOLINE_EXTERN



void __s390_indirect_jump_r1(void);

void __s390_indirect_jump_r2(void);

void __s390_indirect_jump_r3(void);
@@ -44,8 +42,6 @@ void __s390_indirect_jump_r13(void); 
void __s390_indirect_jump_r14(void);

void __s390_indirect_jump_r15(void);



#endif



#endif /* __ASSEMBLY__ */



#endif /* _ASM_S390_EXPOLINE_H */

arch/s390/net/bpf_jit_comp.c

+59 −62
Original line number Diff line number Diff line
@@ -48,8 +48,6 @@ struct bpf_jit { 
	int lit64;		/* Current position in 64-bit literal pool */

	int base_ip;		/* Base address for literal pool */

	int exit_ip;		/* Address of exit */

	int r1_thunk_ip;	/* Address of expoline thunk for 'br %r1' */

	int r14_thunk_ip;	/* Address of expoline thunk for 'br %r14' */

	int tail_call_start;	/* Tail call start offset */

	int excnt;		/* Number of exception table entries */

	int prologue_plt_ret;	/* Return address for prologue hotpatch PLT */
@@ -127,6 +125,18 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 
		jit->seen_regs |= (1 << r1);

}



static s32 off_to_pcrel(struct bpf_jit *jit, u32 off)

{

	return off - jit->prg;

}



static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr)

{

	if (jit->prg_buf)

		return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg);

	return 0;

}



#define REG_SET_SEEN(b1)					\

({								\

	reg_set_seen(jit, b1);					\
@@ -201,7 +211,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 


#define EMIT4_PCREL_RIC(op, mask, target)			\

({								\

	int __rel = ((target) - jit->prg) / 2;			\

	int __rel = off_to_pcrel(jit, target) / 2;		\

	_EMIT4((op) | (mask) << 20 | (__rel & 0xffff));		\

})
@@ -239,7 +249,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 


#define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target)	\

({								\

	unsigned int rel = (int)((target) - jit->prg) / 2;	\

	unsigned int rel = off_to_pcrel(jit, target) / 2;	\

	_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff),	\

	       (op2) | (mask) << 12);				\

	REG_SET_SEEN(b1);					\
@@ -248,7 +258,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 


#define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target)	\

({								\

	unsigned int rel = (int)((target) - jit->prg) / 2;	\

	unsigned int rel = off_to_pcrel(jit, target) / 2;	\

	_EMIT6((op1) | (reg_high(b1) | (mask)) << 16 |		\

		(rel & 0xffff), (op2) | ((imm) & 0xff) << 8);	\

	REG_SET_SEEN(b1);					\
@@ -257,29 +267,41 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 


#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask)		\

({								\

	int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2;	\

	int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\

	_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\

	REG_SET_SEEN(b1);					\

	REG_SET_SEEN(b2);					\

})



static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel)

{

	u32 pc32dbl = (s32)(pcrel / 2);



	_EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff);

}



static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel)

{

	emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel);

	REG_SET_SEEN(b);

}



#define EMIT6_PCREL_RILB(op, b, target)				\

({								\

	unsigned int rel = (int)((target) - jit->prg) / 2;	\

	_EMIT6((op) | reg_high(b) << 16 | rel >> 16, rel & 0xffff);\

	REG_SET_SEEN(b);					\

})

	emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target))



#define EMIT6_PCREL_RIL(op, target)				\

({								\

	unsigned int rel = (int)((target) - jit->prg) / 2;	\

	_EMIT6((op) | rel >> 16, rel & 0xffff);			\

})

#define EMIT6_PCREL_RILB_PTR(op, b, target_ptr)			\

	emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr))



static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel)

{

	emit6_pcrel_ril(jit, op | mask << 20, pcrel);

}



#define EMIT6_PCREL_RILC(op, mask, target)			\

({								\

	EMIT6_PCREL_RIL((op) | (mask) << 20, (target));		\

})

	emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target))



#define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr)		\

	emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr))



#define _EMIT6_IMM(op, imm)					\

({								\
@@ -503,7 +525,7 @@ static void bpf_skip(struct bpf_jit *jit, int size) 
{

	if (size >= 6 && !is_valid_rel(size)) {

		/* brcl 0xf,size */

		EMIT6_PCREL_RIL(0xc0f4000000, size);

		EMIT6_PCREL_RILC(0xc0040000, 0xf, size);

		size -= 6;

	} else if (size >= 4 && is_valid_rel(size)) {

		/* brc 0xf,size */
@@ -618,28 +640,17 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, 
}



/*

 * Emit an expoline for a jump that follows

 */

static void emit_expoline(struct bpf_jit *jit)

{

	/* exrl %r0,.+10 */

	EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);

	/* j . */

	EMIT4_PCREL(0xa7f40000, 0);

}



/*

 * Emit __s390_indirect_jump_r1 thunk if necessary

 * Jump using a register either directly or via an expoline thunk

 */

static void emit_r1_thunk(struct bpf_jit *jit)

{

	if (nospec_uses_trampoline()) {

		jit->r1_thunk_ip = jit->prg;

		emit_expoline(jit);

		/* br %r1 */

		_EMIT2(0x07f1);

	}

}

#define EMIT_JUMP_REG(reg) do {						\

	if (nospec_uses_trampoline())					\

		/* brcl 0xf,__s390_indirect_jump_rN */			\

		EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f,			\

				     __s390_indirect_jump_r ## reg);	\

	else								\

		/* br %rN */						\

		_EMIT2(0x07f0 | reg);					\

} while (0)



/*

 * Call r1 either directly or via __s390_indirect_jump_r1 thunk
@@ -648,7 +659,8 @@ static void call_r1(struct bpf_jit *jit) 
{

	if (nospec_uses_trampoline())

		/* brasl %r14,__s390_indirect_jump_r1 */

		EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);

		EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14,

				     __s390_indirect_jump_r1);

	else

		/* basr %r14,%r1 */

		EMIT2(0x0d00, REG_14, REG_1);
@@ -664,16 +676,7 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) 
	EMIT4(0xb9040000, REG_2, BPF_REG_0);

	/* Restore registers */

	save_restore_regs(jit, REGS_RESTORE, stack_depth, 0);

	if (nospec_uses_trampoline()) {

		jit->r14_thunk_ip = jit->prg;

		/* Generate __s390_indirect_jump_r14 thunk */

		emit_expoline(jit);

	}

	/* br %r14 */

	_EMIT2(0x07fe);



	if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))

		emit_r1_thunk(jit);

	EMIT_JUMP_REG(14);



	jit->prg = ALIGN(jit->prg, 8);

	jit->prologue_plt = jit->prg;
@@ -1875,7 +1878,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
			/* aghi %r1,tail_call_start */

			EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start);

			/* brcl 0xf,__s390_indirect_jump_r1 */

			EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip);

			EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf,

					     __s390_indirect_jump_r1);

		} else {

			/* bc 0xf,tail_call_start(%r1) */

			_EMIT4(0x47f01000 + jit->tail_call_start);
@@ -2844,17 +2848,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 
	       0xf000 | tjit->tccnt_off);

	/* aghi %r15,stack_size */

	EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size);

	/* Emit an expoline for the following indirect jump. */

	if (nospec_uses_trampoline())

		emit_expoline(jit);

	if (flags & BPF_TRAMP_F_SKIP_FRAME)

		/* br %r14 */

		_EMIT2(0x07fe);

		EMIT_JUMP_REG(14);

	else

		/* br %r1 */

		_EMIT2(0x07f1);



	emit_r1_thunk(jit);

		EMIT_JUMP_REG(1);



	return 0;

}