Commit 5bef46ac authored by Anton Protopopov's avatar Anton Protopopov Committed by Alexei Starovoitov
Browse files

bpf, x86: allow indirect jumps to r8...r15 



Currently the emit_indirect_jump() function only accepts one of the
RAX, RCX, ..., RBP registers as the destination. Make it to accept
R8, R9, ..., R15 as well, and make callers to pass BPF registers, not
native registers. This is required to enable indirect jumps support
in eBPF.

Signed-off-by: default avatarAnton Protopopov <a.s.protopopov@gmail.com>
Acked-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251105090410.1250500-8-a.s.protopopov@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent ae48162a

arch/x86/net/bpf_jit_comp.c

+22 −8
Original line number Diff line number Diff line
@@ -660,24 +660,38 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 


#define EMIT_LFENCE()	EMIT3(0x0F, 0xAE, 0xE8)



static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)

static void __emit_indirect_jump(u8 **pprog, int reg, bool ereg)

{

	u8 *prog = *pprog;



	if (ereg)

		EMIT1(0x41);



	EMIT2(0xFF, 0xE0 + reg);



	*pprog = prog;

}



static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)

{

	u8 *prog = *pprog;

	int reg = reg2hex[bpf_reg];

	bool ereg = is_ereg(bpf_reg);



	if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) {

		OPTIMIZER_HIDE_VAR(reg);

		emit_jump(&prog, its_static_thunk(reg), ip);

		emit_jump(&prog, its_static_thunk(reg + 8*ereg), ip);

	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {

		EMIT_LFENCE();

		EMIT2(0xFF, 0xE0 + reg);

		__emit_indirect_jump(&prog, reg, ereg);

	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {

		OPTIMIZER_HIDE_VAR(reg);

		if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))

			emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip);

			emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg + 8*ereg], ip);

		else

			emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);

			emit_jump(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip);

	} else {

		EMIT2(0xFF, 0xE0 + reg);	/* jmp *%\reg */

		__emit_indirect_jump(&prog, reg, ereg);

		if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || IS_ENABLED(CONFIG_MITIGATION_SLS))

			EMIT1(0xCC);		/* int3 */

	}
@@ -797,7 +811,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, 
	 * rdi == ctx (1st arg)

	 * rcx == prog->bpf_func + X86_TAIL_CALL_OFFSET

	 */

	emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start));

	emit_indirect_jump(&prog, BPF_REG_4 /* R4 -> rcx */, ip + (prog - start));



	/* out: */

	ctx->tail_call_indirect_label = prog - start;
@@ -3543,7 +3557,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, 
		if (err)

			return err;



		emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf));

		emit_indirect_jump(&prog, BPF_REG_3 /* R3 -> rdx */, image + (prog - buf));



		*pprog = prog;

		return 0;