Commit 0dfefc2e authored by James Morse's avatar James Morse
Browse files

arm64: bpf: Add BHB mitigation to the epilogue for cBPF programs 



A malicious BPF program may manipulate the branch history to influence
what the hardware speculates will happen next.

On exit from a BPF program, emit the BHB mititgation sequence.

This is only applied for 'classic' cBPF programs that are loaded by
seccomp.

Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent a1152be3

arch/arm64/include/asm/spectre.h

+1 −0
Original line number Diff line number Diff line
@@ -97,6 +97,7 @@ enum mitigation_state arm64_get_meltdown_state(void); 


enum mitigation_state arm64_get_spectre_bhb_state(void);

bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);

extern bool __nospectre_bhb;

u8 get_spectre_bhb_loop_value(void);

bool is_spectre_bhb_fw_mitigated(void);

void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);

arch/arm64/kernel/proton-pack.c

+1 −1
Original line number Diff line number Diff line
@@ -1021,7 +1021,7 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) 
	isb();

}



static bool __read_mostly __nospectre_bhb;

bool __read_mostly __nospectre_bhb;

static int __init parse_spectre_bhb_param(char *str)

{

	__nospectre_bhb = true;

arch/arm64/net/bpf_jit_comp.c

+50 −4
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 


#define pr_fmt(fmt) "bpf_jit: " fmt



#include <linux/arm-smccc.h>

#include <linux/bitfield.h>

#include <linux/bpf.h>

#include <linux/filter.h>
@@ -17,6 +18,7 @@ 
#include <asm/asm-extable.h>

#include <asm/byteorder.h>

#include <asm/cacheflush.h>

#include <asm/cpufeature.h>

#include <asm/debug-monitors.h>

#include <asm/insn.h>

#include <asm/text-patching.h>
@@ -939,7 +941,48 @@ static void build_plt(struct jit_ctx *ctx) 
		plt->target = (u64)&dummy_tramp;

}



static void build_epilogue(struct jit_ctx *ctx)

/* Clobbers BPF registers 1-4, aka x0-x3 */

static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx)

{

	const u8 r1 = bpf2a64[BPF_REG_1]; /* aka x0 */

	u8 k = get_spectre_bhb_loop_value();



	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||

	    cpu_mitigations_off() || __nospectre_bhb ||

	    arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE)

		return;



	if (supports_clearbhb(SCOPE_SYSTEM)) {

		emit(aarch64_insn_gen_hint(AARCH64_INSN_HINT_CLEARBHB), ctx);

		return;

	}



	if (k) {

		emit_a64_mov_i64(r1, k, ctx);

		emit(A64_B(1), ctx);

		emit(A64_SUBS_I(true, r1, r1, 1), ctx);

		emit(A64_B_(A64_COND_NE, -2), ctx);

		emit(aarch64_insn_gen_dsb(AARCH64_INSN_MB_ISH), ctx);

		emit(aarch64_insn_get_isb_value(), ctx);

	}



	if (is_spectre_bhb_fw_mitigated()) {

		emit(A64_ORR_I(false, r1, AARCH64_INSN_REG_ZR,

			       ARM_SMCCC_ARCH_WORKAROUND_3), ctx);

		switch (arm_smccc_1_1_get_conduit()) {

		case SMCCC_CONDUIT_HVC:

			emit(aarch64_insn_get_hvc_value(), ctx);

			break;

		case SMCCC_CONDUIT_SMC:

			emit(aarch64_insn_get_smc_value(), ctx);

			break;

		default:

			pr_err_once("Firmware mitigation enabled with unknown conduit\n");

		}

	}

}



static void build_epilogue(struct jit_ctx *ctx, bool was_classic)

{

	const u8 r0 = bpf2a64[BPF_REG_0];

	const u8 ptr = bpf2a64[TCCNT_PTR];
@@ -952,10 +995,13 @@ static void build_epilogue(struct jit_ctx *ctx) 


	emit(A64_POP(A64_ZR, ptr, A64_SP), ctx);



	if (was_classic)

		build_bhb_mitigation(ctx);



	/* Restore FP/LR registers */

	emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);



	/* Set return value */

	/* Move the return value from bpf:r0 (aka x7) to x0 */

	emit(A64_MOV(1, A64_R(0), r0), ctx);



	/* Authenticate lr */
@@ -1898,7 +1944,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
	}



	ctx.epilogue_offset = ctx.idx;

	build_epilogue(&ctx);

	build_epilogue(&ctx, was_classic);

	build_plt(&ctx);



	extable_align = __alignof__(struct exception_table_entry);
@@ -1961,7 +2007,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
		goto out_free_hdr;

	}



	build_epilogue(&ctx);

	build_epilogue(&ctx, was_classic);

	build_plt(&ctx);



	/* Extra pass to validate JITed code. */