Commit 8754e67a authored by Pawan Gupta's avatar Pawan Gupta Committed by Dave Hansen
Browse files

x86/its: Add support for ITS-safe indirect thunk 



Due to ITS, indirect branches in the lower half of a cacheline may be
vulnerable to branch target injection attack.

Introduce ITS-safe thunks to patch indirect branches in the lower half of
cacheline with the thunk. Also thunk any eBPF generated indirect branches
in emit_indirect_jump().

Below category of indirect branches are not mitigated:

- Indirect branches in the .init section are not mitigated because they are
  discarded after boot.
- Indirect branches that are explicitly marked retpoline-safe.

Note that retpoline also mitigates the indirect branches against ITS. This
is because the retpoline sequence fills an RSB entry before RET, and it
does not suffer from RSB-underflow part of the ITS.

Signed-off-by: default avatarPawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: default avatarJosh Poimboeuf <jpoimboe@kernel.org>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
parent 159013a7

arch/x86/Kconfig

+11 −0
Original line number Diff line number Diff line
@@ -2710,6 +2710,17 @@ config MITIGATION_SSB 
	  of speculative execution in a similar way to the Meltdown and Spectre

	  security vulnerabilities.



config MITIGATION_ITS

	bool "Enable Indirect Target Selection mitigation"

	depends on CPU_SUP_INTEL && X86_64

	depends on MITIGATION_RETPOLINE && MITIGATION_RETHUNK

	default y

	help

	  Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in

	  BPU on some Intel CPUs that may allow Spectre V2 style attacks. If

	  disabled, mitigation cannot be enabled via cmdline.

	  See <file:Documentation/admin-guide/hw-vuln/indirect-target-selection.rst>



endif



config ARCH_HAS_ADD_PAGES

arch/x86/include/asm/cpufeatures.h

+1 −0
Original line number Diff line number Diff line
@@ -481,6 +481,7 @@ 
#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32 + 6) /* Heterogeneous Core Topology */

#define X86_FEATURE_AMD_WORKLOAD_CLASS	(21*32 + 7) /* Workload Classification */

#define X86_FEATURE_PREFER_YMM		(21*32 + 8) /* Avoid ZMM registers due to downclocking */

#define X86_FEATURE_INDIRECT_THUNK_ITS	(21*32 + 9) /* Use thunk for indirect branches in lower half of cacheline */



/*

 * BUG word(s)

arch/x86/include/asm/nospec-branch.h

+4 −0
Original line number Diff line number Diff line
@@ -336,10 +336,14 @@ 


#else /* __ASSEMBLER__ */



#define ITS_THUNK_SIZE	64



typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];

typedef u8 its_thunk_t[ITS_THUNK_SIZE];

extern retpoline_thunk_t __x86_indirect_thunk_array[];

extern retpoline_thunk_t __x86_indirect_call_thunk_array[];

extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];

extern its_thunk_t	 __x86_indirect_its_thunk_array[];



#ifdef CONFIG_MITIGATION_RETHUNK

extern void __x86_return_thunk(void);

arch/x86/kernel/alternative.c

+42 −3
Original line number Diff line number Diff line
@@ -581,7 +581,8 @@ static int emit_indirect(int op, int reg, u8 *bytes) 
	return i;

}



static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)

static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,

			     void *call_dest, void *jmp_dest)

{

	u8 op = insn->opcode.bytes[0];

	int i = 0;
@@ -602,7 +603,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 
	switch (op) {

	case CALL_INSN_OPCODE:

		__text_gen_insn(bytes+i, op, addr+i,

				__x86_indirect_call_thunk_array[reg],

				call_dest,

				CALL_INSN_SIZE);

		i += CALL_INSN_SIZE;

		break;
@@ -610,7 +611,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 
	case JMP32_INSN_OPCODE:

clang_jcc:

		__text_gen_insn(bytes+i, op, addr+i,

				__x86_indirect_jump_thunk_array[reg],

				jmp_dest,

				JMP32_INSN_SIZE);

		i += JMP32_INSN_SIZE;

		break;
@@ -625,6 +626,35 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 
	return i;

}



static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)

{

	return __emit_trampoline(addr, insn, bytes,

				 __x86_indirect_call_thunk_array[reg],

				 __x86_indirect_jump_thunk_array[reg]);

}



#ifdef CONFIG_MITIGATION_ITS

static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)

{

	return __emit_trampoline(addr, insn, bytes,

				 __x86_indirect_its_thunk_array[reg],

				 __x86_indirect_its_thunk_array[reg]);

}



/* Check if an indirect branch is at ITS-unsafe address */

static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)

{

	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))

		return false;



	/* Indirect branch opcode is 2 or 3 bytes depending on reg */

	addr += 1 + reg / 8;



	/* Lower-half of the cacheline? */

	return !(addr & 0x20);

}

#endif



/*

 * Rewrite the compiler generated retpoline thunk calls.

 *
@@ -699,6 +729,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) 
		bytes[i++] = 0xe8; /* LFENCE */

	}



#ifdef CONFIG_MITIGATION_ITS

	/*

	 * Check if the address of last byte of emitted-indirect is in

	 * lower-half of the cacheline. Such branches need ITS mitigation.

	 */

	if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg))

		return emit_its_trampoline(addr, insn, reg, bytes);

#endif



	ret = emit_indirect(op, reg, bytes + i);

	if (ret < 0)

		return ret;

arch/x86/kernel/vmlinux.lds.S

+6 −0
Original line number Diff line number Diff line
@@ -497,6 +497,12 @@ PROVIDE(__ref_stack_chk_guard = __stack_chk_guard); 
		"SRSO function pair won't alias");

#endif



#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)

. = ASSERT(__x86_indirect_its_thunk_rax & 0x20, "__x86_indirect_thunk_rax not in second half of cacheline");

. = ASSERT(((__x86_indirect_its_thunk_rcx - __x86_indirect_its_thunk_rax) % 64) == 0, "Indirect thunks are not cacheline apart");

. = ASSERT(__x86_indirect_its_thunk_array == __x86_indirect_its_thunk_rax, "Gap in ITS thunk array");

#endif



#endif /* CONFIG_X86_64 */



/*