Commit 872df34d authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Dave Hansen
Browse files

x86/its: Use dynamic thunks for indirect branches 



ITS mitigation moves the unsafe indirect branches to a safe thunk. This
could degrade the prediction accuracy as the source address of indirect
branches becomes same for different execution paths.

To improve the predictions, and hence the performance, assign a separate
thunk for each indirect callsite. This is also a defense-in-depth measure
to avoid indirect branches aliasing with each other.

As an example, 5000 dynamic thunks would utilize around 16 bits of the
address space, thereby gaining entropy. For a BTB that uses
32 bits for indexing, dynamic thunks could provide better prediction
accuracy over fixed thunks.

Have ITS thunks be variable sized and use EXECMEM_MODULE_TEXT such that
they are both more flexible (got to extend them later) and live in 2M TLBs,
just like kernel code, avoiding undue TLB pressure.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarPawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
parent ebebe307

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -2714,6 +2714,7 @@ config MITIGATION_ITS 
	bool "Enable Indirect Target Selection mitigation"

	depends on CPU_SUP_INTEL && X86_64

	depends on MITIGATION_RETPOLINE && MITIGATION_RETHUNK

	select EXECMEM

	default y

	help

	  Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in

arch/x86/include/asm/alternative.h

+10 −0
Original line number Diff line number Diff line
@@ -124,6 +124,16 @@ static __always_inline int x86_call_depth_emit_accounting(u8 **pprog, 
}

#endif



#ifdef CONFIG_MITIGATION_ITS

extern void its_init_mod(struct module *mod);

extern void its_fini_mod(struct module *mod);

extern void its_free_mod(struct module *mod);

#else /* CONFIG_MITIGATION_ITS */

static inline void its_init_mod(struct module *mod) { }

static inline void its_fini_mod(struct module *mod) { }

static inline void its_free_mod(struct module *mod) { }

#endif



#if defined(CONFIG_MITIGATION_RETHUNK) && defined(CONFIG_OBJTOOL)

extern bool cpu_wants_rethunk(void);

extern bool cpu_wants_rethunk_at(void *addr);

arch/x86/kernel/alternative.c

+124 −3
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include <linux/mmu_context.h>

#include <linux/bsearch.h>

#include <linux/sync_core.h>

#include <linux/execmem.h>

#include <asm/text-patching.h>

#include <asm/alternative.h>

#include <asm/sections.h>
@@ -32,6 +33,7 @@ 
#include <asm/asm-prototypes.h>

#include <asm/cfi.h>

#include <asm/ibt.h>

#include <asm/set_memory.h>



int __read_mostly alternatives_patched;
@@ -125,6 +127,121 @@ const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 
#endif

};



#ifdef CONFIG_MITIGATION_ITS



static struct module *its_mod;

static void *its_page;

static unsigned int its_offset;



/* Initialize a thunk with the "jmp *reg; int3" instructions. */

static void *its_init_thunk(void *thunk, int reg)

{

	u8 *bytes = thunk;

	int i = 0;



	if (reg >= 8) {

		bytes[i++] = 0x41; /* REX.B prefix */

		reg -= 8;

	}

	bytes[i++] = 0xff;

	bytes[i++] = 0xe0 + reg; /* jmp *reg */

	bytes[i++] = 0xcc;



	return thunk;

}



void its_init_mod(struct module *mod)

{

	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))

		return;



	mutex_lock(&text_mutex);

	its_mod = mod;

	its_page = NULL;

}



void its_fini_mod(struct module *mod)

{

	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))

		return;



	WARN_ON_ONCE(its_mod != mod);



	its_mod = NULL;

	its_page = NULL;

	mutex_unlock(&text_mutex);



	for (int i = 0; i < mod->its_num_pages; i++) {

		void *page = mod->its_page_array[i];

		execmem_restore_rox(page, PAGE_SIZE);

	}

}



void its_free_mod(struct module *mod)

{

	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))

		return;



	for (int i = 0; i < mod->its_num_pages; i++) {

		void *page = mod->its_page_array[i];

		execmem_free(page);

	}

	kfree(mod->its_page_array);

}



static void *its_alloc(void)

{

	void *page __free(execmem) = execmem_alloc(EXECMEM_MODULE_TEXT, PAGE_SIZE);



	if (!page)

		return NULL;



	if (its_mod) {

		void *tmp = krealloc(its_mod->its_page_array,

				     (its_mod->its_num_pages+1) * sizeof(void *),

				     GFP_KERNEL);

		if (!tmp)

			return NULL;



		its_mod->its_page_array = tmp;

		its_mod->its_page_array[its_mod->its_num_pages++] = page;



		execmem_make_temp_rw(page, PAGE_SIZE);

	}



	return no_free_ptr(page);

}



static void *its_allocate_thunk(int reg)

{

	int size = 3 + (reg / 8);

	void *thunk;



	if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {

		its_page = its_alloc();

		if (!its_page) {

			pr_err("ITS page allocation failed\n");

			return NULL;

		}

		memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE);

		its_offset = 32;

	}



	/*

	 * If the indirect branch instruction will be in the lower half

	 * of a cacheline, then update the offset to reach the upper half.

	 */

	if ((its_offset + size - 1) % 64 < 32)

		its_offset = ((its_offset - 1) | 0x3F) + 33;



	thunk = its_page + its_offset;

	its_offset += size;



	return its_init_thunk(thunk, reg);

}



#endif



/*

 * Nomenclature for variable names to simplify and clarify this code and ease

 * any potential staring at it:
@@ -637,9 +754,13 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 
#ifdef CONFIG_MITIGATION_ITS

static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)

{

	return __emit_trampoline(addr, insn, bytes,

				 __x86_indirect_its_thunk_array[reg],

				 __x86_indirect_its_thunk_array[reg]);

	u8 *thunk = __x86_indirect_its_thunk_array[reg];

	u8 *tmp = its_allocate_thunk(reg);



	if (tmp)

		thunk = tmp;



	return __emit_trampoline(addr, insn, bytes, thunk, thunk);

}



/* Check if an indirect branch is at ITS-unsafe address */

arch/x86/kernel/module.c

+6 −0
Original line number Diff line number Diff line
@@ -266,6 +266,8 @@ int module_finalize(const Elf_Ehdr *hdr, 
			ibt_endbr = s;

	}



	its_init_mod(me);



	if (retpolines || cfi) {

		void *rseg = NULL, *cseg = NULL;

		unsigned int rsize = 0, csize = 0;
@@ -286,6 +288,9 @@ int module_finalize(const Elf_Ehdr *hdr, 
		void *rseg = (void *)retpolines->sh_addr;

		apply_retpolines(rseg, rseg + retpolines->sh_size);

	}



	its_fini_mod(me);



	if (returns) {

		void *rseg = (void *)returns->sh_addr;

		apply_returns(rseg, rseg + returns->sh_size);
@@ -326,4 +331,5 @@ int module_finalize(const Elf_Ehdr *hdr, 
void module_arch_cleanup(struct module *mod)

{

	alternatives_smp_module_del(mod);

	its_free_mod(mod);

}

include/linux/execmem.h

+3 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 


#include <linux/types.h>

#include <linux/moduleloader.h>

#include <linux/cleanup.h>



#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \

		!defined(CONFIG_KASAN_VMALLOC)
@@ -176,6 +177,8 @@ void *execmem_alloc(enum execmem_type type, size_t size); 
 */

void execmem_free(void *ptr);



DEFINE_FREE(execmem, void *, if (_T) execmem_free(_T));



#ifdef CONFIG_MMU

/**

 * execmem_vmap - create virtual mapping for EXECMEM_MODULE_DATA memory