Commit 42c5a3b0 authored by Mark Rutland's avatar Mark Rutland Committed by Catalin Marinas
Browse files

arm64: Split kpti_install_ng_mappings() 



The arm64_cpu_capabilities::cpu_enable callbacks are intended for
cpu-local feature enablement (e.g. poking system registers). These get
called for each online CPU when boot/system cpucaps get finalized and
enabled, and get called whenever a CPU is subsequently onlined.

For KPTI with the ARM64_UNMAP_KERNEL_AT_EL0 cpucap, we use the
kpti_install_ng_mappings() function as the cpu_enable callback. This
does a mixture of cpu-local configuration (setting VBAR_EL1 to the
appropriate trampoline vectors) and some global configuration (rewriting
the swapper page tables to sue non-glboal mappings) that must happen at
most once.

This patch splits kpti_install_ng_mappings() into a cpu-local
cpu_enable_kpti() initialization function and a system-wide
kpti_install_ng_mappings() function. The cpu_enable_kpti() function is
responsible for selecting the necessary cpu-local vectors each time a
CPU is onlined, and the kpti_install_ng_mappings() function performs the
one-time rewrite of the translation tables too use non-global mappings.
Splitting the two makes the code a bit easier to follow and also allows
the page table rewriting code to be marked as __init such that it can be
freed after use.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 7f632d33

arch/arm64/kernel/cpufeature.c

+33 −21
Original line number Diff line number Diff line
@@ -1754,16 +1754,15 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, 
			     phys_addr_t size, pgprot_t prot,

			     phys_addr_t (*pgtable_alloc)(int), int flags);



static phys_addr_t kpti_ng_temp_alloc;

static phys_addr_t __initdata kpti_ng_temp_alloc;



static phys_addr_t kpti_ng_pgd_alloc(int shift)

static phys_addr_t __init kpti_ng_pgd_alloc(int shift)

{

	kpti_ng_temp_alloc -= PAGE_SIZE;

	return kpti_ng_temp_alloc;

}



static void

kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)

static int __init __kpti_install_ng_mappings(void *__unused)

{

	typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);

	extern kpti_remap_fn idmap_kpti_install_ng_mappings;
@@ -1776,20 +1775,6 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) 
	pgd_t *kpti_ng_temp_pgd;

	u64 alloc = 0;



	if (__this_cpu_read(this_cpu_vector) == vectors) {

		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);



		__this_cpu_write(this_cpu_vector, v);

	}



	/*

	 * We don't need to rewrite the page-tables if either we've done

	 * it already or we have KASLR enabled and therefore have not

	 * created any global mappings at all.

	 */

	if (arm64_use_ng_mappings)

		return;



	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);



	if (!cpu) {
@@ -1826,14 +1811,39 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) 
		free_pages(alloc, order);

		arm64_use_ng_mappings = true;

	}



	return 0;

}



static void __init kpti_install_ng_mappings(void)

{

	/*

	 * We don't need to rewrite the page-tables if either we've done

	 * it already or we have KASLR enabled and therefore have not

	 * created any global mappings at all.

	 */

	if (arm64_use_ng_mappings)

		return;



	stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask);

}



#else

static void

kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)

static inline void kpti_install_ng_mappings(void)

{

}

#endif	/* CONFIG_UNMAP_KERNEL_AT_EL0 */



static void cpu_enable_kpti(struct arm64_cpu_capabilities const *cap)

{

	if (__this_cpu_read(this_cpu_vector) == vectors) {

		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);



		__this_cpu_write(this_cpu_vector, v);

	}



}



static int __init parse_kpti(char *str)

{

	bool enabled;
@@ -2362,7 +2372,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { 
		.desc = "Kernel page table isolation (KPTI)",

		.capability = ARM64_UNMAP_KERNEL_AT_EL0,

		.type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,

		.cpu_enable = kpti_install_ng_mappings,

		.cpu_enable = cpu_enable_kpti,

		.matches = unmap_kernel_at_el0,

		/*

		 * The ID feature fields below are used to indicate that
@@ -3355,6 +3365,8 @@ void __init setup_system_features(void) 
	 */

	enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);



	kpti_install_ng_mappings();



	sve_setup();

	sme_setup();