Commit 9684ec18 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: Enable LPA2 at boot if supported by the system 



Update the early kernel mapping code to take 52-bit virtual addressing
into account based on the LPA2 feature. This is a bit more involved than
LVA (which is supported with 64k pages only), given that some page table
descriptor bits change meaning in this case.

To keep the handling in asm to a minimum, the initial ID map is still
created with 48-bit virtual addressing, which implies that the kernel
image must be loaded into 48-bit addressable physical memory. This is
currently required by the boot protocol, even though we happen to
support placement outside of that for LVA/64k based configurations.

Enabling LPA2 involves more than setting TCR.T1SZ to a lower value,
there is also a DS bit in TCR that needs to be set, and which changes
the meaning of bits [9:8] in all page table descriptors. Since we cannot
enable DS and every live page table descriptor at the same time, let's
pivot through another temporary mapping. This avoids the need to
reintroduce manipulations of the page tables with the MMU and caches
disabled.

To permit the LPA2 feature to be overridden on the kernel command line,
which may be necessary to work around silicon errata, or to deal with
mismatched features on heterogeneous SoC designs, test for CPU feature
overrides first, and only then enable LPA2.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-78-ardb+git@google.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 2b6c8f96

arch/arm64/include/asm/assembler.h

+7 −1
Original line number Diff line number Diff line
@@ -581,11 +581,17 @@ alternative_endif 
 * but we have to add an offset so that the TTBR1 address corresponds with the

 * pgdir entry that covers the lowest 48-bit addressable VA.

 *

 * Note that this trick is only used for LVA/64k pages - LPA2/4k pages uses an

 * additional paging level, and on LPA2/16k pages, we would end up with a root

 * level table with only 2 entries, which is suboptimal in terms of TLB

 * utilization, so there we fall back to 47 bits of translation if LPA2 is not

 * supported.

 *

 * orr is used as it can cover the immediate value (and is idempotent).

 * 	ttbr: Value of ttbr to set, modified.

 */

	.macro	offset_ttbr1, ttbr, tmp

#ifdef CONFIG_ARM64_VA_BITS_52

#if defined(CONFIG_ARM64_VA_BITS_52) && !defined(CONFIG_ARM64_LPA2)

	mrs	\tmp, tcr_el1

	and	\tmp, \tmp, #TCR_T1SZ_MASK

	cmp	\tmp, #TCR_T1SZ(VA_BITS_MIN)

arch/arm64/include/asm/cpufeature.h

+18 −0
Original line number Diff line number Diff line
@@ -1008,6 +1008,24 @@ static inline bool cpu_has_lva(void) 
						    ID_AA64MMFR2_EL1_VARange_SHIFT);

}



static inline bool cpu_has_lpa2(void)

{

#ifdef CONFIG_ARM64_LPA2

	u64 mmfr0;

	int feat;



	mmfr0 = read_sysreg(id_aa64mmfr0_el1);

	mmfr0 &= ~id_aa64mmfr0_override.mask;

	mmfr0 |= id_aa64mmfr0_override.val;

	feat = cpuid_feature_extract_signed_field(mmfr0,

						  ID_AA64MMFR0_EL1_TGRAN_SHIFT);



	return feat >= ID_AA64MMFR0_EL1_TGRAN_LPA2;

#else

	return false;

#endif

}



#endif /* __ASSEMBLY__ */



#endif

arch/arm64/include/asm/memory.h

+4 −0
Original line number Diff line number Diff line
@@ -54,7 +54,11 @@ 
#define FIXADDR_TOP		(-UL(SZ_8M))



#if VA_BITS > 48

#ifdef CONFIG_ARM64_16K_PAGES

#define VA_BITS_MIN		(47)

#else

#define VA_BITS_MIN		(48)

#endif

#else

#define VA_BITS_MIN		(VA_BITS)

#endif

arch/arm64/kernel/head.S

+8 −0
Original line number Diff line number Diff line
@@ -89,6 +89,7 @@ SYM_CODE_START(primary_entry) 
	mov	sp, x1

	mov	x29, xzr

	adrp	x0, init_idmap_pg_dir

	mov	x1, xzr

	bl	__pi_create_init_idmap



	/*
@@ -473,9 +474,16 @@ SYM_FUNC_END(__enable_mmu) 


#ifdef CONFIG_ARM64_VA_BITS_52

SYM_FUNC_START(__cpu_secondary_check52bitva)

#ifndef CONFIG_ARM64_LPA2

	mrs_s	x0, SYS_ID_AA64MMFR2_EL1

	and	x0, x0, ID_AA64MMFR2_EL1_VARange_MASK

	cbnz	x0, 2f

#else

	mrs	x0, id_aa64mmfr0_el1

	sbfx	x0, x0, #ID_AA64MMFR0_EL1_TGRAN_SHIFT, 4

	cmp	x0, #ID_AA64MMFR0_EL1_TGRAN_LPA2

	b.ge	2f

#endif



	update_early_cpu_boot_status \

		CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_52_BIT_VA, x0, x1

arch/arm64/kernel/image-vars.h

+1 −0
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ PROVIDE(__pi__ctype = _ctype); 
PROVIDE(__pi_memstart_offset_seed	= memstart_offset_seed);



PROVIDE(__pi_init_idmap_pg_dir		= init_idmap_pg_dir);

PROVIDE(__pi_init_idmap_pg_end		= init_idmap_pg_end);

PROVIDE(__pi_init_pg_dir		= init_pg_dir);

PROVIDE(__pi_init_pg_end		= init_pg_end);

PROVIDE(__pi_swapper_pg_dir		= swapper_pg_dir);