Commit 925a0eb4 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: mm: Add LPA2 support to phys<->pte conversion routines 



In preparation for enabling LPA2 support, introduce the mask values for
converting between physical addresses and their representations in a
page table descriptor.

While at it, move the pte_to_phys asm macro into its only user, so that
we can freely modify it to use its input value register as a temp
register.

For LPA2, the PTE_ADDR_MASK contains two non-adjacent sequences of zero
bits, which means it no longer fits into the immediate field of an
ordinary ALU instruction. So let's redefine it to include the bits in
between as well, and only use it when converting from physical address
to PTE representation, where the distinction does not matter. Also
update the name accordingly to emphasize this.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-75-ardb+git@google.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent db95ea78

arch/arm64/include/asm/assembler.h

+2 −14
Original line number Diff line number Diff line
@@ -612,25 +612,13 @@ alternative_endif 


	.macro	phys_to_pte, pte, phys

#ifdef CONFIG_ARM64_PA_BITS_52

	/*

	 * We assume \phys is 64K aligned and this is guaranteed by only

	 * supporting this configuration with 64K pages.

	 */

	orr	\pte, \phys, \phys, lsr #36

	and	\pte, \pte, #PTE_ADDR_MASK

	orr	\pte, \phys, \phys, lsr #PTE_ADDR_HIGH_SHIFT

	and	\pte, \pte, #PHYS_TO_PTE_ADDR_MASK

#else

	mov	\pte, \phys

#endif

	.endm



	.macro	pte_to_phys, phys, pte

	and	\phys, \pte, #PTE_ADDR_MASK

#ifdef CONFIG_ARM64_PA_BITS_52

	orr	\phys, \phys, \phys, lsl #PTE_ADDR_HIGH_SHIFT

	and	\phys, \phys, GENMASK_ULL(PHYS_MASK_SHIFT - 1, PAGE_SHIFT)

#endif

	.endm



/*

 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.

 */

arch/arm64/include/asm/pgtable-hwdef.h

+7 −3
Original line number Diff line number Diff line
@@ -155,13 +155,17 @@ 
#define PTE_PXN			(_AT(pteval_t, 1) << 53)	/* Privileged XN */

#define PTE_UXN			(_AT(pteval_t, 1) << 54)	/* User XN */



#define PTE_ADDR_LOW		(((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)

#define PTE_ADDR_LOW		(((_AT(pteval_t, 1) << (50 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)

#ifdef CONFIG_ARM64_PA_BITS_52

#ifdef CONFIG_ARM64_64K_PAGES

#define PTE_ADDR_HIGH		(_AT(pteval_t, 0xf) << 12)

#define PTE_ADDR_MASK		(PTE_ADDR_LOW | PTE_ADDR_HIGH)

#define PTE_ADDR_HIGH_SHIFT	36

#define PHYS_TO_PTE_ADDR_MASK	(PTE_ADDR_LOW | PTE_ADDR_HIGH)

#else

#define PTE_ADDR_MASK		PTE_ADDR_LOW

#define PTE_ADDR_HIGH		(_AT(pteval_t, 0x3) << 8)

#define PTE_ADDR_HIGH_SHIFT	42

#define PHYS_TO_PTE_ADDR_MASK	GENMASK_ULL(49, 8)

#endif

#endif



/*

arch/arm64/include/asm/pgtable.h

+3 −2
Original line number Diff line number Diff line
@@ -80,15 +80,16 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; 
#ifdef CONFIG_ARM64_PA_BITS_52

static inline phys_addr_t __pte_to_phys(pte_t pte)

{

	pte_val(pte) &= ~PTE_MAYBE_SHARED;

	return (pte_val(pte) & PTE_ADDR_LOW) |

		((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);

}

static inline pteval_t __phys_to_pte_val(phys_addr_t phys)

{

	return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK;

	return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK;

}

#else

#define __pte_to_phys(pte)	(pte_val(pte) & PTE_ADDR_MASK)

#define __pte_to_phys(pte)	(pte_val(pte) & PTE_ADDR_LOW)

#define __phys_to_pte_val(phys)	(phys)

#endif

arch/arm64/mm/proc.S

+8 −0
Original line number Diff line number Diff line
@@ -205,6 +205,14 @@ SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1) 


	.pushsection ".idmap.text", "a"



	.macro	pte_to_phys, phys, pte

	and	\phys, \pte, #PTE_ADDR_LOW

#ifdef CONFIG_ARM64_PA_BITS_52

	and	\pte, \pte, #PTE_ADDR_HIGH

	orr	\phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT

#endif

	.endm



	.macro	kpti_mk_tbl_ng, type, num_entries

	add	end_\type\()p, cur_\type\()p, #\num_entries * 8

.Ldo_\type: