Commit 6ed8a3a0 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: mm: Add 5 level paging support to fixmap and swapper handling 



Add support for using 5 levels of paging in the fixmap, as well as in
the kernel page table handling code which uses fixmaps internally.
This also handles the case where a 5 level build runs on hardware that
only supports 4 levels of paging.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-79-ardb+git@google.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 9684ec18

arch/arm64/include/asm/fixmap.h

+1 −0
Original line number Diff line number Diff line
@@ -87,6 +87,7 @@ enum fixed_addresses { 
	FIX_PTE,

	FIX_PMD,

	FIX_PUD,

	FIX_P4D,

	FIX_PGD,



	__end_of_fixed_addresses

arch/arm64/include/asm/pgtable.h

+40 −5
Original line number Diff line number Diff line
@@ -621,12 +621,12 @@ static inline bool pud_table(pud_t pud) { return true; } 
				 PUD_TYPE_TABLE)

#endif



extern pgd_t init_pg_dir[PTRS_PER_PGD];

extern pgd_t init_pg_dir[];

extern pgd_t init_pg_end[];

extern pgd_t swapper_pg_dir[PTRS_PER_PGD];

extern pgd_t idmap_pg_dir[PTRS_PER_PGD];

extern pgd_t tramp_pg_dir[PTRS_PER_PGD];

extern pgd_t reserved_pg_dir[PTRS_PER_PGD];

extern pgd_t swapper_pg_dir[];

extern pgd_t idmap_pg_dir[];

extern pgd_t tramp_pg_dir[];

extern pgd_t reserved_pg_dir[];



extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
@@ -891,12 +891,47 @@ static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr) 
	return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr);

}



static inline p4d_t *p4d_set_fixmap(unsigned long addr)

{

	if (!pgtable_l5_enabled())

		return NULL;

	return (p4d_t *)set_fixmap_offset(FIX_P4D, addr);

}



static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr)

{

	if (!pgtable_l5_enabled())

		return pgd_to_folded_p4d(pgdp, addr);

	return p4d_set_fixmap(p4d_offset_phys(pgdp, addr));

}



static inline void p4d_clear_fixmap(void)

{

	if (pgtable_l5_enabled())

		clear_fixmap(FIX_P4D);

}



/* use ONLY for statically allocated translation tables */

static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr)

{

	if (!pgtable_l5_enabled())

		return pgd_to_folded_p4d(pgdp, addr);

	return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr));

}



#define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))



#else



static inline bool pgtable_l5_enabled(void) { return false; }



/* Match p4d_offset folding in <asm/generic/pgtable-nop4d.h> */

#define p4d_set_fixmap(addr)		NULL

#define p4d_set_fixmap_offset(p4dp, addr)	((p4d_t *)p4dp)

#define p4d_clear_fixmap()



#define p4d_offset_kimg(dir,addr)	((p4d_t *)dir)



#endif  /* CONFIG_PGTABLE_LEVELS > 4 */



#define pgd_ERROR(e)	\

arch/arm64/mm/fixmap.c

+1 −1
Original line number Diff line number Diff line
@@ -104,7 +104,7 @@ void __init early_fixmap_init(void) 
	unsigned long end = FIXADDR_TOP;



	pgd_t *pgdp = pgd_offset_k(addr);

	p4d_t *p4dp = p4d_offset(pgdp, addr);

	p4d_t *p4dp = p4d_offset_kimg(pgdp, addr);



	early_fixmap_init_pud(p4dp, addr, end);

}

arch/arm64/mm/mmu.c

+43 −4
Original line number Diff line number Diff line
@@ -313,15 +313,14 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, 
	} while (addr = next, addr != end);

}



static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,

static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,

			   phys_addr_t phys, pgprot_t prot,

			   phys_addr_t (*pgtable_alloc)(int),

			   int flags)

{

	unsigned long next;

	pud_t *pudp;

	p4d_t *p4dp = p4d_offset(pgdp, addr);

	p4d_t p4d = READ_ONCE(*p4dp);

	pud_t *pudp;



	if (p4d_none(p4d)) {

		p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
@@ -369,6 +368,46 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, 
	pud_clear_fixmap();

}



static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,

			   phys_addr_t phys, pgprot_t prot,

			   phys_addr_t (*pgtable_alloc)(int),

			   int flags)

{

	unsigned long next;

	pgd_t pgd = READ_ONCE(*pgdp);

	p4d_t *p4dp;



	if (pgd_none(pgd)) {

		pgdval_t pgdval = PGD_TYPE_TABLE | PGD_TABLE_UXN;

		phys_addr_t p4d_phys;



		if (flags & NO_EXEC_MAPPINGS)

			pgdval |= PGD_TABLE_PXN;

		BUG_ON(!pgtable_alloc);

		p4d_phys = pgtable_alloc(P4D_SHIFT);

		__pgd_populate(pgdp, p4d_phys, pgdval);

		pgd = READ_ONCE(*pgdp);

	}

	BUG_ON(pgd_bad(pgd));



	p4dp = p4d_set_fixmap_offset(pgdp, addr);

	do {

		p4d_t old_p4d = READ_ONCE(*p4dp);



		next = p4d_addr_end(addr, end);



		alloc_init_pud(p4dp, addr, next, phys, prot,

			       pgtable_alloc, flags);



		BUG_ON(p4d_val(old_p4d) != 0 &&

		       p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp)));



		phys += next - addr;

	} while (p4dp++, addr = next, addr != end);



	p4d_clear_fixmap();

}



static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,

					unsigned long virt, phys_addr_t size,

					pgprot_t prot,
@@ -391,7 +430,7 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, 


	do {

		next = pgd_addr_end(addr, end);

		alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,

		alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc,

			       flags);

		phys += next - addr;

	} while (pgdp++, addr = next, addr != end);