Commit 98a7e47f authored by Kevin Brodsky's avatar Kevin Brodsky Committed by Andrew Morton
Browse files

asm-generic: pgalloc: provide generic p4d_{alloc_one,free} 

Four architectures currently implement 5-level pgtables: arm64, riscv, x86
and s390.  The first three have essentially the same implementation for
p4d_alloc_one() and p4d_free(), so we've got an opportunity to reduce
duplication like at the lower levels.

Provide a generic version of p4d_alloc_one() and p4d_free(), and make use
of it on those architectures.

Their implementation is the same as at PUD level, except that p4d_free()
performs a runtime check by calling mm_p4d_folded().  5-level pgtables
depend on a runtime-detected hardware feature on all supported
architectures, so we might as well include this check in the generic
implementation.  No runtime check is required in p4d_alloc_one() as the
top-level p4d_alloc() already does the required check.

Link: https://lkml.kernel.org/r/26d69c74a29183ecc335b9b407040d8e4cd70c6a.1736317725.git.zhengqi.arch@bytedance.com


Signed-off-by: default avatarKevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: default avatarQi Zheng <zhengqi.arch@bytedance.com>
Acked-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Acked-by: Arnd Bergmann <arnd@arndb.de>		[asm-generic]
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 5a32443f

arch/arm64/include/asm/pgalloc.h

+0 −17
Original line number Diff line number Diff line
@@ -85,23 +85,6 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, p4d_t *p4dp) 
	__pgd_populate(pgdp, __pa(p4dp), pgdval);

}



static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)

{

	gfp_t gfp = GFP_PGTABLE_USER;



	if (mm == &init_mm)

		gfp = GFP_PGTABLE_KERNEL;

	return (p4d_t *)get_zeroed_page(gfp);

}



static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	if (!pgtable_l5_enabled())

		return;

	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));

	free_page((unsigned long)p4d);

}



#define __p4d_free_tlb(tlb, p4d, addr)  p4d_free((tlb)->mm, p4d)

#else

static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot)

arch/riscv/include/asm/pgalloc.h

+0 −23
Original line number Diff line number Diff line
@@ -105,29 +105,6 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 
	}

}



#define p4d_alloc_one p4d_alloc_one

static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)

{

	gfp_t gfp = GFP_PGTABLE_USER;



	if (mm == &init_mm)

		gfp = GFP_PGTABLE_KERNEL;

	return (p4d_t *)get_zeroed_page(gfp);

}



static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));

	free_page((unsigned long)p4d);

}



#define p4d_free p4d_free

static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	if (pgtable_l5_enabled)

		__p4d_free(mm, p4d);

}



static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,

				  unsigned long addr)

{

arch/x86/include/asm/pgalloc.h

+0 −18
Original line number Diff line number Diff line
@@ -147,24 +147,6 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4 
	set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));

}



static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)

{

	gfp_t gfp = GFP_KERNEL_ACCOUNT;



	if (mm == &init_mm)

		gfp &= ~__GFP_ACCOUNT;

	return (p4d_t *)get_zeroed_page(gfp);

}



static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	if (!pgtable_l5_enabled())

		return;



	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));

	free_page((unsigned long)p4d);

}



extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);



static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,

include/asm-generic/pgalloc.h

+45 −0
Original line number Diff line number Diff line
@@ -215,6 +215,51 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) 


#endif /* CONFIG_PGTABLE_LEVELS > 3 */



#if CONFIG_PGTABLE_LEVELS > 4



static inline p4d_t *__p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long addr)

{

	gfp_t gfp = GFP_PGTABLE_USER;

	struct ptdesc *ptdesc;



	if (mm == &init_mm)

		gfp = GFP_PGTABLE_KERNEL;

	gfp &= ~__GFP_HIGHMEM;



	ptdesc = pagetable_alloc_noprof(gfp, 0);

	if (!ptdesc)

		return NULL;



	return ptdesc_address(ptdesc);

}

#define __p4d_alloc_one(...)	alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS__))



#ifndef __HAVE_ARCH_P4D_ALLOC_ONE

static inline p4d_t *p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long addr)

{

	return __p4d_alloc_one_noprof(mm, addr);

}

#define p4d_alloc_one(...)	alloc_hooks(p4d_alloc_one_noprof(__VA_ARGS__))

#endif



static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	struct ptdesc *ptdesc = virt_to_ptdesc(p4d);



	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));

	pagetable_free(ptdesc);

}



#ifndef __HAVE_ARCH_P4D_FREE

static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	if (!mm_p4d_folded(mm))

		__p4d_free(mm, p4d);

}

#endif



#endif /* CONFIG_PGTABLE_LEVELS > 4 */



#ifndef __HAVE_ARCH_PGD_FREE

static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)

{