Commit 0549e766 authored by Christophe Leroy's avatar Christophe Leroy Committed by Andrew Morton
Browse files

powerpc/8xx: rework support for 8M pages using contiguous PTE entries 

In order to fit better with standard Linux page tables layout, add support
for 8M pages using contiguous PTE entries in a standard page table.  Page
tables will then be populated with 1024 similar entries and two PMD
entries will point to that page table.

The PMD entries also get a flag to tell it is addressing an 8M page, this
is required for the HW tablewalk assistance.

Link: https://lkml.kernel.org/r/8693d9a0408371043ca63bf9e4a9c140667af63e.1719928057.git.christophe.leroy@csgroup.eu


Signed-off-by: default avatarChristophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: default avatarOscar Salvador <osalvador@suse.de>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7ea98107

arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h

+8 −30
Original line number Diff line number Diff line
@@ -4,42 +4,12 @@ 


#define PAGE_SHIFT_8M		23



static inline pte_t *hugepd_page(hugepd_t hpd)

{

	BUG_ON(!hugepd_ok(hpd));



	return (pte_t *)__va(hpd_val(hpd) & ~HUGEPD_SHIFT_MASK);

}



static inline unsigned int hugepd_shift(hugepd_t hpd)

{

	return PAGE_SHIFT_8M;

}



static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,

				    unsigned int pdshift)

{

	unsigned long idx = (addr & (SZ_4M - 1)) >> PAGE_SHIFT;



	return hugepd_page(hpd) + idx;

}



static inline void flush_hugetlb_page(struct vm_area_struct *vma,

				      unsigned long vmaddr)

{

	flush_tlb_page(vma, vmaddr);

}



static inline void hugepd_populate(hugepd_t *hpdp, pte_t *new, unsigned int pshift)

{

	*hpdp = __hugepd(__pa(new) | _PMD_USER | _PMD_PRESENT | _PMD_PAGE_8M);

}



static inline void hugepd_populate_kernel(hugepd_t *hpdp, pte_t *new, unsigned int pshift)

{

	*hpdp = __hugepd(__pa(new) | _PMD_PRESENT | _PMD_PAGE_8M);

}



static inline int check_and_get_huge_psize(int shift)

{

	return shift_to_mmu_psize(shift);
@@ -49,6 +19,14 @@ static inline int check_and_get_huge_psize(int shift) 
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,

		     pte_t pte, unsigned long sz);



#define __HAVE_ARCH_HUGE_PTEP_GET

static inline pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep)

{

	if (ptep_is_8m_pmdp(mm, addr, ptep))

		ptep = pte_offset_kernel((pmd_t *)ptep, ALIGN_DOWN(addr, SZ_8M));

	return ptep_get(ptep);

}



#define __HAVE_ARCH_HUGE_PTE_CLEAR

static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,

				  pte_t *ptep, unsigned long sz)

arch/powerpc/include/asm/nohash/32/pte-8xx.h

+34 −19
Original line number Diff line number Diff line
@@ -119,7 +119,7 @@ static inline pte_t pte_mkhuge(pte_t pte) 


#define pte_mkhuge pte_mkhuge



static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,

static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep,

				     unsigned long clr, unsigned long set, int huge);



static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
@@ -141,19 +141,12 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *pt 
}

#define __ptep_set_access_flags __ptep_set_access_flags



static inline unsigned long pgd_leaf_size(pgd_t pgd)

{

	if (pgd_val(pgd) & _PMD_PAGE_8M)

		return SZ_8M;

	return SZ_4M;

}



#define pgd_leaf_size pgd_leaf_size



static inline unsigned long pte_leaf_size(pte_t pte)

static inline unsigned long __pte_leaf_size(pmd_t pmd, pte_t pte)

{

	pte_basic_t val = pte_val(pte);



	if (pmd_val(pmd) & _PMD_PAGE_8M)

		return SZ_8M;

	if (val & _PAGE_HUGE)

		return SZ_512K;

	if (val & _PAGE_SPS)
@@ -161,30 +154,37 @@ static inline unsigned long pte_leaf_size(pte_t pte) 
	return SZ_4K;

}



#define pte_leaf_size pte_leaf_size

#define __pte_leaf_size __pte_leaf_size



/*

 * On the 8xx, the page tables are a bit special. For 16k pages, we have

 * 4 identical entries. For 512k pages, we have 128 entries as if it was

 * 4k pages, but they are flagged as 512k pages for the hardware.

 * For other page sizes, we have a single entry in the table.

 * For 8M pages, we have 1024 entries as if it was 4M pages (PMD_SIZE)

 * but they are flagged as 8M pages for the hardware.

 * For 4k pages, we have a single entry in the table.

 */

static pmd_t *pmd_off(struct mm_struct *mm, unsigned long addr);

static int hugepd_ok(hugepd_t hpd);

static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address);



static inline bool ptep_is_8m_pmdp(struct mm_struct *mm, unsigned long addr, pte_t *ptep)

{

	return (pmd_t *)ptep == pmd_off(mm, ALIGN_DOWN(addr, SZ_8M));

}



static inline int number_of_cells_per_pte(pmd_t *pmd, pte_basic_t val, int huge)

{

	if (!huge)

		return PAGE_SIZE / SZ_4K;

	else if (hugepd_ok(*((hugepd_t *)pmd)))

		return 1;

	else if ((pmd_val(*pmd) & _PMD_PAGE_MASK) == _PMD_PAGE_8M)

		return SZ_4M / SZ_4K;

	else if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !(val & _PAGE_HUGE))

		return SZ_16K / SZ_4K;

	else

		return SZ_512K / SZ_4K;

}



static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,

static inline pte_basic_t __pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,

				       unsigned long clr, unsigned long set, int huge)

{

	pte_basic_t *entry = (pte_basic_t *)p;
@@ -197,7 +197,7 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p 


	for (i = 0; i < num; i += PAGE_SIZE / SZ_4K, new += PAGE_SIZE) {

		*entry++ = new;

		if (IS_ENABLED(CONFIG_PPC_16K_PAGES) && num != 1) {

		if (IS_ENABLED(CONFIG_PPC_16K_PAGES)) {

			*entry++ = new;

			*entry++ = new;

			*entry++ = new;
@@ -207,6 +207,21 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p 
	return old;

}



static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep,

				     unsigned long clr, unsigned long set, int huge)

{

	pte_basic_t old;



	if (huge && ptep_is_8m_pmdp(mm, addr, ptep)) {

		pmd_t *pmdp = (pmd_t *)ptep;



		old = __pte_update(mm, addr, pte_offset_kernel(pmdp, 0), clr, set, huge);

		__pte_update(mm, addr, pte_offset_kernel(pmdp + 1, 0), clr, set, huge);

	} else {

		old = __pte_update(mm, addr, ptep, clr, set, huge);

	}

	return old;

}

#define pte_update pte_update



#ifdef CONFIG_PPC_16K_PAGES

arch/powerpc/include/asm/nohash/pgtable.h

+0 −4
Original line number Diff line number Diff line
@@ -343,12 +343,8 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, 
#ifdef CONFIG_ARCH_HAS_HUGEPD

static inline int hugepd_ok(hugepd_t hpd)

{

#ifdef CONFIG_PPC_8xx

	return ((hpd_val(hpd) & _PMD_PAGE_MASK) == _PMD_PAGE_8M);

#else

	/* We clear the top bit to indicate hugepd */

	return (hpd_val(hpd) && (hpd_val(hpd) & PD_HUGE) == 0);

#endif

}



#define is_hugepd(hpd)		(hugepd_ok(hpd))

arch/powerpc/include/asm/page.h

+0 −5
Original line number Diff line number Diff line
@@ -293,13 +293,8 @@ static inline const void *pfn_to_kaddr(unsigned long pfn) 
/*

 * Some number of bits at the level of the page table that points to

 * a hugepte are used to encode the size.  This masks those bits.

 * On 8xx, HW assistance requires 4k alignment for the hugepte.

 */

#ifdef CONFIG_PPC_8xx

#define HUGEPD_SHIFT_MASK     0xfff

#else

#define HUGEPD_SHIFT_MASK     0x3f

#endif



#ifndef __ASSEMBLY__

arch/powerpc/include/asm/pgtable.h

+3 −0
Original line number Diff line number Diff line
@@ -106,6 +106,9 @@ unsigned long vmalloc_to_phys(void *vmalloc_addr); 


void pgtable_cache_add(unsigned int shift);



#ifdef CONFIG_PPC32

void __init *early_alloc_pgtable(unsigned long size);

#endif

pte_t *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va);



#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_PPC32)