Commit 7c44202e authored by Christophe Leroy's avatar Christophe Leroy Committed by Andrew Morton
Browse files

powerpc/e500: use contiguous PMD instead of hugepd 

e500 supports many page sizes among which the following size are
implemented in the kernel at the time being: 4M, 16M, 64M, 256M, 1G.

On e500, TLB miss for hugepages is exclusively handled by SW even on e6500
which has HW assistance for 4k pages, so there are no constraints like on
the 8xx.

On e500/32, all are at PGD/PMD level and can be handled as cont-PMD.

On e500/64, smaller ones are on PMD while bigger ones are on PUD.  Again,
they can easily be handled as cont-PMD and cont-PUD instead of hugepd.

On e500/32, use the pagesize bits in PTE to know if it is a PMD or a leaf
entry.  This works because the pagesize bits are in the last 12 bits and
page tables are 4k aligned.

On e500/64, use highest bit which is always 1 on PxD (Because PxD contains
virtual address of a kernel memory) and always 0 on PTEs because not all
bits of RPN are used/possible.

Link: https://lkml.kernel.org/r/dd085987816ed2a0c70adb7e34966cb833fc03e1.1719928057.git.christophe.leroy@csgroup.eu


Signed-off-by: default avatarChristophe Leroy <christophe.leroy@csgroup.eu>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent dc0aa538

arch/powerpc/include/asm/nohash/hugetlb-e500.h

+3 −29
Original line number Diff line number Diff line
@@ -2,38 +2,12 @@ 
#ifndef _ASM_POWERPC_NOHASH_HUGETLB_E500_H

#define _ASM_POWERPC_NOHASH_HUGETLB_E500_H



static inline pte_t *hugepd_page(hugepd_t hpd)

{

	if (WARN_ON(!hugepd_ok(hpd)))

		return NULL;



	return (pte_t *)((hpd_val(hpd) & ~HUGEPD_SHIFT_MASK) | PD_HUGE);

}



static inline unsigned int hugepd_shift(hugepd_t hpd)

{

	return hpd_val(hpd) & HUGEPD_SHIFT_MASK;

}



static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,

				    unsigned int pdshift)

{

	/*

	 * On FSL BookE, we have multiple higher-level table entries that

	 * point to the same hugepte.  Just use the first one since they're all

	 * identical.  So for that case, idx=0.

	 */

	return hugepd_page(hpd);

}

#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,

		     pte_t pte, unsigned long sz);



void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);



static inline void hugepd_populate(hugepd_t *hpdp, pte_t *new, unsigned int pshift)

{

	/* We use the old format for PPC_E500 */

	*hpdp = __hugepd(((unsigned long)new & ~PD_HUGE) | pshift);

}



static inline int check_and_get_huge_psize(int shift)

{

	if (shift & 1)	/* Not a power of 4 */

arch/powerpc/include/asm/nohash/pgalloc.h

+0 −2
Original line number Diff line number Diff line
@@ -44,8 +44,6 @@ static inline void pgtable_free(void *table, int shift) 
	}

}



#define get_hugepd_cache_index(x)	(x)



static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)

{

	unsigned long pgf = (unsigned long)table;

arch/powerpc/include/asm/nohash/pgtable.h

+31 −11
Original line number Diff line number Diff line
@@ -31,6 +31,13 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p 


extern int icache_44x_need_flush;



#ifndef pte_huge_size

static inline unsigned long pte_huge_size(pte_t pte)

{

	return PAGE_SIZE;

}

#endif



/*

 * PTE updates. This function is called whenever an existing

 * valid PTE is updated. This does -not- include set_pte_at()
@@ -52,11 +59,34 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p 
{

	pte_basic_t old = pte_val(*p);

	pte_basic_t new = (old & ~(pte_basic_t)clr) | set;

	unsigned long sz;

	unsigned long pdsize;

	int i;



	if (new == old)

		return old;



	if (huge)

		sz = pte_huge_size(__pte(old));

	else

		sz = PAGE_SIZE;



	if (sz < PMD_SIZE)

		pdsize = PAGE_SIZE;

	else if (sz < PUD_SIZE)

		pdsize = PMD_SIZE;

	else if (sz < P4D_SIZE)

		pdsize = PUD_SIZE;

	else if (sz < PGDIR_SIZE)

		pdsize = P4D_SIZE;

	else

		pdsize = PGDIR_SIZE;



	for (i = 0; i < sz / pdsize; i++, p++) {

		*p = __pte(new);

		if (new)

			new += (unsigned long long)(pdsize / PAGE_SIZE) << PTE_RPN_SHIFT;

	}



	if (IS_ENABLED(CONFIG_44x) && !is_kernel_addr(addr) && (old & _PAGE_EXEC))

		icache_44x_need_flush = 1;
@@ -340,16 +370,6 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, 


#define pgprot_writecombine pgprot_noncached_wc



#ifdef CONFIG_ARCH_HAS_HUGEPD

static inline int hugepd_ok(hugepd_t hpd)

{

	/* We clear the top bit to indicate hugepd */

	return (hpd_val(hpd) && (hpd_val(hpd) & PD_HUGE) == 0);

}



#define is_hugepd(hpd)		(hugepd_ok(hpd))

#endif



int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot);

void unmap_kernel_page(unsigned long va);

arch/powerpc/include/asm/nohash/pte-e500.h

+33 −0
Original line number Diff line number Diff line
@@ -101,6 +101,39 @@ static inline unsigned long pte_huge_size(pte_t pte) 
}

#define pte_huge_size pte_huge_size



static inline int pmd_leaf(pmd_t pmd)

{

	if (IS_ENABLED(CONFIG_PPC64))

		return (long)pmd_val(pmd) > 0;

	else

		return pmd_val(pmd) & _PAGE_PSIZE_MSK;

}

#define pmd_leaf pmd_leaf



static inline unsigned long pmd_leaf_size(pmd_t pmd)

{

	return pte_huge_size(__pte(pmd_val(pmd)));

}

#define pmd_leaf_size pmd_leaf_size



#ifdef CONFIG_PPC64

static inline int pud_leaf(pud_t pud)

{

	if (IS_ENABLED(CONFIG_PPC64))

		return (long)pud_val(pud) > 0;

	else

		return pud_val(pud) & _PAGE_PSIZE_MSK;

}

#define pud_leaf pud_leaf



static inline unsigned long pud_leaf_size(pud_t pud)

{

	return pte_huge_size(__pte(pud_val(pud)));

}

#define pud_leaf_size pud_leaf_size



#endif



#endif /* __ASSEMBLY__ */



#endif /* __KERNEL__ */

arch/powerpc/include/asm/page.h

+1 −14
Original line number Diff line number Diff line
@@ -269,20 +269,7 @@ static inline const void *pfn_to_kaddr(unsigned long pfn) 
#define is_kernel_addr(x)	((x) >= TASK_SIZE)

#endif



#ifndef CONFIG_PPC_BOOK3S_64

/*

 * Use the top bit of the higher-level page table entries to indicate whether

 * the entries we point to contain hugepages.  This works because we know that

 * the page tables live in kernel space.  If we ever decide to support having

 * page tables at arbitrary addresses, this breaks and will have to change.

 */

#ifdef CONFIG_PPC64

#define PD_HUGE 0x8000000000000000UL

#else

#define PD_HUGE 0x80000000

#endif



#else	/* CONFIG_PPC_BOOK3S_64 */

#ifdef CONFIG_PPC_BOOK3S_64

/*

 * Book3S 64 stores real addresses in the hugepd entries to

 * avoid overlaps with _PAGE_PRESENT and _PAGE_PTE.