Commit 57fb15c3 authored by Christophe Leroy's avatar Christophe Leroy Committed by Andrew Morton
Browse files

powerpc/64s: use contiguous PMD/PUD instead of HUGEPD 

On book3s/64, the only user of hugepd is hash in 4k mode.

All other setups (hash-64, radix-4, radix-64) use leaf PMD/PUD.

Rework hash-4k to use contiguous PMD and PUD instead.

In that setup there are only two huge page sizes: 16M and 16G.

16M sits at PMD level and 16G at PUD level.

pte_update doesn't know page size, lets use the same trick as
hpte_need_flush() to get page size from segment properties.  That's not
the most efficient way but let's do that until callers of pte_update()
provide page size instead of just a huge flag.

Link: https://lkml.kernel.org/r/7448f60a9b3efd396595f4f735d1e0babc5ae379.1719928057.git.christophe.leroy@csgroup.eu


Signed-off-by: default avatarChristophe Leroy <christophe.leroy@csgroup.eu>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7c44202e

arch/powerpc/include/asm/book3s/64/hash-4k.h

+0 −15
Original line number Diff line number Diff line
@@ -74,21 +74,6 @@ 
#define remap_4k_pfn(vma, addr, pfn, prot)	\

	remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot))



#ifdef CONFIG_HUGETLB_PAGE

static inline int hash__hugepd_ok(hugepd_t hpd)

{

	unsigned long hpdval = hpd_val(hpd);

	/*

	 * if it is not a pte and have hugepd shift mask

	 * set, then it is a hugepd directory pointer

	 */

	if (!(hpdval & _PAGE_PTE) && (hpdval & _PAGE_PRESENT) &&

	    ((hpdval & HUGEPD_SHIFT_MASK) != 0))

		return true;

	return false;

}

#endif



/*

 * 4K PTE format is different from 64K PTE format. Saving the hash_slot is just

 * a matter of returning the PTE bits that need to be modified. On 64K PTE,

arch/powerpc/include/asm/book3s/64/hash.h

+33 −7
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#ifdef __KERNEL__



#include <asm/asm-const.h>

#include <asm/book3s/64/slice.h>



/*

 * Common bits between 4K and 64K pages in a linux-style PTE.
@@ -161,14 +162,10 @@ extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr, 
			    pte_t *ptep, unsigned long pte, int huge);

unsigned long htab_convert_pte_flags(unsigned long pteflags, unsigned long flags);

/* Atomic PTE updates */

static inline unsigned long hash__pte_update(struct mm_struct *mm,

					 unsigned long addr,

					 pte_t *ptep, unsigned long clr,

					 unsigned long set,

					 int huge)

static inline unsigned long hash__pte_update_one(pte_t *ptep, unsigned long clr,

						 unsigned long set)

{

	__be64 old_be, tmp_be;

	unsigned long old;



	__asm__ __volatile__(

	"1:	ldarx	%0,0,%3		# pte_update\n\
@@ -182,11 +179,40 @@ static inline unsigned long hash__pte_update(struct mm_struct *mm, 
	: "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep),

	  "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))

	: "cc" );



	return be64_to_cpu(old_be);

}



static inline unsigned long hash__pte_update(struct mm_struct *mm,

					 unsigned long addr,

					 pte_t *ptep, unsigned long clr,

					 unsigned long set,

					 int huge)

{

	unsigned long old;



	old = hash__pte_update_one(ptep, clr, set);



	if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && huge) {

		unsigned int psize = get_slice_psize(mm, addr);

		int nb, i;



		if (psize == MMU_PAGE_16M)

			nb = SZ_16M / PMD_SIZE;

		else if (psize == MMU_PAGE_16G)

			nb = SZ_16G / PUD_SIZE;

		else

			nb = 1;



		WARN_ON_ONCE(nb == 1);	/* Should never happen */



		for (i = 1; i < nb; i++)

			hash__pte_update_one(ptep + i, clr, set);

	}

	/* huge pages use the old page table lock */

	if (!huge)

		assert_pte_locked(mm, addr);



	old = be64_to_cpu(old_be);

	if (old & H_PAGE_HASHPTE)

		hpte_need_flush(mm, addr, ptep, old, huge);

arch/powerpc/include/asm/book3s/64/hugetlb.h

+0 −38
Original line number Diff line number Diff line
@@ -49,9 +49,6 @@ static inline bool gigantic_page_runtime_supported(void) 
	return true;

}



/* hugepd entry valid bit */

#define HUGEPD_VAL_BITS		(0x8000000000000000UL)



#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start

extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,

					 unsigned long addr, pte_t *ptep);
@@ -60,29 +57,7 @@ extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, 
extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,

					 unsigned long addr, pte_t *ptep,

					 pte_t old_pte, pte_t new_pte);

/*

 * This should work for other subarchs too. But right now we use the

 * new format only for 64bit book3s

 */

static inline pte_t *hugepd_page(hugepd_t hpd)

{

	BUG_ON(!hugepd_ok(hpd));

	/*

	 * We have only four bits to encode, MMU page size

	 */

	BUILD_BUG_ON((MMU_PAGE_COUNT - 1) > 0xf);

	return __va(hpd_val(hpd) & HUGEPD_ADDR_MASK);

}



static inline unsigned int hugepd_mmu_psize(hugepd_t hpd)

{

	return (hpd_val(hpd) & HUGEPD_SHIFT_MASK) >> 2;

}



static inline unsigned int hugepd_shift(hugepd_t hpd)

{

	return mmu_psize_to_shift(hugepd_mmu_psize(hpd));

}

static inline void flush_hugetlb_page(struct vm_area_struct *vma,

				      unsigned long vmaddr)

{
@@ -90,19 +65,6 @@ static inline void flush_hugetlb_page(struct vm_area_struct *vma, 
		return radix__flush_hugetlb_page(vma, vmaddr);

}



static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,

				    unsigned int pdshift)

{

	unsigned long idx = (addr & ((1UL << pdshift) - 1)) >> hugepd_shift(hpd);



	return hugepd_page(hpd) + idx;

}



static inline void hugepd_populate(hugepd_t *hpdp, pte_t *new, unsigned int pshift)

{

	*hpdp = __hugepd(__pa(new) | HUGEPD_VAL_BITS | (shift_to_mmu_psize(pshift) << 2));

}



void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);



static inline int check_and_get_huge_psize(int shift)

arch/powerpc/include/asm/book3s/64/pgtable-4k.h

deleted100644 → 0
+0 −47
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H

#define _ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H

/*

 * hash 4k can't share hugetlb and also doesn't support THP

 */

#ifndef __ASSEMBLY__

#ifdef CONFIG_HUGETLB_PAGE

/*

 * With radix , we have hugepage ptes in the pud and pmd entries. We don't

 * need to setup hugepage directory for them. Our pte and page directory format

 * enable us to have this enabled.

 */

static inline int hugepd_ok(hugepd_t hpd)

{

	if (radix_enabled())

		return 0;

	return hash__hugepd_ok(hpd);

}

#define is_hugepd(hpd)		(hugepd_ok(hpd))



/*

 * 16M and 16G huge page directory tables are allocated from slab cache

 *

 */

#define H_16M_CACHE_INDEX (PAGE_SHIFT + H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE - 24)

#define H_16G_CACHE_INDEX                                                      \

	(PAGE_SHIFT + H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + H_PUD_INDEX_SIZE - 34)



static inline int get_hugepd_cache_index(int index)

{

	switch (index) {

	case H_16M_CACHE_INDEX:

		return HTLB_16M_INDEX;

	case H_16G_CACHE_INDEX:

		return HTLB_16G_INDEX;

	default:

		BUG();

	}

	/* should not reach */

}



#endif /* CONFIG_HUGETLB_PAGE */



#endif /* __ASSEMBLY__ */



#endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H */

arch/powerpc/include/asm/book3s/64/pgtable-64k.h

+0 −20
Original line number Diff line number Diff line
@@ -5,26 +5,6 @@ 
#ifndef __ASSEMBLY__

#ifdef CONFIG_HUGETLB_PAGE



/*

 * With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't

 * need to setup hugepage directory for them. Our pte and page directory format

 * enable us to have this enabled.

 */

static inline int hugepd_ok(hugepd_t hpd)

{

	return 0;

}



#define is_hugepd(pdep)			0



/*

 * This should never get called

 */

static __always_inline int get_hugepd_cache_index(int index)

{

	BUILD_BUG();

}



#endif /* CONFIG_HUGETLB_PAGE */



static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,