Unverified Commit c3cc2a4a authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Add support for PUD THP 

Add the necessary page table functions to deal with PUD THP, this
enables the use of PUD pfnmap.

Link: https://lore.kernel.org/r/20250321123954.225097-1-alexghiti@rivosinc.com


Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Signed-off-by: default avatarPalmer Dabbelt <palmer@dabbelt.com>
parent 850d7b14

arch/riscv/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -143,6 +143,7 @@ config RISCV 
	select HAVE_ARCH_THREAD_STRUCT_WHITELIST

	select HAVE_ARCH_TRACEHOOK

	select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT && MMU

	select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if 64BIT && MMU

	select HAVE_ARCH_USERFAULTFD_MINOR if 64BIT && USERFAULTFD

	select HAVE_ARCH_VMAP_STACK if MMU && 64BIT

	select HAVE_ASM_MODVERSIONS

arch/riscv/include/asm/pgtable-64.h

+3 −2
Original line number Diff line number Diff line
@@ -184,7 +184,7 @@ static inline int pud_none(pud_t pud) 


static inline int pud_bad(pud_t pud)

{

	return !pud_present(pud);

	return !pud_present(pud) || (pud_val(pud) & _PAGE_LEAF);

}



#define pud_leaf	pud_leaf
@@ -401,6 +401,7 @@ p4d_t *p4d_offset(pgd_t *pgd, unsigned long address); 
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

static inline int pte_devmap(pte_t pte);

static inline pte_t pmd_pte(pmd_t pmd);

static inline pte_t pud_pte(pud_t pud);



static inline int pmd_devmap(pmd_t pmd)

{
@@ -409,7 +410,7 @@ static inline int pmd_devmap(pmd_t pmd) 


static inline int pud_devmap(pud_t pud)

{

	return 0;

	return pte_devmap(pud_pte(pud));

}



static inline int pgd_devmap(pgd_t pgd)

arch/riscv/include/asm/pgtable.h

+97 −0
Original line number Diff line number Diff line
@@ -902,6 +902,103 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, 
#define pmdp_collapse_flush pmdp_collapse_flush

extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,

				 unsigned long address, pmd_t *pmdp);



static inline pud_t pud_wrprotect(pud_t pud)

{

	return pte_pud(pte_wrprotect(pud_pte(pud)));

}



static inline int pud_trans_huge(pud_t pud)

{

	return pud_leaf(pud);

}



static inline int pud_dirty(pud_t pud)

{

	return pte_dirty(pud_pte(pud));

}



static inline pud_t pud_mkyoung(pud_t pud)

{

	return pte_pud(pte_mkyoung(pud_pte(pud)));

}



static inline pud_t pud_mkold(pud_t pud)

{

	return pte_pud(pte_mkold(pud_pte(pud)));

}



static inline pud_t pud_mkdirty(pud_t pud)

{

	return pte_pud(pte_mkdirty(pud_pte(pud)));

}



static inline pud_t pud_mkclean(pud_t pud)

{

	return pte_pud(pte_mkclean(pud_pte(pud)));

}



static inline pud_t pud_mkwrite(pud_t pud)

{

	return pte_pud(pte_mkwrite_novma(pud_pte(pud)));

}



static inline pud_t pud_mkhuge(pud_t pud)

{

	return pud;

}



static inline pud_t pud_mkdevmap(pud_t pud)

{

	return pte_pud(pte_mkdevmap(pud_pte(pud)));

}



static inline int pudp_set_access_flags(struct vm_area_struct *vma,

					unsigned long address, pud_t *pudp,

					pud_t entry, int dirty)

{

	return ptep_set_access_flags(vma, address, (pte_t *)pudp, pud_pte(entry), dirty);

}



static inline int pudp_test_and_clear_young(struct vm_area_struct *vma,

					    unsigned long address, pud_t *pudp)

{

	return ptep_test_and_clear_young(vma, address, (pte_t *)pudp);

}



static inline int pud_young(pud_t pud)

{

	return pte_young(pud_pte(pud));

}



static inline void update_mmu_cache_pud(struct vm_area_struct *vma,

					unsigned long address, pud_t *pudp)

{

	pte_t *ptep = (pte_t *)pudp;



	update_mmu_cache(vma, address, ptep);

}



static inline pud_t pudp_establish(struct vm_area_struct *vma,

				   unsigned long address, pud_t *pudp, pud_t pud)

{

	page_table_check_pud_set(vma->vm_mm, pudp, pud);

	return __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud)));

}



static inline pud_t pud_mkinvalid(pud_t pud)

{

	return __pud(pud_val(pud) & ~(_PAGE_PRESENT | _PAGE_PROT_NONE));

}



extern pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address,

			     pud_t *pudp);



static inline pud_t pud_modify(pud_t pud, pgprot_t newprot)

{

	return pte_pud(pte_modify(pud_pte(pud), newprot));

}



#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



/*

arch/riscv/include/asm/tlbflush.h

+2 −0
Original line number Diff line number Diff line
@@ -56,6 +56,8 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); 
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE

void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,

			unsigned long end);

void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start,

			 unsigned long end);

#endif



bool arch_tlbbatch_should_defer(struct mm_struct *mm);

arch/riscv/mm/pgtable.c

+10 −0
Original line number Diff line number Diff line
@@ -154,4 +154,14 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 
	flush_tlb_mm(vma->vm_mm);

	return pmd;

}



pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address,

		      pud_t *pudp)

{

	VM_WARN_ON_ONCE(!pud_present(*pudp));

	pud_t old = pudp_establish(vma, address, pudp, pud_mkinvalid(*pudp));



	flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);

	return old;

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */