Unverified Commit 9e113064 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Improve flush_tlb_range() for hugetlb pages 



flush_tlb_range() uses a fixed stride of PAGE_SIZE and in its current form,
when a hugetlb mapping needs to be flushed, flush_tlb_range() flushes the
whole tlb: so set a stride of the size of the hugetlb mapping in order to
only flush the hugetlb mapping. However, if the hugepage is a NAPOT region,
all PTEs that constitute this mapping must be invalidated, so the stride
size must actually be the size of the PTE.

Note that THPs are directly handled by flush_pmd_tlb_range().

Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: default avatarSamuel Holland <samuel.holland@sifive.com>
Tested-by: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com> # On RZ/Five SMARC
Link: https://lore.kernel.org/r/20231030133027.19542-3-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 114d5c85

arch/riscv/mm/tlbflush.c

+28 −1
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#include <linux/mm.h>

#include <linux/smp.h>

#include <linux/sched.h>

#include <linux/hugetlb.h>

#include <asm/sbi.h>

#include <asm/mmu_context.h>
@@ -147,7 +148,33 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,

		     unsigned long end)

{

	__flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE);

	unsigned long stride_size;



	if (!is_vm_hugetlb_page(vma)) {

		stride_size = PAGE_SIZE;

	} else {

		stride_size = huge_page_size(hstate_vma(vma));



		/*

		 * As stated in the privileged specification, every PTE in a

		 * NAPOT region must be invalidated, so reset the stride in that

		 * case.

		 */

		if (has_svnapot()) {

			if (stride_size >= PGDIR_SIZE)

				stride_size = PGDIR_SIZE;

			else if (stride_size >= P4D_SIZE)

				stride_size = P4D_SIZE;

			else if (stride_size >= PUD_SIZE)

				stride_size = PUD_SIZE;

			else if (stride_size >= PMD_SIZE)

				stride_size = PMD_SIZE;

			else

				stride_size = PAGE_SIZE;

		}

	}



	__flush_tlb_range(vma->vm_mm, start, end - start, stride_size);

}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,