Unverified Commit 01261e24 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Only flush the mm icache when setting an exec pte 



We used to emit a flush_icache_all() whenever a dirty executable
mapping is set in the page table but we can instead call
flush_icache_mm() which will only send IPIs to cores that currently run
this mm and add a deferred icache flush to the others.

The number of calls to sbi_remote_fence_i() (tested without IPI
support):

With a simple buildroot rootfs:
* Before: ~5k
* After :  4 (!)

Tested on HW, the boot to login is ~4.5% faster.

With an ubuntu rootfs:
* Before: ~24k
* After : ~13k

Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: default avatarCharlie Jenkins <charlie@rivosinc.com>
Link: https://lore.kernel.org/r/20240202124711.256146-1-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 28e4748e

arch/riscv/include/asm/pgtable.h

+7 −7
Original line number Diff line number Diff line
@@ -513,12 +513,12 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) 
	WRITE_ONCE(*ptep, pteval);

}



void flush_icache_pte(pte_t pte);

void flush_icache_pte(struct mm_struct *mm, pte_t pte);



static inline void __set_pte_at(pte_t *ptep, pte_t pteval)

static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval)

{

	if (pte_present(pteval) && pte_exec(pteval))

		flush_icache_pte(pteval);

		flush_icache_pte(mm, pteval);



	set_pte(ptep, pteval);

}
@@ -529,7 +529,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, 
	page_table_check_ptes_set(mm, ptep, pteval, nr);



	for (;;) {

		__set_pte_at(ptep, pteval);

		__set_pte_at(mm, ptep, pteval);

		if (--nr == 0)

			break;

		ptep++;
@@ -541,7 +541,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, 
static inline void pte_clear(struct mm_struct *mm,

	unsigned long addr, pte_t *ptep)

{

	__set_pte_at(ptep, __pte(0));

	__set_pte_at(mm, ptep, __pte(0));

}



#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS	/* defined in mm/pgtable.c */
@@ -713,14 +713,14 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 
				pmd_t *pmdp, pmd_t pmd)

{

	page_table_check_pmd_set(mm, pmdp, pmd);

	return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd));

	return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd));

}



static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,

				pud_t *pudp, pud_t pud)

{

	page_table_check_pud_set(mm, pudp, pud);

	return __set_pte_at((pte_t *)pudp, pud_pte(pud));

	return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud));

}



#ifdef CONFIG_PAGE_TABLE_CHECK

arch/riscv/mm/cacheflush.c

+2 −2
Original line number Diff line number Diff line
@@ -82,12 +82,12 @@ void flush_icache_mm(struct mm_struct *mm, bool local) 
#endif /* CONFIG_SMP */



#ifdef CONFIG_MMU

void flush_icache_pte(pte_t pte)

void flush_icache_pte(struct mm_struct *mm, pte_t pte)

{

	struct folio *folio = page_folio(pte_page(pte));



	if (!test_bit(PG_dcache_clean, &folio->flags)) {

		flush_icache_all();

		flush_icache_mm(mm, false);

		set_bit(PG_dcache_clean, &folio->flags);

	}

}

arch/riscv/mm/pgtable.c

+1 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, 
			  pte_t entry, int dirty)

{

	if (!pte_same(ptep_get(ptep), entry))

		__set_pte_at(ptep, entry);

		__set_pte_at(vma->vm_mm, ptep, entry);

	/*

	 * update_mmu_cache will unconditionally execute, handling both

	 * the case that the PTE changed and the spurious fault case.