Commit 11f6dd8d authored by Ryan Roberts's avatar Ryan Roberts Committed by Catalin Marinas
Browse files

arm64: mm: Refactor __flush_tlb_range() to take flags 



We have function variants with "_nosync", "_local", "_nonotify" as well
as the "last_level" parameter. Let's generalize and simplify by using a
flags parameter to encode all these variants.

As a first step, convert the "last_level" boolean parameter to a flags
parameter and create the first flag, TLBF_NOWALKCACHE. When present,
walk cache entries are not evicted, which is the same as the old
last_level=true.

Reviewed-by: default avatarLinu Cherian <linu.cherian@arm.com>
Reviewed-by: default avatarJonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: default avatarRyan Roberts <ryan.roberts@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 64212d68

arch/arm64/include/asm/hugetlb.h

+6 −6
Original line number Diff line number Diff line
@@ -71,23 +71,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_area_struct *vma, 
					     unsigned long start,

					     unsigned long end,

					     unsigned long stride,

					     bool last_level)

					     tlbf_t flags)

{

	switch (stride) {

#ifndef __PAGETABLE_PMD_FOLDED

	case PUD_SIZE:

		__flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);

		__flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags);

		break;

#endif

	case CONT_PMD_SIZE:

	case PMD_SIZE:

		__flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);

		__flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags);

		break;

	case CONT_PTE_SIZE:

		__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);

		__flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags);

		break;

	default:

		__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);

		__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags);

	}

}
@@ -98,7 +98,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, 
{

	unsigned long stride = huge_page_size(hstate_vma(vma));



	__flush_hugetlb_tlb_range(vma, start, end, stride, false);

	__flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE);

}



#endif /* __ASM_HUGETLB_H */

arch/arm64/include/asm/pgtable.h

+2 −2
Original line number Diff line number Diff line
@@ -89,9 +89,9 @@ static inline void arch_leave_lazy_mmu_mode(void) 


/* Set stride and tlb_level in flush_*_tlb_range */

#define flush_pmd_tlb_range(vma, addr, end)	\

	__flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)

	__flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)

#define flush_pud_tlb_range(vma, addr, end)	\

	__flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)

	__flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



/*

arch/arm64/include/asm/tlb.h

+3 −3
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ static inline int tlb_get_level(struct mmu_gather *tlb) 
static inline void tlb_flush(struct mmu_gather *tlb)

{

	struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);

	bool last_level = !tlb->freed_tables;

	tlbf_t flags = tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE;

	unsigned long stride = tlb_get_unmap_size(tlb);

	int tlb_level = tlb_get_level(tlb);
@@ -63,13 +63,13 @@ static inline void tlb_flush(struct mmu_gather *tlb) 
	 * reallocate our ASID without invalidating the entire TLB.

	 */

	if (tlb->fullmm) {

		if (!last_level)

		if (tlb->freed_tables)

			flush_tlb_mm(tlb->mm);

		return;

	}



	__flush_tlb_range(&vma, tlb->start, tlb->end, stride,

			  last_level, tlb_level);

			  tlb_level, flags);

}



static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,

arch/arm64/include/asm/tlbflush.h

+20 −12
Original line number Diff line number Diff line
@@ -286,16 +286,16 @@ static inline void __tlbi_sync_s1ish_hyp(void) 
 *		CPUs, ensuring that any walk-cache entries associated with the

 *		translation are also invalidated.

 *

 *	__flush_tlb_range(vma, start, end, stride, last_level, tlb_level)

 *	__flush_tlb_range(vma, start, end, stride, tlb_level, flags)

 *		Invalidate the virtual-address range '[start, end)' on all

 *		CPUs for the user address space corresponding to 'vma->mm'.

 *		The invalidation operations are issued at a granularity

 *		determined by 'stride' and only affect any walk-cache entries

 *		if 'last_level' is equal to false. tlb_level is the level at

 *		determined by 'stride'. tlb_level is the level at

 *		which the invalidation must take place. If the level is wrong,

 *		no invalidation may take place. In the case where the level

 *		cannot be easily determined, the value TLBI_TTL_UNKNOWN will

 *		perform a non-hinted invalidation.

 *		perform a non-hinted invalidation. flags may be TLBF_NONE (0) or

 *		TLBF_NOWALKCACHE (elide eviction of walk cache entries).

 *

 *	local_flush_tlb_page(vma, addr)

 *		Local variant of flush_tlb_page().  Stale TLB entries may
@@ -544,10 +544,18 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages, 
	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;

}



typedef unsigned __bitwise tlbf_t;



/* No special behaviour. */

#define TLBF_NONE		((__force tlbf_t)0)



/* Invalidate tlb entries only, leaving the page table walk cache intact. */

#define TLBF_NOWALKCACHE	((__force tlbf_t)BIT(0))



static inline void __flush_tlb_range_nosync(struct mm_struct *mm,

				     unsigned long start, unsigned long end,

				     unsigned long stride, bool last_level,

				     int tlb_level)

				     unsigned long stride, int tlb_level,

				     tlbf_t flags)

{

	unsigned long asid, pages;
@@ -563,7 +571,7 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm, 
	dsb(ishst);

	asid = ASID(mm);



	if (last_level)

	if (flags & TLBF_NOWALKCACHE)

		__flush_s1_tlb_range_op(vale1is, start, pages, stride,

				     asid, tlb_level);

	else
@@ -575,11 +583,11 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm, 


static inline void __flush_tlb_range(struct vm_area_struct *vma,

				     unsigned long start, unsigned long end,

				     unsigned long stride, bool last_level,

				     int tlb_level)

				     unsigned long stride, int tlb_level,

				     tlbf_t flags)

{

	__flush_tlb_range_nosync(vma->vm_mm, start, end, stride,

				 last_level, tlb_level);

				 tlb_level, flags);

	__tlbi_sync_s1ish();

}
@@ -607,7 +615,7 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, 
	 * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough

	 * information here.

	 */

	__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);

	__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);

}



static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -648,7 +656,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) 
static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,

		struct mm_struct *mm, unsigned long start, unsigned long end)

{

	__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);

	__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE);

}



static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)

arch/arm64/mm/contpte.c

+3 −2
Original line number Diff line number Diff line
@@ -225,7 +225,8 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, 
	 */



	if (!system_supports_bbml2_noabort())

		__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3);

		__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3,

				  TLBF_NOWALKCACHE);



	__set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES);

}
@@ -552,7 +553,7 @@ int contpte_clear_flush_young_ptes(struct vm_area_struct *vma, 
		 * eliding the trailing DSB applies here.

		 */

		__flush_tlb_range_nosync(vma->vm_mm, addr, end,

					 PAGE_SIZE, true, 3);

					 PAGE_SIZE, 3, TLBF_NOWALKCACHE);

	}



	return young;