Commit 15397e3c authored by Ryan Roberts's avatar Ryan Roberts Committed by Catalin Marinas
Browse files

arm64: mm: Wrap flush_tlb_page() around __do_flush_tlb_range() 



Flushing a page from the tlb is just a special case of flushing a range.
So let's rework flush_tlb_page() so that it simply wraps
__do_flush_tlb_range(). While at it, let's also update the API to take
the same flags that we use when flushing a range. This allows us to
delete all the ugly "_nosync", "_local" and "_nonotify" variants.

Thanks to constant folding, all of the complex looping and tlbi-by-range
options get eliminated so that the generated code for flush_tlb_page()
looks very similar to the previous version.

Reviewed-by: default avatarLinu Cherian <linu.cherian@arm.com>
Signed-off-by: default avatarRyan Roberts <ryan.roberts@arm.com>
Reviewed-by: default avatarJonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 0477fc56

arch/arm64/include/asm/pgtable.h

+3 −3
Original line number Diff line number Diff line
@@ -101,10 +101,10 @@ static inline void arch_leave_lazy_mmu_mode(void) 
 * entries exist.

 */

#define flush_tlb_fix_spurious_fault(vma, address, ptep)	\

	local_flush_tlb_page_nonotify(vma, address)

	__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)



#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp)	\

	local_flush_tlb_page_nonotify(vma, address)

	__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)



/*

 * ZERO_PAGE is a global shared page that is always zero: used
@@ -1320,7 +1320,7 @@ static inline int __ptep_clear_flush_young(struct vm_area_struct *vma, 
		 * context-switch, which provides a DSB to complete the TLB

		 * invalidation.

		 */

		flush_tlb_page_nosync(vma, address);

		__flush_tlb_page(vma, address, TLBF_NOSYNC);

	}



	return young;

arch/arm64/include/asm/tlbflush.h

+25 −56
Original line number Diff line number Diff line
@@ -269,10 +269,7 @@ static inline void __tlbi_sync_s1ish_hyp(void) 
 *		unmapping pages from vmalloc/io space.

 *

 *	flush_tlb_page(vma, addr)

 *		Invalidate a single user mapping for address 'addr' in the

 *		address space corresponding to 'vma->mm'.  Note that this

 *		operation only invalidates a single, last-level page-table

 *		entry and therefore does not affect any walk-caches.

 *		Equivalent to __flush_tlb_page(..., flags=TLBF_NONE)

 *

 *

 *	Next, we have some undocumented invalidation routines that you probably
@@ -300,13 +297,14 @@ static inline void __tlbi_sync_s1ish_hyp(void) 
 *		TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST

 *		(only perform the invalidation for the local cpu).

 *

 *	local_flush_tlb_page(vma, addr)

 *		Local variant of flush_tlb_page().  Stale TLB entries may

 *		remain in remote CPUs.

 *

 *	local_flush_tlb_page_nonotify(vma, addr)

 *		Same as local_flush_tlb_page() except MMU notifier will not be

 *		called.

 *	__flush_tlb_page(vma, addr, flags)

 *		Invalidate a single user mapping for address 'addr' in the

 *		address space corresponding to 'vma->mm'.  Note that this

 *		operation only invalidates a single, last-level page-table entry

 *		and therefore does not affect any walk-caches. flags may contain

 *		any combination of TLBF_NONOTIFY (don't call mmu notifiers),

 *		TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST

 *		(only perform the invalidation for the local cpu).

 *

 *	Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented

 *	on top of these routines, since that is our interface to the mmu_gather
@@ -340,51 +338,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm) 
	mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);

}



static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct *mm,

							  unsigned long uaddr)

{

	dsb(nshst);

	__tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm));

}



static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vma,

						 unsigned long uaddr)

{

	__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);

	dsb(nsh);

}



static inline void local_flush_tlb_page(struct vm_area_struct *vma,

					unsigned long uaddr)

{

	__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);

	mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK,

						(uaddr & PAGE_MASK) + PAGE_SIZE);

	dsb(nsh);

}



static inline void __flush_tlb_page_nosync(struct mm_struct *mm,

					   unsigned long uaddr)

{

	dsb(ishst);

	__tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm));

	mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,

						(uaddr & PAGE_MASK) + PAGE_SIZE);

}



static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,

					 unsigned long uaddr)

{

	return __flush_tlb_page_nosync(vma->vm_mm, uaddr);

}



static inline void flush_tlb_page(struct vm_area_struct *vma,

				  unsigned long uaddr)

{

	flush_tlb_page_nosync(vma, uaddr);

	__tlbi_sync_s1ish();

}



static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)

{

	return true;
@@ -632,6 +585,22 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, 
	__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);

}



static inline void __flush_tlb_page(struct vm_area_struct *vma,

				    unsigned long uaddr, tlbf_t flags)

{

	unsigned long start = round_down(uaddr, PAGE_SIZE);

	unsigned long end = start + PAGE_SIZE;



	__do_flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN,

			     TLBF_NOWALKCACHE | flags);

}



static inline void flush_tlb_page(struct vm_area_struct *vma,

				  unsigned long uaddr)

{

	__flush_tlb_page(vma, uaddr, TLBF_NONE);

}



static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)

{

	const unsigned long stride = PAGE_SIZE;

arch/arm64/mm/fault.c

+1 −1
Original line number Diff line number Diff line
@@ -239,7 +239,7 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, 
	 * flush_tlb_fix_spurious_fault().

	 */

	if (dirty)

		local_flush_tlb_page(vma, address);

		__flush_tlb_page(vma, address, TLBF_NOBROADCAST);

	return 1;

}