Commit f7edb07a authored by Piotr Jaroszynski's avatar Piotr Jaroszynski Committed by Will Deacon
Browse files

Fix mmu notifiers for range-based invalidates 



Update the __flush_tlb_range_op macro not to modify its parameters as
these are unexepcted semantics. In practice, this fixes the call to
mmu_notifier_arch_invalidate_secondary_tlbs() in
__flush_tlb_range_nosync() to use the correct range instead of an empty
range with start=end. The empty range was (un)lucky as it results in
taking the invalidate-all path that doesn't cause correctness issues,
but can certainly result in suboptimal perf.

This has been broken since commit 6bbd42e2 ("mmu_notifiers: call
invalidate_range() when invalidating TLBs") when the call to the
notifiers was added to __flush_tlb_range(). It predates the addition of
the __flush_tlb_range_op() macro from commit 36083902 ("arm64: tlb:
Refactor the core flush algorithm of __flush_tlb_range") that made the
bug hard to spot.

Fixes: 6bbd42e2 ("mmu_notifiers: call invalidate_range() when invalidating TLBs")

Signed-off-by: default avatarPiotr Jaroszynski <pjaroszynski@nvidia.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Raghavendra Rao Ananta <rananta@google.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Nicolin Chen <nicolinc@nvidia.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: iommu@lists.linux.dev
Cc: linux-mm@kvack.org
Cc: linux-kernel@vger.kernel.org
Cc: stable@vger.kernel.org
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Reviewed-by: default avatarAlistair Popple <apopple@nvidia.com>
Link: https://lore.kernel.org/r/20250304085127.2238030-1-pjaroszynski@nvidia.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent d4234d13

arch/arm64/include/asm/tlbflush.h

+12 −10
Original line number Diff line number Diff line
@@ -396,33 +396,35 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) 
#define __flush_tlb_range_op(op, start, pages, stride,			\

				asid, tlb_level, tlbi_user, lpa2)	\

do {									\

	typeof(start) __flush_start = start;				\

	typeof(pages) __flush_pages = pages;				\

	int num = 0;							\

	int scale = 3;							\

	int shift = lpa2 ? 16 : PAGE_SHIFT;				\

	unsigned long addr;						\

									\

	while (pages > 0) {						\

	while (__flush_pages > 0) {					\

		if (!system_supports_tlb_range() ||			\

		    pages == 1 ||					\

		    (lpa2 && start != ALIGN(start, SZ_64K))) {		\

			addr = __TLBI_VADDR(start, asid);		\

		    __flush_pages == 1 ||				\

		    (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) {	\

			addr = __TLBI_VADDR(__flush_start, asid);	\

			__tlbi_level(op, addr, tlb_level);		\

			if (tlbi_user)					\

				__tlbi_user_level(op, addr, tlb_level);	\

			start += stride;				\

			pages -= stride >> PAGE_SHIFT;			\

			__flush_start += stride;			\

			__flush_pages -= stride >> PAGE_SHIFT;		\

			continue;					\

		}							\

									\

		num = __TLBI_RANGE_NUM(pages, scale);			\

		num = __TLBI_RANGE_NUM(__flush_pages, scale);		\

		if (num >= 0) {						\

			addr = __TLBI_VADDR_RANGE(start >> shift, asid, \

			addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \

						scale, num, tlb_level);	\

			__tlbi(r##op, addr);				\

			if (tlbi_user)					\

				__tlbi_user(r##op, addr);		\

			start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \

			pages -= __TLBI_RANGE_PAGES(num, scale);	\

			__flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \

			__flush_pages -= __TLBI_RANGE_PAGES(num, scale);\

		}							\

		scale--;						\

	}								\