Commit 82378c6c authored by Rik van Riel's avatar Rik van Riel Committed by Ingo Molnar
Browse files

x86/mm: Use INVLPGB for kernel TLB flushes 



Use broadcast TLB invalidation for kernel addresses when available.
Remove the need to send IPIs for kernel TLB flushes.

   [ bp: Integrate dhansen's comments additions, merge the
     flush_tlb_all() change into this one too. ]

Signed-off-by: default avatarRik van Riel <riel@surriel.com>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20250226030129.530345-5-riel@surriel.com
parent b7aa05cb

arch/x86/mm/tlb.c

+44 −4
Original line number Diff line number Diff line
@@ -1064,7 +1064,6 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, 
	mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);

}





static void do_flush_tlb_all(void *info)

{

	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
@@ -1074,9 +1073,34 @@ static void do_flush_tlb_all(void *info) 
void flush_tlb_all(void)

{

	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);



	/* First try (faster) hardware-assisted TLB invalidation. */

	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))

		invlpgb_flush_all();

	else

		/* Fall back to the IPI-based invalidation. */

		on_each_cpu(do_flush_tlb_all, NULL, 1);

}



/* Flush an arbitrarily large range of memory with INVLPGB. */

static void invlpgb_kernel_range_flush(struct flush_tlb_info *info)

{

	unsigned long addr, nr;



	for (addr = info->start; addr < info->end; addr += nr << PAGE_SHIFT) {

		nr = (info->end - addr) >> PAGE_SHIFT;



		/*

		 * INVLPGB has a limit on the size of ranges it can

		 * flush. Break up large flushes.

		 */

		nr = clamp_val(nr, 1, invlpgb_count_max);



		invlpgb_flush_addr_nosync(addr, nr);

	}

	__tlbsync();

}



static void do_kernel_range_flush(void *info)

{

	struct flush_tlb_info *f = info;
@@ -1087,6 +1111,22 @@ static void do_kernel_range_flush(void *info) 
		flush_tlb_one_kernel(addr);

}



static void kernel_tlb_flush_all(struct flush_tlb_info *info)

{

	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))

		invlpgb_flush_all();

	else

		on_each_cpu(do_flush_tlb_all, NULL, 1);

}



static void kernel_tlb_flush_range(struct flush_tlb_info *info)

{

	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))

		invlpgb_kernel_range_flush(info);

	else

		on_each_cpu(do_kernel_range_flush, info, 1);

}



void flush_tlb_kernel_range(unsigned long start, unsigned long end)

{

	struct flush_tlb_info *info;
@@ -1097,9 +1137,9 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) 
				  TLB_GENERATION_INVALID);



	if (info->end == TLB_FLUSH_ALL)

		on_each_cpu(do_flush_tlb_all, NULL, 1);

		kernel_tlb_flush_all(info);

	else

		on_each_cpu(do_kernel_range_flush, info, 1);

		kernel_tlb_flush_range(info);



	put_flush_tlb_info();

}