Commit 4b9ef32c authored by Xie Yuanbin's avatar Xie Yuanbin Committed by Ingo Molnar
Browse files

x86/mm/tlb: Make enter_lazy_tlb() always inline on x86 



enter_lazy_tlb() on x86 is short enough, and is called in context
switching, which is the hot code path.

Make enter_lazy_tlb() always inline on x86 to optimize performance.

Suggested-by: default avatarDave Hansen <dave.hansen@intel.com>
Signed-off-by: default avatarXie Yuanbin <qq570070308@gmail.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://patch.msgid.link/20260216164950.147617-2-qq570070308@gmail.com
parent 12f80691

arch/x86/include/asm/mmu_context.h

+0 −3
Original line number Diff line number Diff line
@@ -136,9 +136,6 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm) 
}

#endif



#define enter_lazy_tlb enter_lazy_tlb

extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);



extern void mm_init_global_asid(struct mm_struct *mm);

extern void mm_free_global_asid(struct mm_struct *mm);

arch/x86/include/asm/tlbflush.h

+26 −0
Original line number Diff line number Diff line
@@ -172,6 +172,28 @@ struct tlb_state_shared { 
};

DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state_shared, cpu_tlbstate_shared);



/*

 * Please ignore the name of this function.  It should be called

 * switch_to_kernel_thread().

 *

 * enter_lazy_tlb() is a hint from the scheduler that we are entering a

 * kernel thread or other context without an mm.  Acceptable implementations

 * include doing nothing whatsoever, switching to init_mm, or various clever

 * lazy tricks to try to minimize TLB flushes.

 *

 * The scheduler reserves the right to call enter_lazy_tlb() several times

 * in a row.  It will notify us that we're going back to a real mm by

 * calling switch_mm_irqs_off().

 */

#define enter_lazy_tlb enter_lazy_tlb

static __always_inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)

{

	if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm)

		return;



	this_cpu_write(cpu_tlbstate_shared.is_lazy, true);

}



bool nmi_uaccess_okay(void);

#define nmi_uaccess_okay nmi_uaccess_okay
@@ -480,6 +502,10 @@ static inline void cpu_tlbstate_update_lam(unsigned long lam, u64 untag_mask) 
{

}

#endif

#else /* !MODULE */

#define enter_lazy_tlb enter_lazy_tlb

extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)

	__compiletime_error("enter_lazy_tlb() should not be used in modules");

#endif /* !MODULE */



static inline void __native_tlb_flush_global(unsigned long cr4)

arch/x86/mm/tlb.c

+0 −21
Original line number Diff line number Diff line
@@ -971,27 +971,6 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, 
	}

}



/*

 * Please ignore the name of this function.  It should be called

 * switch_to_kernel_thread().

 *

 * enter_lazy_tlb() is a hint from the scheduler that we are entering a

 * kernel thread or other context without an mm.  Acceptable implementations

 * include doing nothing whatsoever, switching to init_mm, or various clever

 * lazy tricks to try to minimize TLB flushes.

 *

 * The scheduler reserves the right to call enter_lazy_tlb() several times

 * in a row.  It will notify us that we're going back to a real mm by

 * calling switch_mm_irqs_off().

 */

void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)

{

	if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm)

		return;



	this_cpu_write(cpu_tlbstate_shared.is_lazy, true);

}



/*

 * Using a temporary mm allows to set temporary mappings that are not accessible

 * by other CPUs. Such mappings are needed to perform sensitive memory writes