Commit 26baa1f1 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Add TIF_NEED_RESCHED_LAZY infrastructure 



Add the basic infrastructure to split the TIF_NEED_RESCHED bit in two.
Either bit will cause a resched on return-to-user, but only
TIF_NEED_RESCHED will drive IRQ preemption.

No behavioural change intended.

Suggested-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: https://lkml.kernel.org/r/20241007075055.219540785@infradead.org
parent 0f0d1b8e

include/linux/entry-common.h

+2 −1
Original line number Diff line number Diff line
@@ -64,7 +64,8 @@ 


#define EXIT_TO_USER_MODE_WORK						\

	(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |		\

	 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL |	\

	 _TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY |			\

	 _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL |			\

	 ARCH_EXIT_TO_USER_MODE_WORK)



/**

include/linux/entry-kvm.h

+3 −2
Original line number Diff line number Diff line
@@ -17,8 +17,9 @@ 
#endif



#define XFER_TO_GUEST_MODE_WORK						\

	(_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL |	\

	 _TIF_NOTIFY_RESUME | ARCH_XFER_TO_GUEST_MODE_WORK)

	(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_SIGPENDING | \

	 _TIF_NOTIFY_SIGNAL | _TIF_NOTIFY_RESUME |			\

	 ARCH_XFER_TO_GUEST_MODE_WORK)



struct kvm_vcpu;

include/linux/sched.h

+2 −1
Original line number Diff line number Diff line
@@ -2002,7 +2002,8 @@ static inline void set_tsk_need_resched(struct task_struct *tsk) 


static inline void clear_tsk_need_resched(struct task_struct *tsk)

{

	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);

	atomic_long_andnot(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY,

			   (atomic_long_t *)&task_thread_info(tsk)->flags);

}



static inline int test_tsk_need_resched(struct task_struct *tsk)

include/linux/thread_info.h

+17 −4
Original line number Diff line number Diff line
@@ -59,6 +59,14 @@ enum syscall_work_bit { 


#include <asm/thread_info.h>



#ifndef TIF_NEED_RESCHED_LAZY

#ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY

#error Inconsistent PREEMPT_LAZY

#endif

#define TIF_NEED_RESCHED_LAZY TIF_NEED_RESCHED

#define _TIF_NEED_RESCHED_LAZY _TIF_NEED_RESCHED

#endif



#ifdef __KERNEL__



#ifndef arch_set_restart_data
@@ -179,22 +187,27 @@ static __always_inline unsigned long read_ti_thread_flags(struct thread_info *ti 


#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H



static __always_inline bool tif_need_resched(void)

static __always_inline bool tif_test_bit(int bit)

{

	return arch_test_bit(TIF_NEED_RESCHED,

	return arch_test_bit(bit,

			     (unsigned long *)(&current_thread_info()->flags));

}



#else



static __always_inline bool tif_need_resched(void)

static __always_inline bool tif_test_bit(int bit)

{

	return test_bit(TIF_NEED_RESCHED,

	return test_bit(bit,

			(unsigned long *)(&current_thread_info()->flags));

}



#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */



static __always_inline bool tif_need_resched(void)

{

	return tif_test_bit(TIF_NEED_RESCHED);

}



#ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES

static inline int arch_within_stack_frames(const void * const stack,

					   const void * const stackend,

kernel/entry/common.c

+1 −1
Original line number Diff line number Diff line
@@ -98,7 +98,7 @@ __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 


		local_irq_enable_exit_to_user(ti_work);



		if (ti_work & _TIF_NEED_RESCHED)

		if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY))

			schedule();



		if (ti_work & _TIF_UPROBE)