Commit 3dcac251 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched/core: Introduce SM_IDLE and an idle re-entry fast-path in __schedule() 



Since commit b2a02fc4 ("smp: Optimize
send_call_function_single_ipi()") an idle CPU in TIF_POLLING_NRFLAG mode
can be pulled out of idle by setting TIF_NEED_RESCHED flag to service an
IPI without actually sending an interrupt. Even in cases where the IPI
handler does not queue a task on the idle CPU, do_idle() will call
__schedule() since need_resched() returns true in these cases.

Introduce and use SM_IDLE to identify call to __schedule() from
schedule_idle() and shorten the idle re-entry time by skipping
pick_next_task() when nr_running is 0 and the previous task is the idle
task.

With the SM_IDLE fast-path, the time taken to complete a fixed set of
IPIs using ipistorm improves noticeably. Following are the numbers
from a dual socket Intel Ice Lake Xeon server (2 x 32C/64T) and
3rd Generation AMD EPYC system (2 x 64C/128T) (boost on, C2 disabled)
running ipistorm between CPU8 and CPU16:

cmdline: insmod ipistorm.ko numipi=100000 single=1 offset=8 cpulist=8 wait=1

   ==================================================================
   Test          : ipistorm (modified)
   Units         : Normalized runtime
   Interpretation: Lower is better
   Statistic     : AMean
   ======================= Intel Ice Lake Xeon ======================
   kernel:				time [pct imp]
   tip:sched/core			1.00 [baseline]
   tip:sched/core + SM_IDLE		0.80 [20.51%]
   ==================== 3rd Generation AMD EPYC =====================
   kernel:				time [pct imp]
   tip:sched/core			1.00 [baseline]
   tip:sched/core + SM_IDLE		0.90 [10.17%]
   ==================================================================

[ kprateek: Commit message, SM_RTLOCK_WAIT fix ]

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Not-yet-signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarK Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Link: https://lore.kernel.org/r/20240809092240.6921-1-kprateek.nayak@amd.com
parent b2d70222

kernel/sched/core.c

+26 −19
Original line number Diff line number Diff line
@@ -6410,19 +6410,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
 * Constants for the sched_mode argument of __schedule().

 *

 * The mode argument allows RT enabled kernels to differentiate a

 * preemption from blocking on an 'sleeping' spin/rwlock. Note that

 * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to

 * optimize the AND operation out and just check for zero.

 * preemption from blocking on an 'sleeping' spin/rwlock.

 */

#define SM_NONE			0x0

#define SM_PREEMPT		0x1

#define SM_RTLOCK_WAIT		0x2



#ifndef CONFIG_PREEMPT_RT

# define SM_MASK_PREEMPT	(~0U)

#else

# define SM_MASK_PREEMPT	SM_PREEMPT

#endif

#define SM_IDLE			(-1)

#define SM_NONE			0

#define SM_PREEMPT		1

#define SM_RTLOCK_WAIT		2



/*

 * __schedule() is the main scheduler function.
@@ -6463,9 +6456,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
 *

 * WARNING: must be called with preemption disabled!

 */

static void __sched notrace __schedule(unsigned int sched_mode)

static void __sched notrace __schedule(int sched_mode)

{

	struct task_struct *prev, *next;

	/*

	 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted

	 * as a preemption by schedule_debug() and RCU.

	 */

	bool preempt = sched_mode > SM_NONE;

	unsigned long *switch_count;

	unsigned long prev_state;

	struct rq_flags rf;
@@ -6476,13 +6474,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) 
	rq = cpu_rq(cpu);

	prev = rq->curr;



	schedule_debug(prev, !!sched_mode);

	schedule_debug(prev, preempt);



	if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))

		hrtick_clear(rq);



	local_irq_disable();

	rcu_note_context_switch(!!sched_mode);

	rcu_note_context_switch(preempt);



	/*

	 * Make sure that signal_pending_state()->signal_pending() below
@@ -6511,12 +6509,20 @@ static void __sched notrace __schedule(unsigned int sched_mode) 


	switch_count = &prev->nivcsw;



	/* Task state changes only considers SM_PREEMPT as preemption */

	preempt = sched_mode == SM_PREEMPT;



	/*

	 * We must load prev->state once (task_struct::state is volatile), such

	 * that we form a control dependency vs deactivate_task() below.

	 */

	prev_state = READ_ONCE(prev->__state);

	if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) {

	if (sched_mode == SM_IDLE) {

		if (!rq->nr_running) {

			next = prev;

			goto picked;

		}

	} else if (!preempt && prev_state) {

		if (signal_pending_state(prev_state, prev)) {

			WRITE_ONCE(prev->__state, TASK_RUNNING);

		} else {
@@ -6547,6 +6553,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) 
	}



	next = pick_next_task(rq, prev, &rf);

picked:

	clear_tsk_need_resched(prev);

	clear_preempt_need_resched();

#ifdef CONFIG_SCHED_DEBUG
@@ -6588,7 +6595,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) 
		psi_account_irqtime(rq, prev, next);

		psi_sched_switch(prev, next, !task_on_rq_queued(prev));



		trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);

		trace_sched_switch(preempt, prev, next, prev_state);



		/* Also unlocks the rq: */

		rq = context_switch(rq, prev, next, &rf);
@@ -6664,7 +6671,7 @@ static void sched_update_worker(struct task_struct *tsk) 
	}

}



static __always_inline void __schedule_loop(unsigned int sched_mode)

static __always_inline void __schedule_loop(int sched_mode)

{

	do {

		preempt_disable();
@@ -6709,7 +6716,7 @@ void __sched schedule_idle(void) 
	 */

	WARN_ON_ONCE(current->__state);

	do {

		__schedule(SM_NONE);

		__schedule(SM_IDLE);

	} while (need_resched());

}