Commit fd03c5b8 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Rework pick_next_task() 



The current rule is that:

  pick_next_task() := pick_task() + set_next_task(.first = true)

And many classes implement it directly as such. Change things around
to make pick_next_task() optional while also changing the definition to:

  pick_next_task(prev) := pick_task() + put_prev_task() + set_next_task(.first = true)

The reason is that sched_ext would like to have a 'final' call that
knows the next task. By placing put_prev_task() right next to
set_next_task() (as it already is for sched_core) this becomes
trivial.

As a bonus, this is a nice cleanup on its own.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20240813224016.051225657@infradead.org
parent 260598f1

kernel/sched/core.c

+15 −6
Original line number Diff line number Diff line
@@ -5893,8 +5893,9 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 


		/* Assume the next prioritized class is idle_sched_class */

		if (!p) {

			p = pick_task_idle(rq);

			put_prev_task(rq, prev);

			p = pick_next_task_idle(rq);

			set_next_task_first(rq, p);

		}



		/*
@@ -5916,12 +5917,20 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 


restart:

	prev_balance(rq, prev, rf);

	put_prev_task(rq, prev);



	for_each_class(class) {

		p = class->pick_next_task(rq);

		if (class->pick_next_task) {

			p = class->pick_next_task(rq, prev);

			if (p)

				return p;

		} else {

			p = class->pick_task(rq);

			if (p) {

				put_prev_task(rq, prev);

				set_next_task_first(rq, p);

				return p;

			}

		}

	}



	BUG(); /* The idle class should always have a runnable task. */
@@ -6017,7 +6026,6 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
	}



	prev_balance(rq, prev, rf);

	put_prev_task(rq, prev);



	smt_mask = cpu_smt_mask(cpu);

	need_sync = !!rq->core->core_cookie;
@@ -6184,6 +6192,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
	}



out_set_next:

	put_prev_task(rq, prev);

	set_next_task_first(rq, next);

out:

	if (rq->core->core_forceidle_count && next == rq->idle)

kernel/sched/deadline.c

+1 −20
Original line number Diff line number Diff line
@@ -2431,28 +2431,10 @@ static struct task_struct *__pick_task_dl(struct rq *rq) 
	return p;

}



#ifdef CONFIG_SMP

static struct task_struct *pick_task_dl(struct rq *rq)

{

	return __pick_task_dl(rq);

}

#endif



static struct task_struct *pick_next_task_dl(struct rq *rq)

{

	struct task_struct *p;



	p = __pick_task_dl(rq);

	if (!p)

		return p;



	if (p->dl_server)

		p->sched_class->set_next_task(rq, p, true);

	else

		set_next_task_dl(rq, p, true);



	return p;

}



static void put_prev_task_dl(struct rq *rq, struct task_struct *p)

{
@@ -3146,13 +3128,12 @@ DEFINE_SCHED_CLASS(dl) = { 


	.wakeup_preempt		= wakeup_preempt_dl,



	.pick_next_task		= pick_next_task_dl,

	.pick_task		= pick_task_dl,

	.put_prev_task		= put_prev_task_dl,

	.set_next_task		= set_next_task_dl,



#ifdef CONFIG_SMP

	.balance		= balance_dl,

	.pick_task		= pick_task_dl,

	.select_task_rq		= select_task_rq_dl,

	.migrate_task_rq	= migrate_task_rq_dl,

	.set_cpus_allowed       = set_cpus_allowed_dl,

kernel/sched/fair.c

+5 −6
Original line number Diff line number Diff line
@@ -8777,7 +8777,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	se = &p->se;



#ifdef CONFIG_FAIR_GROUP_SCHED

	if (!prev || prev->sched_class != &fair_sched_class)

	if (prev->sched_class != &fair_sched_class)

		goto simple;



	/*
@@ -8819,7 +8819,6 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 


simple:

#endif

	if (prev)

	put_prev_task(rq, prev);

	set_next_task_fair(rq, p, true);

	return p;
@@ -8850,9 +8849,9 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	return NULL;

}



static struct task_struct *__pick_next_task_fair(struct rq *rq)

static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev)

{

	return pick_next_task_fair(rq, NULL, NULL);

	return pick_next_task_fair(rq, prev, NULL);

}



static bool fair_server_has_tasks(struct sched_dl_entity *dl_se)
@@ -13490,13 +13489,13 @@ DEFINE_SCHED_CLASS(fair) = { 


	.wakeup_preempt		= check_preempt_wakeup_fair,



	.pick_task		= pick_task_fair,

	.pick_next_task		= __pick_next_task_fair,

	.put_prev_task		= put_prev_task_fair,

	.set_next_task          = set_next_task_fair,



#ifdef CONFIG_SMP

	.balance		= balance_fair,

	.pick_task		= pick_task_fair,

	.select_task_rq		= select_task_rq_fair,

	.migrate_task_rq	= migrate_task_rq_fair,

kernel/sched/idle.c

+2 −14
Original line number Diff line number Diff line
@@ -462,21 +462,10 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir 
	next->se.exec_start = rq_clock_task(rq);

}



#ifdef CONFIG_SMP

static struct task_struct *pick_task_idle(struct rq *rq)

struct task_struct *pick_task_idle(struct rq *rq)

{

	return rq->idle;

}

#endif



struct task_struct *pick_next_task_idle(struct rq *rq)

{

	struct task_struct *next = rq->idle;



	set_next_task_idle(rq, next, true);



	return next;

}



/*

 * It is not legal to sleep in the idle task - print a warning
@@ -531,13 +520,12 @@ DEFINE_SCHED_CLASS(idle) = { 


	.wakeup_preempt		= wakeup_preempt_idle,



	.pick_next_task		= pick_next_task_idle,

	.pick_task		= pick_task_idle,

	.put_prev_task		= put_prev_task_idle,

	.set_next_task          = set_next_task_idle,



#ifdef CONFIG_SMP

	.balance		= balance_idle,

	.pick_task		= pick_task_idle,

	.select_task_rq		= select_task_rq_idle,

	.set_cpus_allowed	= set_cpus_allowed_common,

#endif

kernel/sched/rt.c

+1 −12
Original line number Diff line number Diff line
@@ -1748,16 +1748,6 @@ static struct task_struct *pick_task_rt(struct rq *rq) 
	return p;

}



static struct task_struct *pick_next_task_rt(struct rq *rq)

{

	struct task_struct *p = pick_task_rt(rq);



	if (p)

		set_next_task_rt(rq, p, true);



	return p;

}



static void put_prev_task_rt(struct rq *rq, struct task_struct *p)

{

	struct sched_rt_entity *rt_se = &p->rt;
@@ -2645,13 +2635,12 @@ DEFINE_SCHED_CLASS(rt) = { 


	.wakeup_preempt		= wakeup_preempt_rt,



	.pick_next_task		= pick_next_task_rt,

	.pick_task		= pick_task_rt,

	.put_prev_task		= put_prev_task_rt,

	.set_next_task          = set_next_task_rt,



#ifdef CONFIG_SMP

	.balance		= balance_rt,

	.pick_task		= pick_task_rt,

	.select_task_rq		= select_task_rq_rt,

	.set_cpus_allowed       = set_cpus_allowed_common,

	.rq_online              = rq_online_rt,