Commit dae4320b authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Fixup set_next_task() implementations 



The rule is that:

  pick_next_task() := pick_task() + set_next_task(.first = true)

Turns out, there's still a few things in pick_next_task() that are
missing from that combination.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20240813224015.724111109@infradead.org
parent 7d2180d9

kernel/sched/deadline.c

+3 −3
Original line number Diff line number Diff line
@@ -2386,6 +2386,9 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) 
		update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);



	deadline_queue_push_tasks(rq);



	if (hrtick_enabled(rq))

		start_hrtick_dl(rq, &p->dl);

}



static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq)
@@ -2452,9 +2455,6 @@ static struct task_struct *pick_next_task_dl(struct rq *rq) 
	if (!p->dl_server)

		set_next_task_dl(rq, p, true);



	if (hrtick_enabled(rq))

		start_hrtick_dl(rq, &p->dl);



	return p;

}

kernel/sched/fair.c

+31 −31
Original line number Diff line number Diff line
@@ -8760,6 +8760,9 @@ static struct task_struct *pick_task_fair(struct rq *rq) 
	return task_of(se);

}



static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);

static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);



struct task_struct *

pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)

{
@@ -8808,33 +8811,17 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 


		put_prev_entity(cfs_rq, pse);

		set_next_entity(cfs_rq, se);



		__set_next_task_fair(rq, p, true);

	}



	goto done;

	return p;



simple:

#endif

	if (prev)

		put_prev_task(rq, prev);



	for_each_sched_entity(se)

		set_next_entity(cfs_rq_of(se), se);



done: __maybe_unused;

#ifdef CONFIG_SMP

	/*

	 * Move the next running task to the front of

	 * the list, so our cfs_tasks list becomes MRU

	 * one.

	 */

	list_move(&p->se.group_node, &rq->cfs_tasks);

#endif



	if (hrtick_enabled_fair(rq))

		hrtick_start_fair(rq, p);



	update_misfit_status(p, rq);

	sched_fair_update_stop_tick(rq, p);



	set_next_task_fair(rq, p, true);

	return p;



idle:
@@ -13145,12 +13132,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) 
	}

}



/* Account for a task changing its policy or group.

 *

 * This routine is mostly called to set cfs_rq->curr field when a task

 * migrates between groups/classes.

 */

static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)

static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)

{

	struct sched_entity *se = &p->se;
@@ -13163,6 +13145,27 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 
		list_move(&se->group_node, &rq->cfs_tasks);

	}

#endif

	if (!first)

		return;



	SCHED_WARN_ON(se->sched_delayed);



	if (hrtick_enabled_fair(rq))

		hrtick_start_fair(rq, p);



	update_misfit_status(p, rq);

	sched_fair_update_stop_tick(rq, p);

}



/*

 * Account for a task changing its policy or group.

 *

 * This routine is mostly called to set cfs_rq->curr field when a task

 * migrates between groups/classes.

 */

static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)

{

	struct sched_entity *se = &p->se;



	for_each_sched_entity(se) {

		struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -13172,10 +13175,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 
		account_cfs_rq_runtime(cfs_rq, 0);

	}



	if (!first)

		return;



	SCHED_WARN_ON(se->sched_delayed);

	__set_next_task_fair(rq, p, first);

}



void init_cfs_rq(struct cfs_rq *cfs_rq)