Commit 436f3eed authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Combine the last put_prev_task() and the first set_next_task() 



Ensure the last put_prev_task() and the first set_next_task() always
go together.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20240813224016.158454756@infradead.org
parent fd03c5b8

kernel/sched/core.c

+4 −13
Original line number Diff line number Diff line
@@ -5894,8 +5894,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		/* Assume the next prioritized class is idle_sched_class */

		if (!p) {

			p = pick_task_idle(rq);

			put_prev_task(rq, prev);

			set_next_task_first(rq, p);

			put_prev_set_next_task(rq, prev, p);

		}



		/*
@@ -5926,8 +5925,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		} else {

			p = class->pick_task(rq);

			if (p) {

				put_prev_task(rq, prev);

				set_next_task_first(rq, p);

				put_prev_set_next_task(rq, prev, p);

				return p;

			}

		}
@@ -6016,13 +6014,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);



		next = rq->core_pick;

		if (next != prev) {

			put_prev_task(rq, prev);

			set_next_task_first(rq, next);

		}



		rq->core_pick = NULL;

		goto out;

		goto out_set_next;

	}



	prev_balance(rq, prev, rf);
@@ -6192,9 +6185,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
	}



out_set_next:

	put_prev_task(rq, prev);

	set_next_task_first(rq, next);

out:

	put_prev_set_next_task(rq, prev, next);

	if (rq->core->core_forceidle_count && next == rq->idle)

		queue_core_balance(rq);

kernel/sched/fair.c

+1 −2
Original line number Diff line number Diff line
@@ -8819,8 +8819,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 


simple:

#endif

	put_prev_task(rq, prev);

	set_next_task_fair(rq, p, true);

	put_prev_set_next_task(rq, prev, p);

	return p;



idle:

kernel/sched/sched.h

+9 −1
Original line number Diff line number Diff line
@@ -2370,8 +2370,16 @@ static inline void set_next_task(struct rq *rq, struct task_struct *next) 
	next->sched_class->set_next_task(rq, next, false);

}



static inline void set_next_task_first(struct rq *rq, struct task_struct *next)

static inline void put_prev_set_next_task(struct rq *rq,

					  struct task_struct *prev,

					  struct task_struct *next)

{

	WARN_ON_ONCE(rq->curr != prev);



	if (next == prev)

		return;



	prev->sched_class->put_prev_task(rq, prev);

	next->sched_class->set_next_task(rq, next, true);

}