Commit 1e900f41 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Detect per-class runqueue changes 



Have enqueue/dequeue set a per-class bit in rq->queue_mask. This then
enables easy tracking of which runqueues are modified over a
lock-break.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarTejun Heo <tj@kernel.org>
parent 73ec89a1

kernel/sched/core.c

+2 −0
Original line number Diff line number Diff line
@@ -2089,6 +2089,7 @@ void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 
	 */

	uclamp_rq_inc(rq, p, flags);



	rq->queue_mask |= p->sched_class->queue_mask;

	p->sched_class->enqueue_task(rq, p, flags);



	psi_enqueue(p, flags);
@@ -2121,6 +2122,7 @@ inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags) 
	 * and mark the task ->sched_delayed.

	 */

	uclamp_rq_dec(rq, p);

	rq->queue_mask |= p->sched_class->queue_mask;

	return p->sched_class->dequeue_task(rq, p, flags);

}

kernel/sched/deadline.c

+2 −0
Original line number Diff line number Diff line
@@ -3092,6 +3092,8 @@ static int task_is_throttled_dl(struct task_struct *p, int cpu) 


DEFINE_SCHED_CLASS(dl) = {



	.queue_mask		= 8,



	.enqueue_task		= enqueue_task_dl,

	.dequeue_task		= dequeue_task_dl,

	.yield_task		= yield_task_dl,

kernel/sched/ext.c

+2 −0
Original line number Diff line number Diff line
@@ -3234,6 +3234,8 @@ static void scx_cgroup_unlock(void) {} 
 *   their current sched_class. Call them directly from sched core instead.

 */

DEFINE_SCHED_CLASS(ext) = {

	.queue_mask		= 1,



	.enqueue_task		= enqueue_task_scx,

	.dequeue_task		= dequeue_task_scx,

	.yield_task		= yield_task_scx,

kernel/sched/fair.c

+5 −2
Original line number Diff line number Diff line
@@ -12841,6 +12841,7 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 
	}

	rcu_read_unlock();



	rq_modified_clear(this_rq);

	raw_spin_rq_unlock(this_rq);



	t0 = sched_clock_cpu(this_cpu);
@@ -12898,8 +12899,8 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 
	if (this_rq->cfs.h_nr_queued && !pulled_task)

		pulled_task = 1;



	/* Is there a task of a high priority class? */

	if (this_rq->nr_running != this_rq->cfs.h_nr_queued)

	/* If a higher prio class was modified, restart the pick */

	if (rq_modified_above(this_rq, &fair_sched_class))

		pulled_task = -1;



out:
@@ -13633,6 +13634,8 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task 
 */

DEFINE_SCHED_CLASS(fair) = {



	.queue_mask		= 2,



	.enqueue_task		= enqueue_task_fair,

	.dequeue_task		= dequeue_task_fair,

	.yield_task		= yield_task_fair,

kernel/sched/idle.c

+2 −0
Original line number Diff line number Diff line
@@ -521,6 +521,8 @@ static void update_curr_idle(struct rq *rq) 
 */

DEFINE_SCHED_CLASS(idle) = {



	.queue_mask		= 0,



	/* no enqueue/yield_task for idle tasks */



	/* dequeue is not valid, we print a debug message there: */