Commit 53439363 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Audit MOVE vs balance_callbacks 



The {DE,EN}QUEUE_MOVE flag indicates a task is allowed to change
priority, which means there could be balance callbacks queued.

Therefore audit all MOVE users and make sure they do run balance
callbacks before dropping rq-lock.

Fixes: 6455ad53 ("sched: Move sched_class::prio_changed() into the change pattern")
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: default avatarPierre Gondois <pierre.gondois@arm.com>
Tested-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Link: https://patch.msgid.link/20260114130528.GB831285@noisy.programming.kicks-ass.net
parent 49041e87

kernel/sched/core.c

+3 −1
Original line number Diff line number Diff line
@@ -4950,7 +4950,7 @@ struct balance_callback *splice_balance_callbacks(struct rq *rq) 
	return __splice_balance_callbacks(rq, true);

}



static void __balance_callbacks(struct rq *rq, struct rq_flags *rf)

void __balance_callbacks(struct rq *rq, struct rq_flags *rf)

{

	if (rf)

		rq_unpin_lock(rq, rf);
@@ -9126,6 +9126,8 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup) 


	if (resched)

		resched_curr(rq);



	__balance_callbacks(rq, &rq_guard.rf);

}



static struct cgroup_subsys_state *

kernel/sched/ext.c

+1 −0
Original line number Diff line number Diff line
@@ -545,6 +545,7 @@ static void scx_task_iter_start(struct scx_task_iter *iter) 
static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter)

{

	if (iter->locked_task) {

		__balance_callbacks(iter->rq, &iter->rf);

		task_rq_unlock(iter->rq, iter->locked_task, &iter->rf);

		iter->locked_task = NULL;

	}

kernel/sched/sched.h

+4 −1
Original line number Diff line number Diff line
@@ -2388,7 +2388,8 @@ extern const u32 sched_prio_to_wmult[40]; 
 *                should preserve as much state as possible.

 *

 * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location

 *        in the runqueue.

 *        in the runqueue. IOW the priority is allowed to change. Callers

 *        must expect to deal with balance callbacks.

 *

 * NOCLOCK - skip the update_rq_clock() (avoids double updates)

 *
@@ -3969,6 +3970,8 @@ extern void enqueue_task(struct rq *rq, struct task_struct *p, int flags); 
extern bool dequeue_task(struct rq *rq, struct task_struct *p, int flags);



extern struct balance_callback *splice_balance_callbacks(struct rq *rq);



extern void __balance_callbacks(struct rq *rq, struct rq_flags *rf);

extern void balance_callbacks(struct rq *rq, struct balance_callback *head);



/*