Commit f9530b31 authored by John Stultz's avatar John Stultz Committed by Peter Zijlstra
Browse files

sched: Add assert_balance_callbacks_empty helper 



With proxy-exec utilizing pick-again logic, we can end up having
balance callbacks set by the preivous pick_next_task() call left
on the list.

So pull the warning out into a helper function, and make sure we
check it when we pick again.

Suggested-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarJohn Stultz <jstultz@google.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarK Prateek Nayak <kprateek.nayak@amd.com>
Link: https://patch.msgid.link/20260324191337.1841376-8-jstultz@google.com
parent 2d762266

kernel/sched/core.c

+1 −0
Original line number Diff line number Diff line
@@ -6853,6 +6853,7 @@ static void __sched notrace __schedule(int sched_mode) 
	}



pick_again:

	assert_balance_callbacks_empty(rq);

	next = pick_next_task(rq, rq->donor, &rf);

	rq->next_class = next->sched_class;

	if (sched_proxy_exec()) {

kernel/sched/sched.h

+8 −1
Original line number Diff line number Diff line
@@ -1857,6 +1857,13 @@ static inline void scx_rq_clock_update(struct rq *rq, u64 clock) {} 
static inline void scx_rq_clock_invalidate(struct rq *rq) {}

#endif /* !CONFIG_SCHED_CLASS_EXT */



static inline void assert_balance_callbacks_empty(struct rq *rq)

{

	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_LOCKING) &&

		     rq->balance_callback &&

		     rq->balance_callback != &balance_push_callback);

}



/*

 * Lockdep annotation that avoids accidental unlocks; it's like a

 * sticky/continuous lockdep_assert_held().
@@ -1873,7 +1880,7 @@ static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf) 


	rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);

	rf->clock_update_flags = 0;

	WARN_ON_ONCE(rq->balance_callback && rq->balance_callback != &balance_push_callback);

	assert_balance_callbacks_empty(rq);

}



static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)