Commit f47a8189 authored by Tejun Heo's avatar Tejun Heo
Browse files

sched_ext: s/SCX_RQ_BALANCING/SCX_RQ_IN_BALANCE/ and add SCX_RQ_IN_WAKEUP 



SCX_RQ_BALANCING is used to mark that the rq is currently in balance().
Rename it to SCX_RQ_IN_BALANCE and add SCX_RQ_IN_WAKEUP which marks whether
the rq is currently enqueueing for a wakeup. This will be used to implement
direct dispatching to local DSQ of another CPU.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Acked-by: default avatarDavid Vernet <void@manifault.com>
parent 3cf78c5d

kernel/sched/ext.c

+9 −4
Original line number Diff line number Diff line
@@ -1827,6 +1827,9 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags 
{

	int sticky_cpu = p->scx.sticky_cpu;



	if (enq_flags & ENQUEUE_WAKEUP)

		rq->scx.flags |= SCX_RQ_IN_WAKEUP;



	enq_flags |= rq->scx.extra_enq_flags;



	if (sticky_cpu >= 0)
@@ -1843,7 +1846,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags 


	if (p->scx.flags & SCX_TASK_QUEUED) {

		WARN_ON_ONCE(!task_runnable(p));

		return;

		goto out;

	}



	set_task_runnable(rq, p);
@@ -1858,6 +1861,8 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags 
		touch_core_sched(rq, p);



	do_enqueue_task(rq, p, enq_flags, sticky_cpu);

out:

	rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;

}



static void ops_dequeue(struct task_struct *p, u64 deq_flags)
@@ -2420,7 +2425,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev, bool local) 
	bool has_tasks = false;



	lockdep_assert_rq_held(rq);

	rq->scx.flags |= SCX_RQ_BALANCING;

	rq->scx.flags |= SCX_RQ_IN_BALANCE;



	if (static_branch_unlikely(&scx_ops_cpu_preempt) &&

	    unlikely(rq->scx.cpu_released)) {
@@ -2514,7 +2519,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev, bool local) 
has_tasks:

	has_tasks = true;

out:

	rq->scx.flags &= ~SCX_RQ_BALANCING;

	rq->scx.flags &= ~SCX_RQ_IN_BALANCE;

	return has_tasks;

}
@@ -5063,7 +5068,7 @@ static bool can_skip_idle_kick(struct rq *rq) 
	 * The race window is small and we don't and can't guarantee that @rq is

	 * only kicked while idle anyway. Skip only when sure.

	 */

	return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_BALANCING);

	return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE);

}



static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs)

kernel/sched/sched.h

+4 −2
Original line number Diff line number Diff line
@@ -737,8 +737,10 @@ enum scx_rq_flags { 
	 * only while the BPF scheduler considers the CPU to be online.

	 */

	SCX_RQ_ONLINE		= 1 << 0,

	SCX_RQ_BALANCING	= 1 << 1,

	SCX_RQ_CAN_STOP_TICK	= 1 << 2,

	SCX_RQ_CAN_STOP_TICK	= 1 << 1,



	SCX_RQ_IN_WAKEUP	= 1 << 16,

	SCX_RQ_IN_BALANCE	= 1 << 17,

};



struct scx_rq {