Commit d4c64207 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Cleanup the sched_change NOCLOCK usage 



Teach the sched_change pattern how to do update_rq_clock(); this
allows for some simplifications / cleanups.

Suggested-by: default avatarK Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Acked-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
parent 5892cbd8

kernel/sched/core.c

+11 −22
Original line number Diff line number Diff line
@@ -2346,11 +2346,9 @@ static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 
	if (p->cpus_ptr != &p->cpus_mask)

		return;



	scoped_guard (task_rq_lock, p) {

		update_rq_clock(scope.rq);

	scoped_guard (task_rq_lock, p)

		do_set_cpus_allowed(p, &ac);

}

}



void ___migrate_enable(void)

{
@@ -2666,9 +2664,7 @@ void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx 
static void

do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)

{

	u32 flags = DEQUEUE_SAVE | DEQUEUE_NOCLOCK;



	scoped_guard (sched_change, p, flags) {

	scoped_guard (sched_change, p, DEQUEUE_SAVE) {

		p->sched_class->set_cpus_allowed(p, ctx);

		mm_set_cpus_allowed(p->mm, ctx->new_mask);

	}
@@ -2690,10 +2686,8 @@ void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mas 
		struct rcu_head rcu;

	};



	scoped_guard (__task_rq_lock, p) {

		update_rq_clock(scope.rq);

	scoped_guard (__task_rq_lock, p)

		do_set_cpus_allowed(p, &ac);

	}



	/*

	 * Because this is called with p->pi_lock held, it is not possible
@@ -9108,16 +9102,13 @@ static void sched_change_group(struct task_struct *tsk) 
 */

void sched_move_task(struct task_struct *tsk, bool for_autogroup)

{

	unsigned int queue_flags =

		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;

	unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;

	bool resched = false;

	struct rq *rq;



	CLASS(task_rq_lock, rq_guard)(tsk);

	rq = rq_guard.rq;



	update_rq_clock(rq);



	scoped_guard (sched_change, tsk, queue_flags) {

		sched_change_group(tsk);

		if (!for_autogroup)
@@ -10792,17 +10783,15 @@ struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int 


	lockdep_assert_rq_held(rq);



	if (!(flags & DEQUEUE_NOCLOCK)) {

		update_rq_clock(rq);

		flags |= DEQUEUE_NOCLOCK;

	}



	if (flags & DEQUEUE_CLASS) {

		if (p->sched_class->switching_from) {

			/*

			 * switching_from_fair() assumes CLASS implies NOCLOCK;

			 * fixing this assumption would mean switching_from()

			 * would need to be able to change flags.

			 */

			WARN_ON(!(flags & DEQUEUE_NOCLOCK));

		if (p->sched_class->switching_from)

			p->sched_class->switching_from(rq, p);

	}

	}



	*ctx = (struct sched_change_ctx){

		.p = p,
@@ -10840,7 +10829,7 @@ void sched_change_end(struct sched_change_ctx *ctx) 
		p->sched_class->switching_to(rq, p);



	if (ctx->queued)

		enqueue_task(rq, p, ctx->flags | ENQUEUE_NOCLOCK);

		enqueue_task(rq, p, ctx->flags);

	if (ctx->running)

		set_next_task(rq, p);

kernel/sched/ext.c

+1 −3
Original line number Diff line number Diff line
@@ -4654,7 +4654,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) 
	percpu_down_write(&scx_fork_rwsem);

	scx_task_iter_start(&sti);

	while ((p = scx_task_iter_next_locked(&sti))) {

		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;

		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;

		const struct sched_class *old_class = p->sched_class;

		const struct sched_class *new_class =

			__setscheduler_class(p->policy, p->prio);
@@ -4662,8 +4662,6 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) 
		if (!tryget_task_struct(p))

			continue;



		update_rq_clock(task_rq(p));



		if (old_class != new_class)

			queue_flags |= DEQUEUE_CLASS;

kernel/sched/syscalls.c

+2 −6
Original line number Diff line number Diff line
@@ -64,7 +64,6 @@ static int effective_prio(struct task_struct *p) 


void set_user_nice(struct task_struct *p, long nice)

{

	struct rq *rq;

	int old_prio;



	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
@@ -73,10 +72,7 @@ void set_user_nice(struct task_struct *p, long nice) 
	 * We have to be careful, if called from sys_setpriority(),

	 * the task might be in the middle of scheduling on another CPU.

	 */

	CLASS(task_rq_lock, rq_guard)(p);

	rq = rq_guard.rq;



	update_rq_clock(rq);

	guard(task_rq_lock)(p);



	/*

	 * The RT priorities are set via sched_setscheduler(), but we still
@@ -89,7 +85,7 @@ void set_user_nice(struct task_struct *p, long nice) 
		return;

	}



	scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) {

	scoped_guard (sched_change, p, DEQUEUE_SAVE) {

		p->static_prio = NICE_TO_PRIO(nice);

		set_load_weight(p, true);

		old_prio = p->prio;