Commit d7b01aef authored by Tejun Heo's avatar Tejun Heo
Browse files

Merge branch 'tip/sched/core' into for-6.12 



- Resolve trivial context conflicts from dl_server clearing being moved
  around.

- Add @next to put_prev_task_scx() and @prev to pick_next_task_scx() to
  match sched/core.

- Merge sched_class->switch_class() addition from sched_ext with
  tip/sched/core changes in __pick_next_task().

- Make pick_next_task_scx() call put_prev_task_scx() to emulate the previous
  behavior where sched_class->put_prev_task() was called before
  sched_class->pick_next_task().

While this makes sched_ext build and function, the behavior is not in line
with other sched classes. The follow-up patches will address the
discrepancies and remove sched_class->switch_class().

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parents 62607d03 b2d70222

include/linux/sched.h

+0 −1
Original line number Diff line number Diff line
@@ -694,7 +694,6 @@ struct sched_dl_entity { 
	 */

	struct rq			*rq;

	dl_server_has_tasks_f		server_has_tasks;

	dl_server_pick_f		server_pick_next;

	dl_server_pick_f		server_pick_task;



#ifdef CONFIG_RT_MUTEXES

kernel/sched/core.c

+42 −47
Original line number Diff line number Diff line
@@ -3690,8 +3690,6 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 
		rq->idle_stamp = 0;

	}

#endif



	p->dl_server = NULL;

}



/*
@@ -5895,7 +5893,7 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) 
	schedstat_inc(this_rq()->sched_count);

}



static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,

static void prev_balance(struct rq *rq, struct task_struct *prev,

			 struct rq_flags *rf)

{

	const struct sched_class *start_class = prev->sched_class;
@@ -5923,16 +5921,6 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, 
		if (class->balance && class->balance(rq, prev, rf))

			break;

	}



	put_prev_task(rq, prev);



	/*

	 * We've updated @prev and no longer need the server link, clear it.

	 * Must be done before ->pick_next_task() because that can (re)set

	 * ->dl_server.

	 */

	if (prev->dl_server)

		prev->dl_server = NULL;

}



/*
@@ -5944,6 +5932,8 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
	const struct sched_class *class;

	struct task_struct *p;



	rq->dl_server = NULL;



	if (scx_enabled())

		goto restart;
@@ -5962,32 +5952,19 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 


		/* Assume the next prioritized class is idle_sched_class */

		if (!p) {

			put_prev_task(rq, prev);

			p = pick_next_task_idle(rq);

			p = pick_task_idle(rq);

			put_prev_set_next_task(rq, prev, p);

		}



		/*

		 * This is a normal CFS pick, but the previous could be a DL pick.

		 * Clear it as previous is no longer picked.

		 */

		if (prev->dl_server)

			prev->dl_server = NULL;



		/*

		 * This is the fast path; it cannot be a DL server pick;

		 * therefore even if @p == @prev, ->dl_server must be NULL.

		 */

		if (p->dl_server)

			p->dl_server = NULL;



		return p;

	}



restart:

	put_prev_task_balance(rq, prev, rf);

	prev_balance(rq, prev, rf);



	for_each_active_class(class) {

		p = class->pick_next_task(rq);

		if (class->pick_next_task) {

			p = class->pick_next_task(rq, prev);

			if (p) {

				const struct sched_class *prev_class = prev->sched_class;
@@ -5995,6 +5972,18 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
					prev_class->switch_class(rq, p);

				return p;

			}

		} else {

			p = class->pick_task(rq);

			if (p) {

				const struct sched_class *prev_class = prev->sched_class;



				put_prev_set_next_task(rq, prev, p);



				if (class != prev_class && prev_class->switch_class)

					prev_class->switch_class(rq, p);

				return p;

			}

		}

	}



	BUG(); /* The idle class should always have a runnable task. */
@@ -6024,6 +6013,8 @@ static inline struct task_struct *pick_task(struct rq *rq) 
	const struct sched_class *class;

	struct task_struct *p;



	rq->dl_server = NULL;



	for_each_active_class(class) {

		p = class->pick_task(rq);

		if (p)
@@ -6062,6 +6053,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		 * another cpu during offline.

		 */

		rq->core_pick = NULL;

		rq->core_dl_server = NULL;

		return __pick_next_task(rq, prev, rf);

	}
@@ -6080,16 +6072,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);



		next = rq->core_pick;

		if (next != prev) {

			put_prev_task(rq, prev);

			set_next_task(rq, next);

		}



		rq->dl_server = rq->core_dl_server;

		rq->core_pick = NULL;

		goto out;

		rq->core_dl_server = NULL;

		goto out_set_next;

	}



	put_prev_task_balance(rq, prev, rf);

	prev_balance(rq, prev, rf);



	smt_mask = cpu_smt_mask(cpu);

	need_sync = !!rq->core->core_cookie;
@@ -6130,6 +6119,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		next = pick_task(rq);

		if (!next->core_cookie) {

			rq->core_pick = NULL;

			rq->core_dl_server = NULL;

			/*

			 * For robustness, update the min_vruntime_fi for

			 * unconstrained picks as well.
@@ -6157,7 +6147,9 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		if (i != cpu && (rq_i != rq->core || !core_clock_updated))

			update_rq_clock(rq_i);



		p = rq_i->core_pick = pick_task(rq_i);

		rq_i->core_pick = p = pick_task(rq_i);

		rq_i->core_dl_server = rq_i->dl_server;



		if (!max || prio_less(max, p, fi_before))

			max = p;

	}
@@ -6181,6 +6173,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
		}



		rq_i->core_pick = p;

		rq_i->core_dl_server = NULL;



		if (p == rq_i->idle) {

			if (rq_i->nr_running) {
@@ -6241,6 +6234,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 


		if (i == cpu) {

			rq_i->core_pick = NULL;

			rq_i->core_dl_server = NULL;

			continue;

		}
@@ -6249,6 +6243,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 


		if (rq_i->curr == rq_i->core_pick) {

			rq_i->core_pick = NULL;

			rq_i->core_dl_server = NULL;

			continue;

		}
@@ -6256,8 +6251,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 
	}



out_set_next:

	set_next_task(rq, next);

out:

	put_prev_set_next_task(rq, prev, next);

	if (rq->core->core_forceidle_count && next == rq->idle)

		queue_core_balance(rq);
@@ -8487,6 +8481,7 @@ void __init sched_init(void) 
#ifdef CONFIG_SCHED_CORE

		rq->core = rq;

		rq->core_pick = NULL;

		rq->core_dl_server = NULL;

		rq->core_enabled = 0;

		rq->core_tree = RB_ROOT;

		rq->core_forceidle_count = 0;

kernel/sched/deadline.c

+25 −54
Original line number Diff line number Diff line
@@ -1665,12 +1665,10 @@ void dl_server_stop(struct sched_dl_entity *dl_se) 


void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,

		    dl_server_has_tasks_f has_tasks,

		    dl_server_pick_f pick_next,

		    dl_server_pick_f pick_task)

{

	dl_se->rq = rq;

	dl_se->server_has_tasks = has_tasks;

	dl_se->server_pick_next = pick_next;

	dl_se->server_pick_task = pick_task;

}
@@ -1896,45 +1894,39 @@ static inline bool __dl_less(struct rb_node *a, const struct rb_node *b) 
	return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline);

}



static inline struct sched_statistics *

static __always_inline struct sched_statistics *

__schedstats_from_dl_se(struct sched_dl_entity *dl_se)

{

	if (!schedstat_enabled())

		return NULL;



	if (dl_server(dl_se))

		return NULL;



	return &dl_task_of(dl_se)->stats;

}



static inline void

update_stats_wait_start_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)

{

	struct sched_statistics *stats;



	if (!schedstat_enabled())

		return;



	stats = __schedstats_from_dl_se(dl_se);

	struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);

	if (stats)

		__update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);

}



static inline void

update_stats_wait_end_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)

{

	struct sched_statistics *stats;



	if (!schedstat_enabled())

		return;



	stats = __schedstats_from_dl_se(dl_se);

	struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);

	if (stats)

		__update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);

}



static inline void

update_stats_enqueue_sleeper_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)

{

	struct sched_statistics *stats;



	if (!schedstat_enabled())

		return;



	stats = __schedstats_from_dl_se(dl_se);

	struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);

	if (stats)

		__update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);

}
@@ -2392,6 +2384,9 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) 
		update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);



	deadline_queue_push_tasks(rq);



	if (hrtick_enabled(rq))

		start_hrtick_dl(rq, &p->dl);

}



static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq)
@@ -2407,9 +2402,8 @@ static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq) 
/*

 * __pick_next_task_dl - Helper to pick the next -deadline task to run.

 * @rq: The runqueue to pick the next task from.

 * @peek: If true, just peek at the next task. Only relevant for dlserver.

 */

static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek)

static struct task_struct *__pick_task_dl(struct rq *rq)

{

	struct sched_dl_entity *dl_se;

	struct dl_rq *dl_rq = &rq->dl;
@@ -2423,16 +2417,13 @@ static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek) 
	WARN_ON_ONCE(!dl_se);



	if (dl_server(dl_se)) {

		if (IS_ENABLED(CONFIG_SMP) && peek)

		p = dl_se->server_pick_task(dl_se);

		else

			p = dl_se->server_pick_next(dl_se);

		if (!p) {

			dl_se->dl_yielded = 1;

			update_curr_dl_se(rq, dl_se, 0);

			goto again;

		}

		p->dl_server = dl_se;

		rq->dl_server = dl_se;

	} else {

		p = dl_task_of(dl_se);

	}
@@ -2440,31 +2431,12 @@ static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek) 
	return p;

}



#ifdef CONFIG_SMP

static struct task_struct *pick_task_dl(struct rq *rq)

{

	return __pick_next_task_dl(rq, true);

	return __pick_task_dl(rq);

}

#endif



static struct task_struct *pick_next_task_dl(struct rq *rq)

{

	struct task_struct *p;



	p = __pick_next_task_dl(rq, false);

	if (!p)

		return p;



	if (!p->dl_server)

		set_next_task_dl(rq, p, true);



	if (hrtick_enabled(rq))

		start_hrtick_dl(rq, &p->dl);



	return p;

}



static void put_prev_task_dl(struct rq *rq, struct task_struct *p)

static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_struct *next)

{

	struct sched_dl_entity *dl_se = &p->dl;

	struct dl_rq *dl_rq = &rq->dl;
@@ -3156,13 +3128,12 @@ DEFINE_SCHED_CLASS(dl) = { 


	.wakeup_preempt		= wakeup_preempt_dl,



	.pick_next_task		= pick_next_task_dl,

	.pick_task		= pick_task_dl,

	.put_prev_task		= put_prev_task_dl,

	.set_next_task		= set_next_task_dl,



#ifdef CONFIG_SMP

	.balance		= balance_dl,

	.pick_task		= pick_task_dl,

	.select_task_rq		= select_task_rq_dl,

	.migrate_task_rq	= migrate_task_rq_dl,

	.set_cpus_allowed       = set_cpus_allowed_dl,

kernel/sched/ext.c

+10 −2
Original line number Diff line number Diff line
@@ -2719,7 +2719,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) 
	}

}



static void put_prev_task_scx(struct rq *rq, struct task_struct *p)

static void put_prev_task_scx(struct rq *rq, struct task_struct *p,

			      struct task_struct *next)

{

	update_curr_scx(rq);
@@ -2774,14 +2775,21 @@ static struct task_struct *first_local_task(struct rq *rq) 
					struct task_struct, scx.dsq_list.node);

}



static struct task_struct *pick_next_task_scx(struct rq *rq)

static struct task_struct *pick_next_task_scx(struct rq *rq,

					      struct task_struct *prev)

{

	struct task_struct *p;



	if (prev->sched_class == &ext_sched_class)

		put_prev_task_scx(rq, prev, NULL);



	p = first_local_task(rq);

	if (!p)

		return NULL;



	if (prev->sched_class != &ext_sched_class)

		prev->sched_class->put_prev_task(rq, prev, p);



	set_next_task_scx(rq, p, true);



	if (unlikely(!p->scx.slice)) {

kernel/sched/fair.c

+56 −66
Original line number Diff line number Diff line
@@ -5457,6 +5457,13 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 


static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);



static inline void finish_delayed_dequeue_entity(struct sched_entity *se)

{

	se->sched_delayed = 0;

	if (sched_feat(DELAY_ZERO) && se->vlag > 0)

		se->vlag = 0;

}



static bool

dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)

{
@@ -5532,11 +5539,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 
	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE)

		update_min_vruntime(cfs_rq);



	if (flags & DEQUEUE_DELAYED) {

		se->sched_delayed = 0;

		if (sched_feat(DELAY_ZERO) && se->vlag > 0)

			se->vlag = 0;

	}

	if (flags & DEQUEUE_DELAYED)

		finish_delayed_dequeue_entity(se);



	if (cfs_rq->nr_running == 0)

		update_idle_cfs_rq_clock_pelt(cfs_rq);
@@ -8746,17 +8750,12 @@ static struct task_struct *pick_task_fair(struct rq *rq) 
		cfs_rq = group_cfs_rq(se);

	} while (cfs_rq);



	/*

	 * This can be called from directly from CFS's ->pick_task() or indirectly

	 * from DL's ->pick_task when fair server is enabled. In the indirect case,

	 * DL will set ->dl_server just after this function is called, so its Ok to

	 * clear. In the direct case, we are picking directly so we must clear it.

	 */

	task_of(se)->dl_server = NULL;



	return task_of(se);

}



static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);

static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);



struct task_struct *

pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)

{
@@ -8771,9 +8770,11 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	se = &p->se;



#ifdef CONFIG_FAIR_GROUP_SCHED

	if (!prev || prev->sched_class != &fair_sched_class)

	if (prev->sched_class != &fair_sched_class)

		goto simple;



	__put_prev_set_next_dl_server(rq, prev, p);



	/*

	 * Because of the set_next_buddy() in dequeue_task_fair() it is rather

	 * likely that a next task is from the same cgroup as the current.
@@ -8805,33 +8806,15 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 


		put_prev_entity(cfs_rq, pse);

		set_next_entity(cfs_rq, se);

	}



	goto done;

simple:

#endif

	if (prev)

		put_prev_task(rq, prev);

		__set_next_task_fair(rq, p, true);

	}



	for_each_sched_entity(se)

		set_next_entity(cfs_rq_of(se), se);

	return p;



done: __maybe_unused;

#ifdef CONFIG_SMP

	/*

	 * Move the next running task to the front of

	 * the list, so our cfs_tasks list becomes MRU

	 * one.

	 */

	list_move(&p->se.group_node, &rq->cfs_tasks);

simple:

#endif



	if (hrtick_enabled_fair(rq))

		hrtick_start_fair(rq, p);



	update_misfit_status(p, rq);

	sched_fair_update_stop_tick(rq, p);



	put_prev_set_next_task(rq, prev, p);

	return p;



idle:
@@ -8860,9 +8843,9 @@ done: __maybe_unused; 
	return NULL;

}



static struct task_struct *__pick_next_task_fair(struct rq *rq)

static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev)

{

	return pick_next_task_fair(rq, NULL, NULL);

	return pick_next_task_fair(rq, prev, NULL);

}



static bool fair_server_has_tasks(struct sched_dl_entity *dl_se)
@@ -8872,16 +8855,7 @@ static bool fair_server_has_tasks(struct sched_dl_entity *dl_se) 


static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se)

{

#ifdef CONFIG_SMP

	return pick_task_fair(dl_se->rq);

#else

	return NULL;

#endif

}



static struct task_struct *fair_server_pick_next(struct sched_dl_entity *dl_se)

{

	return pick_next_task_fair(dl_se->rq, NULL, NULL);

}



void fair_server_init(struct rq *rq)
@@ -8890,15 +8864,13 @@ void fair_server_init(struct rq *rq) 


	init_dl_entity(dl_se);



	dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_next,

		       fair_server_pick_task);



	dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_task);

}



/*

 * Account for a descheduled task:

 */

static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)

static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next)

{

	struct sched_entity *se = &prev->se;

	struct cfs_rq *cfs_rq;
@@ -13098,11 +13070,16 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) 
	 * and we cannot use DEQUEUE_DELAYED.

	 */

	if (p->se.sched_delayed) {

		/* First, dequeue it from its new class' structures */

		dequeue_task(rq, p, DEQUEUE_NOCLOCK | DEQUEUE_SLEEP);

		p->se.sched_delayed = 0;

		/*

		 * Now, clean up the fair_sched_class side of things

		 * related to sched_delayed being true and that wasn't done

		 * due to the generic dequeue not using DEQUEUE_DELAYED.

		 */

		finish_delayed_dequeue_entity(&p->se);

		p->se.rel_deadline = 0;

		if (sched_feat(DELAY_ZERO) && p->se.vlag > 0)

			p->se.vlag = 0;

		__block_task(rq, p);

	}

}
@@ -13127,12 +13104,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) 
	}

}



/* Account for a task changing its policy or group.

 *

 * This routine is mostly called to set cfs_rq->curr field when a task

 * migrates between groups/classes.

 */

static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)

static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)

{

	struct sched_entity *se = &p->se;
@@ -13145,6 +13117,27 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 
		list_move(&se->group_node, &rq->cfs_tasks);

	}

#endif

	if (!first)

		return;



	SCHED_WARN_ON(se->sched_delayed);



	if (hrtick_enabled_fair(rq))

		hrtick_start_fair(rq, p);



	update_misfit_status(p, rq);

	sched_fair_update_stop_tick(rq, p);

}



/*

 * Account for a task changing its policy or group.

 *

 * This routine is mostly called to set cfs_rq->curr field when a task

 * migrates between groups/classes.

 */

static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)

{

	struct sched_entity *se = &p->se;



	for_each_sched_entity(se) {

		struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -13154,10 +13147,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 
		account_cfs_rq_runtime(cfs_rq, 0);

	}



	if (!first)

		return;



	SCHED_WARN_ON(se->sched_delayed);

	__set_next_task_fair(rq, p, first);

}



void init_cfs_rq(struct cfs_rq *cfs_rq)
@@ -13483,13 +13473,13 @@ DEFINE_SCHED_CLASS(fair) = { 


	.wakeup_preempt		= check_preempt_wakeup_fair,



	.pick_task		= pick_task_fair,

	.pick_next_task		= __pick_next_task_fair,

	.put_prev_task		= put_prev_task_fair,

	.set_next_task          = set_next_task_fair,



#ifdef CONFIG_SMP

	.balance		= balance_fair,

	.pick_task		= pick_task_fair,

	.select_task_rq		= select_task_rq_fair,

	.migrate_task_rq	= migrate_task_rq_fair,