Commit 54a58a78 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched/fair: Implement DELAY_ZERO 



'Extend' DELAY_DEQUEUE by noting that since we wanted to dequeued them
at the 0-lag point, truncate lag (eg. don't let them earn positive
lag).

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarValentin Schneider <vschneid@redhat.com>
Tested-by: default avatarValentin Schneider <vschneid@redhat.com>
Link: https://lkml.kernel.org/r/20240727105030.403750550@infradead.org
parent 152e11f6

kernel/sched/fair.c

+18 −2
Original line number Diff line number Diff line
@@ -5447,8 +5447,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 
	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE)

		update_min_vruntime(cfs_rq);



	if (flags & DEQUEUE_DELAYED)

	if (flags & DEQUEUE_DELAYED) {

		se->sched_delayed = 0;

		if (sched_feat(DELAY_ZERO) && se->vlag > 0)

			se->vlag = 0;

	}



	if (cfs_rq->nr_running == 0)

		update_idle_cfs_rq_clock_pelt(cfs_rq);
@@ -5527,7 +5530,6 @@ pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq) 
		dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED);

		SCHED_WARN_ON(se->sched_delayed);

		SCHED_WARN_ON(se->on_rq);



		return NULL;

	}

	return se;
@@ -6825,6 +6827,20 @@ requeue_delayed_entity(struct sched_entity *se) 
	SCHED_WARN_ON(!se->sched_delayed);

	SCHED_WARN_ON(!se->on_rq);



	if (sched_feat(DELAY_ZERO)) {

		update_entity_lag(cfs_rq, se);

		if (se->vlag > 0) {

			cfs_rq->nr_running--;

			if (se != cfs_rq->curr)

				__dequeue_entity(cfs_rq, se);

			se->vlag = 0;

			place_entity(cfs_rq, se, 0);

			if (se != cfs_rq->curr)

				__enqueue_entity(cfs_rq, se);

			cfs_rq->nr_running++;

		}

	}



	se->sched_delayed = 0;

}

kernel/sched/features.h

+3 −0
Original line number Diff line number Diff line
@@ -34,8 +34,11 @@ SCHED_FEAT(CACHE_HOT_BUDDY, true) 
 * By delaying the dequeue for non-eligible tasks, they remain in the

 * competition and can burn off their negative lag. When they get selected

 * they'll have positive lag by definition.

 *

 * DELAY_ZERO clips the lag on dequeue (or wakeup) to 0.

 */

SCHED_FEAT(DELAY_DEQUEUE, true)

SCHED_FEAT(DELAY_ZERO, true)



/*

 * Allow wakeup-time preemption of the current task: