Commit 4b603f15 authored by Shubhang Kaushik's avatar Shubhang Kaushik Committed by Peter Zijlstra
Browse files

sched: Update rq->avg_idle when a task is moved to an idle CPU 



Currently, rq->idle_stamp is only used to calculate avg_idle during
wakeups. This means other paths that move a task to an idle CPU such as
fork/clone, execve, or migrations, do not end the CPU's idle status in
the scheduler's eyes, leading to an inaccurate avg_idle.

This patch introduces update_rq_avg_idle() to provide a more accurate
measurement of CPU idle duration. By invoking this helper in
put_prev_task_idle(), we ensure avg_idle is updated whenever a CPU
stops being idle, regardless of how the new task arrived.

Testing on an 80-core Ampere Altra (ARMv8) with 6.19-rc5 baseline:
 - Hackbench : +7.2% performance gain at 16 threads.
 - Schbench: Reduced p99.9 tail latencies at high concurrency.

Signed-off-by: default avatarShubhang Kaushik <shubhang@os.amperecomputing.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Tested-by: default avatarShubhang Kaushik <shubhang@os.amperecomputing.com>
Link: https://patch.msgid.link/20260121-v8-patch-series-v8-1-b7f1cbee5055@os.amperecomputing.com
parent bb332a9e

kernel/sched/core.c

+12 −12
Original line number Diff line number Diff line
@@ -3613,6 +3613,18 @@ static inline void ttwu_do_wakeup(struct task_struct *p) 
	trace_sched_wakeup(p);

}



void update_rq_avg_idle(struct rq *rq)

{

	u64 delta = rq_clock(rq) - rq->idle_stamp;

	u64 max = 2*rq->max_idle_balance_cost;



	update_avg(&rq->avg_idle, delta);



	if (rq->avg_idle > max)

		rq->avg_idle = max;

	rq->idle_stamp = 0;

}



static void

ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,

		 struct rq_flags *rf)
@@ -3648,18 +3660,6 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 
		p->sched_class->task_woken(rq, p);

		rq_repin_lock(rq, rf);

	}



	if (rq->idle_stamp) {

		u64 delta = rq_clock(rq) - rq->idle_stamp;

		u64 max = 2*rq->max_idle_balance_cost;



		update_avg(&rq->avg_idle, delta);



		if (rq->avg_idle > max)

			rq->avg_idle = max;



		rq->idle_stamp = 0;

	}

}



/*

kernel/sched/idle.c

+1 −0
Original line number Diff line number Diff line
@@ -460,6 +460,7 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct t 
{

	update_curr_idle(rq);

	scx_update_idle(rq, false, true);

	update_rq_avg_idle(rq);

}



static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)

kernel/sched/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -1670,6 +1670,7 @@ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 


#endif /* !CONFIG_FAIR_GROUP_SCHED */



extern void update_rq_avg_idle(struct rq *rq);

extern void update_rq_clock(struct rq *rq);



/*