Commit f1f8d0a2 authored by Vincent Guittot's avatar Vincent Guittot Committed by Ingo Molnar
Browse files

sched/cpufreq: Take cpufreq feedback into account 



Aggregate the different pressures applied on the capacity of CPUs and
create a new function that returns the actual capacity of the CPU:
get_actual_cpu_capacity().

Signed-off-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Tested-by: default avatarLukasz Luba <lukasz.luba@arm.com>
Reviewed-by: default avatarLukasz Luba <lukasz.luba@arm.com>
Reviewed-by: default avatarQais Yousef <qyousef@layalina.io>
Link: https://lore.kernel.org/r/20240326091616.3696851-3-vincent.guittot@linaro.org
parent 75d65931

kernel/sched/fair.c

+25 −20
Original line number Diff line number Diff line
@@ -4965,13 +4965,22 @@ static inline void util_est_update(struct cfs_rq *cfs_rq, 
	trace_sched_util_est_se_tp(&p->se);

}



static inline unsigned long get_actual_cpu_capacity(int cpu)

{

	unsigned long capacity = arch_scale_cpu_capacity(cpu);



	capacity -= max(thermal_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu));



	return capacity;

}



static inline int util_fits_cpu(unsigned long util,

				unsigned long uclamp_min,

				unsigned long uclamp_max,

				int cpu)

{

	unsigned long capacity_orig, capacity_orig_thermal;

	unsigned long capacity = capacity_of(cpu);

	unsigned long capacity_orig;

	bool fits, uclamp_max_fits;



	/*
@@ -5003,7 +5012,6 @@ static inline int util_fits_cpu(unsigned long util, 
	 * goal is to cap the task. So it's okay if it's getting less.

	 */

	capacity_orig = arch_scale_cpu_capacity(cpu);

	capacity_orig_thermal = capacity_orig - arch_scale_thermal_pressure(cpu);



	/*

	 * We want to force a task to fit a cpu as implied by uclamp_max.
@@ -5078,7 +5086,8 @@ static inline int util_fits_cpu(unsigned long util, 
	 * handle the case uclamp_min > uclamp_max.

	 */

	uclamp_min = min(uclamp_min, uclamp_max);

	if (fits && (util < uclamp_min) && (uclamp_min > capacity_orig_thermal))

	if (fits && (util < uclamp_min) &&

	    (uclamp_min > get_actual_cpu_capacity(cpu)))

		return -1;



	return fits;
@@ -7494,7 +7503,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 
		 * Look for the CPU with best capacity.

		 */

		else if (fits < 0)

			cpu_cap = arch_scale_cpu_capacity(cpu) - thermal_load_avg(cpu_rq(cpu));

			cpu_cap = get_actual_cpu_capacity(cpu);



		/*

		 * First, select CPU which fits better (-1 being better than 0).
@@ -7987,8 +7996,8 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 
	struct root_domain *rd = this_rq()->rd;

	int cpu, best_energy_cpu, target = -1;

	int prev_fits = -1, best_fits = -1;

	unsigned long best_thermal_cap = 0;

	unsigned long prev_thermal_cap = 0;

	unsigned long best_actual_cap = 0;

	unsigned long prev_actual_cap = 0;

	struct sched_domain *sd;

	struct perf_domain *pd;

	struct energy_env eenv;
@@ -8018,7 +8027,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 


	for (; pd; pd = pd->next) {

		unsigned long util_min = p_util_min, util_max = p_util_max;

		unsigned long cpu_cap, cpu_thermal_cap, util;

		unsigned long cpu_cap, cpu_actual_cap, util;

		long prev_spare_cap = -1, max_spare_cap = -1;

		unsigned long rq_util_min, rq_util_max;

		unsigned long cur_delta, base_energy;
@@ -8030,18 +8039,17 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 
		if (cpumask_empty(cpus))

			continue;



		/* Account thermal pressure for the energy estimation */

		/* Account external pressure for the energy estimation */

		cpu = cpumask_first(cpus);

		cpu_thermal_cap = arch_scale_cpu_capacity(cpu);

		cpu_thermal_cap -= arch_scale_thermal_pressure(cpu);

		cpu_actual_cap = get_actual_cpu_capacity(cpu);



		eenv.cpu_cap = cpu_thermal_cap;

		eenv.cpu_cap = cpu_actual_cap;

		eenv.pd_cap = 0;



		for_each_cpu(cpu, cpus) {

			struct rq *rq = cpu_rq(cpu);



			eenv.pd_cap += cpu_thermal_cap;

			eenv.pd_cap += cpu_actual_cap;



			if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))

				continue;
@@ -8112,7 +8120,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 
			if (prev_delta < base_energy)

				goto unlock;

			prev_delta -= base_energy;

			prev_thermal_cap = cpu_thermal_cap;

			prev_actual_cap = cpu_actual_cap;

			best_delta = min(best_delta, prev_delta);

		}
@@ -8127,7 +8135,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 
			 * but best energy cpu has better capacity.

			 */

			if ((max_fits < 0) &&

			    (cpu_thermal_cap <= best_thermal_cap))

			    (cpu_actual_cap <= best_actual_cap))

				continue;



			cur_delta = compute_energy(&eenv, pd, cpus, p,
@@ -8148,14 +8156,14 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 
			best_delta = cur_delta;

			best_energy_cpu = max_spare_cap_cpu;

			best_fits = max_fits;

			best_thermal_cap = cpu_thermal_cap;

			best_actual_cap = cpu_actual_cap;

		}

	}

	rcu_read_unlock();



	if ((best_fits > prev_fits) ||

	    ((best_fits > 0) && (best_delta < prev_delta)) ||

	    ((best_fits < 0) && (best_thermal_cap > prev_thermal_cap)))

	    ((best_fits < 0) && (best_actual_cap > prev_actual_cap)))

		target = best_energy_cpu;



	return target;
@@ -9560,8 +9568,8 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 


static unsigned long scale_rt_capacity(int cpu)

{

	unsigned long max = get_actual_cpu_capacity(cpu);

	struct rq *rq = cpu_rq(cpu);

	unsigned long max = arch_scale_cpu_capacity(cpu);

	unsigned long used, free;

	unsigned long irq;
@@ -9573,12 +9581,9 @@ static unsigned long scale_rt_capacity(int cpu) 
	/*

	 * avg_rt.util_avg and avg_dl.util_avg track binary signals

	 * (running and not running) with weights 0 and 1024 respectively.

	 * avg_thermal.load_avg tracks thermal pressure and the weighted

	 * average uses the actual delta max capacity(load).

	 */

	used = cpu_util_rt(rq);

	used += cpu_util_dl(rq);

	used += thermal_load_avg(rq);



	if (unlikely(used >= max))

		return 1;