Commit 956f2b11 authored by Tejun Heo's avatar Tejun Heo
Browse files

sched_ext: Drop kf_cpu_valid() 



The intention behind kf_cpu_valid() was that when called from kfuncs,
kf_cpu_valid() would be able to implicitly determine the scx_sched instance
being operated on and thus wouldn't need @sch passed in explicitly. This
turned out to be unnecessarily complicated to implement and not have
justifiable practical benefits. Replace kf_cpu_valid() usages with
ops_cpu_valid() which takes explicit @sch.

Callers which don't have $sch available in the context are updated to read
$scx_root under RCU read lock, verify that it's not NULL and pass it in.

scx_bpf_cpu_rq() is restructured to use guard(rcu)() instead of explicit
rcu_read_[un]lock().

Reviewed-by: default avatarAndrea Righi <arighi@nvidia.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 2407bae2

kernel/sched/ext.c

+40 −27
Original line number Diff line number Diff line
@@ -736,23 +736,6 @@ static bool ops_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) 
	}

}



/**

 * kf_cpu_valid - Verify a CPU number, to be used on kfunc input args

 * @cpu: cpu number which came from a BPF ops

 * @where: extra information reported on error

 *

 * The same as ops_cpu_valid() but @sch is implicit.

 */

static bool kf_cpu_valid(u32 cpu, const char *where)

{

	if (__cpu_valid(cpu)) {

		return true;

	} else {

		scx_kf_error("invalid CPU %d%s%s", cpu, where ? " " : "", where ?: "");

		return false;

	}

}



/**

 * ops_sanitize_err - Sanitize a -errno value

 * @sch: scx_sched to error out on error
@@ -5815,7 +5798,7 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) 
	struct rq *this_rq;

	unsigned long irq_flags;



	if (!kf_cpu_valid(cpu, NULL))

	if (!ops_cpu_valid(sch, cpu, NULL))

		return;



	local_irq_save(irq_flags);
@@ -6224,7 +6207,12 @@ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, 
 */

__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu)

{

	if (kf_cpu_valid(cpu, NULL))

	struct scx_sched *sch;



	guard(rcu)();



	sch = rcu_dereference(scx_root);

	if (likely(sch) && ops_cpu_valid(sch, cpu, NULL))

		return arch_scale_cpu_capacity(cpu);

	else

		return SCX_CPUPERF_ONE;
@@ -6246,7 +6234,12 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu) 
 */

__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu)

{

	if (kf_cpu_valid(cpu, NULL))

	struct scx_sched *sch;



	guard(rcu)();



	sch = rcu_dereference(scx_root);

	if (likely(sch) && ops_cpu_valid(sch, cpu, NULL))

		return arch_scale_freq_capacity(cpu);

	else

		return SCX_CPUPERF_ONE;
@@ -6268,12 +6261,20 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu) 
 */

__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf)

{

	struct scx_sched *sch;



	guard(rcu)();



	sch = rcu_dereference(sch);

	if (unlikely(!sch))

		return;



	if (unlikely(perf > SCX_CPUPERF_ONE)) {

		scx_kf_error("Invalid cpuperf target %u for CPU %d", perf, cpu);

		return;

	}



	if (kf_cpu_valid(cpu, NULL)) {

	if (ops_cpu_valid(sch, cpu, NULL)) {

		struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq();

		struct rq_flags rf;
@@ -6379,18 +6380,21 @@ __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu) 
{

	struct scx_sched *sch;



	if (!kf_cpu_valid(cpu, NULL))

		return NULL;

	guard(rcu)();



	rcu_read_lock();

	sch = rcu_dereference(scx_root);

	if (likely(sch) && !sch->warned_deprecated_rq) {

	if (unlikely(!sch))

		return NULL;



	if (!ops_cpu_valid(sch, cpu, NULL))

		return NULL;



	if (!sch->warned_deprecated_rq) {

		printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; "

				"use scx_bpf_locked_rq() when holding rq lock "

				"or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__);

		sch->warned_deprecated_rq = true;

	}

	rcu_read_unlock();



	return cpu_rq(cpu);

}
@@ -6425,8 +6429,17 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(void) 
 */

__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu)

{

	if (!kf_cpu_valid(cpu, NULL))

	struct scx_sched *sch;



	guard(rcu)();



	sch = rcu_dereference(scx_root);

	if (unlikely(!sch))

		return NULL;



	if (!ops_cpu_valid(sch, cpu, NULL))

		return NULL;



	return rcu_dereference(cpu_rq(cpu)->curr);

}

kernel/sched/ext_idle.c

+8 −4
Original line number Diff line number Diff line
@@ -864,7 +864,7 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, 
	struct rq_flags rf;

	s32 cpu;



	if (!kf_cpu_valid(prev_cpu, NULL))

	if (!ops_cpu_valid(sch, prev_cpu, NULL))

		return -EINVAL;



	if (!check_builtin_idle_enabled(sch))
@@ -923,9 +923,13 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, 
 */

__bpf_kfunc int scx_bpf_cpu_node(s32 cpu)

{

	if (!kf_cpu_valid(cpu, NULL))

		return NUMA_NO_NODE;

	struct scx_sched *sch;



	guard(rcu)();



	sch = rcu_dereference(scx_root);

	if (unlikely(!sch) || !ops_cpu_valid(sch, cpu, NULL))

		return NUMA_NO_NODE;

	return cpu_to_node(cpu);

}
@@ -1154,7 +1158,7 @@ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) 
	if (!check_builtin_idle_enabled(sch))

		return false;



	if (!kf_cpu_valid(cpu, NULL))

	if (!ops_cpu_valid(sch, cpu, NULL))

		return false;



	return scx_idle_test_and_clear_cpu(cpu);