Commit a8433f7a authored by Tejun Heo's avatar Tejun Heo
Browse files

sched_ext: Add @sch to SCX_CALL_OP*() 



In preparation of hierarchical scheduling support, add @sch to scx_exit()
and friends:

- scx_exit/error() updated to take explicit @sch instead of assuming
  scx_root.

- scx_kf_exit/error() added. These are to be used from kfuncs, don't take
  @sch and internally determine the scx_sched instance to abort. Currently,
  it's always scx_root but once multiple scheduler support is in place, it
  will be the scx_sched instance that invoked the kfunc. This simplifies
  many callsites and defers scx_sched lookup until error is triggered.

- @sch is propagated to ops_cpu_valid() and ops_sanitize_err(). The CPU
  validity conditions in ops_cpu_valid() are factored into __cpu_valid() to
  implement kf_cpu_valid() which is the counterpart to scx_kf_exit/error().

- All users are converted. Most conversions are straightforward.
  check_rq_for_timeouts() and scx_softlockup() are updated to use explicit
  rcu_dereference*(scx_root) for safety as they may execute asynchronous to
  the exit path. scx_tick() is also updated to use rcu_dereference(). While
  not strictly necessary due to the preceding scx_enabled() test and IRQ
  disabled context, this removes the subtlety at no noticeable cost.

No behavior changes intended.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarAndrea Righi <arighi@nvidia.com>
parent c4c286d7

kernel/sched/ext.c

+175 −109

File changed.

Preview size limit exceeded, changes collapsed.

kernel/sched/ext_idle.c

+12 −12
Original line number Diff line number Diff line
@@ -820,7 +820,7 @@ void scx_idle_disable(void) 
static int validate_node(int node)

{

	if (!static_branch_likely(&scx_builtin_idle_per_node)) {

		scx_error("per-node idle tracking is disabled");

		scx_kf_error("per-node idle tracking is disabled");

		return -EOPNOTSUPP;

	}
@@ -830,13 +830,13 @@ static int validate_node(int node) 


	/* Make sure node is in a valid range */

	if (node < 0 || node >= nr_node_ids) {

		scx_error("invalid node %d", node);

		scx_kf_error("invalid node %d", node);

		return -EINVAL;

	}



	/* Make sure the node is part of the set of possible nodes */

	if (!node_possible(node)) {

		scx_error("unavailable node %d", node);

		scx_kf_error("unavailable node %d", node);

		return -EINVAL;

	}
@@ -850,7 +850,7 @@ static bool check_builtin_idle_enabled(void) 
	if (static_branch_likely(&scx_builtin_idle_enabled))

		return true;



	scx_error("built-in idle tracking is disabled");

	scx_kf_error("built-in idle tracking is disabled");

	return false;

}
@@ -862,7 +862,7 @@ static bool check_builtin_idle_enabled(void) 
__bpf_kfunc int scx_bpf_cpu_node(s32 cpu)

{

#ifdef CONFIG_NUMA

	if (!ops_cpu_valid(cpu, NULL))

	if (!kf_cpu_valid(cpu, NULL))

		return NUMA_NO_NODE;



	return cpu_to_node(cpu);
@@ -891,7 +891,7 @@ __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, 
#ifdef CONFIG_SMP

	s32 cpu;

#endif

	if (!ops_cpu_valid(prev_cpu, NULL))

	if (!kf_cpu_valid(prev_cpu, NULL))

		goto prev_cpu;



	if (!check_builtin_idle_enabled())
@@ -937,7 +937,7 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 
{

	s32 cpu;



	if (!ops_cpu_valid(prev_cpu, NULL))

	if (!kf_cpu_valid(prev_cpu, NULL))

		return -EINVAL;



	if (!check_builtin_idle_enabled())
@@ -999,7 +999,7 @@ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) 
__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void)

{

	if (static_branch_unlikely(&scx_builtin_idle_per_node)) {

		scx_error("SCX_OPS_BUILTIN_IDLE_PER_NODE enabled");

		scx_kf_error("SCX_OPS_BUILTIN_IDLE_PER_NODE enabled");

		return cpu_none_mask;

	}
@@ -1050,7 +1050,7 @@ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) 
__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(void)

{

	if (static_branch_unlikely(&scx_builtin_idle_per_node)) {

		scx_error("SCX_OPS_BUILTIN_IDLE_PER_NODE enabled");

		scx_kf_error("SCX_OPS_BUILTIN_IDLE_PER_NODE enabled");

		return cpu_none_mask;

	}
@@ -1097,7 +1097,7 @@ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) 
	if (!check_builtin_idle_enabled())

		return false;



	if (ops_cpu_valid(cpu, NULL))

	if (kf_cpu_valid(cpu, NULL))

		return scx_idle_test_and_clear_cpu(cpu);

	else

		return false;
@@ -1158,7 +1158,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, 
				      u64 flags)

{

	if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) {

		scx_error("per-node idle tracking is enabled");

		scx_kf_error("per-node idle tracking is enabled");

		return -EBUSY;

	}
@@ -1235,7 +1235,7 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, 
	s32 cpu;



	if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) {

		scx_error("per-node idle tracking is enabled");

		scx_kf_error("per-node idle tracking is enabled");

		return -EBUSY;

	}