Commit cded46d9 authored by Tejun Heo's avatar Tejun Heo
Browse files

sched_ext: Make scx_bpf_dsq_insert*() return bool 



In preparation for hierarchical schedulers, change scx_bpf_dsq_insert() and
scx_bpf_dsq_insert_vtime() to return bool instead of void. With
sub-schedulers, there will be no reliable way to guarantee a task is still
owned by the sub-scheduler at insertion time (e.g., the task may have been
migrated to another scheduler). The bool return value will enable
sub-schedulers to detect and gracefully handle insertion failures.

For the root scheduler, insertion failures will continue to trigger scheduler
abort via scx_error(), so existing code doesn't need to check the return
value. Backward compatibility is maintained through compat wrappers.

Also update scx_bpf_dsq_move() documentation to clarify that it can return
false for sub-schedulers when @dsq_id points to a disallowed local DSQ.

Reviewed-by: default avatarChangwoo Min <changwoo@igalia.com>
Reviewed-by: default avatarEmil Tsalapatis <emil@etsalapatis.com>
Acked-by: default avatarAndrea Righi <arighi@nvidia.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent c0d630ba

kernel/sched/ext.c

+34 −11
Original line number Diff line number Diff line
@@ -5426,8 +5426,12 @@ __bpf_kfunc_start_defs(); 
 * exhaustion. If zero, the current residual slice is maintained. If

 * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with

 * scx_bpf_kick_cpu() to trigger scheduling.

 *

 * Returns %true on successful insertion, %false on failure. On the root

 * scheduler, %false return triggers scheduler abort and the caller doesn't need

 * to check the return value.

 */

__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice,

__bpf_kfunc bool scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice,

				    u64 enq_flags)

{

	struct scx_sched *sch;
@@ -5435,10 +5439,10 @@ __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice 
	guard(rcu)();

	sch = rcu_dereference(scx_root);

	if (unlikely(!sch))

		return;

		return false;



	if (!scx_dsq_insert_preamble(sch, p, enq_flags))

		return;

		return false;



	if (slice)

		p->scx.slice = slice;
@@ -5446,13 +5450,24 @@ __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice 
		p->scx.slice = p->scx.slice ?: 1;



	scx_dsq_insert_commit(sch, p, dsq_id, enq_flags);



	return true;

}



static void scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,

/*

 * COMPAT: Will be removed in v6.23.

 */

__bpf_kfunc void scx_bpf_dsq_insert___compat(struct task_struct *p, u64 dsq_id,

					     u64 slice, u64 enq_flags)

{

	scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags);

}



static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,

				 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)

{

	if (!scx_dsq_insert_preamble(sch, p, enq_flags))

		return;

		return false;



	if (slice)

		p->scx.slice = slice;
@@ -5462,6 +5477,8 @@ static void scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, 
	p->scx.dsq_vtime = vtime;



	scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);



	return true;

}



struct scx_bpf_dsq_insert_vtime_args {
@@ -5497,8 +5514,12 @@ struct scx_bpf_dsq_insert_vtime_args { 
 * function must not be called on a DSQ which already has one or more FIFO tasks

 * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and

 * SCX_DSQ_GLOBAL) cannot be used as priority queues.

 *

 * Returns %true on successful insertion, %false on failure. On the root

 * scheduler, %false return triggers scheduler abort and the caller doesn't need

 * to check the return value.

 */

__bpf_kfunc void

__bpf_kfunc bool

__scx_bpf_dsq_insert_vtime(struct task_struct *p,

			   struct scx_bpf_dsq_insert_vtime_args *args)

{
@@ -5508,10 +5529,10 @@ __scx_bpf_dsq_insert_vtime(struct task_struct *p, 


	sch = rcu_dereference(scx_root);

	if (unlikely(!sch))

		return;

		return false;



	scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, args->vtime,

			     args->enq_flags);

	return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice,

				    args->vtime, args->enq_flags);

}



/*
@@ -5535,6 +5556,7 @@ __bpf_kfunc_end_defs(); 


BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)

BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_RCU)

BTF_ID_FLAGS(func, scx_bpf_dsq_insert___compat, KF_RCU)

BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_RCU)

BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU)

BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
@@ -5789,8 +5811,9 @@ __bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, 
 * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq

 * lock (e.g. BPF timers or SYSCALL programs).

 *

 * Returns %true if @p has been consumed, %false if @p had already been consumed

 * or dequeued.

 * Returns %true if @p has been consumed, %false if @p had already been

 * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local

 * DSQ.

 */

__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,

				  struct task_struct *p, u64 dsq_id,

tools/sched_ext/include/scx/common.bpf.h

+1 −2
Original line number Diff line number Diff line
@@ -62,8 +62,7 @@ s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; 
s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;

s32 __scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed,

			     struct scx_bpf_select_cpu_and_args *args) __ksym __weak;

void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;

void __scx_bpf_dsq_insert_vtime(struct task_struct *p, struct scx_bpf_dsq_insert_vtime_args *args) __ksym __weak;

bool __scx_bpf_dsq_insert_vtime(struct task_struct *p, struct scx_bpf_dsq_insert_vtime_args *args) __ksym __weak;

u32 scx_bpf_dispatch_nr_slots(void) __ksym;

void scx_bpf_dispatch_cancel(void) __ksym;

bool scx_bpf_dsq_move_to_local(u64 dsq_id) __ksym __weak;

tools/sched_ext/include/scx/compat.bpf.h

+21 −2
Original line number Diff line number Diff line
@@ -196,7 +196,7 @@ scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 
 * Inline wrapper that packs scalar arguments into a struct and calls

 * __scx_bpf_dsq_insert_vtime(). See __scx_bpf_dsq_insert_vtime() for details.

 */

static inline void

static inline bool

scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime,

			 u64 enq_flags)

{
@@ -208,10 +208,29 @@ scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime 
			.enq_flags = enq_flags,

		};



		__scx_bpf_dsq_insert_vtime(p, &args);

		return __scx_bpf_dsq_insert_vtime(p, &args);

	} else {

		scx_bpf_dsq_insert_vtime___compat(p, dsq_id, slice, vtime,

						  enq_flags);

		return true;

	}

}



/*

 * v6.19: scx_bpf_dsq_insert() now returns bool instead of void. Move

 * scx_bpf_dsq_insert() decl to common.bpf.h and drop compat helper after v6.22.

 */

bool scx_bpf_dsq_insert___new(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;

void scx_bpf_dsq_insert___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;



static inline bool

scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags)

{

	if (bpf_ksym_exists(scx_bpf_dsq_insert___new)) {

		return scx_bpf_dsq_insert___new(p, dsq_id, slice, enq_flags);

	} else {

		scx_bpf_dsq_insert___compat(p, dsq_id, slice, enq_flags);

		return true;

	}

}