Commit 34ecfb35 authored by Tejun Heo's avatar Tejun Heo
Browse files

sched_ext: Move scx_dsp_ctx and scx_dsp_max_batch into scx_sched 



scx_dsp_ctx and scx_dsp_max_batch are global variables used in the dispatch
path. In prepration for multiple scheduler support, move the former into
scx_sched_pcpu and the latter into scx_sched. No user-visible behavior
changes intended.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarAndrea Righi <arighi@nvidia.com>
parent 0203e0c3

kernel/sched/ext.c

+15 −40
Original line number Diff line number Diff line
@@ -106,25 +106,6 @@ static const struct rhashtable_params dsq_hash_params = { 


static LLIST_HEAD(dsqs_to_free);



/* dispatch buf */

struct scx_dsp_buf_ent {

	struct task_struct	*task;

	unsigned long		qseq;

	u64			dsq_id;

	u64			enq_flags;

};



static u32 scx_dsp_max_batch;



struct scx_dsp_ctx {

	struct rq		*rq;

	u32			cursor;

	u32			nr_tasks;

	struct scx_dsp_buf_ent	buf[];

};



static struct scx_dsp_ctx __percpu *scx_dsp_ctx;



/* string formatting from BPF */

struct scx_bstr_buf {

	u64			data[MAX_BPRINTF_VARARGS];
@@ -2402,7 +2383,7 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, 


static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)

{

	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);

	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;

	u32 u;



	for (u = 0; u < dspc->cursor; u++) {
@@ -2432,7 +2413,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq) 
static bool scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,

			       struct task_struct *prev)

{

	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);

	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;

	int nr_loops = SCX_DSP_MAX_LOOPS;

	s32 cpu = cpu_of(rq);

	bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
@@ -4972,9 +4953,6 @@ static void scx_root_disable(struct scx_sched *sch) 
	 */

	kobject_del(&sch->kobj);



	free_percpu(scx_dsp_ctx);

	scx_dsp_ctx = NULL;

	scx_dsp_max_batch = 0;

	free_kick_syncs();



	mutex_unlock(&scx_enable_mutex);
@@ -5469,7 +5447,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops, 
		sch->global_dsqs[node] = dsq;

	}



	sch->pcpu = alloc_percpu(struct scx_sched_pcpu);

	sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;

	sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu,

						 dsp_ctx.buf, sch->dsp_max_batch),

				   __alignof__(struct scx_sched_pcpu));

	if (!sch->pcpu) {

		ret = -ENOMEM;

		goto err_free_gdsqs;
@@ -5716,16 +5697,6 @@ static void scx_root_enable_workfn(struct kthread_work *work) 
	if (ret)

		goto err_disable;



	WARN_ON_ONCE(scx_dsp_ctx);

	scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;

	scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf,

						   scx_dsp_max_batch),

				     __alignof__(struct scx_dsp_ctx));

	if (!scx_dsp_ctx) {

		ret = -ENOMEM;

		goto err_disable;

	}



	if (ops->timeout_ms)

		timeout = msecs_to_jiffies(ops->timeout_ms);

	else
@@ -6703,7 +6674,7 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p 
static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,

				  u64 dsq_id, u64 enq_flags)

{

	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);

	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;

	struct task_struct *ddsp_task;



	ddsp_task = __this_cpu_read(direct_dispatch_task);
@@ -6712,7 +6683,7 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, 
		return;

	}



	if (unlikely(dspc->cursor >= scx_dsp_max_batch)) {

	if (unlikely(dspc->cursor >= sch->dsp_max_batch)) {

		scx_error(sch, "dispatch buffer overflow");

		return;

	}
@@ -7030,7 +7001,7 @@ __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) 
	if (!scx_kf_allowed(sch, SCX_KF_DISPATCH))

		return 0;



	return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor);

	return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor);

}



/**
@@ -7042,8 +7013,8 @@ __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) 
 */

__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux)

{

	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);

	struct scx_sched *sch;

	struct scx_dsp_ctx *dspc;



	guard(rcu)();
@@ -7054,6 +7025,8 @@ __bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) 
	if (!scx_kf_allowed(sch, SCX_KF_DISPATCH))

		return;



	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;



	if (dspc->cursor > 0)

		dspc->cursor--;

	else
@@ -7077,9 +7050,9 @@ __bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) 
 */

__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux)

{

	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);

	struct scx_dispatch_q *dsq;

	struct scx_sched *sch;

	struct scx_dsp_ctx *dspc;



	guard(rcu)();
@@ -7090,6 +7063,8 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux 
	if (!scx_kf_allowed(sch, SCX_KF_DISPATCH))

		return false;



	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;



	flush_dispatch_buf(sch, dspc->rq);



	dsq = find_user_dsq(sch, dsq_id);

kernel/sched/ext_internal.h

+19 −0
Original line number Diff line number Diff line
@@ -937,6 +937,21 @@ enum scx_sched_pcpu_flags { 
	SCX_SCHED_PCPU_BYPASSING	= 1LLU << 0,

};



/* dispatch buf */

struct scx_dsp_buf_ent {

	struct task_struct	*task;

	unsigned long		qseq;

	u64			dsq_id;

	u64			enq_flags;

};



struct scx_dsp_ctx {

	struct rq		*rq;

	u32			cursor;

	u32			nr_tasks;

	struct scx_dsp_buf_ent	buf[];

};



struct scx_sched_pcpu {

	u64			flags;	/* protected by rq lock */
@@ -951,6 +966,9 @@ struct scx_sched_pcpu { 
#ifdef CONFIG_EXT_SUB_SCHED

	u32			bypass_host_seq;

#endif



	/* must be the last entry - contains flex array */

	struct scx_dsp_ctx	dsp_ctx;

};



struct scx_sched {
@@ -978,6 +996,7 @@ struct scx_sched { 
	atomic_t		bypass_dsp_enable_depth;



	bool			aborting;

	u32			dsp_max_batch;

	s32			level;



	/*