Commit d4ae868c authored by Tejun Heo's avatar Tejun Heo
Browse files

sched_ext: Wrap global DSQs in per-node structure 



Global DSQs are currently stored as an array of scx_dispatch_q pointers,
one per NUMA node. To allow adding more per-node data structures, wrap the
global DSQ in scx_sched_pnode and replace global_dsqs with pnode array.

NUMA-aware allocation is maintained. No functional changes.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarEmil Tsalapatis <emil@etsalapatis.com>
Reviewed-by: default avatarDaniel Jordan <daniel.m.jordan@oracle.com>
Reviewed-by: default avatarAndrea Righi <arighi@nvidia.com>
parent 26b9c7c7

kernel/sched/ext.c

+16 −16
Original line number Diff line number Diff line
@@ -344,7 +344,7 @@ static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) 
static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch,

					      struct task_struct *p)

{

	return sch->global_dsqs[cpu_to_node(task_cpu(p))];

	return &sch->pnode[cpu_to_node(task_cpu(p))]->global_dsq;

}



static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)
@@ -2228,7 +2228,7 @@ static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) 
{

	int node = cpu_to_node(cpu_of(rq));



	return consume_dispatch_q(sch, rq, sch->global_dsqs[node]);

	return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq);

}



/**
@@ -4147,8 +4147,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) 
	free_percpu(sch->pcpu);



	for_each_node_state(node, N_POSSIBLE)

		kfree(sch->global_dsqs[node]);

	kfree(sch->global_dsqs);

		kfree(sch->pnode[node]);

	kfree(sch->pnode);



	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);

	do {
@@ -5706,23 +5706,23 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops, 
	if (ret < 0)

		goto err_free_ei;



	sch->global_dsqs = kzalloc_objs(sch->global_dsqs[0], nr_node_ids);

	if (!sch->global_dsqs) {

	sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids);

	if (!sch->pnode) {

		ret = -ENOMEM;

		goto err_free_hash;

	}



	for_each_node_state(node, N_POSSIBLE) {

		struct scx_dispatch_q *dsq;

		struct scx_sched_pnode *pnode;



		dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node);

		if (!dsq) {

		pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node);

		if (!pnode) {

			ret = -ENOMEM;

			goto err_free_gdsqs;

			goto err_free_pnode;

		}



		init_dsq(dsq, SCX_DSQ_GLOBAL, sch);

		sch->global_dsqs[node] = dsq;

		init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch);

		sch->pnode[node] = pnode;

	}



	sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
@@ -5731,7 +5731,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops, 
				   __alignof__(struct scx_sched_pcpu));

	if (!sch->pcpu) {

		ret = -ENOMEM;

		goto err_free_gdsqs;

		goto err_free_pnode;

	}



	for_each_possible_cpu(cpu)
@@ -5818,10 +5818,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops, 
	kthread_destroy_worker(sch->helper);

err_free_pcpu:

	free_percpu(sch->pcpu);

err_free_gdsqs:

err_free_pnode:

	for_each_node_state(node, N_POSSIBLE)

		kfree(sch->global_dsqs[node]);

	kfree(sch->global_dsqs);

		kfree(sch->pnode[node]);

	kfree(sch->pnode);

err_free_hash:

	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);

err_free_ei:

kernel/sched/ext_internal.h

+5 −1
Original line number Diff line number Diff line
@@ -975,6 +975,10 @@ struct scx_sched_pcpu { 
	struct scx_dsp_ctx	dsp_ctx;

};



struct scx_sched_pnode {

	struct scx_dispatch_q	global_dsq;

};



struct scx_sched {

	struct sched_ext_ops	ops;

	DECLARE_BITMAP(has_op, SCX_OPI_END);
@@ -988,7 +992,7 @@ struct scx_sched { 
	 * per-node split isn't sufficient, it can be further split.

	 */

	struct rhashtable	dsq_hash;

	struct scx_dispatch_q	**global_dsqs;

	struct scx_sched_pnode	**pnode;

	struct scx_sched_pcpu __percpu *pcpu;



	u64			slice_dfl;