Commit c8a85394 authored by Joel Fernandes (Google)'s avatar Joel Fernandes (Google) Committed by Peter Zijlstra
Browse files

sched/core: Fix picking of tasks for core scheduling with DL server 



* Use simple CFS pick_task for DL pick_task

  DL server's pick_task calls CFS's pick_next_task_fair(), this is wrong
  because core scheduling's pick_task only calls CFS's pick_task() for
  evaluation / checking of the CFS task (comparing across CPUs), not for
  actually affirmatively picking the next task. This causes RB tree
  corruption issues in CFS that were found by syzbot.

* Make pick_task_fair clear DL server

  A DL task pick might set ->dl_server, but it is possible the task will
  never run (say the other HT has a stop task). If the CFS task is picked
  in the future directly (say without DL server), ->dl_server will be
  set. So clear it in pick_task_fair().

This fixes the KASAN issue reported by syzbot in set_next_entity().

(DL refactoring suggestions by Vineeth Pillai).

Reported-by: default avatarSuleiman Souhlal <suleiman@google.com>
Signed-off-by: default avatar"Joel Fernandes (Google)" <joel@joelfernandes.org>
Signed-off-by: default avatarDaniel Bristot de Oliveira <bristot@kernel.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarVineeth Pillai <vineeth@bitbyteword.org>
Tested-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Link: https://lore.kernel.org/r/b10489ab1f03d23e08e6097acea47442e7d6466f.1716811044.git.bristot@kernel.org
parent 4b26cfdd

include/linux/sched.h

+2 −1
Original line number Diff line number Diff line
@@ -686,7 +686,8 @@ struct sched_dl_entity { 
	 */

	struct rq			*rq;

	dl_server_has_tasks_f		server_has_tasks;

	dl_server_pick_f		server_pick;

	dl_server_pick_f		server_pick_next;

	dl_server_pick_f		server_pick_task;



#ifdef CONFIG_RT_MUTEXES

	/*

kernel/sched/deadline.c

+22 −5
Original line number Diff line number Diff line
@@ -1664,11 +1664,13 @@ void dl_server_stop(struct sched_dl_entity *dl_se) 


void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,

		    dl_server_has_tasks_f has_tasks,

		    dl_server_pick_f pick)

		    dl_server_pick_f pick_next,

		    dl_server_pick_f pick_task)

{

	dl_se->rq = rq;

	dl_se->server_has_tasks = has_tasks;

	dl_se->server_pick = pick;

	dl_se->server_pick_next = pick_next;

	dl_se->server_pick_task = pick_task;

}



void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq)
@@ -2399,7 +2401,12 @@ static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq) 
	return __node_2_dle(left);

}



static struct task_struct *pick_task_dl(struct rq *rq)

/*

 * __pick_next_task_dl - Helper to pick the next -deadline task to run.

 * @rq: The runqueue to pick the next task from.

 * @peek: If true, just peek at the next task. Only relevant for dlserver.

 */

static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek)

{

	struct sched_dl_entity *dl_se;

	struct dl_rq *dl_rq = &rq->dl;
@@ -2413,7 +2420,10 @@ static struct task_struct *pick_task_dl(struct rq *rq) 
	WARN_ON_ONCE(!dl_se);



	if (dl_server(dl_se)) {

		p = dl_se->server_pick(dl_se);

		if (IS_ENABLED(CONFIG_SMP) && peek)

			p = dl_se->server_pick_task(dl_se);

		else

			p = dl_se->server_pick_next(dl_se);

		if (!p) {

			WARN_ON_ONCE(1);

			dl_se->dl_yielded = 1;
@@ -2428,11 +2438,18 @@ static struct task_struct *pick_task_dl(struct rq *rq) 
	return p;

}



#ifdef CONFIG_SMP

static struct task_struct *pick_task_dl(struct rq *rq)

{

	return __pick_next_task_dl(rq, true);

}

#endif



static struct task_struct *pick_next_task_dl(struct rq *rq)

{

	struct task_struct *p;



	p = pick_task_dl(rq);

	p = __pick_next_task_dl(rq, false);

	if (!p)

		return p;

kernel/sched/fair.c

+21 −2
Original line number Diff line number Diff line
@@ -8479,6 +8479,14 @@ static struct task_struct *pick_task_fair(struct rq *rq) 
		cfs_rq = group_cfs_rq(se);

	} while (cfs_rq);



	/*

	 * This can be called from directly from CFS's ->pick_task() or indirectly

	 * from DL's ->pick_task when fair server is enabled. In the indirect case,

	 * DL will set ->dl_server just after this function is called, so its Ok to

	 * clear. In the direct case, we are picking directly so we must clear it.

	 */

	task_of(se)->dl_server = NULL;



	return task_of(se);

}

#endif
@@ -8638,7 +8646,16 @@ static bool fair_server_has_tasks(struct sched_dl_entity *dl_se) 
	return !!dl_se->rq->cfs.nr_running;

}



static struct task_struct *fair_server_pick(struct sched_dl_entity *dl_se)

static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se)

{

#ifdef CONFIG_SMP

	return pick_task_fair(dl_se->rq);

#else

	return NULL;

#endif

}



static struct task_struct *fair_server_pick_next(struct sched_dl_entity *dl_se)

{

	return pick_next_task_fair(dl_se->rq, NULL, NULL);

}
@@ -8649,7 +8666,9 @@ void fair_server_init(struct rq *rq) 


	init_dl_entity(dl_se);



	dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick);

	dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_next,

		       fair_server_pick_task);



}



/*

kernel/sched/sched.h

+2 −1
Original line number Diff line number Diff line
@@ -361,7 +361,8 @@ extern void dl_server_start(struct sched_dl_entity *dl_se); 
extern void dl_server_stop(struct sched_dl_entity *dl_se);

extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,

		    dl_server_has_tasks_f has_tasks,

		    dl_server_pick_f pick);

		    dl_server_pick_f pick_next,

		    dl_server_pick_f pick_task);



extern void dl_server_update_idle_time(struct rq *rq,

		    struct task_struct *p);