Commit f207dc2d authored by Tejun Heo's avatar Tejun Heo
Browse files

sched/core: Add ENQUEUE_RQ_SELECTED to indicate whether ->select_task_rq() was called 



During ttwu, ->select_task_rq() can be skipped if only one CPU is allowed or
migration is disabled. sched_ext schedulers may perform operations such as
direct dispatch from ->select_task_rq() path and it is useful for them to
know whether ->select_task_rq() was skipped in the ->enqueue_task() path.

Currently, sched_ext schedulers are using ENQUEUE_WAKEUP for this purpose
and end up assuming incorrectly that ->select_task_rq() was called for tasks
that are bound to a single CPU or migration disabled.

Make select_task_rq() indicate whether ->select_task_rq() was called by
setting WF_RQ_SELECTED in *wake_flags and make ttwu_do_activate() map that
to ENQUEUE_RQ_SELECTED for ->enqueue_task().

This will be used by sched_ext to fix ->select_task_rq() skip detection.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Acked-by: default avatarDavid Vernet <void@manifault.com>
parent b62933ee

kernel/sched/core.c

+6 −2
Original line number Diff line number Diff line
@@ -3522,10 +3522,12 @@ int select_task_rq(struct task_struct *p, int cpu, int *wake_flags) 
{

	lockdep_assert_held(&p->pi_lock);



	if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p))

	if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) {

		cpu = p->sched_class->select_task_rq(p, cpu, *wake_flags);

	else

		*wake_flags |= WF_RQ_SELECTED;

	} else {

		cpu = cpumask_any(p->cpus_ptr);

	}



	/*

	 * In order not to call set_task_cpu() on a blocking task we need
@@ -3659,6 +3661,8 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 
		rq->nr_uninterruptible--;



#ifdef CONFIG_SMP

	if (wake_flags & WF_RQ_SELECTED)

		en_flags |= ENQUEUE_RQ_SELECTED;

	if (wake_flags & WF_MIGRATED)

		en_flags |= ENQUEUE_MIGRATED;

	else

kernel/sched/sched.h

+3 −0
Original line number Diff line number Diff line
@@ -2292,6 +2292,7 @@ static inline int task_on_rq_migrating(struct task_struct *p) 
#define WF_SYNC			0x10 /* Waker goes to sleep after wakeup */

#define WF_MIGRATED		0x20 /* Internal use, task got migrated */

#define WF_CURRENT_CPU		0x40 /* Prefer to move the wakee to the current CPU. */

#define WF_RQ_SELECTED		0x80 /* ->select_task_rq() was called */



#ifdef CONFIG_SMP

static_assert(WF_EXEC == SD_BALANCE_EXEC);
@@ -2334,6 +2335,7 @@ extern const u32 sched_prio_to_wmult[40]; 
 * ENQUEUE_HEAD      - place at front of runqueue (tail if not specified)

 * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline)

 * ENQUEUE_MIGRATED  - the task was migrated during wakeup

 * ENQUEUE_RQ_SELECTED - ->select_task_rq() was called

 *

 */
@@ -2360,6 +2362,7 @@ extern const u32 sched_prio_to_wmult[40]; 
#define ENQUEUE_INITIAL		0x80

#define ENQUEUE_MIGRATING	0x100

#define ENQUEUE_DELAYED		0x200

#define ENQUEUE_RQ_SELECTED	0x400



#define RETRY_TASK		((void *)-1UL)