Commit be41bde4 authored by John Stultz's avatar John Stultz Committed by Peter Zijlstra
Browse files

sched: Add an initial sketch of the find_proxy_task() function



Add a find_proxy_task() function which doesn't do much.

When we select a blocked task to run, we will just deactivate it
and pick again. The exception being if it has become unblocked
after find_proxy_task() was called.

This allows us to validate keeping blocked tasks on the runqueue
and later deactivating them is working ok, stressing the failure
cases for when a proxy isn't found.

Greatly simplified from patch by:
  Peter Zijlstra (Intel) <peterz@infradead.org>
  Juri Lelli <juri.lelli@redhat.com>
  Valentin Schneider <valentin.schneider@arm.com>
  Connor O'Brien <connoro@google.com>

[jstultz: Split out from larger proxy patch and simplified
 for review and testing.]
Signed-off-by: default avatarJohn Stultz <jstultz@google.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: default avatarK Prateek Nayak <kprateek.nayak@amd.com>
Link: https://lkml.kernel.org/r/20250712033407.2383110-7-jstultz@google.com
parent aa4f74df
Loading
Loading
Loading
Loading
+112 −5
Original line number Diff line number Diff line
@@ -6522,11 +6522,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
/*
 * Helper function for __schedule()
 *
 * If a task does not have signals pending, deactivate it
 * Otherwise marks the task's __state as RUNNING
 * Tries to deactivate the task, unless the should_block arg
 * is false or if a signal is pending. In the case a signal
 * is pending, marks the task's __state as RUNNING (and clear
 * blocked_on).
 */
static bool try_to_block_task(struct rq *rq, struct task_struct *p,
			      unsigned long *task_state_p)
			      unsigned long *task_state_p, bool should_block)
{
	unsigned long task_state = *task_state_p;
	int flags = DEQUEUE_NOCLOCK;
@@ -6537,6 +6539,16 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
		return false;
	}

	/*
	 * We check should_block after signal_pending because we
	 * will want to wake the task in that case. But if
	 * should_block is false, its likely due to the task being
	 * blocked on a mutex, and we want to keep it on the runqueue
	 * to be selectable for proxy-execution.
	 */
	if (!should_block)
		return false;

	p->sched_contributes_to_load =
		(task_state & TASK_UNINTERRUPTIBLE) &&
		!(task_state & TASK_NOLOAD) &&
@@ -6560,6 +6572,88 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
	return true;
}

#ifdef CONFIG_SCHED_PROXY_EXEC
static inline void proxy_resched_idle(struct rq *rq)
{
	put_prev_set_next_task(rq, rq->donor, rq->idle);
	rq_set_donor(rq, rq->idle);
	set_tsk_need_resched(rq->idle);
}

static bool __proxy_deactivate(struct rq *rq, struct task_struct *donor)
{
	unsigned long state = READ_ONCE(donor->__state);

	/* Don't deactivate if the state has been changed to TASK_RUNNING */
	if (state == TASK_RUNNING)
		return false;
	/*
	 * Because we got donor from pick_next_task(), it is *crucial*
	 * that we call proxy_resched_idle() before we deactivate it.
	 * As once we deactivate donor, donor->on_rq is set to zero,
	 * which allows ttwu() to immediately try to wake the task on
	 * another rq. So we cannot use *any* references to donor
	 * after that point. So things like cfs_rq->curr or rq->donor
	 * need to be changed from next *before* we deactivate.
	 */
	proxy_resched_idle(rq);
	return try_to_block_task(rq, donor, &state, true);
}

static struct task_struct *proxy_deactivate(struct rq *rq, struct task_struct *donor)
{
	if (!__proxy_deactivate(rq, donor)) {
		/*
		 * XXX: For now, if deactivation failed, set donor
		 * as unblocked, as we aren't doing proxy-migrations
		 * yet (more logic will be needed then).
		 */
		donor->blocked_on = NULL;
	}
	return NULL;
}

/*
 * Initial simple sketch that just deactivates the blocked task
 * chosen by pick_next_task() so we can then pick something that
 * isn't blocked.
 */
static struct task_struct *
find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
{
	struct mutex *mutex;

	mutex = donor->blocked_on;
	/* Something changed in the chain, so pick again */
	if (!mutex)
		return NULL;
	/*
	 * By taking mutex->wait_lock we hold off concurrent mutex_unlock()
	 * and ensure @owner sticks around.
	 */
	guard(raw_spinlock)(&mutex->wait_lock);

	/* Check again that donor is blocked with blocked_lock held */
	if (!task_is_blocked(donor) || mutex != __get_task_blocked_on(donor)) {
		/*
		 * Something changed in the blocked_on chain and
		 * we don't know if only at this level. So, let's
		 * just bail out completely and let __schedule()
		 * figure things out (pick_again loop).
		 */
		return NULL; /* do pick_next_task() again */
	}
	return proxy_deactivate(rq, donor);
}
#else /* SCHED_PROXY_EXEC */
static struct task_struct *
find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
{
	WARN_ONCE(1, "This should never be called in the !SCHED_PROXY_EXEC case\n");
	return donor;
}
#endif /* SCHED_PROXY_EXEC */

/*
 * __schedule() is the main scheduler function.
 *
@@ -6672,12 +6766,25 @@ static void __sched notrace __schedule(int sched_mode)
			goto picked;
		}
	} else if (!preempt && prev_state) {
		try_to_block_task(rq, prev, &prev_state);
		/*
		 * We pass task_is_blocked() as the should_block arg
		 * in order to keep mutex-blocked tasks on the runqueue
		 * for slection with proxy-exec (without proxy-exec
		 * task_is_blocked() will always be false).
		 */
		try_to_block_task(rq, prev, &prev_state,
				  !task_is_blocked(prev));
		switch_count = &prev->nvcsw;
	}

	next = pick_next_task(rq, prev, &rf);
pick_again:
	next = pick_next_task(rq, rq->donor, &rf);
	rq_set_donor(rq, next);
	if (unlikely(task_is_blocked(next))) {
		next = find_proxy_task(rq, next, &rf);
		if (!next)
			goto pick_again;
	}
picked:
	clear_tsk_need_resched(prev);
	clear_preempt_need_resched();
+9 −1
Original line number Diff line number Diff line
@@ -2262,6 +2262,14 @@ static inline int task_current_donor(struct rq *rq, struct task_struct *p)
	return rq->donor == p;
}

static inline bool task_is_blocked(struct task_struct *p)
{
	if (!sched_proxy_exec())
		return false;

	return !!p->blocked_on;
}

static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
{
	return p->on_cpu;
@@ -2459,7 +2467,7 @@ static inline void put_prev_set_next_task(struct rq *rq,
					  struct task_struct *prev,
					  struct task_struct *next)
{
	WARN_ON_ONCE(rq->curr != prev);
	WARN_ON_ONCE(rq->donor != prev);

	__put_prev_set_next_dl_server(rq, prev, next);