Commit 5892cbd8 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Match __task_rq_{,un}lock() 



In preparation to adding more rules to __task_rq_lock(), such that
__task_rq_unlock() will no longer be equivalent to rq_unlock(),
make sure every __task_rq_lock() is matched by a __task_rq_unlock()
and vice-versa.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Acked-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
parent 46a177fb

kernel/sched/core.c

+8 −5
Original line number Diff line number Diff line
@@ -2582,7 +2582,8 @@ static int migration_cpu_stop(void *data) 
		 */

		WARN_ON_ONCE(!pending->stop_pending);

		preempt_disable();

		task_rq_unlock(rq, p, &rf);

		rq_unlock(rq, &rf);

		raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);

		stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,

				    &pending->arg, &pending->stop_work);

		preempt_enable();
@@ -2591,7 +2592,8 @@ static int migration_cpu_stop(void *data) 
out:

	if (pending)

		pending->stop_pending = false;

	task_rq_unlock(rq, p, &rf);

	rq_unlock(rq, &rf);

	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);



	if (complete)

		complete_all(&pending->done);
@@ -3708,7 +3710,7 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) 
		ttwu_do_wakeup(p);

		ret = 1;

	}

	__task_rq_unlock(rq, &rf);

	__task_rq_unlock(rq, p, &rf);



	return ret;

}
@@ -4301,7 +4303,7 @@ int task_call_func(struct task_struct *p, task_call_f func, void *arg) 
	ret = func(p, arg);



	if (rq)

		rq_unlock(rq, &rf);

		__task_rq_unlock(rq, p, &rf);



	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);

	return ret;
@@ -7362,7 +7364,8 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 


	rq_unpin_lock(rq, &rf);

	__balance_callbacks(rq);

	raw_spin_rq_unlock(rq);

	rq_repin_lock(rq, &rf);

	__task_rq_unlock(rq, p, &rf);



	preempt_enable();

}

kernel/sched/sched.h

+4 −4
Original line number Diff line number Diff line
@@ -1825,7 +1825,8 @@ struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 
	__acquires(p->pi_lock)

	__acquires(rq->lock);



static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf)

static inline void

__task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)

	__releases(rq->lock)

{

	rq_unpin_lock(rq, rf);
@@ -1837,8 +1838,7 @@ task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 
	__releases(rq->lock)

	__releases(p->pi_lock)

{

	rq_unpin_lock(rq, rf);

	raw_spin_rq_unlock(rq);

	__task_rq_unlock(rq, p, rf);

	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);

}
@@ -1849,7 +1849,7 @@ DEFINE_LOCK_GUARD_1(task_rq_lock, struct task_struct, 


DEFINE_LOCK_GUARD_1(__task_rq_lock, struct task_struct,

		    _T->rq = __task_rq_lock(_T->lock, &_T->rf),

		    __task_rq_unlock(_T->rq, &_T->rf),

		    __task_rq_unlock(_T->rq, _T->lock, &_T->rf),

		    struct rq *rq; struct rq_flags rf)



static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf)

kernel/sched/stats.h

+1 −1
Original line number Diff line number Diff line
@@ -206,7 +206,7 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) 


		rq = __task_rq_lock(p, &rf);

		psi_task_change(p, p->psi_flags, 0);

		__task_rq_unlock(rq, &rf);

		__task_rq_unlock(rq, p, &rf);

	}

}