Commit 5ec58525 authored by Juri Lelli's avatar Juri Lelli Committed by Peter Zijlstra
Browse files

locking/mutex: Make mutex::wait_lock irq safe 



With the proxy-execution series, we traverse the task->mutex->task
blocked_on/owner chain in the scheduler core. We do this while holding
the rq::lock to keep the structures in place while taking and
releasing the alternating lock types.

Since the mutex::wait_lock is one of the locks we will take in this
way under the rq::lock in the scheduler core, we need to make sure
that its usage elsewhere is irq safe.

[rebase & fix {un,}lock_wait_lock helpers in ww_mutex.h]
Signed-off-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Signed-off-by: default avatarConnor O'Brien <connoro@google.com>
Signed-off-by: default avatarJohn Stultz <jstultz@google.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarMetin Kaya <metin.kaya@arm.com>
Reviewed-by: default avatarValentin Schneider <vschneid@redhat.com>
Tested-by: default avatarK Prateek Nayak <kprateek.nayak@amd.com>
Tested-by: default avatarMetin Kaya <metin.kaya@arm.com>
Link: https://lore.kernel.org/r/20241009235352.1614323-3-jstultz@google.com
parent 894d1b3d

kernel/locking/mutex.c

+10 −8
Original line number Diff line number Diff line
@@ -578,6 +578,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas 
	DEFINE_WAKE_Q(wake_q);

	struct mutex_waiter waiter;

	struct ww_mutex *ww;

	unsigned long flags;

	int ret;



	if (!use_ww_ctx)
@@ -620,7 +621,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas 
		return 0;

	}



	raw_spin_lock(&lock->wait_lock);

	raw_spin_lock_irqsave(&lock->wait_lock, flags);

	/*

	 * After waiting to acquire the wait_lock, try again.

	 */
@@ -681,7 +682,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas 
				goto err;

		}



		raw_spin_unlock(&lock->wait_lock);

		raw_spin_unlock_irqrestore(&lock->wait_lock, flags);

		/* Make sure we do wakeups before calling schedule */

		wake_up_q(&wake_q);

		wake_q_init(&wake_q);
@@ -706,9 +707,9 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas 
			trace_contention_begin(lock, LCB_F_MUTEX);

		}



		raw_spin_lock(&lock->wait_lock);

		raw_spin_lock_irqsave(&lock->wait_lock, flags);

	}

	raw_spin_lock(&lock->wait_lock);

	raw_spin_lock_irqsave(&lock->wait_lock, flags);

acquired:

	__set_current_state(TASK_RUNNING);
@@ -734,7 +735,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas 
	if (ww_ctx)

		ww_mutex_lock_acquired(ww, ww_ctx);



	raw_spin_unlock(&lock->wait_lock);

	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);

	wake_up_q(&wake_q);

	preempt_enable();

	return 0;
@@ -744,7 +745,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas 
	__mutex_remove_waiter(lock, &waiter);

err_early_kill:

	trace_contention_end(lock, ret);

	raw_spin_unlock(&lock->wait_lock);

	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);

	debug_mutex_free_waiter(&waiter);

	mutex_release(&lock->dep_map, ip);

	wake_up_q(&wake_q);
@@ -915,6 +916,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne 
	struct task_struct *next = NULL;

	DEFINE_WAKE_Q(wake_q);

	unsigned long owner;

	unsigned long flags;



	mutex_release(&lock->dep_map, ip);
@@ -941,7 +943,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne 
		}

	}



	raw_spin_lock(&lock->wait_lock);

	raw_spin_lock_irqsave(&lock->wait_lock, flags);

	debug_mutex_unlock(lock);

	if (!list_empty(&lock->wait_list)) {

		/* get the first entry from the wait-list: */
@@ -959,7 +961,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne 
		__mutex_handoff(lock, next);



	preempt_disable();

	raw_spin_unlock(&lock->wait_lock);

	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);

	wake_up_q(&wake_q);

	preempt_enable();

}

kernel/locking/ww_mutex.h

+11 −10
Original line number Diff line number Diff line
@@ -70,14 +70,14 @@ __ww_mutex_has_waiters(struct mutex *lock) 
	return atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS;

}



static inline void lock_wait_lock(struct mutex *lock)

static inline void lock_wait_lock(struct mutex *lock, unsigned long *flags)

{

	raw_spin_lock(&lock->wait_lock);

	raw_spin_lock_irqsave(&lock->wait_lock, *flags);

}



static inline void unlock_wait_lock(struct mutex *lock)

static inline void unlock_wait_lock(struct mutex *lock, unsigned long *flags)

{

	raw_spin_unlock(&lock->wait_lock);

	raw_spin_unlock_irqrestore(&lock->wait_lock, *flags);

}



static inline void lockdep_assert_wait_lock_held(struct mutex *lock)
@@ -144,14 +144,14 @@ __ww_mutex_has_waiters(struct rt_mutex *lock) 
	return rt_mutex_has_waiters(&lock->rtmutex);

}



static inline void lock_wait_lock(struct rt_mutex *lock)

static inline void lock_wait_lock(struct rt_mutex *lock, unsigned long *flags)

{

	raw_spin_lock(&lock->rtmutex.wait_lock);

	raw_spin_lock_irqsave(&lock->rtmutex.wait_lock, *flags);

}



static inline void unlock_wait_lock(struct rt_mutex *lock)

static inline void unlock_wait_lock(struct rt_mutex *lock, unsigned long *flags)

{

	raw_spin_unlock(&lock->rtmutex.wait_lock);

	raw_spin_unlock_irqrestore(&lock->rtmutex.wait_lock, *flags);

}



static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock)
@@ -380,6 +380,7 @@ static __always_inline void 
ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)

{

	DEFINE_WAKE_Q(wake_q);

	unsigned long flags;



	ww_mutex_lock_acquired(lock, ctx);
@@ -408,10 +409,10 @@ ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 
	 * Uh oh, we raced in fastpath, check if any of the waiters need to

	 * die or wound us.

	 */

	lock_wait_lock(&lock->base);

	lock_wait_lock(&lock->base, &flags);

	__ww_mutex_check_waiters(&lock->base, ctx, &wake_q);

	preempt_disable();

	unlock_wait_lock(&lock->base);

	unlock_wait_lock(&lock->base, &flags);

	wake_up_q(&wake_q);

	preempt_enable();

}