Commit 6c67f8d8 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

futex: Create futex_hash() get/put class 



This gets us:

  hb = futex_hash(key) /* gets hb and inc users */
  futex_hash_get(hb)   /* inc users */
  futex_hash_put(hb)   /* dec users */

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250416162921.513656-7-bigeasy@linutronix.de
parent 8486d12f

kernel/futex/core.c

+3 −3
Original line number Diff line number Diff line
@@ -122,6 +122,8 @@ struct futex_hash_bucket *futex_hash(union futex_key *key) 
	return &futex_queues[hash & futex_hashmask];

}



void futex_hash_get(struct futex_hash_bucket *hb) { }

void futex_hash_put(struct futex_hash_bucket *hb) { }



/**

 * futex_setup_timer - set up the sleeping hrtimer.
@@ -957,9 +959,7 @@ static void exit_pi_state_list(struct task_struct *curr) 
		pi_state = list_entry(next, struct futex_pi_state, list);

		key = pi_state->key;

		if (1) {

			struct futex_hash_bucket *hb;



			hb = futex_hash(&key);

			CLASS(hb, hb)(&key);



			/*

			 * We can race against put_pi_state() removing itself from the

kernel/futex/futex.h

+7 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
#include <linux/sched/wake_q.h>

#include <linux/compat.h>

#include <linux/uaccess.h>

#include <linux/cleanup.h>



#ifdef CONFIG_PREEMPT_RT

#include <linux/rcuwait.h>
@@ -202,6 +203,12 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 
		  int flags, u64 range_ns);



extern struct futex_hash_bucket *futex_hash(union futex_key *key);

extern void futex_hash_get(struct futex_hash_bucket *hb);

extern void futex_hash_put(struct futex_hash_bucket *hb);



DEFINE_CLASS(hb, struct futex_hash_bucket *,

	     if (_T) futex_hash_put(_T),

	     futex_hash(key), union futex_key *key);



/**

 * futex_match - Check whether two futex keys are equal

kernel/futex/pi.c

+12 −4
Original line number Diff line number Diff line
@@ -939,9 +939,8 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl 


retry_private:

	if (1) {

		struct futex_hash_bucket *hb;

		CLASS(hb, hb)(&q.key);



		hb = futex_hash(&q.key);

		futex_q_lock(&q, hb);



		ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
@@ -994,6 +993,16 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl 
			goto no_block;

		}



		/*

		 * Caution; releasing @hb in-scope. The hb->lock is still locked

		 * while the reference is dropped. The reference can not be dropped

		 * after the unlock because if a user initiated resize is in progress

		 * then we might need to wake him. This can not be done after the

		 * rt_mutex_pre_schedule() invocation. The hb will remain valid because

		 * the thread, performing resize, will block on hb->lock during

		 * the requeue.

		 */

		futex_hash_put(no_free_ptr(hb));

		/*

		 * Must be done before we enqueue the waiter, here is unfortunately

		 * under the hb lock, but that *should* work because it does nothing.
@@ -1119,7 +1128,6 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) 
{

	u32 curval, uval, vpid = task_pid_vnr(current);

	union futex_key key = FUTEX_KEY_INIT;

	struct futex_hash_bucket *hb;

	struct futex_q *top_waiter;

	int ret;
@@ -1139,7 +1147,7 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) 
	if (ret)

		return ret;



	hb = futex_hash(&key);

	CLASS(hb, hb)(&key);

	spin_lock(&hb->lock);

retry_hb:

kernel/futex/requeue.c

+3 −7
Original line number Diff line number Diff line
@@ -444,10 +444,8 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, 


retry_private:

	if (1) {

		struct futex_hash_bucket *hb1, *hb2;



		hb1 = futex_hash(&key1);

		hb2 = futex_hash(&key2);

		CLASS(hb, hb1)(&key1);

		CLASS(hb, hb2)(&key2);



		futex_hb_waiters_inc(hb2);

		double_lock_hb(hb1, hb2);
@@ -817,9 +815,7 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, 
	switch (futex_requeue_pi_wakeup_sync(&q)) {

	case Q_REQUEUE_PI_IGNORE:

		{

			struct futex_hash_bucket *hb;



			hb = futex_hash(&q.key);

			CLASS(hb, hb)(&q.key);

			/* The waiter is still on uaddr1 */

			spin_lock(&hb->lock);

			ret = handle_early_requeue_pi_wakeup(hb, &q, to);

kernel/futex/waitwake.c

+5 −10
Original line number Diff line number Diff line
@@ -154,7 +154,6 @@ void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) 
 */

int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)

{

	struct futex_hash_bucket *hb;

	struct futex_q *this, *next;

	union futex_key key = FUTEX_KEY_INIT;

	DEFINE_WAKE_Q(wake_q);
@@ -170,7 +169,7 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) 
	if ((flags & FLAGS_STRICT) && !nr_wake)

		return 0;



	hb = futex_hash(&key);

	CLASS(hb, hb)(&key);



	/* Make sure we really have tasks to wakeup */

	if (!futex_hb_waiters_pending(hb))
@@ -267,10 +266,8 @@ int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, 


retry_private:

	if (1) {

		struct futex_hash_bucket *hb1, *hb2;



		hb1 = futex_hash(&key1);

		hb2 = futex_hash(&key2);

		CLASS(hb, hb1)(&key1);

		CLASS(hb, hb2)(&key2);



		double_lock_hb(hb1, hb2);

		op_ret = futex_atomic_op_inuser(op, uaddr2);
@@ -444,9 +441,8 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) 
		u32 val = vs[i].w.val;



		if (1) {

			struct futex_hash_bucket *hb;

			CLASS(hb, hb)(&q->key);



			hb = futex_hash(&q->key);

			futex_q_lock(q, hb);

			ret = futex_get_value_locked(&uval, uaddr);
@@ -618,9 +614,8 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 


retry_private:

	if (1) {

		struct futex_hash_bucket *hb;

		CLASS(hb, hb)(&q->key);



		hb = futex_hash(&q->key);

		futex_q_lock(q, hb);



		ret = futex_get_value_locked(&uval, uaddr);