Commit b9bdd4b6 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Peter Zijlstra
Browse files

locking/semaphore: Remove the list_head from struct semaphore 



Instead of embedding a list_head in struct semaphore, store a pointer to
the first waiter.  The list of waiters remains a doubly linked list so
we can efficiently add to the tail of the list and remove from the front
(or middle) of the list.

Some of the list manipulation becomes more complicated, but it's a
reasonable tradeoff on the slow paths to shrink data structures
which embed a semaphore.

Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260305195545.3707590-3-willy@infradead.org
parent 1ea4b473

drivers/acpi/osl.c

+1 −1
Original line number Diff line number Diff line
@@ -1257,7 +1257,7 @@ acpi_status acpi_os_delete_semaphore(acpi_handle handle) 


	ACPI_DEBUG_PRINT((ACPI_DB_MUTEX, "Deleting semaphore[%p].\n", handle));



	BUG_ON(!list_empty(&sem->wait_list));

	BUG_ON(sem->first_waiter);

	kfree(sem);

	sem = NULL;

include/linux/semaphore.h

+2 −2
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 
struct semaphore {

	raw_spinlock_t		lock;

	unsigned int		count;

	struct list_head	wait_list;

	struct semaphore_waiter *first_waiter;



#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER

	unsigned long		last_holder;
@@ -33,7 +33,7 @@ struct semaphore { 
{									\

	.lock		= __RAW_SPIN_LOCK_UNLOCKED((name).lock),	\

	.count		= n,						\

	.wait_list	= LIST_HEAD_INIT((name).wait_list)		\

	.first_waiter	= NULL						\

	__LAST_HOLDER_SEMAPHORE_INITIALIZER				\

}

kernel/locking/semaphore.c

+31 −10
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ 
 * too.

 *

 * The ->count variable represents how many more tasks can acquire this

 * semaphore.  If it's zero, there may be tasks waiting on the wait_list.

 * semaphore.  If it's zero, there may be waiters.

 */



#include <linux/compiler.h>
@@ -226,7 +226,7 @@ void __sched up(struct semaphore *sem) 


	hung_task_sem_clear_if_holder(sem);



	if (likely(list_empty(&sem->wait_list)))

	if (likely(!sem->first_waiter))

		sem->count++;

	else

		__up(sem, &wake_q);
@@ -244,6 +244,21 @@ struct semaphore_waiter { 
	bool up;

};



static inline

void sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter)

{

	if (list_empty(&waiter->list)) {

		sem->first_waiter = NULL;

		return;

	}



	if (sem->first_waiter == waiter) {

		sem->first_waiter = list_first_entry(&waiter->list,

						     struct semaphore_waiter, list);

	}

	list_del(&waiter->list);

}



/*

 * Because this function is inlined, the 'state' parameter will be

 * constant, and thus optimised away by the compiler.  Likewise the
@@ -252,9 +267,15 @@ struct semaphore_waiter { 
static inline int __sched ___down_common(struct semaphore *sem, long state,

								long timeout)

{

	struct semaphore_waiter waiter;



	list_add_tail(&waiter.list, &sem->wait_list);

	struct semaphore_waiter waiter, *first;



	first = sem->first_waiter;

	if (first) {

		list_add_tail(&waiter.list, &first->list);

	} else {

		INIT_LIST_HEAD(&waiter.list);

		sem->first_waiter = &waiter;

	}

	waiter.task = current;

	waiter.up = false;
@@ -274,11 +295,11 @@ static inline int __sched ___down_common(struct semaphore *sem, long state, 
	}



 timed_out:

	list_del(&waiter.list);

	sem_del_waiter(sem, &waiter);

	return -ETIME;



 interrupted:

	list_del(&waiter.list);

	sem_del_waiter(sem, &waiter);

	return -EINTR;

}
@@ -321,9 +342,9 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long timeout) 
static noinline void __sched __up(struct semaphore *sem,

				  struct wake_q_head *wake_q)

{

	struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,

						struct semaphore_waiter, list);

	list_del(&waiter->list);

	struct semaphore_waiter *waiter = sem->first_waiter;



	sem_del_waiter(sem, waiter);

	waiter->up = true;

	wake_q_add(wake_q, waiter->task);

}