Commit 175b45ed authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Boqun Feng
Browse files

srcu: Use raw spinlocks so call_srcu() can be used under preempt_disable() 



Tree SRCU has used non-raw spinlocks for many years, motivated by a desire
to avoid unnecessary real-time latency and the absence of any reason to
use raw spinlocks.  However, the recent use of SRCU in tracing as the
underlying implementation of RCU Tasks Trace means that call_srcu()
is invoked from preemption-disabled regions of code, which in turn
requires that any locks acquired by call_srcu() or its callees must be
raw spinlocks.

This commit therefore converts SRCU's spinlocks to raw spinlocks.

[boqun: Add Fixes tag]

Reported-by: default avatarKumar Kartikeya Dwivedi <memxor@gmail.com>
Fixes: c27cea44 ("rcu: Re-implement RCU Tasks Trace in terms of SRCU-fast")
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Signed-off-by: default avatarBoqun Feng <boqun@kernel.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
parent f338e773

include/linux/srcutree.h

+4 −4
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ struct srcu_data { 
						/* Values: SRCU_READ_FLAVOR_.*  */



	/* Update-side state. */

	spinlock_t __private lock ____cacheline_internodealigned_in_smp;

	raw_spinlock_t __private lock ____cacheline_internodealigned_in_smp;

	struct rcu_segcblist srcu_cblist;	/* List of callbacks.*/

	unsigned long srcu_gp_seq_needed;	/* Furthest future GP needed. */

	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */
@@ -55,7 +55,7 @@ struct srcu_data { 
 * Node in SRCU combining tree, similar in function to rcu_data.

 */

struct srcu_node {

	spinlock_t __private lock;

	raw_spinlock_t __private lock;

	unsigned long srcu_have_cbs[4];		/* GP seq for children having CBs, but only */

						/*  if greater than ->srcu_gp_seq. */

	unsigned long srcu_data_have_cbs[4];	/* Which srcu_data structs have CBs for given GP? */
@@ -74,7 +74,7 @@ struct srcu_usage { 
						/* First node at each level. */

	int srcu_size_state;			/* Small-to-big transition state. */

	struct mutex srcu_cb_mutex;		/* Serialize CB preparation. */

	spinlock_t __private lock;		/* Protect counters and size state. */

	raw_spinlock_t __private lock;		/* Protect counters and size state. */

	struct mutex srcu_gp_mutex;		/* Serialize GP work. */

	unsigned long srcu_gp_seq;		/* Grace-period seq #. */

	unsigned long srcu_gp_seq_needed;	/* Latest gp_seq needed. */
@@ -156,7 +156,7 @@ struct srcu_struct { 


#define __SRCU_USAGE_INIT(name)									\

{												\

	.lock = __SPIN_LOCK_UNLOCKED(name.lock),						\

	.lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock),						\

	.srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL,							\

	.srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE,				\

	.srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL,					\

kernel/rcu/rcu.h

+9 −0
Original line number Diff line number Diff line
@@ -502,6 +502,15 @@ do { \ 
	___locked;							\

})



#define raw_spin_trylock_irqsave_rcu_node(p, flags)			\

({									\

	bool ___locked = raw_spin_trylock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \

									\

	if (___locked)							\

		smp_mb__after_unlock_lock();				\

	___locked;							\

})



#define raw_lockdep_assert_held_rcu_node(p)				\

	lockdep_assert_held(&ACCESS_PRIVATE(p, lock))

kernel/rcu/srcutree.c

+69 −105

File changed.

Preview size limit exceeded, changes collapsed.