Commit 8235bcfd authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Frederic Weisbecker
Browse files

srcu: Require special srcu_struct define/init for SRCU-fast readers 



This commit adds CONFIG_PROVE_RCU=y checking to enforce the new rule that
srcu_struct structures passed to srcu_read_lock_fast() and other SRCU-fast
read-side markers be either initialized with init_srcu_struct_fast()
on the one hand or defined using either DEFINE_SRCU_FAST() or
DEFINE_STATIC_SRCU_FAST().  This will enable removal of the non-debug
read-side checks from srcu_read_lock_fast() and friends, which on my
laptop provides a 25% speedup (which admittedly amounts to about half
a nanosecond, but when tracing fastpaths...)

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
parent e4ed20c1

include/linux/srcu.h

+22 −12
Original line number Diff line number Diff line
@@ -271,17 +271,26 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) 
 * @ssp: srcu_struct in which to register the new reader.

 *

 * Enter an SRCU read-side critical section, but for a light-weight

 * smp_mb()-free reader.  See srcu_read_lock() for more information.

 *

 * If srcu_read_lock_fast() is ever used on an srcu_struct structure,

 * then none of the other flavors may be used, whether before, during,

 * or after.  Note that grace-period auto-expediting is disabled for _fast

 * srcu_struct structures because auto-expedited grace periods invoke

 * synchronize_rcu_expedited(), IPIs and all.

 *

 * Note that srcu_read_lock_fast() can be invoked only from those contexts

 * where RCU is watching, that is, from contexts where it would be legal

 * to invoke rcu_read_lock().  Otherwise, lockdep will complain.

 * smp_mb()-free reader.  See srcu_read_lock() for more information.  This

 * function is NMI-safe, in a manner similar to srcu_read_lock_nmisafe().

 *

 * For srcu_read_lock_fast() to be used on an srcu_struct structure,

 * that structure must have been defined using either DEFINE_SRCU_FAST()

 * or DEFINE_STATIC_SRCU_FAST() on the one hand or initialized with

 * init_srcu_struct_fast() on the other.  Such an srcu_struct structure

 * cannot be passed to any non-fast variant of srcu_read_{,un}lock() or

 * srcu_{down,up}_read().  In kernels built with CONFIG_PROVE_RCU=y,

 * __srcu_check_read_flavor() will complain bitterly if you ignore this

 * restriction.

 *

 * Grace-period auto-expediting is disabled for SRCU-fast srcu_struct

 * structures because SRCU-fast expedited grace periods invoke

 * synchronize_rcu_expedited(), IPIs and all.  If you need expedited

 * SRCU-fast grace periods, use synchronize_srcu_expedited().

 *

 * The srcu_read_lock_fast() function can be invoked only from those

 * contexts where RCU is watching, that is, from contexts where it would

 * be legal to invoke rcu_read_lock().  Otherwise, lockdep will complain.

 */

static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp)

{
@@ -317,7 +326,8 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ 
 * srcu_down_read() for more information.

 *

 * The same srcu_struct may be used concurrently by srcu_down_read_fast()

 * and srcu_read_lock_fast().

 * and srcu_read_lock_fast().  However, the same definition/initialization

 * requirements called out for srcu_read_lock_safe() apply.

 */

static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)

{

kernel/rcu/srcutree.c

+1 −0
Original line number Diff line number Diff line
@@ -766,6 +766,7 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor) 
	WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor);

	WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor &&

		     old_read_flavor != ssp->srcu_reader_flavor);

	WARN_ON_ONCE(read_flavor == SRCU_READ_FLAVOR_FAST && !ssp->srcu_reader_flavor);

	if (!old_read_flavor) {

		old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);

		if (!old_read_flavor)