Commit ac51c40c authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Frederic Weisbecker
Browse files

srcu: Make SRCU-fast readers enforce use of SRCU-fast definition/init 



This commit makes CONFIG_PROVE_RCU=y kernels enforce the new rule
that srcu_struct structures that are passed to srcu_read_lock_fast()
and other SRCU-fast read-side markers be either initialized with
init_srcu_struct_fast() on the one hand or defined with DEFINE_SRCU_FAST()
or DEFINE_STATIC_SRCU_FAST() on the other.

This eliminates the read-side test that was formerly included in
srcu_read_lock_fast() and friends, speeding these primitives up by
about 25% (admittedly only about half of a nanosecond, but when tracing
on fastpaths...)

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
parent 8235bcfd

include/linux/srcu.h

+3 −3
Original line number Diff line number Diff line
@@ -297,7 +297,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct * 
	struct srcu_ctr __percpu *retval;



	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");

	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);

	retval = __srcu_read_lock_fast(ssp);

	rcu_try_lock_acquire(&ssp->dep_map);

	return retval;
@@ -312,7 +312,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ 
{

	struct srcu_ctr __percpu *retval;



	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);

	retval = __srcu_read_lock_fast(ssp);

	return retval;

}
@@ -333,7 +333,7 @@ static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct * 
{

	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");

	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);

	return __srcu_read_lock_fast(ssp);

}

include/linux/srcutiny.h

+0 −1
Original line number Diff line number Diff line
@@ -112,7 +112,6 @@ static inline void srcu_barrier(struct srcu_struct *ssp) 


static inline void srcu_expedite_current(struct srcu_struct *ssp) { }

#define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)

#define srcu_check_read_flavor_force(ssp, read_flavor) do { } while (0)



/* Defined here to avoid size increase for non-torture kernels. */

static inline void srcu_torture_stats_print(struct srcu_struct *ssp,

include/linux/srcutree.h

+1 −15
Original line number Diff line number Diff line
@@ -307,21 +307,7 @@ __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 


void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);



// Record reader usage even for CONFIG_PROVE_RCU=n kernels.  This is

// needed only for flavors that require grace-period smp_mb() calls to be

// promoted to synchronize_rcu().

static inline void srcu_check_read_flavor_force(struct srcu_struct *ssp, int read_flavor)

{

	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);



	if (likely(READ_ONCE(sdp->srcu_reader_flavor) & read_flavor))

		return;



	// Note that the cmpxchg() in __srcu_check_read_flavor() is fully ordered.

	__srcu_check_read_flavor(ssp, read_flavor);

}



// Record non-_lite() usage only for CONFIG_PROVE_RCU=y kernels.

// Record SRCU-reader usage type only for CONFIG_PROVE_RCU=y kernels.

static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)

{

	if (IS_ENABLED(CONFIG_PROVE_RCU))