Commit a590b67d authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

Merge branch 'srcu-next.2025.08.21a' into HEAD 

SRCU updates:
* Create srcu_read_{,un}lock_fast_notrace()
* Add srcu_read_lock_fast_notrace() and srcu_read_unlock_fast_notrace()
* Add guards for notrace variants of SRCU-fast readers
* Document srcu_read_{,un}lock_fast() use of implicit RCU readers
* Document srcu_flip() memory-barrier D relation to SRCU-fast
* Remove preempt_disable/enable() in Tiny SRCU srcu_gp_start_if_needed()
parents ccd0256e e6a43aeb

include/linux/srcu.h

+34 −0
Original line number Diff line number Diff line
@@ -275,12 +275,27 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct * 
{

	struct srcu_ctr __percpu *retval;



	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");

	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);

	retval = __srcu_read_lock_fast(ssp);

	rcu_try_lock_acquire(&ssp->dep_map);

	return retval;

}



/*

 * Used by tracing, cannot be traced and cannot call lockdep.

 * See srcu_read_lock_fast() for more information.

 */

static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp)

	__acquires(ssp)

{

	struct srcu_ctr __percpu *retval;



	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);

	retval = __srcu_read_lock_fast(ssp);

	return retval;

}



/**

 * srcu_down_read_fast - register a new reader for an SRCU-protected structure.

 * @ssp: srcu_struct in which to register the new reader.
@@ -295,6 +310,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct * 
static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)

{

	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");

	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);

	return __srcu_read_lock_fast(ssp);

}
@@ -389,6 +405,18 @@ static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ct 
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);

	srcu_lock_release(&ssp->dep_map);

	__srcu_read_unlock_fast(ssp, scp);

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast().");

}



/*

 * Used by tracing, cannot be traced and cannot call lockdep.

 * See srcu_read_unlock_fast() for more information.

 */

static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp,

						 struct srcu_ctr __percpu *scp) __releases(ssp)

{

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);

	__srcu_read_unlock_fast(ssp, scp);

}



/**
@@ -405,6 +433,7 @@ static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __ 
	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);

	__srcu_read_unlock_fast(ssp, scp);

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast().");

}



/**
@@ -486,4 +515,9 @@ DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct, 
		    srcu_read_unlock_fast(_T->lock, _T->scp),

		    struct srcu_ctr __percpu *scp)



DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct,

		    _T->scp = srcu_read_lock_fast_notrace(_T->lock),

		    srcu_read_unlock_fast_notrace(_T->lock, _T->scp),

		    struct srcu_ctr __percpu *scp)



#endif

include/linux/srcutree.h

+30 −19
Original line number Diff line number Diff line
@@ -232,23 +232,40 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss 
 * srcu_read_unlock_fast().

 *

 * Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side

 * critical sections either because they disables interrupts, because they

 * are a single instruction, or because they are a read-modify-write atomic

 * operation, depending on the whims of the architecture.

 * critical sections either because they disables interrupts, because

 * they are a single instruction, or because they are read-modify-write

 * atomic operations, depending on the whims of the architecture.

 * This matters because the SRCU-fast grace-period mechanism uses either

 * synchronize_rcu() or synchronize_rcu_expedited(), that is, RCU,

 * *not* SRCU, in order to eliminate the need for the read-side smp_mb()

 * invocations that are used by srcu_read_lock() and srcu_read_unlock().

 * The __srcu_read_unlock_fast() function also relies on this same RCU

 * (again, *not* SRCU) trick to eliminate the need for smp_mb().

 *

 * The key point behind this RCU trick is that if any part of a given

 * RCU reader precedes the beginning of a given RCU grace period, then

 * the entirety of that RCU reader and everything preceding it happens

 * before the end of that same RCU grace period.  Similarly, if any part

 * of a given RCU reader follows the end of a given RCU grace period,

 * then the entirety of that RCU reader and everything following it

 * happens after the beginning of that same RCU grace period.  Therefore,

 * the operations labeled Y in __srcu_read_lock_fast() and those labeled Z

 * in __srcu_read_unlock_fast() are ordered against the corresponding SRCU

 * read-side critical section from the viewpoint of the SRCU grace period.

 * This is all the ordering that is required, hence no calls to smp_mb().

 *

 * This means that __srcu_read_lock_fast() is not all that fast

 * on architectures that support NMIs but do not supply NMI-safe

 * implementations of this_cpu_inc().

 */

static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp)

static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct srcu_struct *ssp)

{

	struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);



	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");

	if (!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE))

		this_cpu_inc(scp->srcu_locks.counter); /* Y */

		this_cpu_inc(scp->srcu_locks.counter); // Y, and implicit RCU reader.

	else

		atomic_long_inc(raw_cpu_ptr(&scp->srcu_locks));  /* Z */

		atomic_long_inc(raw_cpu_ptr(&scp->srcu_locks));  // Y, and implicit RCU reader.

	barrier(); /* Avoid leaking the critical section. */

	return scp;

}
@@ -259,23 +276,17 @@ static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct 
 * different CPU than that which was incremented by the corresponding

 * srcu_read_lock_fast(), but it must be within the same task.

 *

 * Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side

 * critical sections either because they disables interrupts, because they

 * are a single instruction, or because they are a read-modify-write atomic

 * operation, depending on the whims of the architecture.

 *

 * This means that __srcu_read_unlock_fast() is not all that fast

 * on architectures that support NMIs but do not supply NMI-safe

 * implementations of this_cpu_inc().

 * Please see the __srcu_read_lock_fast() function's header comment for

 * information on implicit RCU readers and NMI safety.

 */

static inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)

static inline void notrace

__srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)

{

	barrier();  /* Avoid leaking the critical section. */

	if (!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE))

		this_cpu_inc(scp->srcu_unlocks.counter);  /* Z */

		this_cpu_inc(scp->srcu_unlocks.counter);  // Z, and implicit RCU reader.

	else

		atomic_long_inc(raw_cpu_ptr(&scp->srcu_unlocks));  /* Z */

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast().");

		atomic_long_inc(raw_cpu_ptr(&scp->srcu_unlocks));  // Z, and implicit RCU reader.

}



void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);

kernel/rcu/srcutiny.c

+1 −3
Original line number Diff line number Diff line
@@ -176,10 +176,9 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) 
{

	unsigned long cookie;



	preempt_disable();  // Needed for PREEMPT_LAZY

	lockdep_assert_preemption_disabled(); // Needed for PREEMPT_LAZY

	cookie = get_state_synchronize_srcu(ssp);

	if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) {

		preempt_enable();

		return;

	}

	WRITE_ONCE(ssp->srcu_idx_max, cookie);
@@ -189,7 +188,6 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) 
		else if (list_empty(&ssp->srcu_work.entry))

			list_add(&ssp->srcu_work.entry, &srcu_boot_list);

	}

	preempt_enable();

}



/*

kernel/rcu/srcutree.c

+10 −0
Original line number Diff line number Diff line
@@ -1168,6 +1168,16 @@ static void srcu_flip(struct srcu_struct *ssp) 
	 * counter update.  Note that both this memory barrier and the

	 * one in srcu_readers_active_idx_check() provide the guarantee

	 * for __srcu_read_lock().

	 *

	 * Note that this is a performance optimization, in which we spend

	 * an otherwise unnecessary smp_mb() in order to reduce the number

	 * of full per-CPU-variable scans in srcu_readers_lock_idx() and

	 * srcu_readers_unlock_idx().  But this performance optimization

	 * is not so optimal for SRCU-fast, where we would be spending

	 * not smp_mb(), but rather synchronize_rcu().  At the same time,

	 * the overhead of the smp_mb() is in the noise, so there is no

	 * point in omitting it in the SRCU-fast case.  So the same code

	 * is executed either way.

	 */

	smp_mb(); /* D */  /* Pairs with C. */

}