Commit 4937096b authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Boqun Feng
Browse files

srcu: Pull integer-to-pointer conversion into __srcu_ctr_to_ptr() 



This commit abstracts the srcu_read_unlock*() integer-to-pointer
conversion into a new __srcu_ctr_to_ptr().  This will be used
in rcutorture for testing an srcu_read_unlock_fast() that avoids
array-indexing overhead by taking a pointer rather than an integer.

[ paulmck: Apply kernel test robot feedback. ]

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: <bpf@vger.kernel.org>
Signed-off-by: default avatarBoqun Feng <boqun.feng@gmail.com>
parent f4bde41d

include/linux/srcutree.h

+8 −1
Original line number Diff line number Diff line
@@ -218,6 +218,13 @@ static inline bool __srcu_ptr_to_ctr(struct srcu_struct *ssp, struct srcu_ctr __ 
	return scpp - &ssp->sda->srcu_ctrs[0];

}



// Converts an integer to a per-CPU pointer to the corresponding

// ->srcu_ctrs[] array element.

static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssp, int idx)

{

	return &ssp->sda->srcu_ctrs[idx];

}



/*

 * Counts the new reader in the appropriate per-CPU element of the

 * srcu_struct.  Returns an index that must be passed to the matching
@@ -252,7 +259,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp) 
static inline void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)

{

	barrier();  /* Avoid leaking the critical section. */

	this_cpu_inc(ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter);  /* Z */

	this_cpu_inc(__srcu_ctr_to_ptr(ssp, idx)->srcu_unlocks.counter);  /* Z */

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_lite().");

}

kernel/rcu/srcutree.c

+2 −4
Original line number Diff line number Diff line
@@ -765,7 +765,7 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock); 
void __srcu_read_unlock(struct srcu_struct *ssp, int idx)

{

	smp_mb(); /* C */  /* Avoid leaking the critical section. */

	this_cpu_inc(ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter);

	this_cpu_inc(__srcu_ctr_to_ptr(ssp, idx)->srcu_unlocks.counter);

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock);
@@ -794,10 +794,8 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe); 
 */

void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)

{

	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);



	smp_mb__before_atomic(); /* C */  /* Avoid leaking the critical section. */

	atomic_long_inc(&sdp->srcu_ctrs[idx].srcu_unlocks);

	atomic_long_inc(&raw_cpu_ptr(__srcu_ctr_to_ptr(ssp, idx))->srcu_unlocks);

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock_nmisafe);