Commit 795e7efe authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Boqun Feng
Browse files

srcu: Make SRCU readers use ->srcu_ctrs for counter selection 



This commit causes SRCU readers to use ->srcu_ctrs for counter
selection instead of ->srcu_idx.  This takes another step towards
array-indexing-free SRCU readers.

[ paulmck: Apply kernel test robot feedback. ]

Co-developed-by: default avatarZ qiang <qiang.zhang1211@gmail.com>
Signed-off-by: default avatarZ qiang <qiang.zhang1211@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Tested-by: default avatarkernel test robot <oliver.sang@intel.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: <bpf@vger.kernel.org>
Signed-off-by: default avatarBoqun Feng <boqun.feng@gmail.com>
parent 56eb8be1

include/linux/srcutree.h

+5 −4
Original line number Diff line number Diff line
@@ -101,6 +101,7 @@ struct srcu_usage { 
 */

struct srcu_struct {

	unsigned int srcu_idx;			/* Current rdr array element. */

	struct srcu_ctr __percpu *srcu_ctrp;

	struct srcu_data __percpu *sda;		/* Per-CPU srcu_data array. */

	struct lockdep_map dep_map;

	struct srcu_usage *srcu_sup;		/* Update-side data. */
@@ -167,6 +168,7 @@ struct srcu_struct { 
#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name)						\

{												\

	.sda = &pcpu_name,									\

	.srcu_ctrp = &pcpu_name.srcu_ctrs[0],							\

	__SRCU_STRUCT_INIT_COMMON(name, usage_name)						\

}
@@ -222,13 +224,12 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf); 
 */

static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)

{

	int idx;

	struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);



	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_lite().");

	idx = READ_ONCE(ssp->srcu_idx) & 0x1;

	this_cpu_inc(ssp->sda->srcu_ctrs[idx].srcu_locks.counter); /* Y */

	this_cpu_inc(scp->srcu_locks.counter); /* Y */

	barrier(); /* Avoid leaking the critical section. */

	return idx;

	return scp - &ssp->sda->srcu_ctrs[0];

}



/*

kernel/rcu/srcutree.c

+13 −10
Original line number Diff line number Diff line
@@ -253,8 +253,10 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);

	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);

	ssp->srcu_sup->sda_is_static = is_static;

	if (!is_static)

	if (!is_static) {

		ssp->sda = alloc_percpu(struct srcu_data);

		ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0];

	}

	if (!ssp->sda)

		goto err_free_sup;

	init_srcu_struct_data(ssp);
@@ -742,12 +744,11 @@ EXPORT_SYMBOL_GPL(__srcu_check_read_flavor); 
 */

int __srcu_read_lock(struct srcu_struct *ssp)

{

	int idx;

	struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);



	idx = READ_ONCE(ssp->srcu_idx) & 0x1;

	this_cpu_inc(ssp->sda->srcu_ctrs[idx].srcu_locks.counter);

	this_cpu_inc(scp->srcu_locks.counter);

	smp_mb(); /* B */  /* Avoid leaking the critical section. */

	return idx;

	return scp - &ssp->sda->srcu_ctrs[0];

}

EXPORT_SYMBOL_GPL(__srcu_read_lock);
@@ -772,13 +773,12 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock); 
 */

int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)

{

	int idx;

	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);

	struct srcu_ctr __percpu *scpp = READ_ONCE(ssp->srcu_ctrp);

	struct srcu_ctr *scp = raw_cpu_ptr(scpp);



	idx = READ_ONCE(ssp->srcu_idx) & 0x1;

	atomic_long_inc(&sdp->srcu_ctrs[idx].srcu_locks);

	atomic_long_inc(&scp->srcu_locks);

	smp_mb__after_atomic(); /* B */  /* Avoid leaking the critical section. */

	return idx;

	return scpp - &ssp->sda->srcu_ctrs[0];

}

EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe);
@@ -1152,6 +1152,8 @@ static void srcu_flip(struct srcu_struct *ssp) 
	smp_mb(); /* E */  /* Pairs with B and C. */



	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); // Flip the counter.

	WRITE_ONCE(ssp->srcu_ctrp,

		   &ssp->sda->srcu_ctrs[!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0])]);



	/*

	 * Ensure that if the updater misses an __srcu_read_unlock()
@@ -2000,6 +2002,7 @@ static int srcu_module_coming(struct module *mod) 
		ssp->sda = alloc_percpu(struct srcu_data);

		if (WARN_ON_ONCE(!ssp->sda))

			return -ENOMEM;

		ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0];

	}

	return 0;

}