Commit 37827223 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Frederic Weisbecker
Browse files

srcu: Add SRCU_READ_FLAVOR_FAST_UPDOWN CPP macro 



This commit adds the SRCU_READ_FLAVOR_FAST_UPDOWN=0x8 macro
and adjusts rcutorture to make use of it.  In this commit, both
SRCU_READ_FLAVOR_FAST=0x4 and the new SRCU_READ_FLAVOR_FAST_UPDOWN
test SRCU-fast.  When the SRCU-fast-updown is added, the new
SRCU_READ_FLAVOR_FAST_UPDOWN macro will test it when passed to the
rcutorture.reader_flavor module parameter.

The old SRCU_READ_FLAVOR_FAST macro's value changed from 0x8 to 0x4.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
parent 3ed04e3f

include/linux/srcu.h

+9 −7
Original line number Diff line number Diff line
@@ -59,10 +59,12 @@ int init_srcu_struct_fast(struct srcu_struct *ssp); 
#define SRCU_READ_FLAVOR_NORMAL		0x1		// srcu_read_lock().

#define SRCU_READ_FLAVOR_NMI		0x2		// srcu_read_lock_nmisafe().

//					0x4		// SRCU-lite is no longer with us.

#define SRCU_READ_FLAVOR_FAST	0x8		// srcu_read_lock_fast().

#define SRCU_READ_FLAVOR_FAST		0x4		// srcu_read_lock_fast().

#define SRCU_READ_FLAVOR_FAST_UPDOWN	0x8		// srcu_read_lock_fast().

#define SRCU_READ_FLAVOR_ALL		(SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \

				SRCU_READ_FLAVOR_FAST) // All of the above.

#define SRCU_READ_FLAVOR_SLOWGP	SRCU_READ_FLAVOR_FAST

					 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)

						// All of the above.

#define SRCU_READ_FLAVOR_SLOWGP		(SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)

						// Flavors requiring synchronize_rcu()

						// instead of smp_mb().

void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);

kernel/rcu/rcutorture.c

+18 −6
Original line number Diff line number Diff line
@@ -702,6 +702,8 @@ static void srcu_torture_init(void) 
	rcu_sync_torture_init();

	if (reader_flavor & SRCU_READ_FLAVOR_FAST)

		srcu_ctlp = &srcu_ctlf;

	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)

		srcu_ctlp = &srcu_ctlf;

}



static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)
@@ -728,6 +730,12 @@ static int srcu_torture_read_lock(void) 
		ret += idx << 1;

	}

	if (reader_flavor & SRCU_READ_FLAVOR_FAST) {

		scp = srcu_read_lock_fast(srcu_ctlp);

		idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);

		WARN_ON_ONCE(idx & ~0x1);

		ret += idx << 2;

	}

	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {

		scp = srcu_read_lock_fast(srcu_ctlp);

		idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);

		WARN_ON_ONCE(idx & ~0x1);
@@ -758,8 +766,10 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 
static void srcu_torture_read_unlock(int idx)

{

	WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));

	if (reader_flavor & SRCU_READ_FLAVOR_FAST)

	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)

		srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));

	if (reader_flavor & SRCU_READ_FLAVOR_FAST)

		srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 2));

	if (reader_flavor & SRCU_READ_FLAVOR_NMI)

		srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1);

	if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL))
@@ -793,7 +803,7 @@ static int srcu_torture_down_read(void) 
		WARN_ON_ONCE(idx & ~0x1);

		return idx;

	}

	if (reader_flavor & SRCU_READ_FLAVOR_FAST) {

	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {

		scp = srcu_down_read_fast(srcu_ctlp);

		idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);

		WARN_ON_ONCE(idx & ~0x1);
@@ -806,7 +816,7 @@ static int srcu_torture_down_read(void) 
static void srcu_torture_up_read(int idx)

{

	WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));

	if (reader_flavor & SRCU_READ_FLAVOR_FAST)

	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)

		srcu_up_read_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));

	else if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) ||

		 !(reader_flavor & SRCU_READ_FLAVOR_ALL))
@@ -901,14 +911,16 @@ static struct rcu_torture_ops srcu_ops = { 
	.no_pi_lock	= IS_ENABLED(CONFIG_TINY_SRCU),

	.debug_objects	= 1,

	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)

				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST,

				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,

	.name		= "srcu"

};



static void srcud_torture_init(void)

{

	rcu_sync_torture_init();

	if (reader_flavor & SRCU_READ_FLAVOR_FAST)

	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)

		WARN_ON(init_srcu_struct_fast(&srcu_ctld));

	else if (reader_flavor & SRCU_READ_FLAVOR_FAST)

		WARN_ON(init_srcu_struct_fast(&srcu_ctld));

	else

		WARN_ON(init_srcu_struct(&srcu_ctld));
@@ -953,7 +965,7 @@ static struct rcu_torture_ops srcud_ops = { 
	.no_pi_lock	= IS_ENABLED(CONFIG_TINY_SRCU),

	.debug_objects	= 1,

	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)

				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST,

				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,

	.name		= "srcud"

};