Commit bdba8330 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Frederic Weisbecker
Browse files

refscale: Add this_cpu_inc() readers 



This commit adds refscale readers based on this_cpu_inc() and
this_cpu_inc() ("refscale.scale_type=percpuinc").  On my x86 laptop,
these are about 4.5ns per pair.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
parent 057df3ea

kernel/rcu/refscale.c

+32 −4
Original line number Diff line number Diff line
@@ -323,6 +323,9 @@ static const struct ref_scale_ops rcu_trace_ops = { 
// Definitions for reference count

static atomic_t refcnt;



// Definitions acquire-release.

static DEFINE_PER_CPU(unsigned long, test_acqrel);



static void ref_refcnt_section(const int nloops)

{

	int i;
@@ -351,6 +354,34 @@ static const struct ref_scale_ops refcnt_ops = { 
	.name		= "refcnt"

};



static void ref_percpuinc_section(const int nloops)

{

	int i;



	for (i = nloops; i >= 0; i--) {

		this_cpu_inc(test_acqrel);

		this_cpu_dec(test_acqrel);

	}

}



static void ref_percpuinc_delay_section(const int nloops, const int udl, const int ndl)

{

	int i;



	for (i = nloops; i >= 0; i--) {

		this_cpu_inc(test_acqrel);

		un_delay(udl, ndl);

		this_cpu_dec(test_acqrel);

	}

}



static const struct ref_scale_ops percpuinc_ops = {

	.init		= rcu_sync_scale_init,

	.readsection	= ref_percpuinc_section,

	.delaysection	= ref_percpuinc_delay_section,

	.name		= "percpuinc"

};



// Definitions for rwlock

static rwlock_t test_rwlock;
@@ -494,9 +525,6 @@ static const struct ref_scale_ops lock_irq_ops = { 
	.name		= "lock-irq"

};



// Definitions acquire-release.

static DEFINE_PER_CPU(unsigned long, test_acqrel);



static void ref_acqrel_section(const int nloops)

{

	unsigned long x;
@@ -1291,7 +1319,7 @@ ref_scale_init(void) 
	static const struct ref_scale_ops *scale_ops[] = {

		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS

		&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,

		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,

		&percpuinc_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,

		&preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,

		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,

	};