Commit 5d0f5953 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Convert ->srcu_lock_count and ->srcu_unlock_count to atomic 

NMI-safe variants of srcu_read_lock() and srcu_read_unlock() are needed
by printk(), which on many architectures entails read-modify-write
atomic operations.  This commit prepares Tree SRCU for this change by
making both ->srcu_lock_count and ->srcu_unlock_count by atomic_long_t.

[ paulmck: Apply feedback from John Ogness. ]

Link: https://lore.kernel.org/all/20220910221947.171557773@linutronix.de/



Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Reviewed-by: default avatarFrederic Weisbecker <frederic@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: John Ogness <john.ogness@linutronix.de>
Cc: Petr Mladek <pmladek@suse.com>
parent 9abf2313

include/linux/srcutree.h

+2 −2
Original line number Diff line number Diff line
@@ -23,8 +23,8 @@ struct srcu_struct; 
 */

struct srcu_data {

	/* Read-side state. */

	unsigned long srcu_lock_count[2];	/* Locks per CPU. */

	unsigned long srcu_unlock_count[2];	/* Unlocks per CPU. */

	atomic_long_t srcu_lock_count[2];	/* Locks per CPU. */

	atomic_long_t srcu_unlock_count[2];	/* Unlocks per CPU. */



	/* Update-side state. */

	spinlock_t __private lock ____cacheline_internodealigned_in_smp;

kernel/rcu/srcutree.c

+12 −12
Original line number Diff line number Diff line
@@ -417,7 +417,7 @@ static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx) 
	for_each_possible_cpu(cpu) {

		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);



		sum += READ_ONCE(cpuc->srcu_lock_count[idx]);

		sum += atomic_long_read(&cpuc->srcu_lock_count[idx]);

	}

	return sum;

}
@@ -434,7 +434,7 @@ static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx) 
	for_each_possible_cpu(cpu) {

		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);



		sum += READ_ONCE(cpuc->srcu_unlock_count[idx]);

		sum += atomic_long_read(&cpuc->srcu_unlock_count[idx]);

	}

	return sum;

}
@@ -503,10 +503,10 @@ static bool srcu_readers_active(struct srcu_struct *ssp) 
	for_each_possible_cpu(cpu) {

		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);



		sum += READ_ONCE(cpuc->srcu_lock_count[0]);

		sum += READ_ONCE(cpuc->srcu_lock_count[1]);

		sum -= READ_ONCE(cpuc->srcu_unlock_count[0]);

		sum -= READ_ONCE(cpuc->srcu_unlock_count[1]);

		sum += atomic_long_read(&cpuc->srcu_lock_count[0]);

		sum += atomic_long_read(&cpuc->srcu_lock_count[1]);

		sum -= atomic_long_read(&cpuc->srcu_unlock_count[0]);

		sum -= atomic_long_read(&cpuc->srcu_unlock_count[1]);

	}

	return sum;

}
@@ -636,7 +636,7 @@ int __srcu_read_lock(struct srcu_struct *ssp) 
	int idx;



	idx = READ_ONCE(ssp->srcu_idx) & 0x1;

	this_cpu_inc(ssp->sda->srcu_lock_count[idx]);

	this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter);

	smp_mb(); /* B */  /* Avoid leaking the critical section. */

	return idx;

}
@@ -650,7 +650,7 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock); 
void __srcu_read_unlock(struct srcu_struct *ssp, int idx)

{

	smp_mb(); /* C */  /* Avoid leaking the critical section. */

	this_cpu_inc(ssp->sda->srcu_unlock_count[idx]);

	this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter);

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock);
@@ -1687,8 +1687,8 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) 
			struct srcu_data *sdp;



			sdp = per_cpu_ptr(ssp->sda, cpu);

			u0 = data_race(sdp->srcu_unlock_count[!idx]);

			u1 = data_race(sdp->srcu_unlock_count[idx]);

			u0 = data_race(atomic_long_read(&sdp->srcu_unlock_count[!idx]));

			u1 = data_race(atomic_long_read(&sdp->srcu_unlock_count[idx]));



			/*

			 * Make sure that a lock is always counted if the corresponding
@@ -1696,8 +1696,8 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) 
			 */

			smp_rmb();



			l0 = data_race(sdp->srcu_lock_count[!idx]);

			l1 = data_race(sdp->srcu_lock_count[idx]);

			l0 = data_race(atomic_long_read(&sdp->srcu_lock_count[!idx]));

			l1 = data_race(atomic_long_read(&sdp->srcu_lock_count[idx]));



			c0 = l0 - u0;

			c1 = l1 - u1;