Commit 3b46679c authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Move heuristics fields from srcu_struct to srcu_usage 



This commit moves the ->srcu_size_jiffies, ->srcu_n_lock_retries,
and ->srcu_n_exp_nodelay fields from the srcu_struct structure to the
srcu_usage structure to reduce the size of the former in order to improve
cache locality.

Suggested-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarSachin Sant <sachinp@linux.ibm.com>
Tested-by: default avatar"Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 03200b5c

include/linux/srcutree.h

+3 −3
Original line number Diff line number Diff line
@@ -73,6 +73,9 @@ struct srcu_usage { 
	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */

	unsigned long srcu_gp_start;		/* Last GP start timestamp (jiffies) */

	unsigned long srcu_last_gp_end;		/* Last GP end timestamp (ns) */

	unsigned long srcu_size_jiffies;	/* Current contention-measurement interval. */

	unsigned long srcu_n_lock_retries;	/* Contention events in current interval. */

	unsigned long srcu_n_exp_nodelay;	/* # expedited no-delays in current GP phase. */

};



/*
@@ -80,9 +83,6 @@ struct srcu_usage { 
 */

struct srcu_struct {

	unsigned int srcu_idx;			/* Current rdr array element. */

	unsigned long srcu_size_jiffies;	/* Current contention-measurement interval. */

	unsigned long srcu_n_lock_retries;	/* Contention events in current interval. */

	unsigned long srcu_n_exp_nodelay;	/* # expedited no-delays in current GP phase. */

	struct srcu_data __percpu *sda;		/* Per-CPU srcu_data array. */

	bool sda_is_static;			/* May ->sda be passed to free_percpu()? */

	unsigned long srcu_barrier_seq;		/* srcu_barrier seq #. */

kernel/rcu/srcutree.c

+9 −9
Original line number Diff line number Diff line
@@ -348,11 +348,11 @@ static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp) 
	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)

		return;

	j = jiffies;

	if (ssp->srcu_size_jiffies != j) {

		ssp->srcu_size_jiffies = j;

		ssp->srcu_n_lock_retries = 0;

	if (ssp->srcu_sup->srcu_size_jiffies != j) {

		ssp->srcu_sup->srcu_size_jiffies = j;

		ssp->srcu_sup->srcu_n_lock_retries = 0;

	}

	if (++ssp->srcu_n_lock_retries <= small_contention_lim)

	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)

		return;

	__srcu_transition_to_big(ssp);

}
@@ -624,8 +624,8 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp) 
		if (time_after(j, gpstart))

			jbase += j - gpstart;

		if (!jbase) {

			WRITE_ONCE(ssp->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_n_exp_nodelay) + 1);

			if (READ_ONCE(ssp->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)

			WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay) + 1);

			if (READ_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)

				jbase = 1;

		}

	}
@@ -783,7 +783,7 @@ static void srcu_gp_start(struct srcu_struct *ssp) 
				       rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq));

	spin_unlock_rcu_node(sdp);  /* Interrupts remain disabled. */

	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);

	WRITE_ONCE(ssp->srcu_n_exp_nodelay, 0);

	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);

	smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */

	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);

	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);
@@ -1625,7 +1625,7 @@ static void srcu_advance_state(struct srcu_struct *ssp) 
		srcu_flip(ssp);

		spin_lock_irq_rcu_node(ssp->srcu_sup);

		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);

		ssp->srcu_n_exp_nodelay = 0;

		ssp->srcu_sup->srcu_n_exp_nodelay = 0;

		spin_unlock_irq_rcu_node(ssp->srcu_sup);

	}
@@ -1640,7 +1640,7 @@ static void srcu_advance_state(struct srcu_struct *ssp) 
			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);

			return; /* readers present, retry later. */

		}

		ssp->srcu_n_exp_nodelay = 0;

		ssp->srcu_sup->srcu_n_exp_nodelay = 0;

		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */

	}

}