Commit 660349ac authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Move ->sda_is_static from srcu_struct to srcu_usage 



This commit moves the ->sda_is_static field from the srcu_struct structure
to the srcu_usage structure to reduce the size of the former in order
to improve cache locality.

Suggested-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarSachin Sant <sachinp@linux.ibm.com>
Tested-by: default avatar"Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 3b46679c

include/linux/srcutree.h

+1 −1
Original line number Diff line number Diff line
@@ -76,6 +76,7 @@ struct srcu_usage { 
	unsigned long srcu_size_jiffies;	/* Current contention-measurement interval. */

	unsigned long srcu_n_lock_retries;	/* Contention events in current interval. */

	unsigned long srcu_n_exp_nodelay;	/* # expedited no-delays in current GP phase. */

	bool sda_is_static;			/* May ->sda be passed to free_percpu()? */

};



/*
@@ -84,7 +85,6 @@ struct srcu_usage { 
struct srcu_struct {

	unsigned int srcu_idx;			/* Current rdr array element. */

	struct srcu_data __percpu *sda;		/* Per-CPU srcu_data array. */

	bool sda_is_static;			/* May ->sda be passed to free_percpu()? */

	unsigned long srcu_barrier_seq;		/* srcu_barrier seq #. */

	struct mutex srcu_barrier_mutex;	/* Serialize barrier ops. */

	struct completion srcu_barrier_completion;

kernel/rcu/srcutree.c

+4 −4
Original line number Diff line number Diff line
@@ -252,7 +252,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
	mutex_init(&ssp->srcu_barrier_mutex);

	atomic_set(&ssp->srcu_barrier_cpu_cnt, 0);

	INIT_DELAYED_WORK(&ssp->work, process_srcu);

	ssp->sda_is_static = is_static;

	ssp->srcu_sup->sda_is_static = is_static;

	if (!is_static)

		ssp->sda = alloc_percpu(struct srcu_data);

	if (!ssp->sda) {
@@ -265,7 +265,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();

	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {

		if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) {

			if (!ssp->sda_is_static) {

			if (!ssp->srcu_sup->sda_is_static) {

				free_percpu(ssp->sda);

				ssp->sda = NULL;

				kfree(ssp->srcu_sup);
@@ -667,7 +667,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) 
	kfree(ssp->srcu_sup->node);

	ssp->srcu_sup->node = NULL;

	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;

	if (!ssp->sda_is_static) {

	if (!ssp->srcu_sup->sda_is_static) {

		free_percpu(ssp->sda);

		ssp->sda = NULL;

		kfree(ssp->srcu_sup);
@@ -1906,7 +1906,7 @@ static void srcu_module_going(struct module *mod) 
	for (i = 0; i < mod->num_srcu_structs; i++) {

		ssp = *(sspp++);

		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&

		    !WARN_ON_ONCE(!ssp->sda_is_static))

		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))

			cleanup_srcu_struct(ssp);

		free_percpu(ssp->sda);

	}