Commit fd1b3f8e authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Move work-scheduling fields from srcu_struct to srcu_usage 

This commit moves the ->reschedule_jiffies, ->reschedule_count, and
->work fields from the srcu_struct structure to the srcu_usage structure
to reduce the size of the former in order to improve cache locality.

However, this means that the container_of() calls cannot get a pointer
to the srcu_struct because they are no longer in the srcu_struct.
This issue is addressed by adding a ->srcu_ssp field in the srcu_usage
structure that references the corresponding srcu_struct structure.
And given the presence of the sup pointer to the srcu_usage structure,
replace some ssp->srcu_usage-> instances with sup->.

[ paulmck Apply feedback from kernel test robot. ]

Link: https://lore.kernel.org/oe-kbuild-all/202303191400.iO5BOqka-lkp@intel.com/


Suggested-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarSachin Sant <sachinp@linux.ibm.com>
Tested-by: default avatar"Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent d20162e0

include/linux/srcutree.h

+5 −4
Original line number Diff line number Diff line
@@ -84,6 +84,10 @@ struct srcu_usage { 
	atomic_t srcu_barrier_cpu_cnt;		/* # CPUs not yet posting a */

						/*  callback for the barrier */

						/*  operation. */

	unsigned long reschedule_jiffies;

	unsigned long reschedule_count;

	struct delayed_work work;

	struct srcu_struct *srcu_ssp;

};



/*
@@ -92,9 +96,6 @@ struct srcu_usage { 
struct srcu_struct {

	unsigned int srcu_idx;			/* Current rdr array element. */

	struct srcu_data __percpu *sda;		/* Per-CPU srcu_data array. */

	unsigned long reschedule_jiffies;

	unsigned long reschedule_count;

	struct delayed_work work;

	struct lockdep_map dep_map;

	struct srcu_usage *srcu_sup;		/* Update-side data. */

};
@@ -119,10 +120,10 @@ struct srcu_struct { 
{												\

	.lock = __SPIN_LOCK_UNLOCKED(name.lock),						\

	.srcu_gp_seq_needed = -1UL,								\

	.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),					\

}



#define __SRCU_STRUCT_INIT_COMMON(name, usage_name)						\

	.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),					\

	.srcu_sup = &usage_name,								\

	__SRCU_DEP_MAP_INIT(name)

kernel/rcu/srcutree.c

+22 −19
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
	ssp->srcu_sup->srcu_barrier_seq = 0;

	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);

	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);

	INIT_DELAYED_WORK(&ssp->work, process_srcu);

	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);

	ssp->srcu_sup->sda_is_static = is_static;

	if (!is_static)

		ssp->sda = alloc_percpu(struct srcu_data);
@@ -275,6 +275,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
			WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);

		}

	}

	ssp->srcu_sup->srcu_ssp = ssp;

	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */

	return 0;

}
@@ -647,7 +648,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) 
		return; /* Just leak it! */

	if (WARN_ON(srcu_readers_active(ssp)))

		return; /* Just leak it! */

	flush_delayed_work(&ssp->work);

	flush_delayed_work(&ssp->srcu_sup->work);

	for_each_possible_cpu(cpu) {

		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);
@@ -1059,10 +1060,10 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, 
		// can only be executed during early boot when there is only

		// the one boot CPU running with interrupts still disabled.

		if (likely(srcu_init_done))

			queue_delayed_work(rcu_gp_wq, &ssp->work,

			queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work,

					   !!srcu_get_delay(ssp));

		else if (list_empty(&ssp->work.work.entry))

			list_add(&ssp->work.work.entry, &srcu_boot_list);

		else if (list_empty(&ssp->srcu_sup->work.work.entry))

			list_add(&ssp->srcu_sup->work.work.entry, &srcu_boot_list);

	}

	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);

}
@@ -1723,7 +1724,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) 
	spin_unlock_irq_rcu_node(ssp->srcu_sup);



	if (pushgp)

		queue_delayed_work(rcu_gp_wq, &ssp->work, delay);

		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);

}



/*
@@ -1734,22 +1735,24 @@ static void process_srcu(struct work_struct *work) 
	unsigned long curdelay;

	unsigned long j;

	struct srcu_struct *ssp;

	struct srcu_usage *sup;



	ssp = container_of(work, struct srcu_struct, work.work);

	sup = container_of(work, struct srcu_usage, work.work);

	ssp = sup->srcu_ssp;



	srcu_advance_state(ssp);

	curdelay = srcu_get_delay(ssp);

	if (curdelay) {

		WRITE_ONCE(ssp->reschedule_count, 0);

		WRITE_ONCE(sup->reschedule_count, 0);

	} else {

		j = jiffies;

		if (READ_ONCE(ssp->reschedule_jiffies) == j) {

			WRITE_ONCE(ssp->reschedule_count, READ_ONCE(ssp->reschedule_count) + 1);

			if (READ_ONCE(ssp->reschedule_count) > srcu_max_nodelay)

		if (READ_ONCE(sup->reschedule_jiffies) == j) {

			WRITE_ONCE(sup->reschedule_count, READ_ONCE(sup->reschedule_count) + 1);

			if (READ_ONCE(sup->reschedule_count) > srcu_max_nodelay)

				curdelay = 1;

		} else {

			WRITE_ONCE(ssp->reschedule_count, 1);

			WRITE_ONCE(ssp->reschedule_jiffies, j);

			WRITE_ONCE(sup->reschedule_count, 1);

			WRITE_ONCE(sup->reschedule_jiffies, j);

		}

	}

	srcu_reschedule(ssp, curdelay);
@@ -1848,7 +1851,7 @@ early_initcall(srcu_bootup_announce); 


void __init srcu_init(void)

{

	struct srcu_struct *ssp;

	struct srcu_usage *sup;



	/* Decide on srcu_struct-size strategy. */

	if (SRCU_SIZING_IS(SRCU_SIZING_AUTO)) {
@@ -1868,13 +1871,13 @@ void __init srcu_init(void) 
	 */

	srcu_init_done = true;

	while (!list_empty(&srcu_boot_list)) {

		ssp = list_first_entry(&srcu_boot_list, struct srcu_struct,

		sup = list_first_entry(&srcu_boot_list, struct srcu_usage,

				      work.work.entry);

		list_del_init(&ssp->work.work.entry);

		list_del_init(&sup->work.work.entry);

		if (SRCU_SIZING_IS(SRCU_SIZING_INIT) &&

		    ssp->srcu_sup->srcu_size_state == SRCU_SIZE_SMALL)

			ssp->srcu_sup->srcu_size_state = SRCU_SIZE_ALLOC;

		queue_work(rcu_gp_wq, &ssp->work.work);

		    sup->srcu_size_state == SRCU_SIZE_SMALL)

			sup->srcu_size_state = SRCU_SIZE_ALLOC;

		queue_work(rcu_gp_wq, &sup->work.work);

	}

}