Commit a57ffb3c authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Automatically determine size-transition strategy at boot 



This commit adds a srcutree.convert_to_big option of zero that causes
SRCU to decide at boot whether to wait for contention (small systems) or
immediately expand to large (large systems).  A new srcutree.big_cpu_lim
(defaulting to 128) defines how many CPUs constitute a large system.

Co-developed-by: default avatarNeeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: default avatarNeeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 9f2e91d9

Documentation/admin-guide/kernel-parameters.txt

+10 −0
Original line number Diff line number Diff line
@@ -5608,6 +5608,15 @@ 
			off:    Disable mitigation and remove

				performance impact to RDRAND and RDSEED



	srcutree.big_cpu_lim [KNL]

			Specifies the number of CPUs constituting a

			large system, such that srcu_struct structures

			should immediately allocate an srcu_node array.

			This kernel-boot parameter defaults to 128,

			but takes effect only when the low-order four

			bits of srcutree.convert_to_big is equal to 3

			(decide at boot).



	srcutree.convert_to_big [KNL]

			Specifies under what conditions an SRCU tree

			srcu_struct structure will be converted to big
@@ -5616,6 +5625,7 @@ 
				   0:  Never.

				   1:  At init_srcu_struct() time.

				   2:  When rcutorture decides to.

				   3:  Decide at boot time (default).

				0x1X:  Above plus if high contention.



			Either way, the srcu_node tree will be sized based

kernel/rcu/srcutree.c

+20 −3
Original line number Diff line number Diff line
@@ -41,10 +41,10 @@ module_param(counter_wrap_check, ulong, 0444); 


/*

 * Control conversion to SRCU_SIZE_BIG:

 *    0: Don't convert at all (default).

 *    0: Don't convert at all.

 *    1: Convert at init_srcu_struct() time.

 *    2: Convert when rcutorture invokes srcu_torture_stats_print().

 *    3: Decide at boot time based on system shape.

 *    3: Decide at boot time based on system shape (default).

 * 0x1x: Convert when excessive contention encountered.

 */

#define SRCU_SIZING_NONE	0
@@ -57,9 +57,13 @@ module_param(counter_wrap_check, ulong, 0444); 
#define SRCU_SIZING_IS_INIT() (SRCU_SIZING_IS(SRCU_SIZING_INIT))

#define SRCU_SIZING_IS_TORTURE() (SRCU_SIZING_IS(SRCU_SIZING_TORTURE))

#define SRCU_SIZING_IS_CONTEND() (convert_to_big & SRCU_SIZING_CONTEND)

static int convert_to_big = SRCU_SIZING_NONE;

static int convert_to_big = SRCU_SIZING_AUTO;

module_param(convert_to_big, int, 0444);



/* Number of CPUs to trigger init_srcu_struct()-time transition to big. */

static int big_cpu_lim __read_mostly = 128;

module_param(big_cpu_lim, int, 0444);



/* Contention events per jiffy to initiate transition to big. */

static int small_contention_lim __read_mostly = 100;

module_param(small_contention_lim, int, 0444);
@@ -1619,6 +1623,17 @@ void __init srcu_init(void) 
{

	struct srcu_struct *ssp;



	/* Decide on srcu_struct-size strategy. */

	if (SRCU_SIZING_IS(SRCU_SIZING_AUTO)) {

		if (nr_cpu_ids >= big_cpu_lim) {

			convert_to_big = SRCU_SIZING_INIT; // Don't bother waiting for contention.

			pr_info("%s: Setting srcu_struct sizes to big.\n", __func__);

		} else {

			convert_to_big = SRCU_SIZING_NONE | SRCU_SIZING_CONTEND;

			pr_info("%s: Setting srcu_struct sizes based on contention.\n", __func__);

		}

	}



	/*

	 * Once that is set, call_srcu() can follow the normal path and

	 * queue delayed work. This must follow RCU workqueues creation
@@ -1629,6 +1644,8 @@ void __init srcu_init(void) 
		ssp = list_first_entry(&srcu_boot_list, struct srcu_struct,

				      work.work.entry);

		list_del_init(&ssp->work.work.entry);

		if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && ssp->srcu_size_state == SRCU_SIZE_SMALL)

			ssp->srcu_size_state = SRCU_SIZE_ALLOC;

		queue_work(rcu_gp_wq, &ssp->work.work);

	}

}