Merge branches 'rcu/misc-for-6.16', 'rcu/seq-counters-for-6.16' and... (9c80e443) · Commits · git / linux-nf

Documentation/admin-guide/kernel-parameters.txt

+25 −0

Original line number	Diff line number	Diff line
		@@ -5657,6 +5657,31 @@
		are zero, rcutorture acts as if is interpreted
		they are all non-zero.

		rcutorture.gpwrap_lag= [KNL]
		Enable grace-period wrap lag testing. Setting
		to false prevents the gpwrap lag test from
		running. Default is true.

		rcutorture.gpwrap_lag_gps= [KNL]
		Set the value for grace-period wrap lag during
		active lag testing periods. This controls how many
		grace periods differences we tolerate between
		rdp and rnp's gp_seq before setting overflow flag.
		The default is always set to 8.

		rcutorture.gpwrap_lag_cycle_mins= [KNL]
		Set the total cycle duration for gpwrap lag
		testing in minutes. This is the total time for
		one complete cycle of active and inactive
		testing periods. Default is 30 minutes.

		rcutorture.gpwrap_lag_active_mins= [KNL]
		Set the duration for which gpwrap lag is active
		within each cycle, in minutes. During this time,
		the grace-period wrap lag will be set to the
		value specified by gpwrap_lag_gps. Default is
		5 minutes.

		rcutorture.irqreader= [KNL]
		Run RCU readers from irq handlers, or, more
		accurately, from a timer handler. Not all RCU

kernel/rcu/rcu.h

+17 −1

Original line number	Diff line number	Diff line
		@@ -57,6 +57,9 @@
		/* Low-order bit definition for polled grace-period APIs. */
		#define RCU_GET_STATE_COMPLETED 0x1

		/* A complete grace period count */
		#define RCU_SEQ_GP (RCU_SEQ_STATE_MASK + 1)

		extern int sysctl_sched_rt_runtime;

		/*
		@@ -157,12 +160,21 @@ static inline bool rcu_seq_done(unsigned long *sp, unsigned long s)
		* Given a snapshot from rcu_seq_snap(), determine whether or not a
		* full update-side operation has occurred, but do not allow the
		* (ULONG_MAX / 2) safety-factor/guard-band.
		*
		* The token returned by get_state_synchronize_rcu_full() is based on
		* rcu_state.gp_seq but it is tested in poll_state_synchronize_rcu_full()
		* against the root rnp->gp_seq. Since rcu_seq_start() is first called
		* on rcu_state.gp_seq and only later reflected on the root rnp->gp_seq,
		* it is possible that rcu_seq_snap(rcu_state.gp_seq) returns 2 full grace
		* periods ahead of the root rnp->gp_seq. To prevent false-positives with the
		* full polling API that a wrap around instantly completed the GP, when nothing
		* like that happened, adjust for the 2 GPs in the ULONG_CMP_LT().
		*/
		static inline bool rcu_seq_done_exact(unsigned long *sp, unsigned long s)
		{
		unsigned long cur_s = READ_ONCE(*sp);

		return ULONG_CMP_GE(cur_s, s) \|\| ULONG_CMP_LT(cur_s, s - (3 * RCU_SEQ_STATE_MASK + 1));
		return ULONG_CMP_GE(cur_s, s) \|\| ULONG_CMP_LT(cur_s, s - (2 * RCU_SEQ_GP));
		}

		/*
		@@ -572,6 +584,8 @@ void do_trace_rcu_torture_read(const char *rcutorturename,
		unsigned long c_old,
		unsigned long c);
		void rcu_gp_set_torture_wait(int duration);
		void rcu_set_gpwrap_lag(unsigned long lag);
		int rcu_get_gpwrap_count(int cpu);
		#else
		static inline void rcutorture_get_gp_data(int flags, unsigned long gp_seq)
		{
		@@ -589,6 +603,8 @@ void do_trace_rcu_torture_read(const char *rcutorturename,
		do { } while (0)
		#endif
		static inline void rcu_gp_set_torture_wait(int duration) { }
		static inline void rcu_set_gpwrap_lag(unsigned long lag) { }
		static inline int rcu_get_gpwrap_count(int cpu) { return 0; }
		#endif
		unsigned long long rcutorture_gather_gp_seqs(void);
		void rcutorture_format_gp_seqs(unsigned long long seqs, char *cp, size_t len);

kernel/rcu/rcutorture.c

+159 −47

Original line number	Diff line number	Diff line
		@@ -115,6 +115,10 @@ torture_param(int, nreaders, -1, "Number of RCU reader threads");
		torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing");
		torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
		torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable");
		torture_param(bool, gpwrap_lag, true, "Enable grace-period wrap lag testing");
		torture_param(int, gpwrap_lag_gps, 8, "Value to set for set_gpwrap_lag during an active testing period.");
		torture_param(int, gpwrap_lag_cycle_mins, 30, "Total cycle duration for gpwrap lag testing (in minutes)");
		torture_param(int, gpwrap_lag_active_mins, 5, "Duration for which gpwrap lag is active within each cycle (in minutes)");
		torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable");
		torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)");
		torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable");
		@@ -413,6 +417,8 @@ struct rcu_torture_ops {
		bool (*reader_blocked)(void);
		unsigned long long (*gather_gp_seqs)(void);
		void (format_gp_seqs)(unsigned long long seqs, char cp, size_t len);
		void (*set_gpwrap_lag)(unsigned long lag);
		int (*get_gpwrap_count)(int cpu);
		long cbflood_max;
		int irq_capable;
		int can_boost;
		@@ -619,6 +625,8 @@ static struct rcu_torture_ops rcu_ops = {
		: NULL,
		.gather_gp_seqs = rcutorture_gather_gp_seqs,
		.format_gp_seqs = rcutorture_format_gp_seqs,
		.set_gpwrap_lag = rcu_set_gpwrap_lag,
		.get_gpwrap_count = rcu_get_gpwrap_count,
		.irq_capable = 1,
		.can_boost = IS_ENABLED(CONFIG_RCU_BOOST),
		.extendables = RCUTORTURE_MAX_EXTEND,
		@@ -2164,53 +2172,70 @@ rcutorture_loop_extend(int *readstate, bool insoftirq, struct torture_random_sta
		return &rtrsp[j];
		}

		/*
		* Do one read-side critical section, returning false if there was
		* no data to read. Can be invoked both from process context and
		* from a timer handler.
		*/
		static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
		{
		bool checkpolling = !(torture_random(trsp) & 0xfff);
		struct rcu_torture_one_read_state {
		bool checkpolling;
		unsigned long cookie;
		struct rcu_gp_oldstate cookie_full;
		int i;
		unsigned long started;
		unsigned long completed;
		int newstate;
		struct rcu_torture *p;
		int pipe_count;
		bool preempted = false;
		int readstate = 0;
		struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } };
		struct rt_read_seg *rtrsp = &rtseg[0];
		struct rt_read_seg *rtrsp1;
		int readstate;
		struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS];
		struct rt_read_seg *rtrsp;
		unsigned long long ts;
		};

		WARN_ON_ONCE(!rcu_is_watching());
		newstate = rcutorture_extend_mask(readstate, trsp);
		rcutorture_one_extend(&readstate, newstate, myid < 0, trsp, rtrsp++);
		if (checkpolling) {
		static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp,
		struct torture_random_state *trsp)
		{
		memset(rtorsp, 0, sizeof(*rtorsp));
		rtorsp->checkpolling = !(torture_random(trsp) & 0xfff);
		rtorsp->rtrsp = &rtorsp->rtseg[0];
		}

		/*
		* Set up the first segment of a series of overlapping read-side
		* critical sections. The caller must have actually initiated the
		* outermost read-side critical section.
		*/
		static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp,
		struct torture_random_state *trsp, long myid)
		{
		if (rtorsp->checkpolling) {
		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
		cookie = cur_ops->get_gp_state();
		rtorsp->cookie = cur_ops->get_gp_state();
		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
		cur_ops->get_gp_state_full(&cookie_full);
		cur_ops->get_gp_state_full(&rtorsp->cookie_full);
		}
		started = cur_ops->get_gp_seq();
		ts = rcu_trace_clock_local();
		p = rcu_dereference_check(rcu_torture_current,
		rtorsp->started = cur_ops->get_gp_seq();
		rtorsp->ts = rcu_trace_clock_local();
		rtorsp->p = rcu_dereference_check(rcu_torture_current,
		!cur_ops->readlock_held \|\| cur_ops->readlock_held());
		if (p == NULL) {
		if (rtorsp->p == NULL) {
		/* Wait for rcu_torture_writer to get underway */
		rcutorture_one_extend(&readstate, 0, myid < 0, trsp, rtrsp);
		rcutorture_one_extend(&rtorsp->readstate, 0, myid < 0, trsp, rtorsp->rtrsp);
		return false;
		}
		if (p->rtort_mbtest == 0)
		if (rtorsp->p->rtort_mbtest == 0)
		atomic_inc(&n_rcu_torture_mberror);
		rcu_torture_reader_do_mbchk(myid, p, trsp);
		rtrsp = rcutorture_loop_extend(&readstate, myid < 0, trsp, rtrsp);
		rcu_torture_reader_do_mbchk(myid, rtorsp->p, trsp);
		return true;
		}

		/*
		* Complete the last segment of a series of overlapping read-side
		* critical sections and check for errors.
		*/
		static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp,
		struct torture_random_state *trsp, long myid)
		{
		int i;
		unsigned long completed;
		int pipe_count;
		bool preempted = false;
		struct rt_read_seg *rtrsp1;

		preempt_disable();
		pipe_count = READ_ONCE(p->rtort_pipe_count);
		pipe_count = READ_ONCE(rtorsp->p->rtort_pipe_count);
		if (pipe_count > RCU_TORTURE_PIPE_LEN) {
		// Should not happen in a correct RCU implementation,
		// happens quite often for torture_type=busted.
		@@ -2218,28 +2243,28 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
		}
		completed = cur_ops->get_gp_seq();
		if (pipe_count > 1) {
		do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
		ts, started, completed);
		do_trace_rcu_torture_read(cur_ops->name, &rtorsp->p->rtort_rcu,
		rtorsp->ts, rtorsp->started, completed);
		rcu_ftrace_dump(DUMP_ALL);
		}
		__this_cpu_inc(rcu_torture_count[pipe_count]);
		completed = rcutorture_seq_diff(completed, started);
		completed = rcutorture_seq_diff(completed, rtorsp->started);
		if (completed > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		completed = RCU_TORTURE_PIPE_LEN;
		}
		__this_cpu_inc(rcu_torture_batch[completed]);
		preempt_enable();
		if (checkpolling) {
		if (rtorsp->checkpolling) {
		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
		WARN_ONCE(cur_ops->poll_gp_state(cookie),
		WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie),
		"%s: Cookie check 2 failed %s(%d) %lu->%lu\n",
		__func__,
		rcu_torture_writer_state_getname(),
		rcu_torture_writer_state,
		cookie, cur_ops->get_gp_state());
		rtorsp->cookie, cur_ops->get_gp_state());
		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
		WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
		WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full),
		"%s: Cookie check 6 failed %s(%d) online %*pbl\n",
		__func__,
		rcu_torture_writer_state_getname(),
		@@ -2248,21 +2273,42 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
		}
		if (cur_ops->reader_blocked)
		preempted = cur_ops->reader_blocked();
		rcutorture_one_extend(&readstate, 0, myid < 0, trsp, rtrsp);
		WARN_ON_ONCE(readstate);
		rcutorture_one_extend(&rtorsp->readstate, 0, myid < 0, trsp, rtorsp->rtrsp);
		WARN_ON_ONCE(rtorsp->readstate);
		// This next splat is expected behavior if leakpointer, especially
		// for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels.
		WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1);
		WARN_ON_ONCE(leakpointer && READ_ONCE(rtorsp->p->rtort_pipe_count) > 1);

		/* If error or close call, record the sequence of reader protections. */
		if ((pipe_count > 1 \|\| completed > 1) && !xchg(&err_segs_recorded, 1)) {
		i = 0;
		for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++)
		for (rtrsp1 = &rtorsp->rtseg[0]; rtrsp1 < rtorsp->rtrsp; rtrsp1++)
		err_segs[i++] = *rtrsp1;
		rt_read_nsegs = i;
		rt_read_preempted = preempted;
		}
		}

		/*
		* Do one read-side critical section, returning false if there was
		* no data to read. Can be invoked both from process context and
		* from a timer handler.
		*/
		static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
		{
		int newstate;
		struct rcu_torture_one_read_state rtors;

		WARN_ON_ONCE(!rcu_is_watching());
		init_rcu_torture_one_read_state(&rtors, trsp);
		newstate = rcutorture_extend_mask(rtors.readstate, trsp);
		rcutorture_one_extend(&rtors.readstate, newstate, myid < 0, trsp, rtors.rtrsp++);
		if (!rcu_torture_one_read_start(&rtors, trsp, myid)) {
		rcutorture_one_extend(&rtors.readstate, 0, myid < 0, trsp, rtors.rtrsp);
		return false;
		}
		rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, myid < 0, trsp, rtors.rtrsp);
		rcu_torture_one_read_end(&rtors, trsp, myid);
		return true;
		}

		@@ -2307,7 +2353,7 @@ rcu_torture_reader(void *arg)
		set_user_nice(current, MAX_NICE);
		if (irqreader && cur_ops->irq_capable)
		timer_setup_on_stack(&t, rcu_torture_timer, 0);
		tick_dep_set_task(current, TICK_DEP_BIT_RCU);
		tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick.
		do {
		if (irqreader && cur_ops->irq_capable) {
		if (!timer_pending(&t))
		@@ -2394,6 +2440,7 @@ rcu_torture_stats_print(void)
		int i;
		long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
		long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
		long n_gpwraps = 0;
		struct rcu_torture *rtcp;
		static unsigned long rtcv_snap = ULONG_MAX;
		static bool splatted;
		@@ -2404,6 +2451,8 @@ rcu_torture_stats_print(void)
		pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
		batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
		}
		if (cur_ops->get_gpwrap_count)
		n_gpwraps += cur_ops->get_gpwrap_count(cpu);
		}
		for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) {
		if (pipesummary[i] != 0)
		@@ -2435,8 +2484,9 @@ rcu_torture_stats_print(void)
		data_race(n_barrier_attempts),
		data_race(n_rcu_torture_barrier_error));
		pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic.
		pr_cont("nocb-toggles: %ld:%ld\n",
		pr_cont("nocb-toggles: %ld:%ld ",
		atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload));
		pr_cont("gpwraps: %ld\n", n_gpwraps);

		pr_alert("%s%s ", torture_type, TORTURE_FLAG);
		if (atomic_read(&n_rcu_torture_mberror) \|\|
		@@ -3036,7 +3086,7 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)
		cver = READ_ONCE(rcu_torture_current_version);
		gps = cur_ops->get_gp_seq();
		rfp->rcu_launder_gp_seq_start = gps;
		tick_dep_set_task(current, TICK_DEP_BIT_RCU);
		tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick.
		while (time_before(jiffies, stopat) &&
		!shutdown_time_arrived() &&
		!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
		@@ -3607,6 +3657,57 @@ static int rcu_torture_preempt(void *unused)

		static enum cpuhp_state rcutor_hp;

		static struct hrtimer gpwrap_lag_timer;
		static bool gpwrap_lag_active;

		/* Timer handler for toggling RCU grace-period sequence overflow test lag value */
		static enum hrtimer_restart rcu_gpwrap_lag_timer(struct hrtimer *timer)
		{
		ktime_t next_delay;

		if (gpwrap_lag_active) {
		pr_alert("rcu-torture: Disabling gpwrap lag (value=0)\n");
		cur_ops->set_gpwrap_lag(0);
		gpwrap_lag_active = false;
		next_delay = ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0);
		} else {
		pr_alert("rcu-torture: Enabling gpwrap lag (value=%d)\n", gpwrap_lag_gps);
		cur_ops->set_gpwrap_lag(gpwrap_lag_gps);
		gpwrap_lag_active = true;
		next_delay = ktime_set(gpwrap_lag_active_mins * 60, 0);
		}

		if (torture_must_stop_irq())
		return HRTIMER_NORESTART;

		hrtimer_forward_now(timer, next_delay);
		return HRTIMER_RESTART;
		}

		static int rcu_gpwrap_lag_init(void)
		{
		if (!gpwrap_lag)
		return 0;

		if (gpwrap_lag_cycle_mins <= 0 \|\| gpwrap_lag_active_mins <= 0) {
		pr_alert("rcu-torture: lag timing parameters must be positive\n");
		return -EINVAL;
		}

		hrtimer_setup(&gpwrap_lag_timer, rcu_gpwrap_lag_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
		gpwrap_lag_active = false;
		hrtimer_start(&gpwrap_lag_timer,
		ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0), HRTIMER_MODE_REL);

		return 0;
		}

		static void rcu_gpwrap_lag_cleanup(void)
		{
		hrtimer_cancel(&gpwrap_lag_timer);
		cur_ops->set_gpwrap_lag(0);
		gpwrap_lag_active = false;
		}
		static void
		rcu_torture_cleanup(void)
		{
		@@ -3776,6 +3877,9 @@ rcu_torture_cleanup(void)
		torture_cleanup_end();
		if (cur_ops->gp_slow_unregister)
		cur_ops->gp_slow_unregister(NULL);

		if (gpwrap_lag && cur_ops->set_gpwrap_lag)
		rcu_gpwrap_lag_cleanup();
		}

		static void rcu_torture_leak_cb(struct rcu_head *rhp)
		@@ -4272,9 +4376,17 @@ rcu_torture_init(void)
		}
		if (object_debug)
		rcu_test_debug_objects();
		torture_init_end();

		if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister))
		cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay);

		if (gpwrap_lag && cur_ops->set_gpwrap_lag) {
		firsterr = rcu_gpwrap_lag_init();
		if (torture_init_error(firsterr))
		goto unwind;
		}

		torture_init_end();
		return 0;

		unwind:

kernel/rcu/srcutree.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -1589,7 +1589,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
		bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
		{
		if (cookie != SRCU_GET_STATE_COMPLETED &&
		!rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie))
		!rcu_seq_done_exact(&ssp->srcu_sup->srcu_gp_seq, cookie))
		return false;
		// Ensure that the end of the SRCU grace period happens before
		// any subsequent code that the caller might execute.

kernel/rcu/tree.c

+38 −2

Original line number	Diff line number	Diff line
		@@ -80,6 +80,15 @@ static void rcu_sr_normal_gp_cleanup_work(struct work_struct *);
		static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
		.gpwrap = true,
		};

		int rcu_get_gpwrap_count(int cpu)
		{
		struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);

		return READ_ONCE(rdp->gpwrap_count);
		}
		EXPORT_SYMBOL_GPL(rcu_get_gpwrap_count);

		static struct rcu_state rcu_state = {
		.level = { &rcu_state.node[0] },
		.gp_state = RCU_GP_IDLE,
		@@ -757,6 +766,25 @@ void rcu_request_urgent_qs_task(struct task_struct *t)
		smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
		}

		static unsigned long seq_gpwrap_lag = ULONG_MAX / 4;

		/**
		* rcu_set_gpwrap_lag - Set RCU GP sequence overflow lag value.
		* @lag_gps: Set overflow lag to this many grace period worth of counters
		* which is used by rcutorture to quickly force a gpwrap situation.
		* @lag_gps = 0 means we reset it back to the boot-time value.
		*/
		void rcu_set_gpwrap_lag(unsigned long lag_gps)
		{
		unsigned long lag_seq_count;

		lag_seq_count = (lag_gps == 0)
		? ULONG_MAX / 4
		: lag_gps << RCU_SEQ_CTR_SHIFT;
		WRITE_ONCE(seq_gpwrap_lag, lag_seq_count);
		}
		EXPORT_SYMBOL_GPL(rcu_set_gpwrap_lag);

		/*
		* When trying to report a quiescent state on behalf of some other CPU,
		* it is our responsibility to check for and handle potential overflow
		@@ -767,9 +795,11 @@ void rcu_request_urgent_qs_task(struct task_struct *t)
		static void rcu_gpnum_ovf(struct rcu_node rnp, struct rcu_data rdp)
		{
		raw_lockdep_assert_held_rcu_node(rnp);
		if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4,
		rnp->gp_seq))
		if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + seq_gpwrap_lag,
		rnp->gp_seq)) {
		WRITE_ONCE(rdp->gpwrap, true);
		WRITE_ONCE(rdp->gpwrap_count, READ_ONCE(rdp->gpwrap_count) + 1);
		}
		if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
		rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
		}
		@@ -1770,6 +1800,7 @@ static noinline_for_stack bool rcu_gp_init(void)
		struct rcu_data *rdp;
		struct rcu_node *rnp = rcu_get_root();
		bool start_new_poll;
		unsigned long old_gp_seq;

		WRITE_ONCE(rcu_state.gp_activity, jiffies);
		raw_spin_lock_irq_rcu_node(rnp);
		@@ -1797,7 +1828,12 @@ static noinline_for_stack bool rcu_gp_init(void)
		*/
		start_new_poll = rcu_sr_normal_gp_init();
		/* Record GP times before starting GP, hence rcu_seq_start(). */
		old_gp_seq = rcu_state.gp_seq;
		rcu_seq_start(&rcu_state.gp_seq);
		/* Ensure that rcu_seq_done_exact() guardband doesn't give false positives. */
		WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&
		rcu_seq_done_exact(&old_gp_seq, rcu_seq_snap(&rcu_state.gp_seq)));

		ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
		trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
		rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap);