Commit 02219caa authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

Merge branches 'doc.2024.06.06a', 'fixes.2024.07.04a', 'mb.2024.06.28a',... 

Merge branches 'doc.2024.06.06a', 'fixes.2024.07.04a', 'mb.2024.06.28a', 'nocb.2024.06.03a', 'rcu-tasks.2024.06.06a', 'rcutorture.2024.06.06a' and 'srcu.2024.06.18a' into HEAD

doc.2024.06.06a: Documentation updates.
fixes.2024.07.04a: Miscellaneous fixes.
mb.2024.06.28a: Grace-period memory-barrier redundancy removal.
nocb.2024.06.03a: No-CB CPU updates.
rcu-tasks.2024.06.06a: RCU-Tasks updates.
rcutorture.2024.06.06a: Torture-test updates.
srcu.2024.06.18a: SRCU polled-grace-period updates.
parents a3fbf860 55d4669e 677ab23b e4f78057 399ced95 0ac55d09 e206f33e

Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst

+3 −3
Original line number Diff line number Diff line
@@ -149,9 +149,9 @@ This case is handled by calls to the strongly ordered 
``atomic_add_return()`` read-modify-write atomic operation that

is invoked within ``rcu_dynticks_eqs_enter()`` at idle-entry

time and within ``rcu_dynticks_eqs_exit()`` at idle-exit time.

The grace-period kthread invokes ``rcu_dynticks_snap()`` and

``rcu_dynticks_in_eqs_since()`` (both of which invoke

an ``atomic_add_return()`` of zero) to detect idle CPUs.

The grace-period kthread invokes first ``ct_dynticks_cpu_acquire()``

(preceded by a full memory barrier) and ``rcu_dynticks_in_eqs_since()``

(both of which rely on acquire semantics) to detect idle CPUs.



+-----------------------------------------------------------------------+

| **Quick Quiz**:                                                       |

Documentation/admin-guide/kernel-parameters.txt

+8 −0
Original line number Diff line number Diff line
@@ -5018,6 +5018,14 @@ 
			the ->nocb_bypass queue.  The definition of "too

			many" is supplied by this kernel boot parameter.



	rcutree.nohz_full_patience_delay= [KNL]

			On callback-offloaded (rcu_nocbs) CPUs, avoid

			disturbing RCU unless the grace period has

			reached the specified age in milliseconds.

			Defaults to zero.  Large values will be capped

			at five seconds.  All values will be rounded down

			to the nearest value representable by jiffies.



	rcutree.qhimark= [KNL]

			Set threshold of queued RCU callbacks beyond which

			batch limiting is disabled.

MAINTAINERS

+1 −0
Original line number Diff line number Diff line
@@ -18868,6 +18868,7 @@ M: Neeraj Upadhyay <neeraj.upadhyay@kernel.org> (kernel/rcu/tasks.h) 
M:	Joel Fernandes <joel@joelfernandes.org>

M:	Josh Triplett <josh@joshtriplett.org>

M:	Boqun Feng <boqun.feng@gmail.com>

M:	Uladzislau Rezki <urezki@gmail.com>

R:	Steven Rostedt <rostedt@goodmis.org>

R:	Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

R:	Lai Jiangshan <jiangshanlai@gmail.com>

include/linux/rcu_segcblist.h

+38 −50
Original line number Diff line number Diff line
@@ -80,36 +80,35 @@ struct rcu_cblist { 
 *  |       SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED       |

 *  |                                                                          |

 *  | Callbacks processed by rcu_core() from softirqs or local                 |

 *  | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads,     |

 *  | allowing nocb_timer to be armed.                                         |

 *  | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads.     |

 *  ----------------------------------------------------------------------------

 *                                         |

 *                                         v

 *                        -----------------------------------

 *  ----------------------------------------------------------------------------

 *  |        SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED      |

 *  |                              + unparked CB kthread                       |

 *  |                                                                          |

 *                        v                                 v

 *  ---------------------------------------  ----------------------------------|

 *  |        SEGCBLIST_RCU_CORE   |       |  |     SEGCBLIST_RCU_CORE   |      |

 *  |        SEGCBLIST_LOCKING    |       |  |     SEGCBLIST_LOCKING    |      |

 *  |        SEGCBLIST_OFFLOADED  |       |  |     SEGCBLIST_OFFLOADED  |      |

 *  |        SEGCBLIST_KTHREAD_CB         |  |     SEGCBLIST_KTHREAD_GP        |

 *  |                                     |  |                                 |

 *  |                                     |  |                                 |

 *  | CB kthread woke up and              |  | GP kthread woke up and          |

 *  | acknowledged SEGCBLIST_OFFLOADED.   |  | acknowledged SEGCBLIST_OFFLOADED|

 *  | Processes callbacks concurrently    |  |                                 |

 *  | with rcu_core(), holding            |  |                                 |

 *  | nocb_lock.                          |  |                                 |

 *  ---------------------------------------  -----------------------------------

 *  | CB kthread got unparked and processes callbacks concurrently with        |

 *  | rcu_core(), holding nocb_lock.                                           |

 *  ---------------------------------------------------------------------------

 *                                         |

 *                                         v

 *  ---------------------------------------------------------------------------|

 *  |                           SEGCBLIST_RCU_CORE |                           |

 *  |                           SEGCBLIST_LOCKING |                            |

 *  |                           SEGCBLIST_OFFLOADED |                          |

 *  |                           SEGCBLIST_KTHREAD_GP                           |

 *  |                           + unparked CB kthread                          |

 *  |                                                                          |

 *                        -----------------------------------

 *  | GP kthread woke up and acknowledged nocb_lock.                           |

 *  ---------------------------------------- -----------------------------------

 *                                         |

 *                                         v

 *  |--------------------------------------------------------------------------|

 *  |                           SEGCBLIST_LOCKING |                            |

 *  |                           SEGCBLIST_OFFLOADED |                          |

 *  |                           SEGCBLIST_KTHREAD_GP |                         |

 *  |                           SEGCBLIST_KTHREAD_CB                           |

 *  |                           + unparked CB kthread                          |

 *  |                                                                          |

 *  |   Kthreads handle callbacks holding nocb_lock, local rcu_core() stops    |

 *  |   handling callbacks. Enable bypass queueing.                            |
@@ -125,8 +124,8 @@ struct rcu_cblist { 
 *  |--------------------------------------------------------------------------|

 *  |                           SEGCBLIST_LOCKING    |                         |

 *  |                           SEGCBLIST_OFFLOADED  |                         |

 *  |                           SEGCBLIST_KTHREAD_CB |                         |

 *  |                           SEGCBLIST_KTHREAD_GP                           |

 *  |                           + unparked CB kthread                          |

 *  |                                                                          |

 *  |   CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core()    |

 *  |   ignores callbacks. Bypass enqueue is enabled.                          |
@@ -137,11 +136,11 @@ struct rcu_cblist { 
 *  |                           SEGCBLIST_RCU_CORE   |                         |

 *  |                           SEGCBLIST_LOCKING    |                         |

 *  |                           SEGCBLIST_OFFLOADED  |                         |

 *  |                           SEGCBLIST_KTHREAD_CB |                         |

 *  |                           SEGCBLIST_KTHREAD_GP                           |

 *  |                           + unparked CB kthread                          |

 *  |                                                                          |

 *  |   CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core()    |

 *  |   handles callbacks concurrently. Bypass enqueue is enabled.             |

 *  |   handles callbacks concurrently. Bypass enqueue is disabled.            |

 *  |   Invoke RCU core so we make sure not to preempt it in the middle with   |

 *  |   leaving some urgent work unattended within a jiffy.                    |

 *  ----------------------------------------------------------------------------
@@ -150,42 +149,31 @@ struct rcu_cblist { 
 *  |--------------------------------------------------------------------------|

 *  |                           SEGCBLIST_RCU_CORE   |                         |

 *  |                           SEGCBLIST_LOCKING    |                         |

 *  |                           SEGCBLIST_KTHREAD_CB |                         |

 *  |                           SEGCBLIST_KTHREAD_GP                           |

 *  |                           + unparked CB kthread                          |

 *  |                                                                          |

 *  |   CB/GP kthreads and local rcu_core() handle callbacks concurrently      |

 *  |   holding nocb_lock. Wake up CB and GP kthreads if necessary. Disable    |

 *  |   bypass enqueue.                                                        |

 *  |   holding nocb_lock. Wake up GP kthread if necessary.                    |

 *  ----------------------------------------------------------------------------

 *                                      |

 *                                      v

 *                     -----------------------------------

 *  |--------------------------------------------------------------------------|

 *  |                           SEGCBLIST_RCU_CORE   |                         |

 *  |                           SEGCBLIST_LOCKING    |                         |

 *  |                           + unparked CB kthread                          |

 *  |                                                                          |

 *                     v                                 v

 *  ---------------------------------------------------------------------------|

 *  |                                     |                                    |

 *  |        SEGCBLIST_RCU_CORE |         |       SEGCBLIST_RCU_CORE |         |

 *  |        SEGCBLIST_LOCKING  |         |       SEGCBLIST_LOCKING  |         |

 *  |        SEGCBLIST_KTHREAD_CB         |       SEGCBLIST_KTHREAD_GP         |

 *  |                                     |                                    |

 *  | GP kthread woke up and              |   CB kthread woke up and           |

 *  | acknowledged the fact that          |   acknowledged the fact that       |

 *  | SEGCBLIST_OFFLOADED got cleared.    |   SEGCBLIST_OFFLOADED got cleared. |

 *  |                                     |   The CB kthread goes to sleep     |

 *  | The callbacks from the target CPU   |   until it ever gets re-offloaded. |

 *  | will be ignored from the GP kthread |                                    |

 *  | loop.                               |                                    |

 *  |   GP kthread woke up and acknowledged the fact that SEGCBLIST_OFFLOADED  |

 *  |   got cleared. The callbacks from the target CPU will be ignored from the|

 *  |   GP kthread loop.                                                       |

 *  ----------------------------------------------------------------------------

 *                      |                                 |

 *                      -----------------------------------

 *                                      |

 *                                      v

 *  ----------------------------------------------------------------------------

 *  |                SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING                    |

 *  |                          + parked CB kthread                             |

 *  |                                                                          |

 *  | Callbacks processed by rcu_core() from softirqs or local                 |

 *  | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed.    |

 *  | Flush pending nocb_timer. Flush nocb bypass callbacks.                   |

 *  | CB kthread is parked. Callbacks processed by rcu_core() from softirqs or |

 *  | local rcuc kthread, while holding nocb_lock.                             |

 *  ----------------------------------------------------------------------------

 *                                      |

 *                                      v

include/linux/rcupdate.h

+60 −2
Original line number Diff line number Diff line
@@ -209,7 +209,6 @@ void synchronize_rcu_tasks_rude(void); 


#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)

void exit_tasks_rcu_start(void);

void exit_tasks_rcu_stop(void);

void exit_tasks_rcu_finish(void);

#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */

#define rcu_tasks_classic_qs(t, preempt) do { } while (0)
@@ -218,7 +217,6 @@ void exit_tasks_rcu_finish(void); 
#define call_rcu_tasks call_rcu

#define synchronize_rcu_tasks synchronize_rcu

static inline void exit_tasks_rcu_start(void) { }

static inline void exit_tasks_rcu_stop(void) { }

static inline void exit_tasks_rcu_finish(void) { }

#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
@@ -421,11 +419,71 @@ static inline void rcu_preempt_sleep_check(void) { } 
				 "Illegal context switch in RCU-sched read-side critical section"); \

	} while (0)



// See RCU_LOCKDEP_WARN() for an explanation of the double call to

// debug_lockdep_rcu_enabled().

static inline bool lockdep_assert_rcu_helper(bool c)

{

	return debug_lockdep_rcu_enabled() &&

	       (c || !rcu_is_watching() || !rcu_lockdep_current_cpu_online()) &&

	       debug_lockdep_rcu_enabled();

}



/**

 * lockdep_assert_in_rcu_read_lock - WARN if not protected by rcu_read_lock()

 *

 * Splats if lockdep is enabled and there is no rcu_read_lock() in effect.

 */

#define lockdep_assert_in_rcu_read_lock() \

	WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map)))



/**

 * lockdep_assert_in_rcu_read_lock_bh - WARN if not protected by rcu_read_lock_bh()

 *

 * Splats if lockdep is enabled and there is no rcu_read_lock_bh() in effect.

 * Note that local_bh_disable() and friends do not suffice here, instead an

 * actual rcu_read_lock_bh() is required.

 */

#define lockdep_assert_in_rcu_read_lock_bh() \

	WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_bh_lock_map)))



/**

 * lockdep_assert_in_rcu_read_lock_sched - WARN if not protected by rcu_read_lock_sched()

 *

 * Splats if lockdep is enabled and there is no rcu_read_lock_sched()

 * in effect.  Note that preempt_disable() and friends do not suffice here,

 * instead an actual rcu_read_lock_sched() is required.

 */

#define lockdep_assert_in_rcu_read_lock_sched() \

	WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_sched_lock_map)))



/**

 * lockdep_assert_in_rcu_reader - WARN if not within some type of RCU reader

 *

 * Splats if lockdep is enabled and there is no RCU reader of any

 * type in effect.  Note that regions of code protected by things like

 * preempt_disable, local_bh_disable(), and local_irq_disable() all qualify

 * as RCU readers.

 *

 * Note that this will never trigger in PREEMPT_NONE or PREEMPT_VOLUNTARY

 * kernels that are not also built with PREEMPT_COUNT.  But if you have

 * lockdep enabled, you might as well also enable PREEMPT_COUNT.

 */

#define lockdep_assert_in_rcu_reader()								\

	WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map) &&			\

					       !lock_is_held(&rcu_bh_lock_map) &&		\

					       !lock_is_held(&rcu_sched_lock_map) &&		\

					       preemptible()))



#else /* #ifdef CONFIG_PROVE_RCU */



#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))

#define rcu_sleep_check() do { } while (0)



#define lockdep_assert_in_rcu_read_lock() do { } while (0)

#define lockdep_assert_in_rcu_read_lock_bh() do { } while (0)

#define lockdep_assert_in_rcu_read_lock_sched() do { } while (0)

#define lockdep_assert_in_rcu_reader() do { } while (0)



#endif /* #else #ifdef CONFIG_PROVE_RCU */



/*