Commit 91a967fd authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Add full-sized polling for get_completed*() and poll_state*() 



The get_completed_synchronize_rcu() and poll_state_synchronize_rcu()
APIs compress the combined expedited and normal grace-period states into a
single unsigned long, which conserves storage, but can miss grace periods
in certain cases involving overlapping normal and expedited grace periods.
Missing the occasional grace period is usually not a problem, but there
are use cases that care about each and every grace period.

This commit therefore adds the first members of the full-state RCU
grace-period polling API, namely the get_completed_synchronize_rcu_full()
and poll_state_synchronize_rcu_full() functions.  These use up to three
times the storage (rcu_gp_oldstate structure instead of unsigned long),
but which are guaranteed not to miss grace periods, at least in situations
where the single-CPU grace-period optimization does not apply.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 568035b0

include/linux/rcupdate.h

+3 −0
Original line number Diff line number Diff line
@@ -42,7 +42,10 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); 
void rcu_barrier_tasks(void);

void rcu_barrier_tasks_rude(void);

void synchronize_rcu(void);



struct rcu_gp_oldstate;

unsigned long get_completed_synchronize_rcu(void);

void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);



#ifdef CONFIG_PREEMPT_RCU

include/linux/rcutiny.h

+9 −0
Original line number Diff line number Diff line
@@ -14,10 +14,19 @@ 


#include <asm/param.h> /* for HZ */



struct rcu_gp_oldstate {

	unsigned long rgos_norm;

};



unsigned long get_state_synchronize_rcu(void);

unsigned long start_poll_synchronize_rcu(void);

bool poll_state_synchronize_rcu(unsigned long oldstate);



static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

{

	return poll_state_synchronize_rcu(rgosp->rgos_norm);

}



static inline void cond_synchronize_rcu(unsigned long oldstate)

{

	might_sleep();

include/linux/rcutree.h

+8 −0
Original line number Diff line number Diff line
@@ -40,11 +40,19 @@ bool rcu_eqs_special_set(int cpu); 
void rcu_momentary_dyntick_idle(void);

void kfree_rcu_scheduler_running(void);

bool rcu_gp_might_be_stalled(void);



struct rcu_gp_oldstate {

	unsigned long rgos_norm;

	unsigned long rgos_exp;

	unsigned long rgos_polled;

};



unsigned long start_poll_synchronize_rcu_expedited(void);

void cond_synchronize_rcu_expedited(unsigned long oldstate);

unsigned long get_state_synchronize_rcu(void);

unsigned long start_poll_synchronize_rcu(void);

bool poll_state_synchronize_rcu(unsigned long oldstate);

bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

void cond_synchronize_rcu(unsigned long oldstate);



bool rcu_is_idle_cpu(int cpu);

kernel/rcu/rcutorture.c

+9 −0
Original line number Diff line number Diff line
@@ -336,8 +336,10 @@ struct rcu_torture_ops { 
	void (*cond_sync_exp)(unsigned long oldstate);

	unsigned long (*get_gp_state)(void);

	unsigned long (*get_gp_completed)(void);

	void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);

	unsigned long (*start_gp_poll)(void);

	bool (*poll_gp_state)(unsigned long oldstate);

	bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);

	void (*cond_sync)(unsigned long oldstate);

	call_rcu_func_t call;

	void (*cb_barrier)(void);
@@ -503,8 +505,10 @@ static struct rcu_torture_ops rcu_ops = { 
	.exp_sync		= synchronize_rcu_expedited,

	.get_gp_state		= get_state_synchronize_rcu,

	.get_gp_completed	= get_completed_synchronize_rcu,

	.get_gp_completed_full	= get_completed_synchronize_rcu_full,

	.start_gp_poll		= start_poll_synchronize_rcu,

	.poll_gp_state		= poll_state_synchronize_rcu,

	.poll_gp_state_full	= poll_state_synchronize_rcu_full,

	.cond_sync		= cond_synchronize_rcu,

	.get_gp_state_exp	= get_state_synchronize_rcu,

	.start_gp_poll_exp	= start_poll_synchronize_rcu_expedited,
@@ -1212,6 +1216,7 @@ rcu_torture_writer(void *arg) 
	bool boot_ended;

	bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();

	unsigned long cookie;

	struct rcu_gp_oldstate cookie_full;

	int expediting = 0;

	unsigned long gp_snap;

	int i;
@@ -1277,6 +1282,10 @@ rcu_torture_writer(void *arg) 
				}

				cur_ops->readunlock(idx);

			}

			if (cur_ops->get_gp_completed_full && cur_ops->poll_gp_state_full) {

				cur_ops->get_gp_completed_full(&cookie_full);

				WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));

			}

			switch (synctype[torture_random(&rand) % nsynctypes]) {

			case RTWS_DEF_FREE:

				rcu_torture_writer_state = RTWS_DEF_FREE;

kernel/rcu/tiny.c

+10 −0
Original line number Diff line number Diff line
@@ -183,6 +183,16 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) 
}

EXPORT_SYMBOL_GPL(call_rcu);



/*

 * Store a grace-period-counter "cookie".  For more information,

 * see the Tree RCU header comment.

 */

void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

{

	rgosp->rgos_norm = RCU_GET_STATE_COMPLETED;

}

EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full);



/*

 * Return a grace-period-counter "cookie".  For more information,

 * see the Tree RCU header comment.