Commit 53b46303 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Remove rsp parameter from rcu_boot_init_percpu_data() and friends 



There now is only one rcu_state structure in a given build of
the Linux kernel, so there is no need to pass it as a parameter
to RCU's functions.  This commit therefore removes the rsp
parameter from rcu_boot_init_percpu_data(), rcu_init_percpu_data(),
rcu_cleanup_dying_idle_cpu(), and rcu_migrate_callbacks().  While in
the neighborhood, line the last three into rcutree_prepare_cpu(),
rcu_report_dead() and rcutree_migrate_callbacks(), respectively.
This also gets rid of the for_each_rcu_flavor() calls that were in
those tree functions.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 8344b871

kernel/rcu/tree.c

+35 −67
Original line number Diff line number Diff line
@@ -3352,7 +3352,7 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf) 
 * Do boot-time initialization of a CPU's per-CPU RCU data.

 */

static void __init

rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)

rcu_boot_init_percpu_data(int cpu)

{

	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
@@ -3361,23 +3361,25 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) 
	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);

	WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != 1);

	WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));

	rdp->rcu_ofl_gp_seq = rsp->gp_seq;

	rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;

	rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;

	rdp->rcu_onl_gp_seq = rsp->gp_seq;

	rdp->rcu_onl_gp_seq = rcu_state.gp_seq;

	rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;

	rdp->cpu = cpu;

	rdp->rsp = rsp;

	rdp->rsp = &rcu_state;

	rcu_boot_init_nocb_percpu_data(rdp);

}



/*

 * Initialize a CPU's per-CPU RCU data.  Note that only one online or

 * Invoked early in the CPU-online process, when pretty much all services

 * are available.  The incoming CPU is not present.

 *

 * Initializes a CPU's per-CPU RCU data.  Note that only one online or

 * offline event can be happening at a given time.  Note also that we can

 * accept some slop in the rsp->gp_seq access due to the fact that this

 * CPU cannot possibly have any RCU callbacks in flight yet.

 */

static void

rcu_init_percpu_data(int cpu, struct rcu_state *rsp)

int rcutree_prepare_cpu(unsigned int cpu)

{

	unsigned long flags;

	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
@@ -3386,7 +3388,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) 
	/* Set up local state, ensuring consistent view of global state. */

	raw_spin_lock_irqsave_rcu_node(rnp, flags);

	rdp->qlen_last_fqs_check = 0;

	rdp->n_force_qs_snap = rsp->n_force_qs;

	rdp->n_force_qs_snap = rcu_state.n_force_qs;

	rdp->blimit = blimit;

	if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */

	    !init_nocb_callback_list(rdp))
@@ -3410,21 +3412,8 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) 
	rdp->core_needs_qs = false;

	rdp->rcu_iw_pending = false;

	rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;

	trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));

	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));

	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

}



/*

 * Invoked early in the CPU-online process, when pretty much all

 * services are available.  The incoming CPU is not present.

 */

int rcutree_prepare_cpu(unsigned int cpu)

{

	struct rcu_state *rsp;



	for_each_rcu_flavor(rsp)

		rcu_init_percpu_data(cpu, rsp);



	rcu_prepare_kthreads(cpu);

	rcu_spawn_all_nocb_kthreads(cpu);
@@ -3548,23 +3537,32 @@ void rcu_cpu_starting(unsigned int cpu) 


#ifdef CONFIG_HOTPLUG_CPU

/*

 * The CPU is exiting the idle loop into the arch_cpu_idle_dead()

 * function.  We now remove it from the rcu_node tree's ->qsmaskinitnext

 * bit masks.

 * The outgoing function has no further need of RCU, so remove it from

 * the rcu_node tree's ->qsmaskinitnext bit masks.

 *

 * Note that this function is special in that it is invoked directly

 * from the outgoing CPU rather than from the cpuhp_step mechanism.

 * This is because this function must be invoked at a precise location.

 */

static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)

void rcu_report_dead(unsigned int cpu)

{

	unsigned long flags;

	unsigned long mask;

	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);

	struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */



	/* QS for any half-done expedited RCU-sched GP. */

	preempt_disable();

	rcu_report_exp_rdp(&rcu_state, this_cpu_ptr(&rcu_data));

	preempt_enable();

	rcu_preempt_deferred_qs(current);



	/* Remove outgoing CPU from mask in the leaf rcu_node structure. */

	mask = rdp->grpmask;

	spin_lock(&rsp->ofl_lock);

	spin_lock(&rcu_state.ofl_lock);

	raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */

	rdp->rcu_ofl_gp_seq = READ_ONCE(rsp->gp_seq);

	rdp->rcu_ofl_gp_flags = READ_ONCE(rsp->gp_flags);

	rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);

	rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);

	if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */

		/* Report quiescent state -before- changing ->qsmaskinitnext! */

		rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
@@ -3572,34 +3570,17 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) 
	}

	rnp->qsmaskinitnext &= ~mask;

	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

	spin_unlock(&rsp->ofl_lock);

}



/*

 * The outgoing function has no further need of RCU, so remove it from

 * the list of CPUs that RCU must track.

 *

 * Note that this function is special in that it is invoked directly

 * from the outgoing CPU rather than from the cpuhp_step mechanism.

 * This is because this function must be invoked at a precise location.

 */

void rcu_report_dead(unsigned int cpu)

{

	struct rcu_state *rsp;



	/* QS for any half-done expedited RCU-sched GP. */

	preempt_disable();

	rcu_report_exp_rdp(&rcu_state, this_cpu_ptr(&rcu_data));

	preempt_enable();

	rcu_preempt_deferred_qs(current);

	for_each_rcu_flavor(rsp)

		rcu_cleanup_dying_idle_cpu(cpu, rsp);

	spin_unlock(&rcu_state.ofl_lock);



	per_cpu(rcu_cpu_started, cpu) = 0;

}



/* Migrate the dead CPU's callbacks to the current CPU. */

static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)

/*

 * The outgoing CPU has just passed through the dying-idle state, and we

 * are being invoked from the CPU that was IPIed to continue the offline

 * operation.  Migrate the outgoing CPU's callbacks to the current CPU.

 */

void rcutree_migrate_callbacks(int cpu)

{

	unsigned long flags;

	struct rcu_data *my_rdp;
@@ -3632,19 +3613,6 @@ static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp) 
		  cpu, rcu_segcblist_n_cbs(&rdp->cblist),

		  rcu_segcblist_first_cb(&rdp->cblist));

}



/*

 * The outgoing CPU has just passed through the dying-idle state,

 * and we are being invoked from the CPU that was IPIed to continue the

 * offline operation.  We need to migrate the outgoing CPU's callbacks.

 */

void rcutree_migrate_callbacks(int cpu)

{

	struct rcu_state *rsp;



	for_each_rcu_flavor(rsp)

		rcu_migrate_callbacks(cpu, rsp);

}

#endif



/*
@@ -3814,7 +3782,7 @@ static void __init rcu_init_one(struct rcu_state *rsp) 
		while (i > rnp->grphi)

			rnp++;

		per_cpu_ptr(&rcu_data, i)->mynode = rnp;

		rcu_boot_init_percpu_data(i, rsp);

		rcu_boot_init_percpu_data(i);

	}

	list_add(&rsp->flavors, &rcu_struct_flavors);

}