Commit 51038506 authored by Zqiang's avatar Zqiang Committed by Paul E. McKenney
Browse files

rcu: Add nocb_cb_kthread check to rcu_is_callbacks_kthread() 



Callbacks are invoked in RCU kthreads when calbacks are offloaded
(rcu_nocbs boot parameter) or when RCU's softirq handler has been
offloaded to rcuc kthreads (use_softirq==0).  The current code allows
for the rcu_nocbs case but not the use_softirq case.  This commit adds
support for the use_softirq case.

Reported-by: default avatarkernel test robot <lkp@intel.com>
Signed-off-by: default avatarZqiang <qiang1.zhang@intel.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Reviewed-by: default avatarNeeraj Upadhyay <quic_neeraju@quicinc.com>
parent b37a667c

kernel/rcu/tree.c

+2 −2
Original line number Diff line number Diff line
@@ -2530,7 +2530,7 @@ static void rcu_do_batch(struct rcu_data *rdp) 
		trace_rcu_batch_end(rcu_state.name, 0,

				    !rcu_segcblist_empty(&rdp->cblist),

				    need_resched(), is_idle_task(current),

				    rcu_is_callbacks_kthread());

				    rcu_is_callbacks_kthread(rdp));

		return;

	}
@@ -2608,7 +2608,7 @@ static void rcu_do_batch(struct rcu_data *rdp) 
	rcu_nocb_lock_irqsave(rdp, flags);

	rdp->n_cbs_invoked += count;

	trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),

			    is_idle_task(current), rcu_is_callbacks_kthread());

			    is_idle_task(current), rcu_is_callbacks_kthread(rdp));



	/* Update counts and requeue any remaining callbacks. */

	rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);

kernel/rcu/tree.h

+1 −1
Original line number Diff line number Diff line
@@ -426,7 +426,7 @@ static void rcu_flavor_sched_clock_irq(int user); 
static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck);

static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);

static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);

static bool rcu_is_callbacks_kthread(void);

static bool rcu_is_callbacks_kthread(struct rcu_data *rdp);

static void rcu_cpu_kthread_setup(unsigned int cpu);

static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp);

static bool rcu_preempt_has_tasks(struct rcu_node *rnp);

kernel/rcu/tree_plugin.h

+19 −14
Original line number Diff line number Diff line
@@ -1012,6 +1012,25 @@ static void rcu_cpu_kthread_setup(unsigned int cpu) 
	WRITE_ONCE(rdp->rcuc_activity, jiffies);

}



static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp)

{

#ifdef CONFIG_RCU_NOCB_CPU

	return rdp->nocb_cb_kthread == current;

#else

	return false;

#endif

}



/*

 * Is the current CPU running the RCU-callbacks kthread?

 * Caller must have preemption disabled.

 */

static bool rcu_is_callbacks_kthread(struct rcu_data *rdp)

{

	return rdp->rcu_cpu_kthread_task == current ||

			rcu_is_callbacks_nocb_kthread(rdp);

}



#ifdef CONFIG_RCU_BOOST



/*
@@ -1151,15 +1170,6 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) 
	}

}



/*

 * Is the current CPU running the RCU-callbacks kthread?

 * Caller must have preemption disabled.

 */

static bool rcu_is_callbacks_kthread(void)

{

	return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current;

}



#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)



/*
@@ -1242,11 +1252,6 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) 
	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

}



static bool rcu_is_callbacks_kthread(void)

{

	return false;

}



static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)

{

}