Commit 29721b85 authored by Frederic Weisbecker's avatar Frederic Weisbecker Committed by Ingo Molnar
Browse files

tick/nohz: Only wake up a single target cpu when kicking a task 



When adding a tick dependency to a task, its necessary to
wake up the CPU where the task resides to reevaluate tick
dependencies on that CPU.

However the current code wakes up all nohz_full CPUs, which
is unnecessary.

Switch to waking up a single CPU, by using ordering of writes
to task->cpu and task->tick_dep_mask.

[ mingo: Minor readability edit. ]

Suggested-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
Signed-off-by: default avatarMarcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarPeter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210512232924.150322-7-frederic@kernel.org
parent 176b8906

kernel/time/tick-sched.c

+27 −13
Original line number Diff line number Diff line
@@ -322,6 +322,31 @@ void tick_nohz_full_kick_cpu(int cpu) 
	irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);

}



static void tick_nohz_kick_task(struct task_struct *tsk)

{

	int cpu = task_cpu(tsk);



	/*

	 * If the task concurrently migrates to another CPU,

	 * we guarantee it sees the new tick dependency upon

	 * schedule.

	 *

	 *

	 * set_task_cpu(p, cpu);

	 *   STORE p->cpu = @cpu

	 * __schedule() (switch to task 'p')

	 *   LOCK rq->lock

	 *   smp_mb__after_spin_lock()          STORE p->tick_dep_mask

	 *   tick_nohz_task_switch()            smp_mb() (atomic_fetch_or())

	 *      LOAD p->tick_dep_mask           LOAD p->cpu

	 */



	preempt_disable();

	if (cpu_online(cpu))

		tick_nohz_full_kick_cpu(cpu);

	preempt_enable();

}



/*

 * Kick all full dynticks CPUs in order to force these to re-evaluate

 * their dependency on the tick and restart it if necessary.
@@ -404,19 +429,8 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu); 
 */

void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)

{

	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {

		if (tsk == current) {

			preempt_disable();

			tick_nohz_full_kick();

			preempt_enable();

		} else {

			/*

			 * Some future tick_nohz_full_kick_task()

			 * should optimize this.

			 */

			tick_nohz_full_kick_all();

		}

	}

	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask))

		tick_nohz_kick_task(tsk);

}

EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);