Commit 978fcca9 authored by James Morse's avatar James Morse Committed by Borislav Petkov (AMD)
Browse files

x86/resctrl: Allow overflow/limbo handlers to be scheduled on any-but CPU 



When a CPU is taken offline resctrl may need to move the overflow or limbo
handlers to run on a different CPU.

Once the offline callbacks have been split, cqm_setup_limbo_handler() will be
called while the CPU that is going offline is still present in the CPU mask.

Pass the CPU to exclude to cqm_setup_limbo_handler() and
mbm_setup_overflow_handler(). These functions can use a variant of
cpumask_any_but() when selecting the CPU. -1 is used to indicate no CPUs need
excluding.

Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: default avatarShaopeng Tan <tan.shaopeng@fujitsu.com>
Reviewed-by: default avatarBabu Moger <babu.moger@amd.com>
Reviewed-by: default avatarReinette Chatre <reinette.chatre@intel.com>
Tested-by: default avatarShaopeng Tan <tan.shaopeng@fujitsu.com>
Tested-by: default avatarPeter Newman <peternewman@google.com>
Tested-by: default avatarBabu Moger <babu.moger@amd.com>
Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
Link: https://lore.kernel.org/r/20240213184438.16675-22-james.morse@arm.com


Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
parent 1b3e50ce

arch/x86/kernel/cpu/resctrl/core.c

+6 −2
Original line number Diff line number Diff line
@@ -584,12 +584,16 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) 
	if (r == &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl) {

		if (is_mbm_enabled() && cpu == d->mbm_work_cpu) {

			cancel_delayed_work(&d->mbm_over);

			mbm_setup_overflow_handler(d, 0);

			/*

			 * temporary: exclude_cpu=-1 as this CPU has already

			 * been removed by cpumask_clear_cpu()d

			 */

			mbm_setup_overflow_handler(d, 0, RESCTRL_PICK_ANY_CPU);

		}

		if (is_llc_occupancy_enabled() && cpu == d->cqm_work_cpu &&

		    has_busy_rmid(d)) {

			cancel_delayed_work(&d->cqm_limbo);

			cqm_setup_limbo_handler(d, 0);

			cqm_setup_limbo_handler(d, 0, RESCTRL_PICK_ANY_CPU);

		}

	}

}

arch/x86/kernel/cpu/resctrl/ctrlmondata.c

+1 −1
Original line number Diff line number Diff line
@@ -552,7 +552,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, 
		return;

	}



	cpu = cpumask_any_housekeeping(&d->cpu_mask);

	cpu = cpumask_any_housekeeping(&d->cpu_mask, RESCTRL_PICK_ANY_CPU);



	/*

	 * cpumask_any_housekeeping() prefers housekeeping CPUs, but

arch/x86/kernel/cpu/resctrl/internal.h

+26 −7
Original line number Diff line number Diff line
@@ -60,19 +60,36 @@ 
 * cpumask_any_housekeeping() - Choose any CPU in @mask, preferring those that

 *			        aren't marked nohz_full

 * @mask:	The mask to pick a CPU from.

 * @exclude_cpu:The CPU to avoid picking.

 *

 * Returns a CPU in @mask. If there are housekeeping CPUs that don't use

 * nohz_full, these are preferred.

 * Returns a CPU from @mask, but not @exclude_cpu. If there are housekeeping

 * CPUs that don't use nohz_full, these are preferred. Pass

 * RESCTRL_PICK_ANY_CPU to avoid excluding any CPUs.

 *

 * When a CPU is excluded, returns >= nr_cpu_ids if no CPUs are available.

 */

static inline unsigned int cpumask_any_housekeeping(const struct cpumask *mask)

static inline unsigned int

cpumask_any_housekeeping(const struct cpumask *mask, int exclude_cpu)

{

	unsigned int cpu, hk_cpu;



	if (exclude_cpu == RESCTRL_PICK_ANY_CPU)

		cpu = cpumask_any(mask);

	if (!tick_nohz_full_cpu(cpu))

	else

		cpu = cpumask_any_but(mask, exclude_cpu);



	if (!IS_ENABLED(CONFIG_NO_HZ_FULL))

		return cpu;



	/* If the CPU picked isn't marked nohz_full nothing more needs doing. */

	if (cpu < nr_cpu_ids && !tick_nohz_full_cpu(cpu))

		return cpu;



	/* Try to find a CPU that isn't nohz_full to use in preference */

	hk_cpu = cpumask_nth_andnot(0, mask, tick_nohz_full_mask);

	if (hk_cpu == exclude_cpu)

		hk_cpu = cpumask_nth_andnot(1, mask, tick_nohz_full_mask);



	if (hk_cpu < nr_cpu_ids)

		cpu = hk_cpu;
@@ -573,11 +590,13 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, 
		    struct rdt_domain *d, struct rdtgroup *rdtgrp,

		    int evtid, int first);

void mbm_setup_overflow_handler(struct rdt_domain *dom,

				unsigned long delay_ms);

				unsigned long delay_ms,

				int exclude_cpu);

void mbm_handle_overflow(struct work_struct *work);

void __init intel_rdt_mbm_apply_quirk(void);

bool is_mba_sc(struct rdt_resource *r);

void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);

void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms,

			     int exclude_cpu);

void cqm_handle_limbo(struct work_struct *work);

bool has_busy_rmid(struct rdt_domain *d);

void __check_limbo(struct rdt_domain *d, bool force_free);

arch/x86/kernel/cpu/resctrl/monitor.c

+33 −9
Original line number Diff line number Diff line
@@ -481,7 +481,8 @@ static void add_rmid_to_limbo(struct rmid_entry *entry) 
		 * setup up the limbo worker.

		 */

		if (!has_busy_rmid(d))

			cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL);

			cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL,

						RESCTRL_PICK_ANY_CPU);

		set_bit(idx, d->rmid_busy_llc);

		entry->busy++;

	}
@@ -784,7 +785,8 @@ void cqm_handle_limbo(struct work_struct *work) 
	__check_limbo(d, false);



	if (has_busy_rmid(d)) {

		d->cqm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask);

		d->cqm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask,

							   RESCTRL_PICK_ANY_CPU);

		schedule_delayed_work_on(d->cqm_work_cpu, &d->cqm_limbo,

					 delay);

	}
@@ -792,14 +794,24 @@ void cqm_handle_limbo(struct work_struct *work) 
	mutex_unlock(&rdtgroup_mutex);

}



void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms)

/**

 * cqm_setup_limbo_handler() - Schedule the limbo handler to run for this

 *                             domain.

 * @dom:           The domain the limbo handler should run for.

 * @delay_ms:      How far in the future the handler should run.

 * @exclude_cpu:   Which CPU the handler should not run on,

 *		   RESCTRL_PICK_ANY_CPU to pick any CPU.

 */

void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms,

			     int exclude_cpu)

{

	unsigned long delay = msecs_to_jiffies(delay_ms);

	int cpu;



	cpu = cpumask_any_housekeeping(&dom->cpu_mask);

	cpu = cpumask_any_housekeeping(&dom->cpu_mask, exclude_cpu);

	dom->cqm_work_cpu = cpu;



	if (cpu < nr_cpu_ids)

		schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay);

}
@@ -838,14 +850,24 @@ void mbm_handle_overflow(struct work_struct *work) 
	 * Re-check for housekeeping CPUs. This allows the overflow handler to

	 * move off a nohz_full CPU quickly.

	 */

	d->mbm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask);

	d->mbm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask,

						   RESCTRL_PICK_ANY_CPU);

	schedule_delayed_work_on(d->mbm_work_cpu, &d->mbm_over, delay);



out_unlock:

	mutex_unlock(&rdtgroup_mutex);

}



void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms)

/**

 * mbm_setup_overflow_handler() - Schedule the overflow handler to run for this

 *                                domain.

 * @dom:           The domain the overflow handler should run for.

 * @delay_ms:      How far in the future the handler should run.

 * @exclude_cpu:   Which CPU the handler should not run on,

 *		   RESCTRL_PICK_ANY_CPU to pick any CPU.

 */

void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms,

				int exclude_cpu)

{

	unsigned long delay = msecs_to_jiffies(delay_ms);

	int cpu;
@@ -856,8 +878,10 @@ void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms) 
	 */

	if (!resctrl_mounted || !resctrl_arch_mon_capable())

		return;

	cpu = cpumask_any_housekeeping(&dom->cpu_mask);

	cpu = cpumask_any_housekeeping(&dom->cpu_mask, exclude_cpu);

	dom->mbm_work_cpu = cpu;



	if (cpu < nr_cpu_ids)

		schedule_delayed_work_on(cpu, &dom->mbm_over, delay);

}

arch/x86/kernel/cpu/resctrl/rdtgroup.c

+4 −2
Original line number Diff line number Diff line
@@ -2678,7 +2678,8 @@ static int rdt_get_tree(struct fs_context *fc) 
	if (is_mbm_enabled()) {

		r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;

		list_for_each_entry(dom, &r->domains, list)

			mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL);

			mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL,

						   RESCTRL_PICK_ANY_CPU);

	}



	goto out;
@@ -3989,7 +3990,8 @@ int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d) 


	if (is_mbm_enabled()) {

		INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow);

		mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL);

		mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL,

					   RESCTRL_PICK_ANY_CPU);

	}



	if (is_llc_occupancy_enabled())