Commit cd84f72b authored by Tony Luck's avatar Tony Luck Committed by Borislav Petkov (AMD)
Browse files

x86/resctrl: Prepare for different scope for control/monitor operations 



Resctrl assumes that control and monitor operations on a resource are
performed at the same scope.

Prepare for systems that use different scope (specifically Intel needs
to split the RDT_RESOURCE_L3 resource to use L3 scope for cache control
and NODE scope for cache occupancy and memory bandwidth monitoring).

Create separate domain lists for control and monitor operations.

Note that errors during initialization of either control or monitor
functions on a domain would previously result in that domain being
excluded from both control and monitor operations. Now the domains are
allocated independently it is no longer required to disable both control
and monitor operations if either fail.

Signed-off-by: default avatarTony Luck <tony.luck@intel.com>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: default avatarReinette Chatre <reinette.chatre@intel.com>
Tested-by: default avatarBabu Moger <babu.moger@amd.com>
Link: https://lore.kernel.org/r/20240628215619.76401-4-tony.luck@intel.com
parent c103d4d4

arch/x86/kernel/cpu/resctrl/core.c

+171 −53
Original line number Diff line number Diff line
@@ -60,7 +60,8 @@ static void mba_wrmsr_intel(struct msr_param *m); 
static void cat_wrmsr(struct msr_param *m);

static void mba_wrmsr_amd(struct msr_param *m);



#define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains)

#define ctrl_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.ctrl_domains)

#define mon_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.mon_domains)



struct rdt_hw_resource rdt_resources_all[] = {

	[RDT_RESOURCE_L3] =
@@ -68,8 +69,10 @@ struct rdt_hw_resource rdt_resources_all[] = { 
		.r_resctrl = {

			.rid			= RDT_RESOURCE_L3,

			.name			= "L3",

			.scope			= RESCTRL_L3_CACHE,

			.domains		= domain_init(RDT_RESOURCE_L3),

			.ctrl_scope		= RESCTRL_L3_CACHE,

			.mon_scope		= RESCTRL_L3_CACHE,

			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_L3),

			.mon_domains		= mon_domain_init(RDT_RESOURCE_L3),

			.parse_ctrlval		= parse_cbm,

			.format_str		= "%d=%0*x",

			.fflags			= RFTYPE_RES_CACHE,
@@ -82,8 +85,8 @@ struct rdt_hw_resource rdt_resources_all[] = { 
		.r_resctrl = {

			.rid			= RDT_RESOURCE_L2,

			.name			= "L2",

			.scope			= RESCTRL_L2_CACHE,

			.domains		= domain_init(RDT_RESOURCE_L2),

			.ctrl_scope		= RESCTRL_L2_CACHE,

			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_L2),

			.parse_ctrlval		= parse_cbm,

			.format_str		= "%d=%0*x",

			.fflags			= RFTYPE_RES_CACHE,
@@ -96,8 +99,8 @@ struct rdt_hw_resource rdt_resources_all[] = { 
		.r_resctrl = {

			.rid			= RDT_RESOURCE_MBA,

			.name			= "MB",

			.scope			= RESCTRL_L3_CACHE,

			.domains		= domain_init(RDT_RESOURCE_MBA),

			.ctrl_scope		= RESCTRL_L3_CACHE,

			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_MBA),

			.parse_ctrlval		= parse_bw,

			.format_str		= "%d=%*u",

			.fflags			= RFTYPE_RES_MB,
@@ -108,8 +111,8 @@ struct rdt_hw_resource rdt_resources_all[] = { 
		.r_resctrl = {

			.rid			= RDT_RESOURCE_SMBA,

			.name			= "SMBA",

			.scope			= RESCTRL_L3_CACHE,

			.domains		= domain_init(RDT_RESOURCE_SMBA),

			.ctrl_scope		= RESCTRL_L3_CACHE,

			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_SMBA),

			.parse_ctrlval		= parse_bw,

			.format_str		= "%d=%*u",

			.fflags			= RFTYPE_RES_MB,
@@ -349,13 +352,28 @@ static void cat_wrmsr(struct msr_param *m) 
		wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]);

}



struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r)

struct rdt_domain *get_ctrl_domain_from_cpu(int cpu, struct rdt_resource *r)

{

	struct rdt_domain *d;



	lockdep_assert_cpus_held();



	list_for_each_entry(d, &r->domains, hdr.list) {

	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {

		/* Find the domain that contains this CPU */

		if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))

			return d;

	}



	return NULL;

}



struct rdt_domain *get_mon_domain_from_cpu(int cpu, struct rdt_resource *r)

{

	struct rdt_domain *d;



	lockdep_assert_cpus_held();



	list_for_each_entry(d, &r->mon_domains, hdr.list) {

		/* Find the domain that contains this CPU */

		if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))

			return d;
@@ -379,26 +397,26 @@ void rdt_ctrl_update(void *arg) 
}



/*

 * rdt_find_domain - Find a domain in a resource that matches input resource id

 * rdt_find_domain - Search for a domain id in a resource domain list.

 *

 * Search resource r's domain list to find the resource id. If the resource

 * id is found in a domain, return the domain. Otherwise, if requested by

 * caller, return the first domain whose id is bigger than the input id.

 * The domain list is sorted by id in ascending order.

 * Search the domain list to find the domain id. If the domain id is

 * found, return the domain. NULL otherwise.  If the domain id is not

 * found (and NULL returned) then the first domain with id bigger than

 * the input id can be returned to the caller via @pos.

 */

struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,

struct rdt_domain_hdr *rdt_find_domain(struct list_head *h, int id,

				       struct list_head **pos)

{

	struct rdt_domain *d;

	struct rdt_domain_hdr *d;

	struct list_head *l;



	list_for_each(l, &r->domains) {

		d = list_entry(l, struct rdt_domain, hdr.list);

	list_for_each(l, h) {

		d = list_entry(l, struct rdt_domain_hdr, list);

		/* When id is found, return its domain. */

		if (id == d->hdr.id)

		if (id == d->id)

			return d;

		/* Stop searching when finding id's position in sorted list. */

		if (id < d->hdr.id)

		if (id < d->id)

			break;

	}
@@ -494,38 +512,29 @@ static int get_domain_id_from_scope(int cpu, enum resctrl_scope scope) 
	return -EINVAL;

}



/*

 * domain_add_cpu - Add a cpu to a resource's domain list.

 *

 * If an existing domain in the resource r's domain list matches the cpu's

 * resource id, add the cpu in the domain.

 *

 * Otherwise, a new domain is allocated and inserted into the right position

 * in the domain list sorted by id in ascending order.

 *

 * The order in the domain list is visible to users when we print entries

 * in the schemata file and schemata input is validated to have the same order

 * as this list.

 */

static void domain_add_cpu(int cpu, struct rdt_resource *r)

static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r)

{

	int id = get_domain_id_from_scope(cpu, r->scope);

	int id = get_domain_id_from_scope(cpu, r->ctrl_scope);

	struct list_head *add_pos = NULL;

	struct rdt_hw_domain *hw_dom;

	struct rdt_domain_hdr *hdr;

	struct rdt_domain *d;

	int err;



	lockdep_assert_held(&domain_list_lock);



	if (id < 0) {

		pr_warn_once("Can't find domain id for CPU:%d scope:%d for resource %s\n",

			     cpu, r->scope, r->name);

		pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n",

			     cpu, r->ctrl_scope, r->name);

		return;

	}



	d = rdt_find_domain(r, id, &add_pos);

	hdr = rdt_find_domain(&r->ctrl_domains, id, &add_pos);

	if (hdr) {

		if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN))

			return;

		d = container_of(hdr, struct rdt_domain, hdr);



	if (d) {

		cpumask_set_cpu(cpu, &d->hdr.cpu_mask);

		if (r->cache.arch_has_per_cpu_cfg)

			rdt_domain_reconfigure_cdp(r);
@@ -538,23 +547,70 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r) 


	d = &hw_dom->d_resctrl;

	d->hdr.id = id;

	d->hdr.type = RESCTRL_CTRL_DOMAIN;

	cpumask_set_cpu(cpu, &d->hdr.cpu_mask);



	rdt_domain_reconfigure_cdp(r);



	if (r->alloc_capable && domain_setup_ctrlval(r, d)) {

	if (domain_setup_ctrlval(r, d)) {

		domain_free(hw_dom);

		return;

	}



	if (r->mon_capable && arch_domain_mbm_alloc(r->num_rmid, hw_dom)) {

	list_add_tail_rcu(&d->hdr.list, add_pos);



	err = resctrl_online_ctrl_domain(r, d);

	if (err) {

		list_del_rcu(&d->hdr.list);

		synchronize_rcu();

		domain_free(hw_dom);

	}

}



static void domain_add_cpu_mon(int cpu, struct rdt_resource *r)

{

	int id = get_domain_id_from_scope(cpu, r->mon_scope);

	struct list_head *add_pos = NULL;

	struct rdt_hw_domain *hw_dom;

	struct rdt_domain_hdr *hdr;

	struct rdt_domain *d;

	int err;



	lockdep_assert_held(&domain_list_lock);



	if (id < 0) {

		pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n",

			     cpu, r->mon_scope, r->name);

		return;

	}



	hdr = rdt_find_domain(&r->mon_domains, id, &add_pos);

	if (hdr) {

		if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN))

			return;

		d = container_of(hdr, struct rdt_domain, hdr);



		cpumask_set_cpu(cpu, &d->hdr.cpu_mask);

		return;

	}



	hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu));

	if (!hw_dom)

		return;



	d = &hw_dom->d_resctrl;

	d->hdr.id = id;

	d->hdr.type = RESCTRL_MON_DOMAIN;

	cpumask_set_cpu(cpu, &d->hdr.cpu_mask);



	if (arch_domain_mbm_alloc(r->num_rmid, hw_dom)) {

		domain_free(hw_dom);

		return;

	}



	list_add_tail_rcu(&d->hdr.list, add_pos);



	err = resctrl_online_domain(r, d);

	err = resctrl_online_mon_domain(r, d);

	if (err) {

		list_del_rcu(&d->hdr.list);

		synchronize_rcu();
@@ -562,30 +618,45 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r) 
	}

}



static void domain_remove_cpu(int cpu, struct rdt_resource *r)

static void domain_add_cpu(int cpu, struct rdt_resource *r)

{

	if (r->alloc_capable)

		domain_add_cpu_ctrl(cpu, r);

	if (r->mon_capable)

		domain_add_cpu_mon(cpu, r);

}



static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r)

{

	int id = get_domain_id_from_scope(cpu, r->scope);

	int id = get_domain_id_from_scope(cpu, r->ctrl_scope);

	struct rdt_hw_domain *hw_dom;

	struct rdt_domain_hdr *hdr;

	struct rdt_domain *d;



	lockdep_assert_held(&domain_list_lock);



	if (id < 0) {

		pr_warn_once("Can't find domain id for CPU:%d scope:%d for resource %s\n",

			     cpu, r->scope, r->name);

		pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n",

			     cpu, r->ctrl_scope, r->name);

		return;

	}



	d = rdt_find_domain(r, id, NULL);

	if (!d) {

		pr_warn("Couldn't find domain with id=%d for CPU %d\n", id, cpu);

	hdr = rdt_find_domain(&r->ctrl_domains, id, NULL);

	if (!hdr) {

		pr_warn("Can't find control domain for id=%d for CPU %d for resource %s\n",

			id, cpu, r->name);

		return;

	}



	if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN))

		return;



	d = container_of(hdr, struct rdt_domain, hdr);

	hw_dom = resctrl_to_arch_dom(d);



	cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);

	if (cpumask_empty(&d->hdr.cpu_mask)) {

		resctrl_offline_domain(r, d);

		resctrl_offline_ctrl_domain(r, d);

		list_del_rcu(&d->hdr.list);

		synchronize_rcu();
@@ -601,6 +672,53 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) 
	}

}



static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r)

{

	int id = get_domain_id_from_scope(cpu, r->mon_scope);

	struct rdt_hw_domain *hw_dom;

	struct rdt_domain_hdr *hdr;

	struct rdt_domain *d;



	lockdep_assert_held(&domain_list_lock);



	if (id < 0) {

		pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n",

			     cpu, r->mon_scope, r->name);

		return;

	}



	hdr = rdt_find_domain(&r->mon_domains, id, NULL);

	if (!hdr) {

		pr_warn("Can't find monitor domain for id=%d for CPU %d for resource %s\n",

			id, cpu, r->name);

		return;

	}



	if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN))

		return;



	d = container_of(hdr, struct rdt_domain, hdr);

	hw_dom = resctrl_to_arch_dom(d);



	cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);

	if (cpumask_empty(&d->hdr.cpu_mask)) {

		resctrl_offline_mon_domain(r, d);

		list_del_rcu(&d->hdr.list);

		synchronize_rcu();

		domain_free(hw_dom);



		return;

	}

}



static void domain_remove_cpu(int cpu, struct rdt_resource *r)

{

	if (r->alloc_capable)

		domain_remove_cpu_ctrl(cpu, r);

	if (r->mon_capable)

		domain_remove_cpu_mon(cpu, r);

}



static void clear_closid_rmid(int cpu)

{

	struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);

arch/x86/kernel/cpu/resctrl/ctrlmondata.c

+7 −5
Original line number Diff line number Diff line
@@ -231,7 +231,7 @@ static int parse_line(char *line, struct resctrl_schema *s, 
		return -EINVAL;

	}

	dom = strim(dom);

	list_for_each_entry(d, &r->domains, hdr.list) {

	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {

		if (d->hdr.id == dom_id) {

			data.buf = dom;

			data.rdtgrp = rdtgrp;
@@ -306,7 +306,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) 
	/* Walking r->domains, ensure it can't race with cpuhp */

	lockdep_assert_cpus_held();



	list_for_each_entry(d, &r->domains, hdr.list) {

	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {

		hw_dom = resctrl_to_arch_dom(d);

		msr_param.res = NULL;

		for (t = 0; t < CDP_NUM_TYPES; t++) {
@@ -450,7 +450,7 @@ static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int clo 
	lockdep_assert_cpus_held();



	seq_printf(s, "%*s:", max_name_width, schema->name);

	list_for_each_entry(dom, &r->domains, hdr.list) {

	list_for_each_entry(dom, &r->ctrl_domains, hdr.list) {

		if (sep)

			seq_puts(s, ";");
@@ -556,6 +556,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, 
int rdtgroup_mondata_show(struct seq_file *m, void *arg)

{

	struct kernfs_open_file *of = m->private;

	struct rdt_domain_hdr *hdr;

	u32 resid, evtid, domid;

	struct rdtgroup *rdtgrp;

	struct rdt_resource *r;
@@ -576,11 +577,12 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) 
	evtid = md.u.evtid;



	r = &rdt_resources_all[resid].r_resctrl;

	d = rdt_find_domain(r, domid, NULL);

	if (!d) {

	hdr = rdt_find_domain(&r->mon_domains, domid, NULL);

	if (!hdr || WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) {

		ret = -ENOENT;

		goto out;

	}

	d = container_of(hdr, struct rdt_domain, hdr);



	mon_event_read(&rr, r, d, rdtgrp, evtid, false);

arch/x86/kernel/cpu/resctrl/internal.h

+4 −3
Original line number Diff line number Diff line
@@ -558,7 +558,7 @@ void rdtgroup_kn_unlock(struct kernfs_node *kn); 
int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name);

int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name,

			     umode_t mask);

struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,

struct rdt_domain_hdr *rdt_find_domain(struct list_head *h, int id,

				       struct list_head **pos);

ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,

				char *buf, size_t nbytes, loff_t off);
@@ -578,7 +578,8 @@ int rdt_pseudo_lock_init(void); 
void rdt_pseudo_lock_release(void);

int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp);

void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp);

struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);

struct rdt_domain *get_ctrl_domain_from_cpu(int cpu, struct rdt_resource *r);

struct rdt_domain *get_mon_domain_from_cpu(int cpu, struct rdt_resource *r);

int closids_supported(void);

void closid_free(int closid);

int alloc_rmid(u32 closid);

arch/x86/kernel/cpu/resctrl/monitor.c

+2 −2
Original line number Diff line number Diff line
@@ -490,7 +490,7 @@ static void add_rmid_to_limbo(struct rmid_entry *entry) 
	idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid);



	entry->busy = 0;

	list_for_each_entry(d, &r->domains, hdr.list) {

	list_for_each_entry(d, &r->mon_domains, hdr.list) {

		/*

		 * For the first limbo RMID in the domain,

		 * setup up the limbo worker.
@@ -687,7 +687,7 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) 
	idx = resctrl_arch_rmid_idx_encode(closid, rmid);

	pmbm_data = &dom_mbm->mbm_local[idx];



	dom_mba = get_domain_from_cpu(smp_processor_id(), r_mba);

	dom_mba = get_ctrl_domain_from_cpu(smp_processor_id(), r_mba);

	if (!dom_mba) {

		pr_warn_once("Failure to get domain for MBA update\n");

		return;

arch/x86/kernel/cpu/resctrl/pseudo_lock.c

+2 −2
Original line number Diff line number Diff line
@@ -292,7 +292,7 @@ static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) 
 */

static int pseudo_lock_region_init(struct pseudo_lock_region *plr)

{

	enum resctrl_scope scope = plr->s->res->scope;

	enum resctrl_scope scope = plr->s->res->ctrl_scope;

	struct cacheinfo *ci;

	int ret;
@@ -854,7 +854,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) 
	 * associated with them.

	 */

	for_each_alloc_capable_rdt_resource(r) {

		list_for_each_entry(d_i, &r->domains, hdr.list) {

		list_for_each_entry(d_i, &r->ctrl_domains, hdr.list) {

			if (d_i->plr)

				cpumask_or(cpu_with_psl, cpu_with_psl,

					   &d_i->hdr.cpu_mask);