Commit d1604112 authored by Vasant Hegde's avatar Vasant Hegde Committed by Joerg Roedel
Browse files

iommu/amd: xarray to track protection_domain->iommu list 



Use xarray to track IOMMU attached to protection domain instead of
static array of MAX_IOMMUS. Also add lockdep assertion.

Signed-off-by: default avatarVasant Hegde <vasant.hegde@amd.com>
Reviewed-by: default avatarJoerg Roedel <jroedel@suse.de>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20241030063556.6104-5-vasant.hegde@amd.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 743a4bae

drivers/iommu/amd/amd_iommu_types.h

+7 −1
Original line number Diff line number Diff line
@@ -565,6 +565,12 @@ struct pdom_dev_data { 
	struct list_head list;

};



/* Keeps track of the IOMMUs attached to protection domain */

struct pdom_iommu_info {

	struct amd_iommu *iommu; /* IOMMUs attach to protection domain */

	u32 refcnt;	/* Count of attached dev/pasid per domain/IOMMU */

};



/*

 * This structure contains generic data for  IOMMU protection domains

 * independent of their use.
@@ -578,7 +584,7 @@ struct protection_domain { 
	u16 id;			/* the domain id written to the device table */

	enum protection_domain_mode pd_mode; /* Track page table type */

	bool dirty_tracking;	/* dirty tracking is enabled in the domain */

	unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */

	struct xarray iommu_array;	/* per-IOMMU reference count */



	struct mmu_notifier mn;	/* mmu notifier for the SVA domain */

	struct list_head dev_data_list; /* List of pdom_dev_data */

drivers/iommu/amd/iommu.c

+70 −19
Original line number Diff line number Diff line
@@ -1257,18 +1257,17 @@ static int iommu_completion_wait(struct amd_iommu *iommu) 


static void domain_flush_complete(struct protection_domain *domain)

{

	int i;

	struct pdom_iommu_info *pdom_iommu_info;

	unsigned long i;



	for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {

		if (domain && !domain->dev_iommu[i])

			continue;

	lockdep_assert_held(&domain->lock);



	/*

	 * Devices of this domain are behind this IOMMU

	 * We need to wait for completion of all commands.

	 */

		iommu_completion_wait(amd_iommus[i]);

	}

	 xa_for_each(&domain->iommu_array, i, pdom_iommu_info)

		iommu_completion_wait(pdom_iommu_info->iommu);

}



static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
@@ -1450,21 +1449,22 @@ static int domain_flush_pages_v2(struct protection_domain *pdom, 
static int domain_flush_pages_v1(struct protection_domain *pdom,

				 u64 address, size_t size)

{

	struct pdom_iommu_info *pdom_iommu_info;

	struct iommu_cmd cmd;

	int ret = 0, i;

	int ret = 0;

	unsigned long i;



	lockdep_assert_held(&pdom->lock);



	build_inv_iommu_pages(&cmd, address, size,

			      pdom->id, IOMMU_NO_PASID, false);



	for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {

		if (!pdom->dev_iommu[i])

			continue;



	xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) {

		/*

		 * Devices of this domain are behind this IOMMU

		 * We need a TLB flush

		 */

		ret |= iommu_queue_command(amd_iommus[i], &cmd);

		ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);

	}



	return ret;
@@ -1503,6 +1503,8 @@ static void __domain_flush_pages(struct protection_domain *domain, 
void amd_iommu_domain_flush_pages(struct protection_domain *domain,

				  u64 address, size_t size)

{

	lockdep_assert_held(&domain->lock);



	if (likely(!amd_iommu_np_cache)) {

		__domain_flush_pages(domain, address, size);
@@ -2014,6 +2016,50 @@ static void destroy_gcr3_table(struct iommu_dev_data *dev_data, 
	free_gcr3_table(gcr3_info);

}



static int pdom_attach_iommu(struct amd_iommu *iommu,

			     struct protection_domain *pdom)

{

	struct pdom_iommu_info *pdom_iommu_info, *curr;



	pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);

	if (pdom_iommu_info) {

		pdom_iommu_info->refcnt++;

		return 0;

	}



	pdom_iommu_info = kzalloc(sizeof(*pdom_iommu_info), GFP_ATOMIC);

	if (!pdom_iommu_info)

		return -ENOMEM;



	pdom_iommu_info->iommu = iommu;

	pdom_iommu_info->refcnt = 1;



	curr = xa_cmpxchg(&pdom->iommu_array, iommu->index,

			  NULL, pdom_iommu_info, GFP_ATOMIC);

	if (curr) {

		kfree(pdom_iommu_info);

		return -ENOSPC;

	}



	return 0;

}



static void pdom_detach_iommu(struct amd_iommu *iommu,

			      struct protection_domain *pdom)

{

	struct pdom_iommu_info *pdom_iommu_info;



	pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);

	if (!pdom_iommu_info)

		return;



	pdom_iommu_info->refcnt--;

	if (pdom_iommu_info->refcnt == 0) {

		xa_erase(&pdom->iommu_array, iommu->index);

		kfree(pdom_iommu_info);

	}

}



static int do_attach(struct iommu_dev_data *dev_data,

		     struct protection_domain *domain)

{
@@ -2030,14 +2076,18 @@ static int do_attach(struct iommu_dev_data *dev_data, 
		cfg->amd.nid = dev_to_node(dev_data->dev);



	/* Do reference counting */

	domain->dev_iommu[iommu->index] += 1;

	ret = pdom_attach_iommu(iommu, domain);

	if (ret)

		return ret;



	/* Setup GCR3 table */

	if (pdom_is_sva_capable(domain)) {

		ret = init_gcr3_table(dev_data, domain);

		if (ret)

		if (ret) {

			pdom_detach_iommu(iommu, domain);

			return ret;

		}

	}



	return ret;

}
@@ -2062,7 +2112,7 @@ static void do_detach(struct iommu_dev_data *dev_data) 
	list_del(&dev_data->list);



	/* decrease reference counters - needs to happen after the flushes */

	domain->dev_iommu[iommu->index] -= 1;

	pdom_detach_iommu(iommu, domain);

}



/*
@@ -2258,6 +2308,7 @@ static void protection_domain_init(struct protection_domain *domain, int nid) 
	spin_lock_init(&domain->lock);

	INIT_LIST_HEAD(&domain->dev_list);

	INIT_LIST_HEAD(&domain->dev_data_list);

	xa_init(&domain->iommu_array);

	domain->iop.pgtbl.cfg.amd.nid = nid;

}