Commit e8e1aac3 authored by Suravee Suthikulpanit's avatar Suravee Suthikulpanit Committed by Joerg Roedel
Browse files

iommu/amd: Refactor helper function for setting / clearing GCR3 



Refactor GCR3 helper functions in preparation to use per device
GCR3 table.
  * Add new function update_gcr3 to update per device GCR3 table

  * Remove per domain default GCR3 setup during v2 page table allocation.
    Subsequent patch will add support to setup default gcr3 while
    attaching device to domain.

  * Remove amd_iommu_domain_update() from V2 page table path as device
    detach path will take care of updating the domain.

  * Consolidate GCR3 table related code in one place so that its easy
    to maintain.

  * Rename functions to reflect its usage.

Signed-off-by: default avatarSuravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Co-developed-by: default avatarVasant Hegde <vasant.hegde@amd.com>
Signed-off-by: default avatarVasant Hegde <vasant.hegde@amd.com>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20240205115615.6053-11-vasant.hegde@amd.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent bf8aff29

drivers/iommu/amd/amd_iommu.h

+5 −3
Original line number Diff line number Diff line
@@ -44,6 +44,11 @@ bool amd_iommu_v2_supported(void); 
int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev);

void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev);



/* GCR3 setup */

int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data,

		       ioasid_t pasid, unsigned long gcr3);

int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid);



int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address);

/*

 * This function flushes all internal caches of
@@ -61,9 +66,6 @@ void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, 
				   ioasid_t pasid);



int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid);

int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,

			      unsigned long cr3);

int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid);



#ifdef CONFIG_IRQ_REMAP

int amd_iommu_create_irq_domain(struct amd_iommu *iommu);

drivers/iommu/amd/io_pgtable_v2.c

+2 −19
Original line number Diff line number Diff line
@@ -350,38 +350,26 @@ static const struct iommu_flush_ops v2_flush_ops = { 


static void v2_free_pgtable(struct io_pgtable *iop)

{

	struct protection_domain *pdom;

	struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop);



	pdom = container_of(pgtable, struct protection_domain, iop);

	if (!(pdom->flags & PD_IOMMUV2_MASK))

	if (!pgtable || !pgtable->pgd)

		return;



	/* Clear gcr3 entry */

	amd_iommu_domain_clear_gcr3(&pdom->domain, 0);



	/* Make changes visible to IOMMUs */

	amd_iommu_domain_update(pdom);



	/* Free page table */

	free_pgtable(pgtable->pgd, get_pgtable_level());

	pgtable->pgd = NULL;

}



static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)

{

	struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);

	struct protection_domain *pdom = (struct protection_domain *)cookie;

	int ret;

	int ias = IOMMU_IN_ADDR_BIT_SIZE;



	pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC);

	if (!pgtable->pgd)

		return NULL;



	ret = amd_iommu_domain_set_gcr3(&pdom->domain, 0, iommu_virt_to_phys(pgtable->pgd));

	if (ret)

		goto err_free_pgd;



	if (get_pgtable_level() == PAGE_MODE_5_LEVEL)

		ias = 57;
@@ -395,11 +383,6 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo 
	cfg->tlb           = &v2_flush_ops;



	return &pgtable->iop;



err_free_pgd:

	free_pgtable_page(pgtable->pgd);



	return NULL;

}



struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = {

drivers/iommu/amd/iommu.c

+54 −61
Original line number Diff line number Diff line
@@ -1714,10 +1714,13 @@ static int setup_gcr3_table(struct protection_domain *domain, int pasids) 
	return 0;

}



static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)

static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info,

			   ioasid_t pasid, bool alloc)

{

	int index;

	u64 *pte;

	u64 *root = gcr3_info->gcr3_tbl;

	int level = gcr3_info->glx;



	while (true) {
@@ -1746,6 +1749,56 @@ static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) 
	return pte;

}



static int update_gcr3(struct iommu_dev_data *dev_data,

		       ioasid_t pasid, unsigned long gcr3, bool set)

{

	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;

	u64 *pte;



	pte = __get_gcr3_pte(gcr3_info, pasid, true);

	if (pte == NULL)

		return -ENOMEM;



	if (set)

		*pte = (gcr3 & PAGE_MASK) | GCR3_VALID;

	else

		*pte = 0;



	amd_iommu_dev_flush_pasid_all(dev_data, pasid);

	return 0;

}



int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid,

		       unsigned long gcr3)

{

	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;

	int ret;



	iommu_group_mutex_assert(dev_data->dev);



	ret = update_gcr3(dev_data, pasid, gcr3, true);

	if (ret)

		return ret;



	gcr3_info->pasid_cnt++;

	return ret;

}



int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)

{

	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;

	int ret;



	iommu_group_mutex_assert(dev_data->dev);



	ret = update_gcr3(dev_data, pasid, 0, false);

	if (ret)

		return ret;



	gcr3_info->pasid_cnt--;

	return ret;

}



static void set_dte_entry(struct amd_iommu *iommu,

			  struct iommu_dev_data *dev_data)

{
@@ -2769,66 +2822,6 @@ int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid) 
	return ret;

}



static int __set_gcr3(struct protection_domain *domain, u32 pasid,

		      unsigned long cr3)

{

	u64 *pte;



	if (domain->iop.mode != PAGE_MODE_NONE)

		return -EINVAL;



	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);

	if (pte == NULL)

		return -ENOMEM;



	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;



	return __amd_iommu_flush_tlb(domain, pasid);

}



static int __clear_gcr3(struct protection_domain *domain, u32 pasid)

{

	u64 *pte;



	if (domain->iop.mode != PAGE_MODE_NONE)

		return -EINVAL;



	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);

	if (pte == NULL)

		return 0;



	*pte = 0;



	return __amd_iommu_flush_tlb(domain, pasid);

}



int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,

			      unsigned long cr3)

{

	struct protection_domain *domain = to_pdomain(dom);

	unsigned long flags;

	int ret;



	spin_lock_irqsave(&domain->lock, flags);

	ret = __set_gcr3(domain, pasid, cr3);

	spin_unlock_irqrestore(&domain->lock, flags);



	return ret;

}



int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)

{

	struct protection_domain *domain = to_pdomain(dom);

	unsigned long flags;

	int ret;



	spin_lock_irqsave(&domain->lock, flags);

	ret = __clear_gcr3(domain, pasid);

	spin_unlock_irqrestore(&domain->lock, flags);



	return ret;

}



int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,

			   int status, int tag)

{