Commit 06792d06 authored by Lu Baolu's avatar Lu Baolu Committed by Joerg Roedel
Browse files

iommu/vt-d: Cleanup use of iommu_flush_iotlb_psi() 



Use cache_tag_flush_range() in switch_to_super_page() to invalidate the
necessary caches when switching mappings from normal to super pages. The
iommu_flush_iotlb_psi() call in intel_iommu_memory_notifier() is
unnecessary since there should be no cache invalidation for the identity
domain.

Clean up iommu_flush_iotlb_psi() after the last call site is removed.

Signed-off-by: default avatarLu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: default avatarKevin Tian <kevin.tian@intel.com>
Link: https://lore.kernel.org/r/20240416080656.60968-8-baolu.lu@linux.intel.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 129dab6e

drivers/iommu/intel/iommu.c

+2 −169
Original line number Diff line number Diff line
@@ -1390,157 +1390,6 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, 
	quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep);

}



static void iommu_flush_dev_iotlb(struct dmar_domain *domain,

				  u64 addr, unsigned mask)

{

	struct dev_pasid_info *dev_pasid;

	struct device_domain_info *info;

	unsigned long flags;



	if (!domain->has_iotlb_device)

		return;



	spin_lock_irqsave(&domain->lock, flags);

	list_for_each_entry(info, &domain->devices, link)

		__iommu_flush_dev_iotlb(info, addr, mask);



	list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) {

		info = dev_iommu_priv_get(dev_pasid->dev);



		if (!info->ats_enabled)

			continue;



		qi_flush_dev_iotlb_pasid(info->iommu,

					 PCI_DEVID(info->bus, info->devfn),

					 info->pfsid, dev_pasid->pasid,

					 info->ats_qdep, addr,

					 mask);

	}

	spin_unlock_irqrestore(&domain->lock, flags);

}



static void domain_flush_pasid_iotlb(struct intel_iommu *iommu,

				     struct dmar_domain *domain, u64 addr,

				     unsigned long npages, bool ih)

{

	u16 did = domain_id_iommu(domain, iommu);

	struct dev_pasid_info *dev_pasid;

	unsigned long flags;



	spin_lock_irqsave(&domain->lock, flags);

	list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain)

		qi_flush_piotlb(iommu, did, dev_pasid->pasid, addr, npages, ih);



	if (!list_empty(&domain->devices))

		qi_flush_piotlb(iommu, did, IOMMU_NO_PASID, addr, npages, ih);

	spin_unlock_irqrestore(&domain->lock, flags);

}



static void __iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,

				    unsigned long pfn, unsigned int pages,

				    int ih)

{

	unsigned int aligned_pages = __roundup_pow_of_two(pages);

	unsigned long bitmask = aligned_pages - 1;

	unsigned int mask = ilog2(aligned_pages);

	u64 addr = (u64)pfn << VTD_PAGE_SHIFT;



	/*

	 * PSI masks the low order bits of the base address. If the

	 * address isn't aligned to the mask, then compute a mask value

	 * needed to ensure the target range is flushed.

	 */

	if (unlikely(bitmask & pfn)) {

		unsigned long end_pfn = pfn + pages - 1, shared_bits;



		/*

		 * Since end_pfn <= pfn + bitmask, the only way bits

		 * higher than bitmask can differ in pfn and end_pfn is

		 * by carrying. This means after masking out bitmask,

		 * high bits starting with the first set bit in

		 * shared_bits are all equal in both pfn and end_pfn.

		 */

		shared_bits = ~(pfn ^ end_pfn) & ~bitmask;

		mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG;

	}



	/*

	 * Fallback to domain selective flush if no PSI support or

	 * the size is too big.

	 */

	if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))

		iommu->flush.flush_iotlb(iommu, did, 0, 0,

					 DMA_TLB_DSI_FLUSH);

	else

		iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,

					 DMA_TLB_PSI_FLUSH);

}



static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,

				  struct dmar_domain *domain,

				  unsigned long pfn, unsigned int pages,

				  int ih, int map)

{

	unsigned int aligned_pages = __roundup_pow_of_two(pages);

	unsigned int mask = ilog2(aligned_pages);

	uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;

	u16 did = domain_id_iommu(domain, iommu);



	if (WARN_ON(!pages))

		return;



	if (ih)

		ih = 1 << 6;



	if (domain->use_first_level)

		domain_flush_pasid_iotlb(iommu, domain, addr, pages, ih);

	else

		__iommu_flush_iotlb_psi(iommu, did, pfn, pages, ih);



	if (!map)

		iommu_flush_dev_iotlb(domain, addr, mask);

}



/*

 * Flush the relevant caches in nested translation if the domain

 * also serves as a parent

 */

static void parent_domain_flush(struct dmar_domain *domain,

				unsigned long pfn,

				unsigned long pages, int ih)

{

	struct dmar_domain *s1_domain;



	spin_lock(&domain->s1_lock);

	list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {

		struct device_domain_info *device_info;

		struct iommu_domain_info *info;

		unsigned long flags;

		unsigned long i;



		xa_for_each(&s1_domain->iommu_array, i, info)

			__iommu_flush_iotlb_psi(info->iommu, info->did,

						pfn, pages, ih);



		if (!s1_domain->has_iotlb_device)

			continue;



		spin_lock_irqsave(&s1_domain->lock, flags);

		list_for_each_entry(device_info, &s1_domain->devices, link)

			/*

			 * Address translation cache in device side caches the

			 * result of nested translation. There is no easy way

			 * to identify the exact set of nested translations

			 * affected by a change in S2. So just flush the entire

			 * device cache.

			 */

			__iommu_flush_dev_iotlb(device_info, 0,

						MAX_AGAW_PFN_WIDTH);

		spin_unlock_irqrestore(&s1_domain->lock, flags);

	}

	spin_unlock(&domain->s1_lock);

}



static void intel_flush_iotlb_all(struct iommu_domain *domain)

{

	cache_tag_flush_all(to_dmar_domain(domain));
@@ -1991,9 +1840,7 @@ static void switch_to_super_page(struct dmar_domain *domain, 
				 unsigned long end_pfn, int level)

{

	unsigned long lvl_pages = lvl_to_nr_pages(level);

	struct iommu_domain_info *info;

	struct dma_pte *pte = NULL;

	unsigned long i;



	while (start_pfn <= end_pfn) {

		if (!pte)
@@ -2005,13 +1852,8 @@ static void switch_to_super_page(struct dmar_domain *domain, 
					       start_pfn + lvl_pages - 1,

					       level + 1);



			xa_for_each(&domain->iommu_array, i, info)

				iommu_flush_iotlb_psi(info->iommu, domain,

						      start_pfn, lvl_pages,

						      0, 0);

			if (domain->nested_parent)

				parent_domain_flush(domain, start_pfn,

						    lvl_pages, 0);

			cache_tag_flush_range(domain, start_pfn << VTD_PAGE_SHIFT,

					      end_pfn << VTD_PAGE_SHIFT, 0);

		}



		pte++;
@@ -3381,18 +3223,9 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, 
	case MEM_OFFLINE:

	case MEM_CANCEL_ONLINE:

		{

			struct dmar_drhd_unit *drhd;

			struct intel_iommu *iommu;

			LIST_HEAD(freelist);



			domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);



			rcu_read_lock();

			for_each_active_iommu(iommu, drhd)

				iommu_flush_iotlb_psi(iommu, si_domain,

					start_vpfn, mhp->nr_pages,

					list_empty(&freelist), 0);

			rcu_read_unlock();

			put_pages_list(&freelist);

		}

		break;