Commit f701c9f3 authored by Tina Zhang's avatar Tina Zhang Committed by Joerg Roedel
Browse files

iommu/vt-d: Factor out invalidation descriptor composition 



Separate the logic for constructing IOTLB and device TLB invalidation
descriptors from the qi_flush interfaces. New helpers, qi_desc(), are
introduced to encapsulate this common functionality.

Moving descriptor composition code to new helpers enables its reuse in
the upcoming qi_batch interfaces.

No functional changes are intended.

Signed-off-by: default avatarTina Zhang <tina.zhang@intel.com>
Link: https://lore.kernel.org/r/20240815065221.50328-2-tina.zhang@intel.com


Signed-off-by: default avatarLu Baolu <baolu.lu@linux.intel.com>
Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 1f5e307c

drivers/iommu/intel/dmar.c

+6 −87
Original line number Diff line number Diff line
@@ -1526,24 +1526,9 @@ void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm, 
void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

		    unsigned int size_order, u64 type)

{

	u8 dw = 0, dr = 0;



	struct qi_desc desc;

	int ih = 0;



	if (cap_write_drain(iommu->cap))

		dw = 1;



	if (cap_read_drain(iommu->cap))

		dr = 1;



	desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)

		| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;

	desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)

		| QI_IOTLB_AM(size_order);

	desc.qw2 = 0;

	desc.qw3 = 0;



	qi_desc_iotlb(iommu, did, addr, size_order, type, &desc);

	qi_submit_sync(iommu, &desc, 1, 0);

}
@@ -1561,20 +1546,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, 
	if (!(iommu->gcmd & DMA_GCMD_TE))

		return;



	if (mask) {

		addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;

		desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;

	} else

		desc.qw1 = QI_DEV_IOTLB_ADDR(addr);



	if (qdep >= QI_DEV_IOTLB_MAX_INVS)

		qdep = 0;



	desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |

		   QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);

	desc.qw2 = 0;

	desc.qw3 = 0;



	qi_desc_dev_iotlb(sid, pfsid, qdep, addr, mask, &desc);

	qi_submit_sync(iommu, &desc, 1, 0);

}
@@ -1594,28 +1566,7 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, 
		return;

	}



	if (npages == -1) {

		desc.qw0 = QI_EIOTLB_PASID(pasid) |

				QI_EIOTLB_DID(did) |

				QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |

				QI_EIOTLB_TYPE;

		desc.qw1 = 0;

	} else {

		int mask = ilog2(__roundup_pow_of_two(npages));

		unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));



		if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))

			addr = ALIGN_DOWN(addr, align);



		desc.qw0 = QI_EIOTLB_PASID(pasid) |

				QI_EIOTLB_DID(did) |

				QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |

				QI_EIOTLB_TYPE;

		desc.qw1 = QI_EIOTLB_ADDR(addr) |

				QI_EIOTLB_IH(ih) |

				QI_EIOTLB_AM(mask);

	}



	qi_desc_piotlb(did, pasid, addr, npages, ih, &desc);

	qi_submit_sync(iommu, &desc, 1, 0);

}
@@ -1623,7 +1574,6 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, 
void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,

			      u32 pasid,  u16 qdep, u64 addr, unsigned int size_order)

{

	unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);

	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};



	/*
@@ -1635,40 +1585,9 @@ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, 
	if (!(iommu->gcmd & DMA_GCMD_TE))

		return;



	desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |

		QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |

		QI_DEV_IOTLB_PFSID(pfsid);



	/*

	 * If S bit is 0, we only flush a single page. If S bit is set,

	 * The least significant zero bit indicates the invalidation address

	 * range. VT-d spec 6.5.2.6.

	 * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.

	 * size order = 0 is PAGE_SIZE 4KB

	 * Max Invs Pending (MIP) is set to 0 for now until we have DIT in

	 * ECAP.

	 */

	if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))

		pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",

				    addr, size_order);



	/* Take page address */

	desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);



	if (size_order) {

		/*

		 * Existing 0s in address below size_order may be the least

		 * significant bit, we must set them to 1s to avoid having

		 * smaller size than desired.

		 */

		desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,

					VTD_PAGE_SHIFT);

		/* Clear size_order bit to indicate size */

		desc.qw1 &= ~mask;

		/* Set the S bit to indicate flushing more than 1 page */

		desc.qw1 |= QI_DEV_EIOTLB_SIZE;

	}



	qi_desc_dev_iotlb_pasid(sid, pfsid, pasid,

				qdep, addr, size_order,

				&desc);

	qi_submit_sync(iommu, &desc, 1, 0);

}

drivers/iommu/intel/iommu.h

+109 −0
Original line number Diff line number Diff line
@@ -1066,6 +1066,115 @@ static inline unsigned long nrpages_to_size(unsigned long npages) 
	return npages << VTD_PAGE_SHIFT;

}



static inline void qi_desc_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

				 unsigned int size_order, u64 type,

				 struct qi_desc *desc)

{

	u8 dw = 0, dr = 0;

	int ih = 0;



	if (cap_write_drain(iommu->cap))

		dw = 1;



	if (cap_read_drain(iommu->cap))

		dr = 1;



	desc->qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)

		| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;

	desc->qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)

		| QI_IOTLB_AM(size_order);

	desc->qw2 = 0;

	desc->qw3 = 0;

}



static inline void qi_desc_dev_iotlb(u16 sid, u16 pfsid, u16 qdep, u64 addr,

				     unsigned int mask, struct qi_desc *desc)

{

	if (mask) {

		addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;

		desc->qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;

	} else {

		desc->qw1 = QI_DEV_IOTLB_ADDR(addr);

	}



	if (qdep >= QI_DEV_IOTLB_MAX_INVS)

		qdep = 0;



	desc->qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |

		   QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);

	desc->qw2 = 0;

	desc->qw3 = 0;

}



static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,

				  unsigned long npages, bool ih,

				  struct qi_desc *desc)

{

	if (npages == -1) {

		desc->qw0 = QI_EIOTLB_PASID(pasid) |

				QI_EIOTLB_DID(did) |

				QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |

				QI_EIOTLB_TYPE;

		desc->qw1 = 0;

	} else {

		int mask = ilog2(__roundup_pow_of_two(npages));

		unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));



		if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))

			addr = ALIGN_DOWN(addr, align);



		desc->qw0 = QI_EIOTLB_PASID(pasid) |

				QI_EIOTLB_DID(did) |

				QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |

				QI_EIOTLB_TYPE;

		desc->qw1 = QI_EIOTLB_ADDR(addr) |

				QI_EIOTLB_IH(ih) |

				QI_EIOTLB_AM(mask);

	}

}



static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,

					   u16 qdep, u64 addr,

					   unsigned int size_order,

					   struct qi_desc *desc)

{

	unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);



	desc->qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |

		QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |

		QI_DEV_IOTLB_PFSID(pfsid);



	/*

	 * If S bit is 0, we only flush a single page. If S bit is set,

	 * The least significant zero bit indicates the invalidation address

	 * range. VT-d spec 6.5.2.6.

	 * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.

	 * size order = 0 is PAGE_SIZE 4KB

	 * Max Invs Pending (MIP) is set to 0 for now until we have DIT in

	 * ECAP.

	 */

	if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))

		pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",

				    addr, size_order);



	/* Take page address */

	desc->qw1 = QI_DEV_EIOTLB_ADDR(addr);



	if (size_order) {

		/*

		 * Existing 0s in address below size_order may be the least

		 * significant bit, we must set them to 1s to avoid having

		 * smaller size than desired.

		 */

		desc->qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,

					VTD_PAGE_SHIFT);

		/* Clear size_order bit to indicate size */

		desc->qw1 &= ~mask;

		/* Set the S bit to indicate flushing more than 1 page */

		desc->qw1 |= QI_DEV_EIOTLB_SIZE;

	}

}



/* Convert value to context PASID directory size field coding. */

#define context_pdts(pds)	(((pds) & 0x7) << 9)