Commit 19e5cc15 authored by Kishon Vijay Abraham I's avatar Kishon Vijay Abraham I Committed by Joerg Roedel
Browse files

iommu/amd: Enable support for up to 2K interrupts per function 



AMD IOMMU optionally supports up to 2K interrupts per function on newer
platforms. Support for this feature is indicated through Extended
Feature 2 Register (MMIO Offset 01A0h[NumIntRemapSup]). Allocate 2K IRTEs
per device when this support is available.

Co-developed-by: default avatarSairaj Kodilkar <sarunkod@amd.com>
Signed-off-by: default avatarSairaj Kodilkar <sarunkod@amd.com>
Signed-off-by: default avatarKishon Vijay Abraham I <kvijayab@amd.com>
Reviewed-by: default avatarVasant Hegde <vasant.hegde@amd.com>
Link: https://lore.kernel.org/r/20250307095822.2274-5-sarunkod@amd.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 950865c1

drivers/iommu/amd/amd_iommu_types.h

+11 −4
Original line number Diff line number Diff line
@@ -112,6 +112,10 @@ 
#define FEATURE_SNPAVICSUP_GAM(x) \

	(FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)



#define FEATURE_NUM_INT_REMAP_SUP	GENMASK_ULL(9, 8)

#define FEATURE_NUM_INT_REMAP_SUP_2K(x) \

	(FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)



/* Note:

 * The current driver only support 16-bit PASID.

 * Currently, hardware only implement upto 16-bit PASID
@@ -175,6 +179,9 @@ 
#define CONTROL_GAM_EN		25

#define CONTROL_GALOG_EN	28

#define CONTROL_GAINT_EN	29

#define CONTROL_NUM_INT_REMAP_MODE	43

#define CONTROL_NUM_INT_REMAP_MODE_MASK	0x03

#define CONTROL_NUM_INT_REMAP_MODE_2K	0x01

#define CONTROL_EPH_EN		45

#define CONTROL_XT_EN		50

#define CONTROL_INTCAPXT_EN	51
@@ -309,14 +316,13 @@ 
#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)

#define DTE_IRQ_REMAP_ENABLE    1ULL



/*

 * AMD IOMMU hardware only support 512 IRTEs despite

 * the architectural limitation of 2048 entries.

 */

#define DTE_INTTABLEN_MASK      (0xfULL << 1)

#define DTE_INTTABLEN_VALUE_512 9ULL

#define DTE_INTTABLEN_512       (DTE_INTTABLEN_VALUE_512 << 1)

#define MAX_IRQS_PER_TABLE_512  BIT(DTE_INTTABLEN_VALUE_512)

#define DTE_INTTABLEN_VALUE_2K	11ULL

#define DTE_INTTABLEN_2K	(DTE_INTTABLEN_VALUE_2K << 1)

#define MAX_IRQS_PER_TABLE_2K	BIT(DTE_INTTABLEN_VALUE_2K)



#define PAGE_MODE_NONE    0x00

#define PAGE_MODE_1_LEVEL 0x01
@@ -847,6 +853,7 @@ struct iommu_dev_data { 
	struct device *dev;

	u16 devid;			  /* PCI Device ID */



	unsigned int max_irqs;		  /* Maximum IRQs supported by device */

	u32 max_pasids;			  /* Max supported PASIDs */

	u32 flags;			  /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */

	int ats_qdep;

drivers/iommu/amd/init.c

+15 −1
Original line number Diff line number Diff line
@@ -1060,7 +1060,8 @@ static bool __copy_device_table(struct amd_iommu *iommu) 
		int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK;

		if (irq_v && (int_ctl || int_tab_len)) {

			if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||

			    (int_tab_len != DTE_INTTABLEN_512)) {

			    (int_tab_len != DTE_INTTABLEN_512 &&

			     int_tab_len != DTE_INTTABLEN_2K)) {

				pr_err("Wrong old irq remapping flag: %#x\n", devid);

				memunmap(old_devtb);

				return false;
@@ -2736,6 +2737,17 @@ static void iommu_enable_irtcachedis(struct amd_iommu *iommu) 
		iommu->irtcachedis_enabled ? "disabled" : "enabled");

}



static void iommu_enable_2k_int(struct amd_iommu *iommu)

{

	if (!FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))

		return;



	iommu_feature_set(iommu,

			  CONTROL_NUM_INT_REMAP_MODE_2K,

			  CONTROL_NUM_INT_REMAP_MODE_MASK,

			  CONTROL_NUM_INT_REMAP_MODE);

}



static void early_enable_iommu(struct amd_iommu *iommu)

{

	iommu_disable(iommu);
@@ -2748,6 +2760,7 @@ static void early_enable_iommu(struct amd_iommu *iommu) 
	iommu_enable_ga(iommu);

	iommu_enable_xt(iommu);

	iommu_enable_irtcachedis(iommu);

	iommu_enable_2k_int(iommu);

	iommu_enable(iommu);

	amd_iommu_flush_all_caches(iommu);

}
@@ -2804,6 +2817,7 @@ static void early_enable_iommus(void) 
			iommu_enable_ga(iommu);

			iommu_enable_xt(iommu);

			iommu_enable_irtcachedis(iommu);

			iommu_enable_2k_int(iommu);

			iommu_set_device_table(iommu);

			amd_iommu_flush_all_caches(iommu);

		}

drivers/iommu/amd/iommu.c

+39 −9
Original line number Diff line number Diff line
@@ -2394,8 +2394,14 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) 
	}



out_err:



	iommu_completion_wait(iommu);



	if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))

		dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K;

	else

		dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;



	if (dev_is_pci(dev))

		pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
@@ -3076,6 +3082,13 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) 
	raw_spin_unlock_irqrestore(&iommu->lock, flags);

}



static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data)

{

	if (dev_data && dev_data->max_irqs == MAX_IRQS_PER_TABLE_2K)

		return DTE_INTTABLEN_2K;

	return DTE_INTTABLEN_512;

}



static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,

			      struct irq_remap_table *table)

{
@@ -3090,7 +3103,7 @@ static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid, 
	new &= ~DTE_IRQ_PHYS_ADDR_MASK;

	new |= iommu_virt_to_phys(table->table);

	new |= DTE_IRQ_REMAP_INTCTL;

	new |= DTE_INTTABLEN_512;

	new |= iommu_get_int_tablen(dev_data);

	new |= DTE_IRQ_REMAP_ENABLE;

	WRITE_ONCE(dte->data[2], new);
@@ -3171,13 +3184,14 @@ static inline size_t get_irq_table_size(unsigned int max_irqs) 
}



static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,

					       u16 devid, struct pci_dev *pdev)

					       u16 devid, struct pci_dev *pdev,

					       unsigned int max_irqs)

{

	struct irq_remap_table *table = NULL;

	struct irq_remap_table *new_table = NULL;

	struct amd_iommu_pci_seg *pci_seg;

	unsigned long flags;

	int order = get_order(get_irq_table_size(MAX_IRQS_PER_TABLE));

	int order = get_order(get_irq_table_size(max_irqs));

	int nid = iommu && iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

	u16 alias;
@@ -3239,13 +3253,14 @@ static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu, 
}



static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,

			   bool align, struct pci_dev *pdev)

			   bool align, struct pci_dev *pdev,

			   unsigned long max_irqs)

{

	struct irq_remap_table *table;

	int index, c, alignment = 1;

	unsigned long flags;



	table = alloc_irq_table(iommu, devid, pdev);

	table = alloc_irq_table(iommu, devid, pdev, max_irqs);

	if (!table)

		return -ENODEV;
@@ -3256,7 +3271,7 @@ static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count, 


	/* Scan table for free entries */

	for (index = ALIGN(table->min_index, alignment), c = 0;

	     index < MAX_IRQS_PER_TABLE;) {

	     index < max_irqs;) {

		if (!iommu->irte_ops->is_allocated(table, index)) {

			c += 1;

		} else {
@@ -3526,6 +3541,14 @@ static void fill_msi_msg(struct msi_msg *msg, u32 index) 
	msg->data = index;

	msg->address_lo = 0;

	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;

	/*

	 * The struct msi_msg.dest_mode_logical is used to set the DM bit

	 * in MSI Message Address Register. For device w/ 2K int-remap support,

	 * this is bit must be set to 1 regardless of the actual destination

	 * mode, which is signified by the IRTE[DM].

	 */

	if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))

		msg->arch_addr_lo.dest_mode_logical = true;

	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;

}
@@ -3588,6 +3611,8 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 
	struct amd_ir_data *data = NULL;

	struct amd_iommu *iommu;

	struct irq_cfg *cfg;

	struct iommu_dev_data *dev_data;

	unsigned long max_irqs;

	int i, ret, devid, seg, sbdf;

	int index;
@@ -3606,6 +3631,9 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 
	if (!iommu)

		return -EINVAL;



	dev_data = search_dev_data(iommu, devid);

	max_irqs = dev_data ? dev_data->max_irqs : MAX_IRQS_PER_TABLE_512;



	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);

	if (ret < 0)

		return ret;
@@ -3613,7 +3641,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 
	if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {

		struct irq_remap_table *table;



		table = alloc_irq_table(iommu, devid, NULL);

		table = alloc_irq_table(iommu, devid, NULL, max_irqs);

		if (table) {

			if (!table->min_index) {

				/*
@@ -3634,9 +3662,11 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 
		bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);



		index = alloc_irq_index(iommu, devid, nr_irqs, align,

					msi_desc_to_pci_dev(info->desc));

					msi_desc_to_pci_dev(info->desc),

					max_irqs);

	} else {

		index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);

		index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL,

					max_irqs);

	}



	if (index < 0) {