Commit 48552153 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd 

Pull iommufd updates from Jason Gunthorpe:
 "Two significant new items:

   - Allow reporting IOMMU HW events to userspace when the events are
     clearly linked to a device.

     This is linked to the VIOMMU object and is intended to be used by a
     VMM to forward HW events to the virtual machine as part of
     emulating a vIOMMU. ARM SMMUv3 is the first driver to use this
     mechanism. Like the existing fault events the data is delivered
     through a simple FD returning event records on read().

   - PASID support in VFIO.

     The "Process Address Space ID" is a PCI feature that allows the
     device to tag all PCI DMA operations with an ID. The IOMMU will
     then use the ID to select a unique translation for those DMAs. This
     is part of Intel's vIOMMU support as VT-D HW requires the
     hypervisor to manage each PASID entry.

     The support is generic so any VFIO user could attach any
     translation to a PASID, and the support should work on ARM SMMUv3
     as well. AMD requires additional driver work.

  Some minor updates, along with fixes:

   - Prevent using nested parents with fault's, no driver support today

   - Put a single "cookie_type" value in the iommu_domain to indicate
     what owns the various opaque owner fields"

* tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (49 commits)
  iommufd: Test attach before detaching pasid
  iommufd: Fix iommu_vevent_header tables markup
  iommu: Convert unreachable() to BUG()
  iommufd: Balance veventq->num_events inc/dec
  iommufd: Initialize the flags of vevent in iommufd_viommu_report_event()
  iommufd/selftest: Add coverage for reporting max_pasid_log2 via IOMMU_HW_INFO
  iommufd: Extend IOMMU_GET_HW_INFO to report PASID capability
  vfio: VFIO_DEVICE_[AT|DE]TACH_IOMMUFD_PT support pasid
  vfio-iommufd: Support pasid [at|de]tach for physical VFIO devices
  ida: Add ida_find_first_range()
  iommufd/selftest: Add coverage for iommufd pasid attach/detach
  iommufd/selftest: Add test ops to test pasid attach/detach
  iommufd/selftest: Add a helper to get test device
  iommufd/selftest: Add set_dev_pasid in mock iommu
  iommufd: Allow allocating PASID-compatible domain
  iommu/vt-d: Add IOMMU_HWPT_ALLOC_PASID support
  iommufd: Enforce PASID-compatible domain for RID
  iommufd: Support pasid attach/replace
  iommufd: Enforce PASID-compatible domain in PASID path
  iommufd/device: Add pasid_attach array to track per-PASID attach
  ...
parents 792b8307 7be11d34

Documentation/userspace-api/iommufd.rst

+17 −0
Original line number Diff line number Diff line
@@ -63,6 +63,13 @@ Following IOMMUFD objects are exposed to userspace: 
  space usually has mappings from guest-level I/O virtual addresses to guest-

  level physical addresses.



- IOMMUFD_FAULT, representing a software queue for an HWPT reporting IO page

  faults using the IOMMU HW's PRI (Page Request Interface). This queue object

  provides user space an FD to poll the page fault events and also to respond

  to those events. A FAULT object must be created first to get a fault_id that

  could be then used to allocate a fault-enabled HWPT via the IOMMU_HWPT_ALLOC

  command by setting the IOMMU_HWPT_FAULT_ID_VALID bit in its flags field.



- IOMMUFD_OBJ_VIOMMU, representing a slice of the physical IOMMU instance,

  passed to or shared with a VM. It may be some HW-accelerated virtualization

  features and some SW resources used by the VM. For examples:
@@ -109,6 +116,14 @@ Following IOMMUFD objects are exposed to userspace: 
  vIOMMU, which is a separate ioctl call from attaching the same device to an

  HWPT_PAGING that the vIOMMU holds.



- IOMMUFD_OBJ_VEVENTQ, representing a software queue for a vIOMMU to report its

  events such as translation faults occurred to a nested stage-1 (excluding I/O

  page faults that should go through IOMMUFD_OBJ_FAULT) and HW-specific events.

  This queue object provides user space an FD to poll/read the vIOMMU events. A

  vIOMMU object must be created first to get its viommu_id, which could be then

  used to allocate a vEVENTQ. Each vIOMMU can support multiple types of vEVENTS,

  but is confined to one vEVENTQ per vEVENTQ type.



All user-visible objects are destroyed via the IOMMU_DESTROY uAPI.



The diagrams below show relationships between user-visible objects and kernel
@@ -251,8 +266,10 @@ User visible objects are backed by following datastructures: 
- iommufd_device for IOMMUFD_OBJ_DEVICE.

- iommufd_hwpt_paging for IOMMUFD_OBJ_HWPT_PAGING.

- iommufd_hwpt_nested for IOMMUFD_OBJ_HWPT_NESTED.

- iommufd_fault for IOMMUFD_OBJ_FAULT.

- iommufd_viommu for IOMMUFD_OBJ_VIOMMU.

- iommufd_vdevice for IOMMUFD_OBJ_VDEVICE.

- iommufd_veventq for IOMMUFD_OBJ_VEVENTQ.



Several terminologies when looking at these datastructures:

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c

+60 −0
Original line number Diff line number Diff line
@@ -43,6 +43,8 @@ static void arm_smmu_make_nested_cd_table_ste( 
	target->data[0] |= nested_domain->ste[0] &

			   ~cpu_to_le64(STRTAB_STE_0_CFG);

	target->data[1] |= nested_domain->ste[1];

	/* Merge events for DoS mitigations on eventq */

	target->data[1] |= cpu_to_le64(STRTAB_STE_1_MEV);

}



/*
@@ -85,6 +87,47 @@ static void arm_smmu_make_nested_domain_ste( 
	}

}



int arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,

				    struct arm_smmu_nested_domain *nested_domain)

{

	struct arm_smmu_vmaster *vmaster;

	unsigned long vsid;

	int ret;



	iommu_group_mutex_assert(state->master->dev);



	ret = iommufd_viommu_get_vdev_id(&nested_domain->vsmmu->core,

					 state->master->dev, &vsid);

	if (ret)

		return ret;



	vmaster = kzalloc(sizeof(*vmaster), GFP_KERNEL);

	if (!vmaster)

		return -ENOMEM;

	vmaster->vsmmu = nested_domain->vsmmu;

	vmaster->vsid = vsid;

	state->vmaster = vmaster;



	return 0;

}



void arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state)

{

	struct arm_smmu_master *master = state->master;



	mutex_lock(&master->smmu->streams_mutex);

	kfree(master->vmaster);

	master->vmaster = state->vmaster;

	mutex_unlock(&master->smmu->streams_mutex);

}



void arm_smmu_master_clear_vmaster(struct arm_smmu_master *master)

{

	struct arm_smmu_attach_state state = { .master = master };



	arm_smmu_attach_commit_vmaster(&state);

}



static int arm_smmu_attach_dev_nested(struct iommu_domain *domain,

				      struct device *dev)

{
@@ -392,4 +435,21 @@ struct iommufd_viommu *arm_vsmmu_alloc(struct device *dev, 
	return &vsmmu->core;

}



int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster, u64 *evt)

{

	struct iommu_vevent_arm_smmuv3 vevt;

	int i;



	lockdep_assert_held(&vmaster->vsmmu->smmu->streams_mutex);



	vevt.evt[0] = cpu_to_le64((evt[0] & ~EVTQ_0_SID) |

				  FIELD_PREP(EVTQ_0_SID, vmaster->vsid));

	for (i = 1; i < EVTQ_ENT_DWORDS; i++)

		vevt.evt[i] = cpu_to_le64(evt[i]);



	return iommufd_viommu_report_event(&vmaster->vsmmu->core,

					   IOMMU_VEVENTQ_TYPE_ARM_SMMUV3, &vevt,

					   sizeof(vevt));

}



MODULE_IMPORT_NS("IOMMUFD");

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

+52 −28
Original line number Diff line number Diff line
@@ -1052,7 +1052,7 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) 
			cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |

				    STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |

				    STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |

				    STRTAB_STE_1_EATS);

				    STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);

		used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);



		/*
@@ -1068,7 +1068,7 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) 
	if (cfg & BIT(1)) {

		used_bits[1] |=

			cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |

				    STRTAB_STE_1_SHCFG);

				    STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);

		used_bits[2] |=

			cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |

				    STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
@@ -1813,7 +1813,7 @@ static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw, 
	mutex_unlock(&smmu->streams_mutex);

}



static int arm_smmu_handle_event(struct arm_smmu_device *smmu,

static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,

				 struct arm_smmu_event *event)

{

	int ret = 0;
@@ -1823,6 +1823,10 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu, 
	struct iommu_fault *flt = &fault_evt.fault;



	switch (event->id) {

	case EVT_ID_BAD_STE_CONFIG:

	case EVT_ID_STREAM_DISABLED_FAULT:

	case EVT_ID_BAD_SUBSTREAMID_CONFIG:

	case EVT_ID_BAD_CD_CONFIG:

	case EVT_ID_TRANSLATION_FAULT:

	case EVT_ID_ADDR_SIZE_FAULT:

	case EVT_ID_ACCESS_FAULT:
@@ -1832,9 +1836,7 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu, 
		return -EOPNOTSUPP;

	}



	if (!event->stall)

		return -EOPNOTSUPP;



	if (event->stall) {

		if (event->read)

			perm |= IOMMU_FAULT_PERM_READ;

		else
@@ -1858,6 +1860,7 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu, 
			flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;

			flt->prm.pasid = event->ssid;

		}

	}



	mutex_lock(&smmu->streams_mutex);

	master = arm_smmu_find_master(smmu, event->sid);
@@ -1866,7 +1869,12 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu, 
		goto out_unlock;

	}



	if (event->stall)

		ret = iommu_report_device_fault(master->dev, &fault_evt);

	else if (master->vmaster && !event->s2)

		ret = arm_vmaster_report_event(master->vmaster, evt);

	else

		ret = -EOPNOTSUPP; /* Unhandled events should be pinned */

out_unlock:

	mutex_unlock(&smmu->streams_mutex);

	return ret;
@@ -1944,7 +1952,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) 
	do {

		while (!queue_remove_raw(q, evt)) {

			arm_smmu_decode_event(smmu, evt, &event);

			if (arm_smmu_handle_event(smmu, &event))

			if (arm_smmu_handle_event(smmu, evt, &event))

				arm_smmu_dump_event(smmu, evt, &event, &rs);



			put_device(event.dev);
@@ -2803,6 +2811,7 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, 
	struct arm_smmu_domain *smmu_domain =

		to_smmu_domain_devices(new_domain);

	unsigned long flags;

	int ret;



	/*

	 * arm_smmu_share_asid() must not see two domains pointing to the same
@@ -2832,9 +2841,18 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, 
	}



	if (smmu_domain) {

		if (new_domain->type == IOMMU_DOMAIN_NESTED) {

			ret = arm_smmu_attach_prepare_vmaster(

				state, to_smmu_nested_domain(new_domain));

			if (ret)

				return ret;

		}



		master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);

		if (!master_domain)

		if (!master_domain) {

			kfree(state->vmaster);

			return -ENOMEM;

		}

		master_domain->master = master;

		master_domain->ssid = state->ssid;

		if (new_domain->type == IOMMU_DOMAIN_NESTED)
@@ -2861,6 +2879,7 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, 
			spin_unlock_irqrestore(&smmu_domain->devices_lock,

					       flags);

			kfree(master_domain);

			kfree(state->vmaster);

			return -EINVAL;

		}
@@ -2893,6 +2912,8 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state) 


	lockdep_assert_held(&arm_smmu_asid_lock);



	arm_smmu_attach_commit_vmaster(state);



	if (state->ats_enabled && !master->ats_enabled) {

		arm_smmu_enable_ats(master);

	} else if (state->ats_enabled && master->ats_enabled) {
@@ -3162,6 +3183,7 @@ static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, 
	struct arm_smmu_ste ste;

	struct arm_smmu_master *master = dev_iommu_priv_get(dev);



	arm_smmu_master_clear_vmaster(master);

	arm_smmu_make_bypass_ste(master->smmu, &ste);

	arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);

	return 0;
@@ -3180,7 +3202,9 @@ static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, 
					struct device *dev)

{

	struct arm_smmu_ste ste;

	struct arm_smmu_master *master = dev_iommu_priv_get(dev);



	arm_smmu_master_clear_vmaster(master);

	arm_smmu_make_abort_ste(&ste);

	arm_smmu_attach_dev_ste(domain, dev, &ste,

				STRTAB_STE_1_S1DSS_TERMINATE);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

+36 −0
Original line number Diff line number Diff line
@@ -266,6 +266,7 @@ static inline u32 arm_smmu_strtab_l2_idx(u32 sid) 
#define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)

#define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)



#define STRTAB_STE_1_MEV		(1UL << 19)

#define STRTAB_STE_1_S2FWB		(1UL << 25)

#define STRTAB_STE_1_S1STALLD		(1UL << 27)
@@ -799,6 +800,11 @@ struct arm_smmu_stream { 
	struct rb_node			node;

};



struct arm_smmu_vmaster {

	struct arm_vsmmu		*vsmmu;

	unsigned long			vsid;

};



struct arm_smmu_event {

	u8				stall : 1,

					ssv : 1,
@@ -824,6 +830,7 @@ struct arm_smmu_master { 
	struct arm_smmu_device		*smmu;

	struct device			*dev;

	struct arm_smmu_stream		*streams;

	struct arm_smmu_vmaster		*vmaster; /* use smmu->streams_mutex */

	/* Locked by the iommu core using the group mutex */

	struct arm_smmu_ctx_desc_cfg	cd_table;

	unsigned int			num_streams;
@@ -972,6 +979,7 @@ struct arm_smmu_attach_state { 
	bool disable_ats;

	ioasid_t ssid;

	/* Resulting state */

	struct arm_smmu_vmaster *vmaster;

	bool ats_enabled;

};
@@ -1055,9 +1063,37 @@ struct iommufd_viommu *arm_vsmmu_alloc(struct device *dev, 
				       struct iommu_domain *parent,

				       struct iommufd_ctx *ictx,

				       unsigned int viommu_type);

int arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,

				    struct arm_smmu_nested_domain *nested_domain);

void arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state);

void arm_smmu_master_clear_vmaster(struct arm_smmu_master *master);

int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster, u64 *evt);

#else

#define arm_smmu_hw_info NULL

#define arm_vsmmu_alloc NULL



static inline int

arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,

				struct arm_smmu_nested_domain *nested_domain)

{

	return 0;

}



static inline void

arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state)

{

}



static inline void

arm_smmu_master_clear_vmaster(struct arm_smmu_master *master)

{

}



static inline int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster,

					   u64 *evt)

{

	return -EOPNOTSUPP;

}

#endif /* CONFIG_ARM_SMMU_V3_IOMMUFD */



#endif /* _ARM_SMMU_V3_H */

drivers/iommu/dma-iommu.c

+104 −100
Original line number Diff line number Diff line
@@ -42,11 +42,6 @@ struct iommu_dma_msi_page { 
	phys_addr_t		phys;

};



enum iommu_dma_cookie_type {

	IOMMU_DMA_IOVA_COOKIE,

	IOMMU_DMA_MSI_COOKIE,

};



enum iommu_dma_queue_type {

	IOMMU_DMA_OPTS_PER_CPU_QUEUE,

	IOMMU_DMA_OPTS_SINGLE_QUEUE,
@@ -59,11 +54,8 @@ struct iommu_dma_options { 
};



struct iommu_dma_cookie {

	enum iommu_dma_cookie_type	type;

	union {

		/* Full allocator for IOMMU_DMA_IOVA_COOKIE */

		struct {

	struct iova_domain iovad;

	struct list_head msi_page_list;

	/* Flush queue */

	union {

		struct iova_fq *single_fq;
@@ -77,18 +69,17 @@ struct iommu_dma_cookie { 
	struct timer_list fq_timer;

	/* 1 when timer is active, 0 when not */

	atomic_t fq_timer_on;

		};

		/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */

		dma_addr_t		msi_iova;

	};

	struct list_head		msi_page_list;



	/* Domain for flush queue callback; NULL if flush queue not in use */

	struct iommu_domain *fq_domain;

	/* Options for dma-iommu use */

	struct iommu_dma_options options;

};



struct iommu_dma_msi_cookie {

	dma_addr_t msi_iova;

	struct list_head msi_page_list;

};



static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);

bool iommu_dma_forcedac __read_mostly;
@@ -102,9 +93,6 @@ static int __init iommu_dma_forcedac_setup(char *str) 
}

early_param("iommu.forcedac", iommu_dma_forcedac_setup);



static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

			    phys_addr_t msi_addr);



/* Number of entries per flush queue */

#define IOVA_DEFAULT_FQ_SIZE	256

#define IOVA_SINGLE_FQ_SIZE	32768
@@ -368,39 +356,24 @@ int iommu_dma_init_fq(struct iommu_domain *domain) 
	return 0;

}



static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)

{

	if (cookie->type == IOMMU_DMA_IOVA_COOKIE)

		return cookie->iovad.granule;

	return PAGE_SIZE;

}



static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)

{

	struct iommu_dma_cookie *cookie;



	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);

	if (cookie) {

		INIT_LIST_HEAD(&cookie->msi_page_list);

		cookie->type = type;

	}

	return cookie;

}



/**

 * iommu_get_dma_cookie - Acquire DMA-API resources for a domain

 * @domain: IOMMU domain to prepare for DMA-API usage

 */

int iommu_get_dma_cookie(struct iommu_domain *domain)

{

	if (domain->iova_cookie)

	struct iommu_dma_cookie *cookie;



	if (domain->cookie_type != IOMMU_COOKIE_NONE)

		return -EEXIST;



	domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);

	if (!domain->iova_cookie)

	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);

	if (!cookie)

		return -ENOMEM;



	iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);

	INIT_LIST_HEAD(&cookie->msi_page_list);

	domain->cookie_type = IOMMU_COOKIE_DMA_IOVA;

	domain->iova_cookie = cookie;

	return 0;

}
@@ -418,54 +391,56 @@ int iommu_get_dma_cookie(struct iommu_domain *domain) 
 */

int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)

{

	struct iommu_dma_cookie *cookie;

	struct iommu_dma_msi_cookie *cookie;



	if (domain->type != IOMMU_DOMAIN_UNMANAGED)

		return -EINVAL;



	if (domain->iova_cookie)

	if (domain->cookie_type != IOMMU_COOKIE_NONE)

		return -EEXIST;



	cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);

	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);

	if (!cookie)

		return -ENOMEM;



	cookie->msi_iova = base;

	domain->iova_cookie = cookie;

	iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);

	INIT_LIST_HEAD(&cookie->msi_page_list);

	domain->cookie_type = IOMMU_COOKIE_DMA_MSI;

	domain->msi_cookie = cookie;

	return 0;

}

EXPORT_SYMBOL(iommu_get_msi_cookie);



/**

 * iommu_put_dma_cookie - Release a domain's DMA mapping resources

 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or

 *          iommu_get_msi_cookie()

 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()

 */

void iommu_put_dma_cookie(struct iommu_domain *domain)

{

	struct iommu_dma_cookie *cookie = domain->iova_cookie;

	struct iommu_dma_msi_page *msi, *tmp;



#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)

	if (domain->sw_msi != iommu_dma_sw_msi)

		return;

#endif



	if (!cookie)

		return;



	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {

	if (cookie->iovad.granule) {

		iommu_dma_free_fq(cookie);

		put_iova_domain(&cookie->iovad);

	}



	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {

		list_del(&msi->list);

	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list)

		kfree(msi);

	kfree(cookie);

}



/**

 * iommu_put_msi_cookie - Release a domain's MSI mapping resources

 * @domain: IOMMU domain previously prepared by iommu_get_msi_cookie()

 */

void iommu_put_msi_cookie(struct iommu_domain *domain)

{

	struct iommu_dma_msi_cookie *cookie = domain->msi_cookie;

	struct iommu_dma_msi_page *msi, *tmp;



	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list)

		kfree(msi);

	kfree(cookie);

	domain->iova_cookie = NULL;

}



/**
@@ -685,7 +660,7 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev 
	struct iova_domain *iovad;

	int ret;



	if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)

	if (!cookie || domain->cookie_type != IOMMU_COOKIE_DMA_IOVA)

		return -EINVAL;



	iovad = &cookie->iovad;
@@ -768,9 +743,9 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, 
	struct iova_domain *iovad = &cookie->iovad;

	unsigned long shift, iova_len, iova;



	if (cookie->type == IOMMU_DMA_MSI_COOKIE) {

		cookie->msi_iova += size;

		return cookie->msi_iova - size;

	if (domain->cookie_type == IOMMU_COOKIE_DMA_MSI) {

		domain->msi_cookie->msi_iova += size;

		return domain->msi_cookie->msi_iova - size;

	}



	shift = iova_shift(iovad);
@@ -807,16 +782,16 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, 
	return (dma_addr_t)iova << shift;

}



static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,

		dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)

static void iommu_dma_free_iova(struct iommu_domain *domain, dma_addr_t iova,

				size_t size, struct iommu_iotlb_gather *gather)

{

	struct iova_domain *iovad = &cookie->iovad;

	struct iova_domain *iovad = &domain->iova_cookie->iovad;



	/* The MSI case is only ever cleaning up its most recent allocation */

	if (cookie->type == IOMMU_DMA_MSI_COOKIE)

		cookie->msi_iova -= size;

	if (domain->cookie_type == IOMMU_COOKIE_DMA_MSI)

		domain->msi_cookie->msi_iova -= size;

	else if (gather && gather->queued)

		queue_iova(cookie, iova_pfn(iovad, iova),

		queue_iova(domain->iova_cookie, iova_pfn(iovad, iova),

				size >> iova_shift(iovad),

				&gather->freelist);

	else
@@ -844,7 +819,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, 


	if (!iotlb_gather.queued)

		iommu_iotlb_sync(domain, &iotlb_gather);

	iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);

	iommu_dma_free_iova(domain, dma_addr, size, &iotlb_gather);

}



static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -872,7 +847,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, 
		return DMA_MAPPING_ERROR;



	if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) {

		iommu_dma_free_iova(cookie, iova, size, NULL);

		iommu_dma_free_iova(domain, iova, size, NULL);

		return DMA_MAPPING_ERROR;

	}

	return iova + iova_off;
@@ -1009,7 +984,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, 
out_free_sg:

	sg_free_table(sgt);

out_free_iova:

	iommu_dma_free_iova(cookie, iova, size, NULL);

	iommu_dma_free_iova(domain, iova, size, NULL);

out_free_pages:

	__iommu_dma_free_pages(pages, count);

	return NULL;
@@ -1486,7 +1461,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 
	return __finalise_sg(dev, sg, nents, iova);



out_free_iova:

	iommu_dma_free_iova(cookie, iova, iova_len, NULL);

	iommu_dma_free_iova(domain, iova, iova_len, NULL);

out_restore_sg:

	__invalidate_sg(sg, nents);

out:
@@ -1764,17 +1739,47 @@ void iommu_setup_dma_ops(struct device *dev) 
	dev->dma_iommu = false;

}



static bool has_msi_cookie(const struct iommu_domain *domain)

{

	return domain && (domain->cookie_type == IOMMU_COOKIE_DMA_IOVA ||

			  domain->cookie_type == IOMMU_COOKIE_DMA_MSI);

}



static size_t cookie_msi_granule(const struct iommu_domain *domain)

{

	switch (domain->cookie_type) {

	case IOMMU_COOKIE_DMA_IOVA:

		return domain->iova_cookie->iovad.granule;

	case IOMMU_COOKIE_DMA_MSI:

		return PAGE_SIZE;

	default:

		BUG();

	};

}



static struct list_head *cookie_msi_pages(const struct iommu_domain *domain)

{

	switch (domain->cookie_type) {

	case IOMMU_COOKIE_DMA_IOVA:

		return &domain->iova_cookie->msi_page_list;

	case IOMMU_COOKIE_DMA_MSI:

		return &domain->msi_cookie->msi_page_list;

	default:

		BUG();

	};

}



static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,

		phys_addr_t msi_addr, struct iommu_domain *domain)

{

	struct iommu_dma_cookie *cookie = domain->iova_cookie;

	struct list_head *msi_page_list = cookie_msi_pages(domain);

	struct iommu_dma_msi_page *msi_page;

	dma_addr_t iova;

	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;

	size_t size = cookie_msi_granule(cookie);

	size_t size = cookie_msi_granule(domain);



	msi_addr &= ~(phys_addr_t)(size - 1);

	list_for_each_entry(msi_page, &cookie->msi_page_list, list)

	list_for_each_entry(msi_page, msi_page_list, list)

		if (msi_page->phys == msi_addr)

			return msi_page;
@@ -1792,23 +1797,23 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, 
	INIT_LIST_HEAD(&msi_page->list);

	msi_page->phys = msi_addr;

	msi_page->iova = iova;

	list_add(&msi_page->list, &cookie->msi_page_list);

	list_add(&msi_page->list, msi_page_list);

	return msi_page;



out_free_iova:

	iommu_dma_free_iova(cookie, iova, size, NULL);

	iommu_dma_free_iova(domain, iova, size, NULL);

out_free_page:

	kfree(msi_page);

	return NULL;

}



static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

		     phys_addr_t msi_addr)

{

	struct device *dev = msi_desc_to_dev(desc);

	const struct iommu_dma_msi_page *msi_page;



	if (!domain->iova_cookie) {

	if (!has_msi_cookie(domain)) {

		msi_desc_set_iommu_msi_iova(desc, 0, 0);

		return 0;

	}
@@ -1818,9 +1823,8 @@ static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc, 
	if (!msi_page)

		return -ENOMEM;



	msi_desc_set_iommu_msi_iova(

		desc, msi_page->iova,

		ilog2(cookie_msi_granule(domain->iova_cookie)));

	msi_desc_set_iommu_msi_iova(desc, msi_page->iova,

				    ilog2(cookie_msi_granule(domain)));

	return 0;

}