Merge branch 'iommu/iommufd/attach-handles' into iommu/next (74e54d53) · Commits · git / linux-net

drivers/dma/idxd/init.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -584,7 +584,7 @@ static int idxd_enable_system_pasid(struct idxd_device *idxd)
		* DMA domain is owned by the driver, it should support all valid
		* types such as DMA-FQ, identity, etc.
		*/
		ret = iommu_attach_device_pasid(domain, dev, pasid);
		ret = iommu_attach_device_pasid(domain, dev, pasid, NULL);
		if (ret) {
		dev_err(dev, "failed to attach device pasid %d, domain type %d",
		pasid, domain->type);

drivers/iommu/io-pgfault.c

+34 −27

Original line number	Diff line number	Diff line
		@@ -59,30 +59,6 @@ void iopf_free_group(struct iopf_group *group)
		}
		EXPORT_SYMBOL_GPL(iopf_free_group);

		static struct iommu_domain get_domain_for_iopf(struct device dev,
		struct iommu_fault *fault)
		{
		struct iommu_domain *domain;

		if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
		domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0);
		if (IS_ERR(domain))
		domain = NULL;
		} else {
		domain = iommu_get_domain_for_dev(dev);
		}

		if (!domain \|\| !domain->iopf_handler) {
		dev_warn_ratelimited(dev,
		"iopf (pasid %d) without domain attached or handler installed\n",
		fault->prm.pasid);

		return NULL;
		}

		return domain;
		}

		/* Non-last request of a group. Postpone until the last one. */
		static int report_partial_fault(struct iommu_fault_param *fault_param,
		struct iommu_fault *fault)
		@@ -206,20 +182,51 @@ void iommu_report_device_fault(struct device dev, struct iopf_fault evt)
		if (group == &abort_group)
		goto err_abort;

		group->domain = get_domain_for_iopf(dev, fault);
		if (!group->domain)
		if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
		group->attach_handle = iommu_attach_handle_get(dev->iommu_group,
		fault->prm.pasid,
		0);
		if (IS_ERR(group->attach_handle)) {
		const struct iommu_ops *ops = dev_iommu_ops(dev);

		if (!ops->user_pasid_table)
		goto err_abort;

		/*
		* The iommu driver for this device supports user-
		* managed PASID table. Therefore page faults for
		* any PASID should go through the NESTING domain
		* attached to the device RID.
		*/
		group->attach_handle =
		iommu_attach_handle_get(dev->iommu_group,
		IOMMU_NO_PASID,
		IOMMU_DOMAIN_NESTED);
		if (IS_ERR(group->attach_handle))
		goto err_abort;
		}
		} else {
		group->attach_handle =
		iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
		if (IS_ERR(group->attach_handle))
		goto err_abort;
		}

		if (!group->attach_handle->domain->iopf_handler)
		goto err_abort;

		/*
		* On success iopf_handler must call iopf_group_response() and
		* iopf_free_group()
		*/
		if (group->domain->iopf_handler(group))
		if (group->attach_handle->domain->iopf_handler(group))
		goto err_abort;

		return;

		err_abort:
		dev_warn_ratelimited(dev, "iopf with pasid %d aborted\n",
		fault->prm.pasid);
		iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE);
		if (group == &abort_group)
		__iopf_free_group(group);

drivers/iommu/iommu-priv.h

+11 −0

Original line number	Diff line number	Diff line
		@@ -35,4 +35,15 @@ void iommu_device_unregister_bus(struct iommu_device *iommu,
		const struct bus_type *bus,
		struct notifier_block *nb);

		struct iommu_attach_handle iommu_attach_handle_get(struct iommu_group group,
		ioasid_t pasid,
		unsigned int type);
		int iommu_attach_group_handle(struct iommu_domain *domain,
		struct iommu_group *group,
		struct iommu_attach_handle *handle);
		void iommu_detach_group_handle(struct iommu_domain *domain,
		struct iommu_group *group);
		int iommu_replace_group_handle(struct iommu_group *group,
		struct iommu_domain *new_domain,
		struct iommu_attach_handle *handle);
		#endif /* __LINUX_IOMMU_PRIV_H */

drivers/iommu/iommu-sva.c

+28 −14

Original line number	Diff line number	Diff line
		@@ -43,7 +43,6 @@ static struct iommu_mm_data iommu_alloc_mm_data(struct mm_struct mm, struct de
		}
		iommu_mm->pasid = pasid;
		INIT_LIST_HEAD(&iommu_mm->sva_domains);
		INIT_LIST_HEAD(&iommu_mm->sva_handles);
		/*
		* Make sure the write to mm->iommu_mm is not reordered in front of
		* initialization to iommu_mm fields. If it does, readers may see a
		@@ -71,11 +70,16 @@ static struct iommu_mm_data iommu_alloc_mm_data(struct mm_struct mm, struct de
		*/
		struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm)
		{
		struct iommu_group *group = dev->iommu_group;
		struct iommu_attach_handle *attach_handle;
		struct iommu_mm_data *iommu_mm;
		struct iommu_domain *domain;
		struct iommu_sva *handle;
		int ret;

		if (!group)
		return ERR_PTR(-ENODEV);

		mutex_lock(&iommu_sva_lock);

		/* Allocate mm->pasid if necessary. */
		@@ -85,12 +89,22 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm
		goto out_unlock;
		}

		list_for_each_entry(handle, &mm->iommu_mm->sva_handles, handle_item) {
		if (handle->dev == dev) {
		/* A bond already exists, just take a reference`. */
		attach_handle = iommu_attach_handle_get(group, iommu_mm->pasid, IOMMU_DOMAIN_SVA);
		if (!IS_ERR(attach_handle)) {
		handle = container_of(attach_handle, struct iommu_sva, handle);
		if (attach_handle->domain->mm != mm) {
		ret = -EBUSY;
		goto out_unlock;
		}
		refcount_inc(&handle->users);
		mutex_unlock(&iommu_sva_lock);
		return handle;
		}

		if (PTR_ERR(attach_handle) != -ENOENT) {
		ret = PTR_ERR(attach_handle);
		goto out_unlock;
		}

		handle = kzalloc(sizeof(*handle), GFP_KERNEL);
		@@ -101,7 +115,8 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm

		/* Search for an existing domain. */
		list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) {
		ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
		ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
		&handle->handle);
		if (!ret) {
		domain->users++;
		goto out;
		@@ -115,7 +130,8 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm
		goto out_free_handle;
		}

		ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
		ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
		&handle->handle);
		if (ret)
		goto out_free_domain;
		domain->users = 1;
		@@ -123,10 +139,8 @@ struct iommu_sva iommu_sva_bind_device(struct device dev, struct mm_struct *mm

		out:
		refcount_set(&handle->users, 1);
		list_add(&handle->handle_item, &mm->iommu_mm->sva_handles);
		mutex_unlock(&iommu_sva_lock);
		handle->dev = dev;
		handle->domain = domain;
		return handle;

		out_free_domain:
		@@ -149,7 +163,7 @@ EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
		*/
		void iommu_sva_unbind_device(struct iommu_sva *handle)
		{
		struct iommu_domain *domain = handle->domain;
		struct iommu_domain *domain = handle->handle.domain;
		struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm;
		struct device *dev = handle->dev;

		@@ -158,7 +172,6 @@ void iommu_sva_unbind_device(struct iommu_sva *handle)
		mutex_unlock(&iommu_sva_lock);
		return;
		}
		list_del(&handle->handle_item);

		iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
		if (--domain->users == 0) {
		@@ -172,7 +185,7 @@ EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);

		u32 iommu_sva_get_pasid(struct iommu_sva *handle)
		{
		struct iommu_domain *domain = handle->domain;
		struct iommu_domain *domain = handle->handle.domain;

		return mm_get_enqcmd_pasid(domain->mm);
		}
		@@ -261,7 +274,8 @@ static void iommu_sva_handle_iopf(struct work_struct *work)
		if (status != IOMMU_PAGE_RESP_SUCCESS)
		break;

		status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm);
		status = iommu_sva_handle_mm(&iopf->fault,
		group->attach_handle->domain->mm);
		}

		iopf_group_response(group, status);

drivers/iommu/iommu.c

+143 −42

Original line number	Diff line number	Diff line
		@@ -3348,16 +3348,17 @@ static void __iommu_remove_group_pasid(struct iommu_group *group,
		* @domain: the iommu domain.
		* @dev: the attached device.
		* @pasid: the pasid of the device.
		* @handle: the attach handle.
		*
		* Return: 0 on success, or an error.
		*/
		int iommu_attach_device_pasid(struct iommu_domain *domain,
		struct device *dev, ioasid_t pasid)
		struct device *dev, ioasid_t pasid,
		struct iommu_attach_handle *handle)
		{
		/* Caller must be a probed driver on dev */
		struct iommu_group *group = dev->iommu_group;
		struct group_device *device;
		void *curr;
		int ret;

		if (!domain->ops->set_dev_pasid)
		@@ -3378,11 +3379,12 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
		}
		}

		curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
		if (curr) {
		ret = xa_err(curr) ? : -EBUSY;
		if (handle)
		handle->domain = domain;

		ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);
		if (ret)
		goto out_unlock;
		}

		ret = __iommu_set_group_pasid(domain, group, pasid);
		if (ret)
		@@ -3410,46 +3412,11 @@ void iommu_detach_device_pasid(struct iommu_domain domain, struct device dev,

		mutex_lock(&group->mutex);
		__iommu_remove_group_pasid(group, pasid, domain);
		WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
		xa_erase(&group->pasid_array, pasid);
		mutex_unlock(&group->mutex);
		}
		EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);

		/*
		* iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
		* @dev: the queried device
		* @pasid: the pasid of the device
		* @type: matched domain type, 0 for any match
		*
		* This is a variant of iommu_get_domain_for_dev(). It returns the existing
		* domain attached to pasid of a device. Callers must hold a lock around this
		* function, and both iommu_attach/detach_dev_pasid() whenever a domain of
		* type is being manipulated. This API does not internally resolve races with
		* attach/detach.
		*
		* Return: attached domain on success, NULL otherwise.
		*/
		struct iommu_domain iommu_get_domain_for_dev_pasid(struct device dev,
		ioasid_t pasid,
		unsigned int type)
		{
		/* Caller must be a probed driver on dev */
		struct iommu_group *group = dev->iommu_group;
		struct iommu_domain *domain;

		if (!group)
		return NULL;

		xa_lock(&group->pasid_array);
		domain = xa_load(&group->pasid_array, pasid);
		if (type && domain && domain->type != type)
		domain = ERR_PTR(-EBUSY);
		xa_unlock(&group->pasid_array);

		return domain;
		}
		EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);

		ioasid_t iommu_alloc_global_pasid(struct device *dev)
		{
		int ret;
		@@ -3476,3 +3443,137 @@ void iommu_free_global_pasid(ioasid_t pasid)
		ida_free(&iommu_global_pasid_ida, pasid);
		}
		EXPORT_SYMBOL_GPL(iommu_free_global_pasid);

		/**
		* iommu_attach_handle_get - Return the attach handle
		* @group: the iommu group that domain was attached to
		* @pasid: the pasid within the group
		* @type: matched domain type, 0 for any match
		*
		* Return handle or ERR_PTR(-ENOENT) on none, ERR_PTR(-EBUSY) on mismatch.
		*
		* Return the attach handle to the caller. The life cycle of an iommu attach
		* handle is from the time when the domain is attached to the time when the
		* domain is detached. Callers are required to synchronize the call of
		* iommu_attach_handle_get() with domain attachment and detachment. The attach
		* handle can only be used during its life cycle.
		*/
		struct iommu_attach_handle *
		iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
		{
		struct iommu_attach_handle *handle;

		xa_lock(&group->pasid_array);
		handle = xa_load(&group->pasid_array, pasid);
		if (!handle)
		handle = ERR_PTR(-ENOENT);
		else if (type && handle->domain->type != type)
		handle = ERR_PTR(-EBUSY);
		xa_unlock(&group->pasid_array);

		return handle;
		}
		EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, IOMMUFD_INTERNAL);

		/**
		* iommu_attach_group_handle - Attach an IOMMU domain to an IOMMU group
		* @domain: IOMMU domain to attach
		* @group: IOMMU group that will be attached
		* @handle: attach handle
		*
		* Returns 0 on success and error code on failure.
		*
		* This is a variant of iommu_attach_group(). It allows the caller to provide
		* an attach handle and use it when the domain is attached. This is currently
		* used by IOMMUFD to deliver the I/O page faults.
		*/
		int iommu_attach_group_handle(struct iommu_domain *domain,
		struct iommu_group *group,
		struct iommu_attach_handle *handle)
		{
		int ret;

		if (handle)
		handle->domain = domain;

		mutex_lock(&group->mutex);
		ret = xa_insert(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
		if (ret)
		goto err_unlock;

		ret = __iommu_attach_group(domain, group);
		if (ret)
		goto err_erase;
		mutex_unlock(&group->mutex);

		return 0;
		err_erase:
		xa_erase(&group->pasid_array, IOMMU_NO_PASID);
		err_unlock:
		mutex_unlock(&group->mutex);
		return ret;
		}
		EXPORT_SYMBOL_NS_GPL(iommu_attach_group_handle, IOMMUFD_INTERNAL);

		/**
		* iommu_detach_group_handle - Detach an IOMMU domain from an IOMMU group
		* @domain: IOMMU domain to attach
		* @group: IOMMU group that will be attached
		*
		* Detach the specified IOMMU domain from the specified IOMMU group.
		* It must be used in conjunction with iommu_attach_group_handle().
		*/
		void iommu_detach_group_handle(struct iommu_domain *domain,
		struct iommu_group *group)
		{
		mutex_lock(&group->mutex);
		__iommu_group_set_core_domain(group);
		xa_erase(&group->pasid_array, IOMMU_NO_PASID);
		mutex_unlock(&group->mutex);
		}
		EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, IOMMUFD_INTERNAL);

		/**
		* iommu_replace_group_handle - replace the domain that a group is attached to
		* @group: IOMMU group that will be attached to the new domain
		* @new_domain: new IOMMU domain to replace with
		* @handle: attach handle
		*
		* This is a variant of iommu_group_replace_domain(). It allows the caller to
		* provide an attach handle for the new domain and use it when the domain is
		* attached.
		*/
		int iommu_replace_group_handle(struct iommu_group *group,
		struct iommu_domain *new_domain,
		struct iommu_attach_handle *handle)
		{
		void *curr;
		int ret;

		if (!new_domain)
		return -EINVAL;

		mutex_lock(&group->mutex);
		if (handle) {
		ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
		if (ret)
		goto err_unlock;
		}

		ret = __iommu_group_set_domain(group, new_domain);
		if (ret)
		goto err_release;

		curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
		WARN_ON(xa_is_err(curr));

		mutex_unlock(&group->mutex);

		return 0;
		err_release:
		xa_release(&group->pasid_array, IOMMU_NO_PASID);
		err_unlock:
		mutex_unlock(&group->mutex);
		return ret;
		}
		EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, IOMMUFD_INTERNAL);