Commit 59b6c346 authored by Joerg Roedel's avatar Joerg Roedel
Browse files

Merge tag 'for-joerg' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd into core 

iommu shared branch with iommufd

The three dependent series on a shared branch:

- Change the iommufd fault handle into an always present hwpt handle in
  the domain

- Give iommufd its own SW_MSI implementation along with some IRQ layer
  rework

- Improvements to the handle attach API
parents 0ad2507d 5e9f822c

drivers/iommu/Kconfig

+0 −1
Original line number Diff line number Diff line
@@ -154,7 +154,6 @@ config IOMMU_DMA 
	select DMA_OPS_HELPERS

	select IOMMU_API

	select IOMMU_IOVA

	select IRQ_MSI_IOMMU

	select NEED_SG_DMA_LENGTH

	select NEED_SG_DMA_FLAGS if SWIOTLB

drivers/iommu/dma-iommu.c

+21 −44
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <linux/memremap.h>

#include <linux/mm.h>

#include <linux/mutex.h>

#include <linux/msi.h>

#include <linux/of_iommu.h>

#include <linux/pci.h>

#include <linux/scatterlist.h>
@@ -102,6 +103,9 @@ static int __init iommu_dma_forcedac_setup(char *str) 
}

early_param("iommu.forcedac", iommu_dma_forcedac_setup);



static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

			    phys_addr_t msi_addr);



/* Number of entries per flush queue */

#define IOVA_DEFAULT_FQ_SIZE	256

#define IOVA_SINGLE_FQ_SIZE	32768
@@ -398,6 +402,7 @@ int iommu_get_dma_cookie(struct iommu_domain *domain) 
		return -ENOMEM;



	mutex_init(&domain->iova_cookie->mutex);

	iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);

	return 0;

}
@@ -429,6 +434,7 @@ int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) 


	cookie->msi_iova = base;

	domain->iova_cookie = cookie;

	iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);

	return 0;

}

EXPORT_SYMBOL(iommu_get_msi_cookie);
@@ -443,6 +449,11 @@ void iommu_put_dma_cookie(struct iommu_domain *domain) 
	struct iommu_dma_cookie *cookie = domain->iova_cookie;

	struct iommu_dma_msi_page *msi, *tmp;



#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)

	if (domain->sw_msi != iommu_dma_sw_msi)

		return;

#endif



	if (!cookie)

		return;
@@ -1800,60 +1811,26 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, 
	return NULL;

}



/**

 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain

 * @desc: MSI descriptor, will store the MSI page

 * @msi_addr: MSI target address to be mapped

 *

 * Return: 0 on success or negative error code if the mapping failed.

 */

int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)

static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

			    phys_addr_t msi_addr)

{

	struct device *dev = msi_desc_to_dev(desc);

	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);

	struct iommu_dma_msi_page *msi_page;

	static DEFINE_MUTEX(msi_prepare_lock); /* see below */

	const struct iommu_dma_msi_page *msi_page;



	if (!domain || !domain->iova_cookie) {

		desc->iommu_cookie = NULL;

	if (!domain->iova_cookie) {

		msi_desc_set_iommu_msi_iova(desc, 0, 0);

		return 0;

	}



	/*

	 * In fact the whole prepare operation should already be serialised by

	 * irq_domain_mutex further up the callchain, but that's pretty subtle

	 * on its own, so consider this locking as failsafe documentation...

	 */

	mutex_lock(&msi_prepare_lock);

	iommu_group_mutex_assert(dev);

	msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);

	mutex_unlock(&msi_prepare_lock);



	msi_desc_set_iommu_cookie(desc, msi_page);



	if (!msi_page)

		return -ENOMEM;

	return 0;

}



/**

 * iommu_dma_compose_msi_msg() - Apply translation to an MSI message

 * @desc: MSI descriptor prepared by iommu_dma_prepare_msi()

 * @msg: MSI message containing target physical address

 */

void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg)

{

	struct device *dev = msi_desc_to_dev(desc);

	const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);

	const struct iommu_dma_msi_page *msi_page;



	msi_page = msi_desc_get_iommu_cookie(desc);



	if (!domain || !domain->iova_cookie || WARN_ON(!msi_page))

		return;



	msg->address_hi = upper_32_bits(msi_page->iova);

	msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1;

	msg->address_lo += lower_32_bits(msi_page->iova);

	msi_desc_set_iommu_msi_iova(

		desc, msi_page->iova,

		ilog2(cookie_msi_granule(domain->iova_cookie)));

	return 0;

}



static int iommu_dma_init(void)

drivers/iommu/iommu-priv.h

+0 −3
Original line number Diff line number Diff line
@@ -24,9 +24,6 @@ static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwsp 
	return iommu_ops_from_fwnode(fwspec ? fwspec->iommu_fwnode : NULL);

}



int iommu_group_replace_domain(struct iommu_group *group,

			       struct iommu_domain *new_domain);



int iommu_device_register_bus(struct iommu_device *iommu,

			      const struct iommu_ops *ops,

			      const struct bus_type *bus,

drivers/iommu/iommu.c

+106 −58
Original line number Diff line number Diff line
@@ -45,6 +45,9 @@ static unsigned int iommu_def_domain_type __read_mostly; 
static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);

static u32 iommu_cmd_line __read_mostly;



/* Tags used with xa_tag_pointer() in group->pasid_array */

enum { IOMMU_PASID_ARRAY_DOMAIN = 0, IOMMU_PASID_ARRAY_HANDLE = 1 };



struct iommu_group {

	struct kobject kobj;

	struct kobject *devices_kobj;
@@ -2147,6 +2150,17 @@ struct iommu_domain *iommu_get_dma_domain(struct device *dev) 
	return dev->iommu_group->default_domain;

}



static void *iommu_make_pasid_array_entry(struct iommu_domain *domain,

					  struct iommu_attach_handle *handle)

{

	if (handle) {

		handle->domain = domain;

		return xa_tag_pointer(handle, IOMMU_PASID_ARRAY_HANDLE);

	}



	return xa_tag_pointer(domain, IOMMU_PASID_ARRAY_DOMAIN);

}



static int __iommu_attach_group(struct iommu_domain *domain,

				struct iommu_group *group)

{
@@ -2187,32 +2201,6 @@ int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 
}

EXPORT_SYMBOL_GPL(iommu_attach_group);



/**

 * iommu_group_replace_domain - replace the domain that a group is attached to

 * @group: IOMMU group that will be attached to the new domain

 * @new_domain: new IOMMU domain to replace with

 *

 * This API allows the group to switch domains without being forced to go to

 * the blocking domain in-between.

 *

 * If the currently attached domain is a core domain (e.g. a default_domain),

 * it will act just like the iommu_attach_group().

 */

int iommu_group_replace_domain(struct iommu_group *group,

			       struct iommu_domain *new_domain)

{

	int ret;



	if (!new_domain)

		return -EINVAL;



	mutex_lock(&group->mutex);

	ret = __iommu_group_set_domain(group, new_domain);

	mutex_unlock(&group->mutex);

	return ret;

}

EXPORT_SYMBOL_NS_GPL(iommu_group_replace_domain, "IOMMUFD_INTERNAL");



static int __iommu_device_set_domain(struct iommu_group *group,

				     struct device *dev,

				     struct iommu_domain *new_domain,
@@ -3374,6 +3362,7 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, 
	struct iommu_group *group = dev->iommu_group;

	struct group_device *device;

	const struct iommu_ops *ops;

	void *entry;

	int ret;



	if (!group)
@@ -3397,16 +3386,31 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, 
		}

	}



	if (handle)

		handle->domain = domain;

	entry = iommu_make_pasid_array_entry(domain, handle);



	ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);

	/*

	 * Entry present is a failure case. Use xa_insert() instead of

	 * xa_reserve().

	 */

	ret = xa_insert(&group->pasid_array, pasid, XA_ZERO_ENTRY, GFP_KERNEL);

	if (ret)

		goto out_unlock;



	ret = __iommu_set_group_pasid(domain, group, pasid);

	if (ret)

		xa_erase(&group->pasid_array, pasid);

	if (ret) {

		xa_release(&group->pasid_array, pasid);

		goto out_unlock;

	}



	/*

	 * The xa_insert() above reserved the memory, and the group->mutex is

	 * held, this cannot fail. The new domain cannot be visible until the

	 * operation succeeds as we cannot tolerate PRIs becoming concurrently

	 * queued and then failing attach.

	 */

	WARN_ON(xa_is_err(xa_store(&group->pasid_array,

				   pasid, entry, GFP_KERNEL)));



out_unlock:

	mutex_unlock(&group->mutex);

	return ret;
@@ -3480,13 +3484,17 @@ struct iommu_attach_handle * 
iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)

{

	struct iommu_attach_handle *handle;

	void *entry;



	xa_lock(&group->pasid_array);

	handle = xa_load(&group->pasid_array, pasid);

	if (!handle)

	entry = xa_load(&group->pasid_array, pasid);

	if (!entry || xa_pointer_tag(entry) != IOMMU_PASID_ARRAY_HANDLE) {

		handle = ERR_PTR(-ENOENT);

	else if (type && handle->domain->type != type)

	} else {

		handle = xa_untag_pointer(entry);

		if (type && handle->domain->type != type)

			handle = ERR_PTR(-EBUSY);

	}

	xa_unlock(&group->pasid_array);



	return handle;
@@ -3509,25 +3517,35 @@ int iommu_attach_group_handle(struct iommu_domain *domain, 
			      struct iommu_group *group,

			      struct iommu_attach_handle *handle)

{

	void *entry;

	int ret;



	if (handle)

		handle->domain = domain;

	if (!handle)

		return -EINVAL;



	mutex_lock(&group->mutex);

	ret = xa_insert(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);

	entry = iommu_make_pasid_array_entry(domain, handle);

	ret = xa_insert(&group->pasid_array,

			IOMMU_NO_PASID, XA_ZERO_ENTRY, GFP_KERNEL);

	if (ret)

		goto err_unlock;

		goto out_unlock;



	ret = __iommu_attach_group(domain, group);

	if (ret)

		goto err_erase;

	mutex_unlock(&group->mutex);

	if (ret) {

		xa_release(&group->pasid_array, IOMMU_NO_PASID);

		goto out_unlock;

	}



	return 0;

err_erase:

	xa_erase(&group->pasid_array, IOMMU_NO_PASID);

err_unlock:

	/*

	 * The xa_insert() above reserved the memory, and the group->mutex is

	 * held, this cannot fail. The new domain cannot be visible until the

	 * operation succeeds as we cannot tolerate PRIs becoming concurrently

	 * queued and then failing attach.

	 */

	WARN_ON(xa_is_err(xa_store(&group->pasid_array,

				   IOMMU_NO_PASID, entry, GFP_KERNEL)));



out_unlock:

	mutex_unlock(&group->mutex);

	return ret;

}
@@ -3557,33 +3575,34 @@ EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, "IOMMUFD_INTERNAL"); 
 * @new_domain: new IOMMU domain to replace with

 * @handle: attach handle

 *

 * This is a variant of iommu_group_replace_domain(). It allows the caller to

 * provide an attach handle for the new domain and use it when the domain is

 * attached.

 * This API allows the group to switch domains without being forced to go to

 * the blocking domain in-between. It allows the caller to provide an attach

 * handle for the new domain and use it when the domain is attached.

 *

 * If the currently attached domain is a core domain (e.g. a default_domain),

 * it will act just like the iommu_attach_group_handle().

 */

int iommu_replace_group_handle(struct iommu_group *group,

			       struct iommu_domain *new_domain,

			       struct iommu_attach_handle *handle)

{

	void *curr;

	void *curr, *entry;

	int ret;



	if (!new_domain)

	if (!new_domain || !handle)

		return -EINVAL;



	mutex_lock(&group->mutex);

	if (handle) {

	entry = iommu_make_pasid_array_entry(new_domain, handle);

	ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);

	if (ret)

		goto err_unlock;

		handle->domain = new_domain;

	}



	ret = __iommu_group_set_domain(group, new_domain);

	if (ret)

		goto err_release;



	curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);

	curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, entry, GFP_KERNEL);

	WARN_ON(xa_is_err(curr));



	mutex_unlock(&group->mutex);
@@ -3596,3 +3615,32 @@ int iommu_replace_group_handle(struct iommu_group *group, 
	return ret;

}

EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");



#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)

/**

 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain

 * @desc: MSI descriptor, will store the MSI page

 * @msi_addr: MSI target address to be mapped

 *

 * The implementation of sw_msi() should take msi_addr and map it to

 * an IOVA in the domain and call msi_desc_set_iommu_msi_iova() with the

 * mapping information.

 *

 * Return: 0 on success or negative error code if the mapping failed.

 */

int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)

{

	struct device *dev = msi_desc_to_dev(desc);

	struct iommu_group *group = dev->iommu_group;

	int ret = 0;



	if (!group)

		return 0;



	mutex_lock(&group->mutex);

	if (group->domain && group->domain->sw_msi)

		ret = group->domain->sw_msi(group->domain, desc, msi_addr);

	mutex_unlock(&group->mutex);

	return ret;

}

#endif /* CONFIG_IRQ_MSI_IOMMU */

drivers/iommu/iommufd/device.c

+244 −22
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#include <linux/iommufd.h>

#include <linux/slab.h>

#include <uapi/linux/iommufd.h>

#include <linux/msi.h>



#include "../iommu-priv.h"

#include "io_pagetable.h"
@@ -293,36 +294,152 @@ u32 iommufd_device_to_id(struct iommufd_device *idev) 
}

EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, "IOMMUFD");



static int iommufd_group_setup_msi(struct iommufd_group *igroup,

				   struct iommufd_hwpt_paging *hwpt_paging)

/*

 * Get a iommufd_sw_msi_map for the msi physical address requested by the irq

 * layer. The mapping to IOVA is global to the iommufd file descriptor, every

 * domain that is attached to a device using the same MSI parameters will use

 * the same IOVA.

 */

static __maybe_unused struct iommufd_sw_msi_map *

iommufd_sw_msi_get_map(struct iommufd_ctx *ictx, phys_addr_t msi_addr,

		       phys_addr_t sw_msi_start)

{

	phys_addr_t sw_msi_start = igroup->sw_msi_start;

	struct iommufd_sw_msi_map *cur;

	unsigned int max_pgoff = 0;



	lockdep_assert_held(&ictx->sw_msi_lock);



	list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) {

		if (cur->sw_msi_start != sw_msi_start)

			continue;

		max_pgoff = max(max_pgoff, cur->pgoff + 1);

		if (cur->msi_addr == msi_addr)

			return cur;

	}



	if (ictx->sw_msi_id >=

	    BITS_PER_BYTE * sizeof_field(struct iommufd_sw_msi_maps, bitmap))

		return ERR_PTR(-EOVERFLOW);



	cur = kzalloc(sizeof(*cur), GFP_KERNEL);

	if (!cur)

		return ERR_PTR(-ENOMEM);



	cur->sw_msi_start = sw_msi_start;

	cur->msi_addr = msi_addr;

	cur->pgoff = max_pgoff;

	cur->id = ictx->sw_msi_id++;

	list_add_tail(&cur->sw_msi_item, &ictx->sw_msi_list);

	return cur;

}



static int iommufd_sw_msi_install(struct iommufd_ctx *ictx,

				  struct iommufd_hwpt_paging *hwpt_paging,

				  struct iommufd_sw_msi_map *msi_map)

{

	unsigned long iova;



	lockdep_assert_held(&ictx->sw_msi_lock);



	iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;

	if (!test_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap)) {

		int rc;



		rc = iommu_map(hwpt_paging->common.domain, iova,

			       msi_map->msi_addr, PAGE_SIZE,

			       IOMMU_WRITE | IOMMU_READ | IOMMU_MMIO,

			       GFP_KERNEL_ACCOUNT);

		if (rc)

			return rc;

		__set_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap);

	}

	return 0;

}



/*

	 * If the IOMMU driver gives a IOMMU_RESV_SW_MSI then it is asking us to

	 * call iommu_get_msi_cookie() on its behalf. This is necessary to setup

	 * the MSI window so iommu_dma_prepare_msi() can install pages into our

	 * domain after request_irq(). If it is not done interrupts will not

	 * work on this domain.

	 *

	 * FIXME: This is conceptually broken for iommufd since we want to allow

	 * userspace to change the domains, eg switch from an identity IOAS to a

	 * DMA IOAS. There is currently no way to create a MSI window that

	 * matches what the IRQ layer actually expects in a newly created

	 * domain.

 * Called by the irq code if the platform translates the MSI address through the

 * IOMMU. msi_addr is the physical address of the MSI page. iommufd will

 * allocate a fd global iova for the physical page that is the same on all

 * domains and devices.

 */

#ifdef CONFIG_IRQ_MSI_IOMMU

int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

		   phys_addr_t msi_addr)

{

	struct device *dev = msi_desc_to_dev(desc);

	struct iommufd_hwpt_paging *hwpt_paging;

	struct iommu_attach_handle *raw_handle;

	struct iommufd_attach_handle *handle;

	struct iommufd_sw_msi_map *msi_map;

	struct iommufd_ctx *ictx;

	unsigned long iova;

	int rc;



	/*

	 * It is safe to call iommu_attach_handle_get() here because the iommu

	 * core code invokes this under the group mutex which also prevents any

	 * change of the attach handle for the duration of this function.

	 */

	if (sw_msi_start != PHYS_ADDR_MAX && !hwpt_paging->msi_cookie) {

		rc = iommu_get_msi_cookie(hwpt_paging->common.domain,

					  sw_msi_start);

	iommu_group_mutex_assert(dev);



	raw_handle =

		iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);

	if (IS_ERR(raw_handle))

		return 0;

	hwpt_paging = find_hwpt_paging(domain->iommufd_hwpt);



	handle = to_iommufd_handle(raw_handle);

	/* No IOMMU_RESV_SW_MSI means no change to the msi_msg */

	if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX)

		return 0;



	ictx = handle->idev->ictx;

	guard(mutex)(&ictx->sw_msi_lock);

	/*

	 * The input msi_addr is the exact byte offset of the MSI doorbell, we

	 * assume the caller has checked that it is contained with a MMIO region

	 * that is secure to map at PAGE_SIZE.

	 */

	msi_map = iommufd_sw_msi_get_map(handle->idev->ictx,

					 msi_addr & PAGE_MASK,

					 handle->idev->igroup->sw_msi_start);

	if (IS_ERR(msi_map))

		return PTR_ERR(msi_map);



	rc = iommufd_sw_msi_install(ictx, hwpt_paging, msi_map);

	if (rc)

		return rc;

	__set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap);



	iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;

	msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT);

	return 0;

}

#endif



static int iommufd_group_setup_msi(struct iommufd_group *igroup,

				   struct iommufd_hwpt_paging *hwpt_paging)

{

	struct iommufd_ctx *ictx = igroup->ictx;

	struct iommufd_sw_msi_map *cur;



	if (igroup->sw_msi_start == PHYS_ADDR_MAX)

		return 0;



	/*

		 * iommu_get_msi_cookie() can only be called once per domain,

		 * it returns -EBUSY on later calls.

	 * Install all the MSI pages the device has been using into the domain

	 */

		hwpt_paging->msi_cookie = true;

	guard(mutex)(&ictx->sw_msi_lock);

	list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) {

		int rc;



		if (cur->sw_msi_start != igroup->sw_msi_start ||

		    !test_bit(cur->id, igroup->required_sw_msi.bitmap))

			continue;



		rc = iommufd_sw_msi_install(ictx, hwpt_paging, cur);

		if (rc)

			return rc;

	}

	return 0;

}
@@ -352,6 +469,111 @@ iommufd_device_attach_reserved_iova(struct iommufd_device *idev, 
	return 0;

}



/* The device attach/detach/replace helpers for attach_handle */



static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,

				      struct iommufd_device *idev)

{

	struct iommufd_attach_handle *handle;

	int rc;



	lockdep_assert_held(&idev->igroup->lock);



	handle = kzalloc(sizeof(*handle), GFP_KERNEL);

	if (!handle)

		return -ENOMEM;



	if (hwpt->fault) {

		rc = iommufd_fault_iopf_enable(idev);

		if (rc)

			goto out_free_handle;

	}



	handle->idev = idev;

	rc = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,

				       &handle->handle);

	if (rc)

		goto out_disable_iopf;



	return 0;



out_disable_iopf:

	if (hwpt->fault)

		iommufd_fault_iopf_disable(idev);

out_free_handle:

	kfree(handle);

	return rc;

}



static struct iommufd_attach_handle *

iommufd_device_get_attach_handle(struct iommufd_device *idev)

{

	struct iommu_attach_handle *handle;



	lockdep_assert_held(&idev->igroup->lock);



	handle =

		iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);

	if (IS_ERR(handle))

		return NULL;

	return to_iommufd_handle(handle);

}



static void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,

				       struct iommufd_device *idev)

{

	struct iommufd_attach_handle *handle;



	handle = iommufd_device_get_attach_handle(idev);

	iommu_detach_group_handle(hwpt->domain, idev->igroup->group);

	if (hwpt->fault) {

		iommufd_auto_response_faults(hwpt, handle);

		iommufd_fault_iopf_disable(idev);

	}

	kfree(handle);

}



static int iommufd_hwpt_replace_device(struct iommufd_device *idev,

				       struct iommufd_hw_pagetable *hwpt,

				       struct iommufd_hw_pagetable *old)

{

	struct iommufd_attach_handle *handle, *old_handle =

		iommufd_device_get_attach_handle(idev);

	int rc;



	handle = kzalloc(sizeof(*handle), GFP_KERNEL);

	if (!handle)

		return -ENOMEM;



	if (hwpt->fault && !old->fault) {

		rc = iommufd_fault_iopf_enable(idev);

		if (rc)

			goto out_free_handle;

	}



	handle->idev = idev;

	rc = iommu_replace_group_handle(idev->igroup->group, hwpt->domain,

					&handle->handle);

	if (rc)

		goto out_disable_iopf;



	if (old->fault) {

		iommufd_auto_response_faults(hwpt, old_handle);

		if (!hwpt->fault)

			iommufd_fault_iopf_disable(idev);

	}

	kfree(old_handle);



	return 0;



out_disable_iopf:

	if (hwpt->fault && !old->fault)

		iommufd_fault_iopf_disable(idev);

out_free_handle:

	kfree(handle);

	return rc;

}



int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,

				struct iommufd_device *idev)

{