Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd (bee0e776) · Commits · git / linux-net

drivers/iommu/iommufd/device.c

+7 −7

Original line number	Diff line number	Diff line
		@@ -571,7 +571,7 @@ iommufd_device_auto_get_domain(struct iommufd_device *idev,
		continue;
		destroy_hwpt = (*do_attach)(idev, hwpt);
		if (IS_ERR(destroy_hwpt)) {
		iommufd_put_object(&hwpt->obj);
		iommufd_put_object(idev->ictx, &hwpt->obj);
		/*
		* -EINVAL means the domain is incompatible with the
		* device. Other error codes should propagate to
		@@ -583,7 +583,7 @@ iommufd_device_auto_get_domain(struct iommufd_device *idev,
		goto out_unlock;
		}
		*pt_id = hwpt->obj.id;
		iommufd_put_object(&hwpt->obj);
		iommufd_put_object(idev->ictx, &hwpt->obj);
		goto out_unlock;
		}

		@@ -652,7 +652,7 @@ static int iommufd_device_change_pt(struct iommufd_device idev, u32 pt_id,
		destroy_hwpt = ERR_PTR(-EINVAL);
		goto out_put_pt_obj;
		}
		iommufd_put_object(pt_obj);
		iommufd_put_object(idev->ictx, pt_obj);

		/* This destruction has to be after we unlock everything */
		if (destroy_hwpt)
		@@ -660,7 +660,7 @@ static int iommufd_device_change_pt(struct iommufd_device idev, u32 pt_id,
		return 0;

		out_put_pt_obj:
		iommufd_put_object(pt_obj);
		iommufd_put_object(idev->ictx, pt_obj);
		return PTR_ERR(destroy_hwpt);
		}

		@@ -792,7 +792,7 @@ static int iommufd_access_change_ioas_id(struct iommufd_access *access, u32 id)
		if (IS_ERR(ioas))
		return PTR_ERR(ioas);
		rc = iommufd_access_change_ioas(access, ioas);
		iommufd_put_object(&ioas->obj);
		iommufd_put_object(access->ictx, &ioas->obj);
		return rc;
		}

		@@ -941,7 +941,7 @@ void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,

		access->ops->unmap(access->data, iova, length);

		iommufd_put_object(&access->obj);
		iommufd_put_object(access->ictx, &access->obj);
		xa_lock(&ioas->iopt.access_list);
		}
		xa_unlock(&ioas->iopt.access_list);
		@@ -1243,6 +1243,6 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
		out_free:
		kfree(data);
		out_put:
		iommufd_put_object(&idev->obj);
		iommufd_put_object(ucmd->ictx, &idev->obj);
		return rc;
		}

drivers/iommu/iommufd/hw_pagetable.c

+4 −4

Original line number	Diff line number	Diff line
		@@ -318,9 +318,9 @@ int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
		if (ioas)
		mutex_unlock(&ioas->mutex);
		out_put_pt:
		iommufd_put_object(pt_obj);
		iommufd_put_object(ucmd->ictx, pt_obj);
		out_put_idev:
		iommufd_put_object(&idev->obj);
		iommufd_put_object(ucmd->ictx, &idev->obj);
		return rc;
		}

		@@ -345,7 +345,7 @@ int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd)
		rc = iopt_set_dirty_tracking(&ioas->iopt, hwpt_paging->common.domain,
		enable);

		iommufd_put_object(&hwpt_paging->common.obj);
		iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
		return rc;
		}

		@@ -368,6 +368,6 @@ int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd)
		rc = iopt_read_and_clear_dirty_data(
		&ioas->iopt, hwpt_paging->common.domain, cmd->flags, cmd);

		iommufd_put_object(&hwpt_paging->common.obj);
		iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
		return rc;
		}

drivers/iommu/iommufd/ioas.c

+7 −7

Original line number	Diff line number	Diff line
		@@ -105,7 +105,7 @@ int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd)
		rc = -EMSGSIZE;
		out_put:
		up_read(&ioas->iopt.iova_rwsem);
		iommufd_put_object(&ioas->obj);
		iommufd_put_object(ucmd->ictx, &ioas->obj);
		return rc;
		}

		@@ -175,7 +175,7 @@ int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd)
		interval_tree_remove(node, &allowed_iova);
		kfree(container_of(node, struct iopt_allowed, node));
		}
		iommufd_put_object(&ioas->obj);
		iommufd_put_object(ucmd->ictx, &ioas->obj);
		return rc;
		}

		@@ -228,7 +228,7 @@ int iommufd_ioas_map(struct iommufd_ucmd *ucmd)
		cmd->iova = iova;
		rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
		out_put:
		iommufd_put_object(&ioas->obj);
		iommufd_put_object(ucmd->ictx, &ioas->obj);
		return rc;
		}

		@@ -258,7 +258,7 @@ int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
		return PTR_ERR(src_ioas);
		rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length,
		&pages_list);
		iommufd_put_object(&src_ioas->obj);
		iommufd_put_object(ucmd->ictx, &src_ioas->obj);
		if (rc)
		return rc;

		@@ -279,7 +279,7 @@ int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
		cmd->dst_iova = iova;
		rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
		out_put_dst:
		iommufd_put_object(&dst_ioas->obj);
		iommufd_put_object(ucmd->ictx, &dst_ioas->obj);
		out_pages:
		iopt_free_pages_list(&pages_list);
		return rc;
		@@ -315,7 +315,7 @@ int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
		rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));

		out_put:
		iommufd_put_object(&ioas->obj);
		iommufd_put_object(ucmd->ictx, &ioas->obj);
		return rc;
		}

		@@ -393,6 +393,6 @@ int iommufd_ioas_option(struct iommufd_ucmd *ucmd)
		rc = -EOPNOTSUPP;
		}

		iommufd_put_object(&ioas->obj);
		iommufd_put_object(ucmd->ictx, &ioas->obj);
		return rc;
		}

drivers/iommu/iommufd/iommufd_private.h

+57 −13

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ struct iommufd_ctx {
		struct file *file;
		struct xarray objects;
		struct xarray groups;
		wait_queue_head_t destroy_wait;

		u8 account_mode;
		/* Compatibility with VFIO no iommu */
		@@ -135,7 +136,7 @@ enum iommufd_object_type {

		/* Base struct for all objects with a userspace ID handle. */
		struct iommufd_object {
		struct rw_semaphore destroy_rwsem;
		refcount_t shortterm_users;
		refcount_t users;
		enum iommufd_object_type type;
		unsigned int id;
		@@ -143,10 +144,15 @@ struct iommufd_object {

		static inline bool iommufd_lock_obj(struct iommufd_object *obj)
		{
		if (!down_read_trylock(&obj->destroy_rwsem))
		if (!refcount_inc_not_zero(&obj->users))
		return false;
		if (!refcount_inc_not_zero(&obj->users)) {
		up_read(&obj->destroy_rwsem);
		if (!refcount_inc_not_zero(&obj->shortterm_users)) {
		/*
		* If the caller doesn't already have a ref on obj this must be
		* called under the xa_lock. Otherwise the caller is holding a
		* ref on users. Thus it cannot be one before this decrement.
		*/
		refcount_dec(&obj->users);
		return false;
		}
		return true;
		@@ -154,10 +160,16 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj)

		struct iommufd_object iommufd_get_object(struct iommufd_ctx ictx, u32 id,
		enum iommufd_object_type type);
		static inline void iommufd_put_object(struct iommufd_object *obj)
		static inline void iommufd_put_object(struct iommufd_ctx *ictx,
		struct iommufd_object *obj)
		{
		/*
		* Users first, then shortterm so that REMOVE_WAIT_SHORTTERM never sees
		* a spurious !0 users with a 0 shortterm_users.
		*/
		refcount_dec(&obj->users);
		up_read(&obj->destroy_rwsem);
		if (refcount_dec_and_test(&obj->shortterm_users))
		wake_up_interruptible_all(&ictx->destroy_wait);
		}

		void iommufd_object_abort(struct iommufd_ctx ictx, struct iommufd_object obj);
		@@ -165,17 +177,49 @@ void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
		struct iommufd_object *obj);
		void iommufd_object_finalize(struct iommufd_ctx *ictx,
		struct iommufd_object *obj);
		void __iommufd_object_destroy_user(struct iommufd_ctx *ictx,
		struct iommufd_object *obj, bool allow_fail);

		enum {
		REMOVE_WAIT_SHORTTERM = 1,
		};
		int iommufd_object_remove(struct iommufd_ctx *ictx,
		struct iommufd_object *to_destroy, u32 id,
		unsigned int flags);

		/*
		* The caller holds a users refcount and wants to destroy the object. At this
		* point the caller has no shortterm_users reference and at least the xarray
		* will be holding one.
		*/
		static inline void iommufd_object_destroy_user(struct iommufd_ctx *ictx,
		struct iommufd_object *obj)
		{
		__iommufd_object_destroy_user(ictx, obj, false);
		int ret;

		ret = iommufd_object_remove(ictx, obj, obj->id, REMOVE_WAIT_SHORTTERM);

		/*
		* If there is a bug and we couldn't destroy the object then we did put
		* back the caller's users refcount and will eventually try to free it
		* again during close.
		*/
		WARN_ON(ret);
		}
		static inline void iommufd_object_deref_user(struct iommufd_ctx *ictx,

		/*
		* The HWPT allocated by autodomains is used in possibly many devices and
		* is automatically destroyed when its refcount reaches zero.
		*
		* If userspace uses the HWPT manually, even for a short term, then it will
		* disrupt this refcounting and the auto-free in the kernel will not work.
		* Userspace that tries to use the automatically allocated HWPT must be careful
		* to ensure that it is consistently destroyed, eg by not racing accesses
		* and by not attaching an automatic HWPT to a device manually.
		*/
		static inline void
		iommufd_object_put_and_try_destroy(struct iommufd_ctx *ictx,
		struct iommufd_object *obj)
		{
		__iommufd_object_destroy_user(ictx, obj, true);
		iommufd_object_remove(ictx, obj, obj->id, 0);
		}

		struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx,
		@@ -311,7 +355,7 @@ static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
		lockdep_assert_not_held(&hwpt_paging->ioas->mutex);

		if (hwpt_paging->auto_domain) {
		iommufd_object_deref_user(ictx, &hwpt->obj);
		iommufd_object_put_and_try_destroy(ictx, &hwpt->obj);
		return;
		}
		}

drivers/iommu/iommufd/main.c

+80 −66

Original line number	Diff line number	Diff line
		@@ -33,7 +33,6 @@ struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx,
		size_t size,
		enum iommufd_object_type type)
		{
		static struct lock_class_key obj_keys[IOMMUFD_OBJ_MAX];
		struct iommufd_object *obj;
		int rc;

		@@ -41,15 +40,8 @@ struct iommufd_object _iommufd_object_alloc(struct iommufd_ctx ictx,
		if (!obj)
		return ERR_PTR(-ENOMEM);
		obj->type = type;
		/*
		* In most cases the destroy_rwsem is obtained with try so it doesn't
		* interact with lockdep, however on destroy we have to sleep. This
		* means if we have to destroy an object while holding a get on another
		* object it triggers lockdep. Using one locking class per object type
		* is a simple and reasonable way to avoid this.
		*/
		__init_rwsem(&obj->destroy_rwsem, "iommufd_object::destroy_rwsem",
		&obj_keys[type]);
		/* Starts out bias'd by 1 until it is removed from the xarray */
		refcount_set(&obj->shortterm_users, 1);
		refcount_set(&obj->users, 1);

		/*
		@@ -129,92 +121,113 @@ struct iommufd_object iommufd_get_object(struct iommufd_ctx ictx, u32 id,
		return obj;
		}

		static int iommufd_object_dec_wait_shortterm(struct iommufd_ctx *ictx,
		struct iommufd_object *to_destroy)
		{
		if (refcount_dec_and_test(&to_destroy->shortterm_users))
		return 0;

		if (wait_event_timeout(ictx->destroy_wait,
		refcount_read(&to_destroy->shortterm_users) ==
		0,
		msecs_to_jiffies(10000)))
		return 0;

		pr_crit("Time out waiting for iommufd object to become free\n");
		refcount_inc(&to_destroy->shortterm_users);
		return -EBUSY;
		}

		/*
		* Remove the given object id from the xarray if the only reference to the
		* object is held by the xarray. The caller must call ops destroy().
		* object is held by the xarray.
		*/
		static struct iommufd_object iommufd_object_remove(struct iommufd_ctx ictx,
		u32 id, bool extra_put)
		int iommufd_object_remove(struct iommufd_ctx *ictx,
		struct iommufd_object *to_destroy, u32 id,
		unsigned int flags)
		{
		struct iommufd_object *obj;
		XA_STATE(xas, &ictx->objects, id);
		bool zerod_shortterm = false;
		int ret;

		xa_lock(&ictx->objects);
		obj = xas_load(&xas);
		if (xa_is_zero(obj) \|\| !obj) {
		obj = ERR_PTR(-ENOENT);
		goto out_xa;
		/*
		* The purpose of the shortterm_users is to ensure deterministic
		* destruction of objects used by external drivers and destroyed by this
		* function. Any temporary increment of the refcount must increment
		* shortterm_users, such as during ioctl execution.
		*/
		if (flags & REMOVE_WAIT_SHORTTERM) {
		ret = iommufd_object_dec_wait_shortterm(ictx, to_destroy);
		if (ret) {
		/*
		* We have a bug. Put back the callers reference and
		* defer cleaning this object until close.
		*/
		refcount_dec(&to_destroy->users);
		return ret;
		}
		zerod_shortterm = true;
		}

		xa_lock(&ictx->objects);
		obj = xas_load(&xas);
		if (to_destroy) {
		/*
		* If the caller is holding a ref on obj we put it here under the
		* spinlock.
		* If the caller is holding a ref on obj we put it here under
		* the spinlock.
		*/
		if (extra_put)
		refcount_dec(&obj->users);

		if (WARN_ON(obj != to_destroy)) {
		ret = -ENOENT;
		goto err_xa;
		}
		} else if (xa_is_zero(obj) \|\| !obj) {
		ret = -ENOENT;
		goto err_xa;
		}

		if (!refcount_dec_if_one(&obj->users)) {
		obj = ERR_PTR(-EBUSY);
		goto out_xa;
		ret = -EBUSY;
		goto err_xa;
		}

		xas_store(&xas, NULL);
		if (ictx->vfio_ioas == container_of(obj, struct iommufd_ioas, obj))
		ictx->vfio_ioas = NULL;

		out_xa:
		xa_unlock(&ictx->objects);

		/* The returned object reference count is zero */
		return obj;
		}

		/*
		* The caller holds a users refcount and wants to destroy the object. Returns
		* true if the object was destroyed. In all cases the caller no longer has a
		* reference on obj.
		*/
		void __iommufd_object_destroy_user(struct iommufd_ctx *ictx,
		struct iommufd_object *obj, bool allow_fail)
		{
		struct iommufd_object *ret;

		/*
		* The purpose of the destroy_rwsem is to ensure deterministic
		* destruction of objects used by external drivers and destroyed by this
		* function. Any temporary increment of the refcount must hold the read
		* side of this, such as during ioctl execution.
		*/
		down_write(&obj->destroy_rwsem);
		ret = iommufd_object_remove(ictx, obj->id, true);
		up_write(&obj->destroy_rwsem);

		if (allow_fail && IS_ERR(ret))
		return;

		/*
		* If there is a bug and we couldn't destroy the object then we did put
		* back the caller's refcount and will eventually try to free it again
		* during close.
		* Since users is zero any positive users_shortterm must be racing
		* iommufd_put_object(), or we have a bug.
		*/
		if (WARN_ON(IS_ERR(ret)))
		return;
		if (!zerod_shortterm) {
		ret = iommufd_object_dec_wait_shortterm(ictx, obj);
		if (WARN_ON(ret))
		return ret;
		}

		iommufd_object_ops[obj->type].destroy(obj);
		kfree(obj);
		return 0;

		err_xa:
		if (zerod_shortterm) {
		/* Restore the xarray owned reference */
		refcount_set(&obj->shortterm_users, 1);
		}
		xa_unlock(&ictx->objects);

		/* The returned object reference count is zero */
		return ret;
		}

		static int iommufd_destroy(struct iommufd_ucmd *ucmd)
		{
		struct iommu_destroy *cmd = ucmd->cmd;
		struct iommufd_object *obj;

		obj = iommufd_object_remove(ucmd->ictx, cmd->id, false);
		if (IS_ERR(obj))
		return PTR_ERR(obj);
		iommufd_object_ops[obj->type].destroy(obj);
		kfree(obj);
		return 0;
		return iommufd_object_remove(ucmd->ictx, NULL, cmd->id, 0);
		}

		static int iommufd_fops_open(struct inode inode, struct file filp)
		@@ -238,6 +251,7 @@ static int iommufd_fops_open(struct inode inode, struct file filp)
		xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 \| XA_FLAGS_ACCOUNT);
		xa_init(&ictx->groups);
		ictx->file = filp;
		init_waitqueue_head(&ictx->destroy_wait);
		filp->private_data = ictx;
		return 0;
		}