Commit 7a9164dc authored by Paolo Bonzini's avatar Paolo Bonzini
Browse files

Merge tag 'kvm-x86-vcpu_array-6.14' of https://github.com/kvm-x86/linux into HEAD 

KVM vcpu_array fixes and cleanups for 6.14:

 - Explicitly verify the target vCPU is online in kvm_get_vcpu() to fix a bug
   where KVM would return a pointer to a vCPU prior to it being fully online,
   and give kvm_for_each_vcpu() similar treatment to fix a similar flaw.

 - Wait for a vCPU to come online prior to executing a vCPU ioctl to fix a
   bug where userspace could coerce KVM into handling the ioctl on a vCPU that
   isn't yet onlined.

 - Gracefully handle xa_insert() failures even though such failuires should be
   impossible in practice.
parents 4e4f38f8 01528db6

include/linux/kvm_host.h

+13 −3
Original line number Diff line number Diff line
@@ -963,6 +963,15 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) 
static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)

{

	int num_vcpus = atomic_read(&kvm->online_vcpus);



	/*

	 * Explicitly verify the target vCPU is online, as the anti-speculation

	 * logic only limits the CPU's ability to speculate, e.g. given a "bad"

	 * index, clamping the index to 0 would return vCPU0, not NULL.

	 */

	if (i >= num_vcpus)

		return NULL;



	i = array_index_nospec(i, num_vcpus);



	/* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu.  */
@@ -971,6 +980,7 @@ static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) 
}



#define kvm_for_each_vcpu(idx, vcpup, kvm)				\

	if (atomic_read(&kvm->online_vcpus))				\

		xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0,	\

				  (atomic_read(&kvm->online_vcpus) - 1))

virt/kvm/kvm_main.c

+52 −16
Original line number Diff line number Diff line
@@ -4111,32 +4111,30 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) 


	mutex_lock(&kvm->lock);



#ifdef CONFIG_LOCKDEP

	/* Ensure that lockdep knows vcpu->mutex is taken *inside* kvm->lock */

	mutex_lock(&vcpu->mutex);

	mutex_unlock(&vcpu->mutex);

#endif



	if (kvm_get_vcpu_by_id(kvm, id)) {

		r = -EEXIST;

		goto unlock_vcpu_destroy;

	}



	vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);

	r = xa_reserve(&kvm->vcpu_array, vcpu->vcpu_idx, GFP_KERNEL_ACCOUNT);

	r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT);

	WARN_ON_ONCE(r == -EBUSY);

	if (r)

		goto unlock_vcpu_destroy;



	/* Now it's all set up, let userspace reach it */

	/*

	 * Now it's all set up, let userspace reach it.  Grab the vCPU's mutex

	 * so that userspace can't invoke vCPU ioctl()s until the vCPU is fully

	 * visible (per online_vcpus), e.g. so that KVM doesn't get tricked

	 * into a NULL-pointer dereference because KVM thinks the _current_

	 * vCPU doesn't exist.  As a bonus, taking vcpu->mutex ensures lockdep

	 * knows it's taken *inside* kvm->lock.

	 */

	mutex_lock(&vcpu->mutex);

	kvm_get_kvm(kvm);

	r = create_vcpu_fd(vcpu);

	if (r < 0)

		goto kvm_put_xa_release;



	if (KVM_BUG_ON(xa_store(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, 0), kvm)) {

		r = -EINVAL;

		goto kvm_put_xa_release;

	}

		goto kvm_put_xa_erase;



	/*

	 * Pairs with smp_rmb() in kvm_get_vcpu.  Store the vcpu
@@ -4144,15 +4142,17 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) 
	 */

	smp_wmb();

	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&vcpu->mutex);



	mutex_unlock(&kvm->lock);

	kvm_arch_vcpu_postcreate(vcpu);

	kvm_create_vcpu_debugfs(vcpu);

	return r;



kvm_put_xa_release:

kvm_put_xa_erase:

	mutex_unlock(&vcpu->mutex);

	kvm_put_kvm_no_destroy(kvm);

	xa_release(&kvm->vcpu_array, vcpu->vcpu_idx);

	xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx);

unlock_vcpu_destroy:

	mutex_unlock(&kvm->lock);

	kvm_dirty_ring_free(&vcpu->dirty_ring);
@@ -4277,6 +4277,33 @@ static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, 
}

#endif



static int kvm_wait_for_vcpu_online(struct kvm_vcpu *vcpu)

{

	struct kvm *kvm = vcpu->kvm;



	/*

	 * In practice, this happy path will always be taken, as a well-behaved

	 * VMM will never invoke a vCPU ioctl() before KVM_CREATE_VCPU returns.

	 */

	if (likely(vcpu->vcpu_idx < atomic_read(&kvm->online_vcpus)))

		return 0;



	/*

	 * Acquire and release the vCPU's mutex to wait for vCPU creation to

	 * complete (kvm_vm_ioctl_create_vcpu() holds the mutex until the vCPU

	 * is fully online).

	 */

	if (mutex_lock_killable(&vcpu->mutex))

		return -EINTR;



	mutex_unlock(&vcpu->mutex);



	if (WARN_ON_ONCE(!kvm_get_vcpu(kvm, vcpu->vcpu_idx)))

		return -EIO;



	return 0;

}



static long kvm_vcpu_ioctl(struct file *filp,

			   unsigned int ioctl, unsigned long arg)

{
@@ -4292,6 +4319,15 @@ static long kvm_vcpu_ioctl(struct file *filp, 
	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))

		return -EINVAL;



	/*

	 * Wait for the vCPU to be online before handling the ioctl(), as KVM

	 * assumes the vCPU is reachable via vcpu_array, i.e. may dereference

	 * a NULL pointer if userspace invokes an ioctl() before KVM is ready.

	 */

	r = kvm_wait_for_vcpu_online(vcpu);

	if (r)

		return r;



	/*

	 * Some architectures have vcpu ioctls that are asynchronous to vcpu

	 * execution; mutex_lock() would break them.