Commit aa856775 authored by Paolo Bonzini's avatar Paolo Bonzini
Browse files

Merge tag 'kvm-x86-mmio-7.1' of https://github.com/kvm-x86/linux into HEAD 

KVM x86 emulated MMIO changes for 7.1

Copy single-chunk MMIO write values into a persistent (per-fragment) field to
fix use-after-free stack bugs due to KVM dereferencing a stack pointer after an
exit to userspace.

Clean up and comment the emulated MMIO code to try to make it easier to
maintain (not necessarily "easy", but "easier").
parents c13008ed e2138c4a

arch/x86/include/asm/kvm_host.h

+0 −3
Original line number Diff line number Diff line
@@ -2097,9 +2097,6 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end); 


int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);



int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,

			  const void *val, int bytes);



extern bool tdp_enabled;



u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);

arch/x86/kvm/emulate.c

+13 −0
Original line number Diff line number Diff line
@@ -1297,12 +1297,25 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt, 
	int rc;

	struct read_cache *mc = &ctxt->mem_read;



	/*

	 * If the read gets a cache hit, simply copy the value from the cache.

	 * A "hit" here means that there is unused data in the cache, i.e. when

	 * re-emulating an instruction to complete a userspace exit, KVM relies

	 * on "no decode" to ensure the instruction is re-emulated in the same

	 * sequence, so that multiple reads are fulfilled in the correct order.

	 */

	if (mc->pos < mc->end)

		goto read_cached;



	if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))

		return X86EMUL_UNHANDLEABLE;



	/*

	 * Route all reads to the cache.  This allows @dest to be an on-stack

	 * variable without triggering use-after-free if KVM needs to exit to

	 * userspace to handle an MMIO read (the MMIO fragment will point at

	 * the current location in the cache).

	 */

	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,

				      &ctxt->exception);

	if (rc != X86EMUL_CONTINUE)

arch/x86/kvm/svm/sev.c

+6 −14
Original line number Diff line number Diff line
@@ -4434,25 +4434,17 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu) 


	switch (control->exit_code) {

	case SVM_VMGEXIT_MMIO_READ:

		ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);

		if (ret)

			break;

	case SVM_VMGEXIT_MMIO_WRITE: {

		bool is_write = control->exit_code == SVM_VMGEXIT_MMIO_WRITE;



		ret = kvm_sev_es_mmio_read(vcpu,

					   control->exit_info_1,

					   control->exit_info_2,

					   svm->sev_es.ghcb_sa);

		break;

	case SVM_VMGEXIT_MMIO_WRITE:

		ret = setup_vmgexit_scratch(svm, false, control->exit_info_2);

		ret = setup_vmgexit_scratch(svm, !is_write, control->exit_info_2);

		if (ret)

			break;



		ret = kvm_sev_es_mmio_write(vcpu,

					    control->exit_info_1,

					    control->exit_info_2,

					    svm->sev_es.ghcb_sa);

		ret = kvm_sev_es_mmio(vcpu, is_write, control->exit_info_1,

				      control->exit_info_2, svm->sev_es.ghcb_sa);

		break;

	}

	case SVM_VMGEXIT_NMI_COMPLETE:

		++vcpu->stat.nmi_window_exits;

		svm->nmi_masked = false;

arch/x86/kvm/vmx/tdx.c

+3 −9
Original line number Diff line number Diff line
@@ -1467,17 +1467,11 @@ static int tdx_emulate_mmio(struct kvm_vcpu *vcpu) 


	/* Request the device emulation to userspace device model. */

	vcpu->mmio_is_write = write;

	if (!write)

		vcpu->arch.complete_userspace_io = tdx_complete_mmio_read;



	vcpu->run->mmio.phys_addr = gpa;

	vcpu->run->mmio.len = size;

	vcpu->run->mmio.is_write = write;

	vcpu->run->exit_reason = KVM_EXIT_MMIO;

	__kvm_prepare_emulated_mmio_exit(vcpu, gpa, size, &val, write);



	if (write) {

		memcpy(vcpu->run->mmio.data, &val, size);

	} else {

	if (!write) {

		vcpu->arch.complete_userspace_io = tdx_complete_mmio_read;

		vcpu->mmio_fragments[0].gpa = gpa;

		vcpu->mmio_fragments[0].len = size;

		trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, size, gpa, NULL);

arch/x86/kvm/x86.c

+127 −160
Original line number Diff line number Diff line
@@ -7768,11 +7768,14 @@ static void kvm_init_msr_lists(void) 
}



static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,

			   const void *v)

			   void *__v)

{

	const void *v = __v;

	int handled = 0;

	int n;



	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, addr, __v);



	do {

		n = min(len, 8);

		if (!(lapic_in_kernel(vcpu) &&
@@ -7807,6 +7810,9 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) 
		v += n;

	} while (len);



	if (len)

		trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len, addr, NULL);



	return handled;

}
@@ -8095,90 +8101,32 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);

}



int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,

			const void *val, int bytes)

{

	int ret;



	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);

	if (ret < 0)

		return 0;

	kvm_page_track_write(vcpu, gpa, val, bytes);

	return 1;

}



struct read_write_emulator_ops {

	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,

				  int bytes);

	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,

	int (*read_write_guest)(struct kvm_vcpu *vcpu, gpa_t gpa,

				void *val, int bytes);

	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,

			       int bytes, void *val);

	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,

				    void *val, int bytes);

	bool write;

};



static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)

{

	if (vcpu->mmio_read_completed) {

		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,

			       vcpu->mmio_fragments[0].gpa, val);

		vcpu->mmio_read_completed = 0;

		return 1;

	}



	return 0;

}



static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,

static int emulator_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa,

			       void *val, int bytes)

{

	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);

}



static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,

			 void *val, int bytes)

{

	return emulator_write_phys(vcpu, gpa, val, bytes);

}



static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)

{

	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);

	return vcpu_mmio_write(vcpu, gpa, bytes, val);

}



static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,

static int emulator_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa,

				void *val, int bytes)

{

	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);

	return X86EMUL_IO_NEEDED;

}



static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,

			   void *val, int bytes)

{

	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

	int ret;



	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));

	return X86EMUL_CONTINUE;

	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);

	if (ret < 0)

		return 0;

	kvm_page_track_write(vcpu, gpa, val, bytes);

	return 1;

}



static const struct read_write_emulator_ops read_emultor = {

	.read_write_prepare = read_prepare,

	.read_write_emulate = read_emulate,

	.read_write_mmio = vcpu_mmio_read,

	.read_write_exit_mmio = read_exit_mmio,

};



static const struct read_write_emulator_ops write_emultor = {

	.read_write_emulate = write_emulate,

	.read_write_mmio = write_mmio,

	.read_write_exit_mmio = write_exit_mmio,

	.write = true,

};



static int emulator_read_write_onepage(unsigned long addr, void *val,

				       unsigned int bytes,

				       struct x86_exception *exception,
@@ -8208,11 +8156,22 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, 
			return X86EMUL_PROPAGATE_FAULT;

	}



	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))

	/*

	 * If the memory is not _known_ to be emulated MMIO, attempt to access

	 * guest memory.  If accessing guest memory fails, e.g. because there's

	 * no memslot, then handle the access as MMIO.  Note, treating the

	 * access as emulated MMIO is technically wrong if there is a memslot,

	 * i.e. if accessing host user memory failed, but this has been KVM's

	 * historical ABI for decades.

	 */

	if (!ret && ops->read_write_guest(vcpu, gpa, val, bytes))

		return X86EMUL_CONTINUE;



	/*

	 * Is this MMIO handled locally?

	 * Attempt to handle emulated MMIO within the kernel, e.g. for accesses

	 * to an in-kernel local or I/O APIC, or to an ioeventfd range attached

	 * to MMIO bus.  If the access isn't fully resolved, insert an MMIO

	 * fragment with the relevant details.

	 */

	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);

	if (handled == bytes)
@@ -8225,8 +8184,21 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, 
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);

	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];

	frag->gpa = gpa;

	if (write && bytes <= 8u) {

		frag->val = 0;

		frag->data = &frag->val;

		memcpy(&frag->val, val, bytes);

	} else {

		frag->data = val;

	}

	frag->len = bytes;



	/*

	 * Continue emulating, even though KVM needs to (eventually) do an MMIO

	 * exit to userspace.  If the access splits multiple pages, then KVM

	 * needs to exit to userspace only after emulating both parts of the

	 * access.

	 */

	return X86EMUL_CONTINUE;

}
@@ -8237,12 +8209,33 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt, 
			const struct read_write_emulator_ops *ops)

{

	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	gpa_t gpa;

	int rc;



	if (ops->read_write_prepare &&

		  ops->read_write_prepare(vcpu, val, bytes))

	if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val)))

		return X86EMUL_UNHANDLEABLE;



	/*

	 * If the read was already completed via a userspace MMIO exit, there's

	 * nothing left to do except trace the MMIO read.  When completing MMIO

	 * reads, KVM re-emulates the instruction to propagate the value into

	 * the correct destination, e.g. into the correct register, but the

	 * value itself has already been copied to the read cache.

	 *

	 * Note!  This is *tightly* coupled to read_emulated() satisfying reads

	 * from the emulator's mem_read cache, so that the MMIO fragment data

	 * is copied to the correct chunk of the correct operand.

	 */

	if (!ops->write && vcpu->mmio_read_completed) {

		/*

		 * For simplicity, trace the entire MMIO read in one shot, even

		 * though the GPA might be incorrect if there are two fragments

		 * that aren't contiguous in the GPA space.

		 */

		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,

			       vcpu->mmio_fragments[0].gpa, val);

		vcpu->mmio_read_completed = 0;

		return X86EMUL_CONTINUE;

	}



	vcpu->mmio_nr_fragments = 0;
@@ -8271,17 +8264,21 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt, 
	if (!vcpu->mmio_nr_fragments)

		return X86EMUL_CONTINUE;



	gpa = vcpu->mmio_fragments[0].gpa;



	vcpu->mmio_needed = 1;

	vcpu->mmio_cur_fragment = 0;

	vcpu->mmio_is_write = ops->write;



	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);

	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;

	vcpu->run->exit_reason = KVM_EXIT_MMIO;

	vcpu->run->mmio.phys_addr = gpa;

	kvm_prepare_emulated_mmio_exit(vcpu, &vcpu->mmio_fragments[0]);



	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);

	/*

	 * For MMIO reads, stop emulating and immediately exit to userspace, as

	 * KVM needs the value to correctly emulate the instruction.  For MMIO

	 * writes, continue emulating as the write to MMIO is a side effect for

	 * all intents and purposes.  KVM will still exit to userspace, but

	 * after completing emulation (see the check on vcpu->mmio_needed in

	 * x86_emulate_instruction()).

	 */

	return ops->write ? X86EMUL_CONTINUE : X86EMUL_IO_NEEDED;

}



static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
@@ -8290,8 +8287,13 @@ static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt, 
				  unsigned int bytes,

				  struct x86_exception *exception)

{

	return emulator_read_write(ctxt, addr, val, bytes,

				   exception, &read_emultor);

	static const struct read_write_emulator_ops ops = {

		.read_write_guest = emulator_read_guest,

		.read_write_mmio = vcpu_mmio_read,

		.write = false,

	};



	return emulator_read_write(ctxt, addr, val, bytes, exception, &ops);

}



static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
@@ -8300,8 +8302,13 @@ static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 
			    unsigned int bytes,

			    struct x86_exception *exception)

{

	return emulator_read_write(ctxt, addr, (void *)val, bytes,

				   exception, &write_emultor);

	static const struct read_write_emulator_ops ops = {

		.read_write_guest = emulator_write_guest,

		.read_write_mmio = vcpu_mmio_write,

		.write = true,

	};



	return emulator_read_write(ctxt, addr, (void *)val, bytes, exception, &ops);

}



#define emulator_try_cmpxchg_user(t, ptr, old, new) \
@@ -9694,7 +9701,8 @@ static int complete_fast_pio_in(struct kvm_vcpu *vcpu) 
	unsigned long val;



	/* We should only ever be called with arch.pio.count equal to 1 */

	BUG_ON(vcpu->arch.pio.count != 1);

	if (KVM_BUG_ON(vcpu->arch.pio.count != 1, vcpu->kvm))

		return -EIO;



	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) {

		vcpu->arch.pio.count = 0;
@@ -11816,7 +11824,8 @@ static inline int complete_emulated_io(struct kvm_vcpu *vcpu) 


static int complete_emulated_pio(struct kvm_vcpu *vcpu)

{

	BUG_ON(!vcpu->arch.pio.count);

	if (KVM_BUG_ON(!vcpu->arch.pio.count, vcpu->kvm))

		return -EIO;



	return complete_emulated_io(vcpu);

}
@@ -11845,7 +11854,8 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 
	struct kvm_mmio_fragment *frag;

	unsigned len;



	BUG_ON(!vcpu->mmio_needed);

	if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))

		return -EIO;



	/* Complete previous fragment */

	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
@@ -11858,6 +11868,9 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 
		frag++;

		vcpu->mmio_cur_fragment++;

	} else {

		if (WARN_ON_ONCE(frag->data == &frag->val))

			return -EIO;



		/* Go forward to the next mmio piece. */

		frag->data += len;

		frag->gpa += len;
@@ -11874,12 +11887,7 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 
		return complete_emulated_io(vcpu);

	}



	run->exit_reason = KVM_EXIT_MMIO;

	run->mmio.phys_addr = frag->gpa;

	if (vcpu->mmio_is_write)

		memcpy(run->mmio.data, frag->data, min(8u, frag->len));

	run->mmio.len = min(8u, frag->len);

	run->mmio.is_write = vcpu->mmio_is_write;

	kvm_prepare_emulated_mmio_exit(vcpu, frag);

	vcpu->arch.complete_userspace_io = complete_emulated_mmio;

	return 0;

}
@@ -14255,7 +14263,8 @@ static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu) 
	struct kvm_mmio_fragment *frag;

	unsigned int len;



	BUG_ON(!vcpu->mmio_needed);

	if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))

		return -EIO;



	/* Complete previous fragment */

	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
@@ -14277,73 +14286,32 @@ static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu) 
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {

		vcpu->mmio_needed = 0;



		// VMG change, at this point, we're always done

		// RIP has already been advanced

		/*

		 * All done, as frag->data always points at the GHCB scratch

		 * area and VMGEXIT is trap-like (RIP is advanced by hardware).

		 */

		return 1;

	}



	// More MMIO is needed

	run->mmio.phys_addr = frag->gpa;

	run->mmio.len = min(8u, frag->len);

	run->mmio.is_write = vcpu->mmio_is_write;

	if (run->mmio.is_write)

		memcpy(run->mmio.data, frag->data, min(8u, frag->len));

	run->exit_reason = KVM_EXIT_MMIO;



	kvm_prepare_emulated_mmio_exit(vcpu, frag);

	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;



	return 0;

}



int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,

			  void *data)

int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,

		    unsigned int bytes, void *data)

{

	int handled;

	struct kvm_mmio_fragment *frag;



	if (!data)

		return -EINVAL;



	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);

	if (handled == bytes)

		return 1;



	bytes -= handled;

	gpa += handled;

	data += handled;



	/*TODO: Check if need to increment number of frags */

	frag = vcpu->mmio_fragments;

	vcpu->mmio_nr_fragments = 1;

	frag->len = bytes;

	frag->gpa = gpa;

	frag->data = data;



	vcpu->mmio_needed = 1;

	vcpu->mmio_cur_fragment = 0;



	vcpu->run->mmio.phys_addr = gpa;

	vcpu->run->mmio.len = min(8u, frag->len);

	vcpu->run->mmio.is_write = 1;

	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));

	vcpu->run->exit_reason = KVM_EXIT_MMIO;



	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;



	return 0;

}

EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio_write);



int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,

			 void *data)

{

	int handled;

	struct kvm_mmio_fragment *frag;



	if (!data)

	if (!data || WARN_ON_ONCE(object_is_on_stack(data)))

		return -EINVAL;



	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);

	if (is_write)

		handled = vcpu_mmio_write(vcpu, gpa, bytes, data);

	else

		handled = vcpu_mmio_read(vcpu, gpa, bytes, data);

	if (handled == bytes)

		return 1;
@@ -14351,26 +14319,25 @@ int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes, 
	gpa += handled;

	data += handled;



	/*TODO: Check if need to increment number of frags */

	/*

	 * TODO: Determine whether or not userspace plays nice with MMIO

	 *       requests that split a page boundary.

	 */

	frag = vcpu->mmio_fragments;

	vcpu->mmio_nr_fragments = 1;

	frag->len = bytes;

	frag->gpa = gpa;

	frag->data = data;



	vcpu->mmio_needed = 1;

	vcpu->mmio_cur_fragment = 0;

	vcpu->mmio_nr_fragments = 1;

	vcpu->mmio_is_write = is_write;



	vcpu->run->mmio.phys_addr = gpa;

	vcpu->run->mmio.len = min(8u, frag->len);

	vcpu->run->mmio.is_write = 0;

	vcpu->run->exit_reason = KVM_EXIT_MMIO;



	kvm_prepare_emulated_mmio_exit(vcpu, frag);

	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;



	return 0;

}

EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio_read);

EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio);



static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)

{