Commit b50cb2b1 authored by Sean Christopherson's avatar Sean Christopherson
Browse files

KVM: x86: Use a dedicated flow for queueing re-injected exceptions 



Open code the filling of vcpu->arch.exception in kvm_requeue_exception()
instead of bouncing through kvm_multiple_exception(), as re-injection
doesn't actually share that much code with "normal" injection, e.g. the
VM-Exit interception check, payload delivery, and nested exception code
is all bypassed as those flows only apply during initial injection.

When FRED comes along, the special casing will only get worse, as FRED
explicitly tracks nested exceptions and essentially delivers the payload
on the stack frame, i.e. re-injection will need more inputs, and normal
injection will have yet more code that needs to be bypassed when KVM is
re-injecting an exception.

No functional change intended.

Signed-off-by: default avatarXin Li (Intel) <xin@zytor.com>
Tested-by: default avatarShan Kang <shan.kang@intel.com>
Link: https://lore.kernel.org/r/20241001050110.3643764-2-xin@zytor.com


Signed-off-by: default avatarSean Christopherson <seanjc@google.com>
parent 4fa0efb4

arch/x86/include/asm/kvm_host.h

+2 −2
Original line number Diff line number Diff line
@@ -2161,8 +2161,8 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu); 
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);

void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);

void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);

void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);

void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);

void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,

			   bool has_error_code, u32 error_code);

void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);

void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,

				    struct x86_exception *fault);

arch/x86/kvm/svm/svm.c

+9 −6
Original line number Diff line number Diff line
@@ -4116,20 +4116,23 @@ static void svm_complete_interrupts(struct kvm_vcpu *vcpu) 
		vcpu->arch.nmi_injected = true;

		svm->nmi_l1_to_l2 = nmi_l1_to_l2;

		break;

	case SVM_EXITINTINFO_TYPE_EXEPT:

	case SVM_EXITINTINFO_TYPE_EXEPT: {

		u32 error_code = 0;



		/*

		 * Never re-inject a #VC exception.

		 */

		if (vector == X86_TRAP_VC)

			break;



		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {

			u32 err = svm->vmcb->control.exit_int_info_err;

			kvm_requeue_exception_e(vcpu, vector, err);

		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR)

			error_code = svm->vmcb->control.exit_int_info_err;



		} else

			kvm_requeue_exception(vcpu, vector);

		kvm_requeue_exception(vcpu, vector,

				      exitintinfo & SVM_EXITINTINFO_VALID_ERR,

				      error_code);

		break;

	}

	case SVM_EXITINTINFO_TYPE_INTR:

		kvm_queue_interrupt(vcpu, vector, false);

		break;

arch/x86/kvm/vmx/vmx.c

+10 −6
Original line number Diff line number Diff line
@@ -7155,13 +7155,17 @@ static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 
	case INTR_TYPE_SOFT_EXCEPTION:

		vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);

		fallthrough;

	case INTR_TYPE_HARD_EXCEPTION:

		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {

			u32 err = vmcs_read32(error_code_field);

			kvm_requeue_exception_e(vcpu, vector, err);

		} else

			kvm_requeue_exception(vcpu, vector);

	case INTR_TYPE_HARD_EXCEPTION: {

		u32 error_code = 0;



		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK)

			error_code = vmcs_read32(error_code_field);



		kvm_requeue_exception(vcpu, vector,

				      idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK,

				      error_code);

		break;

	}

	case INTR_TYPE_SOFT_INTR:

		vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);

		fallthrough;

arch/x86/kvm/x86.c

+42 −47
Original line number Diff line number Diff line
@@ -800,9 +800,9 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto 
	ex->payload = payload;

}



static void kvm_multiple_exception(struct kvm_vcpu *vcpu,

		unsigned nr, bool has_error, u32 error_code,

	        bool has_payload, unsigned long payload, bool reinject)

static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr,

				   bool has_error, u32 error_code,

				   bool has_payload, unsigned long payload)

{

	u32 prev_nr;

	int class1, class2;
@@ -810,13 +810,10 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, 
	kvm_make_request(KVM_REQ_EVENT, vcpu);



	/*

	 * If the exception is destined for L2 and isn't being reinjected,

	 * morph it to a VM-Exit if L1 wants to intercept the exception.  A

	 * previously injected exception is not checked because it was checked

	 * when it was original queued, and re-checking is incorrect if _L1_

	 * injected the exception, in which case it's exempt from interception.

	 * If the exception is destined for L2, morph it to a VM-Exit if L1

	 * wants to intercept the exception.

	 */

	if (!reinject && is_guest_mode(vcpu) &&

	if (is_guest_mode(vcpu) &&

	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {

		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,

					   has_payload, payload);
@@ -825,28 +822,9 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, 


	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {

	queue:

		if (reinject) {

			/*

			 * On VM-Entry, an exception can be pending if and only

			 * if event injection was blocked by nested_run_pending.

			 * In that case, however, vcpu_enter_guest() requests an

			 * immediate exit, and the guest shouldn't proceed far

			 * enough to need reinjection.

			 */

			WARN_ON_ONCE(kvm_is_exception_pending(vcpu));

			vcpu->arch.exception.injected = true;

			if (WARN_ON_ONCE(has_payload)) {

				/*

				 * A reinjected event has already

				 * delivered its payload.

				 */

				has_payload = false;

				payload = 0;

			}

		} else {

		vcpu->arch.exception.pending = true;

		vcpu->arch.exception.injected = false;

		}



		vcpu->arch.exception.has_error_code = has_error;

		vcpu->arch.exception.vector = nr;

		vcpu->arch.exception.error_code = error_code;
@@ -887,29 +865,52 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, 


void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)

{

	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);

	kvm_multiple_exception(vcpu, nr, false, 0, false, 0);

}

EXPORT_SYMBOL_GPL(kvm_queue_exception);



void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)

{

	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);

}

EXPORT_SYMBOL_GPL(kvm_requeue_exception);



void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,

			   unsigned long payload)

{

	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);

	kvm_multiple_exception(vcpu, nr, false, 0, true, payload);

}

EXPORT_SYMBOL_GPL(kvm_queue_exception_p);



static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,

				    u32 error_code, unsigned long payload)

{

	kvm_multiple_exception(vcpu, nr, true, error_code,

			       true, payload, false);

	kvm_multiple_exception(vcpu, nr, true, error_code, true, payload);

}



void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,

			   bool has_error_code, u32 error_code)

{



	/*

	 * On VM-Entry, an exception can be pending if and only if event

	 * injection was blocked by nested_run_pending.  In that case, however,

	 * vcpu_enter_guest() requests an immediate exit, and the guest

	 * shouldn't proceed far enough to need reinjection.

	 */

	WARN_ON_ONCE(kvm_is_exception_pending(vcpu));



	/*

	 * Do not check for interception when injecting an event for L2, as the

	 * exception was checked for intercept when it was original queued, and

	 * re-checking is incorrect if _L1_ injected the exception, in which

	 * case it's exempt from interception.

	 */

	kvm_make_request(KVM_REQ_EVENT, vcpu);



	vcpu->arch.exception.injected = true;

	vcpu->arch.exception.has_error_code = has_error_code;

	vcpu->arch.exception.vector = nr;

	vcpu->arch.exception.error_code = error_code;

	vcpu->arch.exception.has_payload = false;

	vcpu->arch.exception.payload = 0;

}

EXPORT_SYMBOL_GPL(kvm_requeue_exception);



int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)

{
@@ -980,16 +981,10 @@ void kvm_inject_nmi(struct kvm_vcpu *vcpu) 


void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)

{

	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);

	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0);

}

EXPORT_SYMBOL_GPL(kvm_queue_exception_e);



void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)

{

	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);

}

EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);



/*

 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue

 * a #GP and return false.