Commit c92be2fd authored by Sean Christopherson's avatar Sean Christopherson
Browse files

KVM: SVM: Save/restore non-volatile GPRs in SEV-ES VMRUN via host save area 



Use the host save area to save/restore non-volatile (callee-saved)
registers in __svm_sev_es_vcpu_run() to take advantage of hardware loading
all registers from the save area on #VMEXIT.  KVM still needs to save the
registers it wants restored, but the loads are handled automatically by
hardware.

Aside from less assembly code, letting hardware do the restoration means
stack frames are preserved for the entirety of __svm_sev_es_vcpu_run().

Opportunistically add a comment to call out why @svm needs to be saved
across VMRUN->#VMEXIT, as it's not easy to decipher that from the macro
hell.

Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Michael Roth <michael.roth@amd.com>
Cc: Alexey Kardashevskiy <aik@amd.com>
Reviewed-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/20240223204233.3337324-6-seanjc@google.com


Signed-off-by: default avatarSean Christopherson <seanjc@google.com>
parent 87e8e360

arch/x86/kvm/svm/svm.c

+10 −7
Original line number Diff line number Diff line
@@ -1503,6 +1503,11 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) 
	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));

}



static struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)

{

	return page_address(sd->save_area) + 0x400;

}



static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)

{

	struct vcpu_svm *svm = to_svm(vcpu);
@@ -1519,12 +1524,8 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 
	 * or subsequent vmload of host save area.

	 */

	vmsave(sd->save_area_pa);

	if (sev_es_guest(vcpu->kvm)) {

		struct sev_es_save_area *hostsa;

		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);



		sev_es_prepare_switch_to_guest(hostsa);

	}

	if (sev_es_guest(vcpu->kvm))

		sev_es_prepare_switch_to_guest(sev_es_host_save_area(sd));



	if (tsc_scaling)

		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
@@ -4101,6 +4102,7 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 


static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)

{

	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);

	struct vcpu_svm *svm = to_svm(vcpu);



	guest_state_enter_irqoff();
@@ -4108,7 +4110,8 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in 
	amd_clear_divider();



	if (sev_es_guest(vcpu->kvm))

		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted);

		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,

				      sev_es_host_save_area(sd));

	else

		__svm_vcpu_run(svm, spec_ctrl_intercepted);

arch/x86/kvm/svm/svm.h

+2 −1
Original line number Diff line number Diff line
@@ -698,7 +698,8 @@ struct page *snp_safe_alloc_page(struct kvm_vcpu *vcpu); 


/* vmenter.S */



void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);

void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted,

			   struct sev_es_save_area *hostsa);

void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);



#define DEFINE_KVM_GHCB_ACCESSORS(field)						\

arch/x86/kvm/svm/vmenter.S

+23 −18
Original line number Diff line number Diff line
@@ -292,23 +292,35 @@ SYM_FUNC_START(__svm_vcpu_run) 
SYM_FUNC_END(__svm_vcpu_run)



#ifdef CONFIG_KVM_AMD_SEV





#ifdef CONFIG_X86_64

#define SEV_ES_GPRS_BASE 0x300

#define SEV_ES_RBX	(SEV_ES_GPRS_BASE + __VCPU_REGS_RBX * WORD_SIZE)

#define SEV_ES_RBP	(SEV_ES_GPRS_BASE + __VCPU_REGS_RBP * WORD_SIZE)

#define SEV_ES_R12	(SEV_ES_GPRS_BASE + __VCPU_REGS_R12 * WORD_SIZE)

#define SEV_ES_R13	(SEV_ES_GPRS_BASE + __VCPU_REGS_R13 * WORD_SIZE)

#define SEV_ES_R14	(SEV_ES_GPRS_BASE + __VCPU_REGS_R14 * WORD_SIZE)

#define SEV_ES_R15	(SEV_ES_GPRS_BASE + __VCPU_REGS_R15 * WORD_SIZE)

#endif



/**

 * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode

 * @svm:	struct vcpu_svm *

 * @spec_ctrl_intercepted: bool

 */

SYM_FUNC_START(__svm_sev_es_vcpu_run)

	push %rbp

	push %r15

	push %r14

	push %r13

	push %r12

	push %rbx



	/*

	 * Save variables needed after vmexit on the stack, in inverse

	 * order compared to when they are needed.

	 * Save non-volatile (callee-saved) registers to the host save area.

	 * Except for RAX and RSP, all GPRs are restored on #VMEXIT, but not

	 * saved on VMRUN.

	 */

	mov %rbp, SEV_ES_RBP (%rdx)

	mov %r15, SEV_ES_R15 (%rdx)

	mov %r14, SEV_ES_R14 (%rdx)

	mov %r13, SEV_ES_R13 (%rdx)

	mov %r12, SEV_ES_R12 (%rdx)

	mov %rbx, SEV_ES_RBX (%rdx)



	/* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL.  */

	push %rsi
@@ -316,7 +328,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) 
	/* Save @svm. */

	push %rdi



	/* Clobbers RAX, RCX, RDX.  */

	/* Clobbers RAX, RCX, RDX (@hostsa). */

	RESTORE_GUEST_SPEC_CTRL



	/* Get svm->current_vmcb->pa into RAX. */
@@ -338,7 +350,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) 
	FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE

#endif



	/* Clobbers RAX, RCX, RDX.  */

	/* Clobbers RAX, RCX, RDX, consumes RDI (@svm). */

	RESTORE_HOST_SPEC_CTRL



	/*
@@ -353,13 +365,6 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) 
	/* "Pop" and discard @spec_ctrl_intercepted. */

	pop %rax



	pop %rbx



	pop %r12

	pop %r13

	pop %r14

	pop %r15

	pop %rbp

	RET



	RESTORE_GUEST_SPEC_CTRL_BODY