Commit 5dca3808 authored by Sean Christopherson's avatar Sean Christopherson
Browse files

KVM: x86: Merge 'svm' into 'cet' to pick up GHCB dependencies 

Merge the queue of SVM changes for 6.18 to pick up the KVM-defined GHCB
helpers so that kvm_ghcb_get_xss() can be used to virtualize CET for
SEV-ES+ guests.
parents 86bcd23d 4135a9a8

arch/x86/include/asm/cpufeatures.h

+1 −0
Original line number Diff line number Diff line
@@ -444,6 +444,7 @@ 
#define X86_FEATURE_VM_PAGE_FLUSH	(19*32+ 2) /* VM Page Flush MSR is supported */

#define X86_FEATURE_SEV_ES		(19*32+ 3) /* "sev_es" Secure Encrypted Virtualization - Encrypted State */

#define X86_FEATURE_SEV_SNP		(19*32+ 4) /* "sev_snp" Secure Encrypted Virtualization - Secure Nested Paging */

#define X86_FEATURE_SNP_SECURE_TSC	(19*32+ 8) /* SEV-SNP Secure TSC */

#define X86_FEATURE_V_TSC_AUX		(19*32+ 9) /* Virtual TSC_AUX */

#define X86_FEATURE_SME_COHERENT	(19*32+10) /* hardware-enforced cache coherency */

#define X86_FEATURE_DEBUG_SWAP		(19*32+14) /* "debug_swap" SEV-ES full debug state swap support */

arch/x86/include/asm/kvm_host.h

+1 −0
Original line number Diff line number Diff line
@@ -2196,6 +2196,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); 
unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr);

unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);

void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);

int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);

int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu);



int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);

arch/x86/include/asm/svm.h

+1 −0
Original line number Diff line number Diff line
@@ -299,6 +299,7 @@ static_assert((X2AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_ 
#define SVM_SEV_FEAT_RESTRICTED_INJECTION		BIT(3)

#define SVM_SEV_FEAT_ALTERNATE_INJECTION		BIT(4)

#define SVM_SEV_FEAT_DEBUG_SWAP				BIT(5)

#define SVM_SEV_FEAT_SECURE_TSC				BIT(9)



#define VMCB_ALLOWED_SEV_FEATURES_VALID			BIT_ULL(63)

arch/x86/kvm/svm/nested.c

+9 −11
Original line number Diff line number Diff line
@@ -1798,17 +1798,15 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, 
	if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE)

		return -EINVAL;



	ret  = -ENOMEM;

	ctl  = kzalloc(sizeof(*ctl),  GFP_KERNEL);

	save = kzalloc(sizeof(*save), GFP_KERNEL);

	if (!ctl || !save)

		goto out_free;

	ctl = memdup_user(&user_vmcb->control, sizeof(*ctl));

	if (IS_ERR(ctl))

		return PTR_ERR(ctl);



	ret = -EFAULT;

	if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl)))

		goto out_free;

	if (copy_from_user(save, &user_vmcb->save, sizeof(*save)))

		goto out_free;

	save = memdup_user(&user_vmcb->save, sizeof(*save));

	if (IS_ERR(save)) {

		kfree(ctl);

		return PTR_ERR(save);

	}



	ret = -EINVAL;

	__nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl);

arch/x86/kvm/svm/sev.c

+94 −50
Original line number Diff line number Diff line
@@ -37,7 +37,6 @@ 
#include "trace.h"



#define GHCB_VERSION_MAX	2ULL

#define GHCB_VERSION_DEFAULT	2ULL

#define GHCB_VERSION_MIN	1ULL



#define GHCB_HV_FT_SUPPORTED	(GHCB_HV_FT_SNP | GHCB_HV_FT_SNP_AP_CREATION)
@@ -147,6 +146,14 @@ static bool sev_vcpu_has_debug_swap(struct vcpu_svm *svm) 
	return sev->vmsa_features & SVM_SEV_FEAT_DEBUG_SWAP;

}



static bool snp_is_secure_tsc_enabled(struct kvm *kvm)

{

	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);



	return (sev->vmsa_features & SVM_SEV_FEAT_SECURE_TSC) &&

	       !WARN_ON_ONCE(!sev_snp_guest(kvm));

}



/* Must be called with the sev_bitmap_lock held */

static bool __sev_recycle_asids(unsigned int min_asid, unsigned int max_asid)

{
@@ -406,6 +413,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, 
	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);

	struct sev_platform_init_args init_args = {0};

	bool es_active = vm_type != KVM_X86_SEV_VM;

	bool snp_active = vm_type == KVM_X86_SNP_VM;

	u64 valid_vmsa_features = es_active ? sev_supported_vmsa_features : 0;

	int ret;
@@ -415,12 +423,26 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, 
	if (data->flags)

		return -EINVAL;



	if (!snp_active)

		valid_vmsa_features &= ~SVM_SEV_FEAT_SECURE_TSC;



	if (data->vmsa_features & ~valid_vmsa_features)

		return -EINVAL;



	if (data->ghcb_version > GHCB_VERSION_MAX || (!es_active && data->ghcb_version))

		return -EINVAL;



	/*

	 * KVM supports the full range of mandatory features defined by version

	 * 2 of the GHCB protocol, so default to that for SEV-ES guests created

	 * via KVM_SEV_INIT2 (KVM_SEV_INIT forces version 1).

	 */

	if (es_active && !data->ghcb_version)

		data->ghcb_version = 2;



	if (snp_active && data->ghcb_version < 2)

		return -EINVAL;



	if (unlikely(sev->active))

		return -EINVAL;
@@ -429,15 +451,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, 
	sev->vmsa_features = data->vmsa_features;

	sev->ghcb_version = data->ghcb_version;



	/*

	 * Currently KVM supports the full range of mandatory features defined

	 * by version 2 of the GHCB protocol, so default to that for SEV-ES

	 * guests created via KVM_SEV_INIT2.

	 */

	if (sev->es_active && !sev->ghcb_version)

		sev->ghcb_version = GHCB_VERSION_DEFAULT;



	if (vm_type == KVM_X86_SNP_VM)

	if (snp_active)

		sev->vmsa_features |= SVM_SEV_FEAT_SNP_ACTIVE;



	ret = sev_asid_new(sev);
@@ -455,7 +469,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, 
	}



	/* This needs to happen after SEV/SNP firmware initialization. */

	if (vm_type == KVM_X86_SNP_VM) {

	if (snp_active) {

		ret = snp_guest_req_init(kvm);

		if (ret)

			goto e_free;
@@ -569,8 +583,6 @@ static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 
	if (copy_from_user(&params, u64_to_user_ptr(argp->data), sizeof(params)))

		return -EFAULT;



	sev->policy = params.policy;



	memset(&start, 0, sizeof(start));



	dh_blob = NULL;
@@ -618,6 +630,7 @@ static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 
		goto e_free_session;

	}



	sev->policy = params.policy;

	sev->handle = start.handle;

	sev->fd = argp->sev_fd;
@@ -1972,7 +1985,7 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) 
	kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) {

		dst_svm = to_svm(dst_vcpu);



		sev_init_vmcb(dst_svm);

		sev_init_vmcb(dst_svm, false);



		if (!dst->es_active)

			continue;
@@ -2184,7 +2197,12 @@ static int snp_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 
	if (!(params.policy & SNP_POLICY_MASK_RSVD_MBO))

		return -EINVAL;



	sev->policy = params.policy;

	if (snp_is_secure_tsc_enabled(kvm)) {

		if (WARN_ON_ONCE(!kvm->arch.default_tsc_khz))

			return -EINVAL;



		start.desired_tsc_khz = kvm->arch.default_tsc_khz;

	}



	sev->snp_context = snp_context_create(kvm, argp);

	if (!sev->snp_context)
@@ -2192,6 +2210,7 @@ static int snp_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 


	start.gctx_paddr = __psp_pa(sev->snp_context);

	start.policy = params.policy;



	memcpy(start.gosvw, params.gosvw, sizeof(params.gosvw));

	rc = __sev_issue_cmd(argp->sev_fd, SEV_CMD_SNP_LAUNCH_START, &start, &argp->error);

	if (rc) {
@@ -2200,6 +2219,7 @@ static int snp_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 
		goto e_free_context;

	}



	sev->policy = params.policy;

	sev->fd = argp->sev_fd;

	rc = snp_bind_asid(kvm, &argp->error);

	if (rc) {
@@ -3082,6 +3102,9 @@ void __init sev_hardware_setup(void) 
	sev_supported_vmsa_features = 0;

	if (sev_es_debug_swap_enabled)

		sev_supported_vmsa_features |= SVM_SEV_FEAT_DEBUG_SWAP;



	if (sev_snp_enabled && tsc_khz && cpu_feature_enabled(X86_FEATURE_SNP_SECURE_TSC))

		sev_supported_vmsa_features |= SVM_SEV_FEAT_SECURE_TSC;

}



void sev_hardware_unsetup(void)
@@ -3197,7 +3220,7 @@ void sev_free_vcpu(struct kvm_vcpu *vcpu) 
		kvfree(svm->sev_es.ghcb_sa);

}



static u64 kvm_ghcb_get_sw_exit_code(struct vmcb_control_area *control)

static u64 kvm_get_cached_sw_exit_code(struct vmcb_control_area *control)

{

	return (((u64)control->exit_code_hi) << 32) | control->exit_code;

}
@@ -3223,7 +3246,7 @@ static void dump_ghcb(struct vcpu_svm *svm) 
	 */

	pr_err("GHCB (GPA=%016llx) snapshot:\n", svm->vmcb->control.ghcb_gpa);

	pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code",

	       kvm_ghcb_get_sw_exit_code(control), kvm_ghcb_sw_exit_code_is_valid(svm));

	       kvm_get_cached_sw_exit_code(control), kvm_ghcb_sw_exit_code_is_valid(svm));

	pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1",

	       control->exit_info_1, kvm_ghcb_sw_exit_info_1_is_valid(svm));

	pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2",
@@ -3276,26 +3299,24 @@ static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) 
	BUILD_BUG_ON(sizeof(svm->sev_es.valid_bitmap) != sizeof(ghcb->save.valid_bitmap));

	memcpy(&svm->sev_es.valid_bitmap, &ghcb->save.valid_bitmap, sizeof(ghcb->save.valid_bitmap));



	vcpu->arch.regs[VCPU_REGS_RAX] = kvm_ghcb_get_rax_if_valid(svm, ghcb);

	vcpu->arch.regs[VCPU_REGS_RBX] = kvm_ghcb_get_rbx_if_valid(svm, ghcb);

	vcpu->arch.regs[VCPU_REGS_RCX] = kvm_ghcb_get_rcx_if_valid(svm, ghcb);

	vcpu->arch.regs[VCPU_REGS_RDX] = kvm_ghcb_get_rdx_if_valid(svm, ghcb);

	vcpu->arch.regs[VCPU_REGS_RSI] = kvm_ghcb_get_rsi_if_valid(svm, ghcb);

	vcpu->arch.regs[VCPU_REGS_RAX] = kvm_ghcb_get_rax_if_valid(svm);

	vcpu->arch.regs[VCPU_REGS_RBX] = kvm_ghcb_get_rbx_if_valid(svm);

	vcpu->arch.regs[VCPU_REGS_RCX] = kvm_ghcb_get_rcx_if_valid(svm);

	vcpu->arch.regs[VCPU_REGS_RDX] = kvm_ghcb_get_rdx_if_valid(svm);

	vcpu->arch.regs[VCPU_REGS_RSI] = kvm_ghcb_get_rsi_if_valid(svm);



	svm->vmcb->save.cpl = kvm_ghcb_get_cpl_if_valid(svm, ghcb);

	svm->vmcb->save.cpl = kvm_ghcb_get_cpl_if_valid(svm);



	if (kvm_ghcb_xcr0_is_valid(svm)) {

		vcpu->arch.xcr0 = ghcb_get_xcr0(ghcb);

		vcpu->arch.cpuid_dynamic_bits_dirty = true;

	}

	if (kvm_ghcb_xcr0_is_valid(svm))

		__kvm_set_xcr(vcpu, 0, kvm_ghcb_get_xcr0(svm));



	/* Copy the GHCB exit information into the VMCB fields */

	exit_code = ghcb_get_sw_exit_code(ghcb);

	exit_code = kvm_ghcb_get_sw_exit_code(svm);

	control->exit_code = lower_32_bits(exit_code);

	control->exit_code_hi = upper_32_bits(exit_code);

	control->exit_info_1 = ghcb_get_sw_exit_info_1(ghcb);

	control->exit_info_2 = ghcb_get_sw_exit_info_2(ghcb);

	svm->sev_es.sw_scratch = kvm_ghcb_get_sw_scratch_if_valid(svm, ghcb);

	control->exit_info_1 = kvm_ghcb_get_sw_exit_info_1(svm);

	control->exit_info_2 = kvm_ghcb_get_sw_exit_info_2(svm);

	svm->sev_es.sw_scratch = kvm_ghcb_get_sw_scratch_if_valid(svm);



	/* Clear the valid entries fields */

	memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
@@ -3312,7 +3333,7 @@ static int sev_es_validate_vmgexit(struct vcpu_svm *svm) 
	 * Retrieve the exit code now even though it may not be marked valid

	 * as it could help with debugging.

	 */

	exit_code = kvm_ghcb_get_sw_exit_code(control);

	exit_code = kvm_get_cached_sw_exit_code(control);



	/* Only GHCB Usage code 0 is supported */

	if (svm->sev_es.ghcb->ghcb_usage) {
@@ -3884,7 +3905,7 @@ static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc) 
/*

 * Invoked as part of svm_vcpu_reset() processing of an init event.

 */

void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)

static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)

{

	struct vcpu_svm *svm = to_svm(vcpu);

	struct kvm_memory_slot *slot;
@@ -3892,9 +3913,6 @@ void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) 
	kvm_pfn_t pfn;

	gfn_t gfn;



	if (!sev_snp_guest(vcpu->kvm))

		return;



	guard(mutex)(&svm->sev_es.snp_vmsa_mutex);



	if (!svm->sev_es.snp_ap_waiting_for_reset)
@@ -4320,7 +4338,7 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu) 


	svm_vmgexit_success(svm, 0);



	exit_code = kvm_ghcb_get_sw_exit_code(control);

	exit_code = kvm_get_cached_sw_exit_code(control);

	switch (exit_code) {

	case SVM_VMGEXIT_MMIO_READ:

		ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
@@ -4452,6 +4470,9 @@ void sev_es_recalc_msr_intercepts(struct kvm_vcpu *vcpu) 
					  !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&

					  !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID));



	svm_set_intercept_for_msr(vcpu, MSR_AMD64_GUEST_TSC_FREQ, MSR_TYPE_R,

				  !snp_is_secure_tsc_enabled(vcpu->kvm));



	/*

	 * For SEV-ES, accesses to MSR_IA32_XSS should not be intercepted if

	 * the host/guest supports its use.
@@ -4480,7 +4501,7 @@ void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm) 
		vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));

}



static void sev_es_init_vmcb(struct vcpu_svm *svm)

static void sev_es_init_vmcb(struct vcpu_svm *svm, bool init_event)

{

	struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm);

	struct vmcb *vmcb = svm->vmcb01.ptr;
@@ -4541,10 +4562,21 @@ static void sev_es_init_vmcb(struct vcpu_svm *svm) 


	/* Can't intercept XSETBV, HV can't modify XCR0 directly */

	svm_clr_intercept(svm, INTERCEPT_XSETBV);



	/*

	 * Set the GHCB MSR value as per the GHCB specification when emulating

	 * vCPU RESET for an SEV-ES guest.

	 */

	if (!init_event)

		set_ghcb_msr(svm, GHCB_MSR_SEV_INFO((__u64)sev->ghcb_version,

						    GHCB_VERSION_MIN,

						    sev_enc_bit));

}



void sev_init_vmcb(struct vcpu_svm *svm)

void sev_init_vmcb(struct vcpu_svm *svm, bool init_event)

{

	struct kvm_vcpu *vcpu = &svm->vcpu;



	svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;

	clr_exception_intercept(svm, UD_VECTOR);
@@ -4554,24 +4586,36 @@ void sev_init_vmcb(struct vcpu_svm *svm) 
	 */

	clr_exception_intercept(svm, GP_VECTOR);



	if (sev_es_guest(svm->vcpu.kvm))

		sev_es_init_vmcb(svm);

	if (init_event && sev_snp_guest(vcpu->kvm))

		sev_snp_init_protected_guest_state(vcpu);



	if (sev_es_guest(vcpu->kvm))

		sev_es_init_vmcb(svm, init_event);

}



void sev_es_vcpu_reset(struct vcpu_svm *svm)

int sev_vcpu_create(struct kvm_vcpu *vcpu)

{

	struct kvm_vcpu *vcpu = &svm->vcpu;

	struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm);

	struct vcpu_svm *svm = to_svm(vcpu);

	struct page *vmsa_page;



	mutex_init(&svm->sev_es.snp_vmsa_mutex);



	if (!sev_es_guest(vcpu->kvm))

		return 0;



	/*

	 * Set the GHCB MSR value as per the GHCB specification when emulating

	 * vCPU RESET for an SEV-ES guest.

	 * SEV-ES guests require a separate (from the VMCB) VMSA page used to

	 * contain the encrypted register state of the guest.

	 */

	set_ghcb_msr(svm, GHCB_MSR_SEV_INFO((__u64)sev->ghcb_version,

					    GHCB_VERSION_MIN,

					    sev_enc_bit));

	vmsa_page = snp_safe_alloc_page();

	if (!vmsa_page)

		return -ENOMEM;



	mutex_init(&svm->sev_es.snp_vmsa_mutex);

	svm->sev_es.vmsa = page_address(vmsa_page);



	vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm);



	return 0;

}



void sev_es_prepare_switch_to_guest(struct vcpu_svm *svm, struct sev_es_save_area *hostsa)