Commit e44eb583 authored by Sean Christopherson's avatar Sean Christopherson
Browse files

KVM: x86: Load guest FPU state when access XSAVE-managed MSRs 



Load the guest's FPU state if userspace is accessing MSRs whose values
are managed by XSAVES. Introduce two helpers, kvm_{get,set}_xstate_msr(),
to facilitate access to such kind of MSRs.

If MSRs supported in kvm_caps.supported_xss are passed through to guest,
the guest MSRs are swapped with host's before vCPU exits to userspace and
after it reenters kernel before next VM-entry.

Because the modified code is also used for the KVM_GET_MSRS device ioctl(),
explicitly check @vcpu is non-null before attempting to load guest state.
The XSAVE-managed MSRs cannot be retrieved via the device ioctl() without
loading guest FPU state (which doesn't exist).

Note that guest_cpuid_has() is not queried as host userspace is allowed to
access MSRs that have not been exposed to the guest, e.g. it might do
KVM_SET_MSRS prior to KVM_SET_CPUID2.

The two helpers are put here in order to manifest accessing xsave-managed
MSRs requires special check and handling to guarantee the correctness of
read/write to the MSRs.

Co-developed-by: default avatarYang Weijiang <weijiang.yang@intel.com>
Signed-off-by: default avatarYang Weijiang <weijiang.yang@intel.com>
Reviewed-by: default avatarMaxim Levitsky <mlevitsk@redhat.com>
Tested-by: default avatarMathias Krause <minipli@grsecurity.net>
Tested-by: default avatarJohn Allen <john.allen@amd.com>
Tested-by: default avatarRick Edgecombe <rick.p.edgecombe@intel.com>
Signed-off-by: default avatarChao Gao <chao.gao@intel.com>
[sean: drop S_CET, add big comment, move accessors to x86.c]
Reviewed-by: default avatarBinbin Wu <binbin.wu@linux.intel.com>
Reviewed-by: default avatarXiaoyao Li <xiaoyao.li@intel.com>
Reviewed-by: default avatarXin Li (Intel) <xin@zytor.com>
Link: https://lore.kernel.org/r/20250919223258.1604852-10-seanjc@google.com


Signed-off-by: default avatarSean Christopherson <seanjc@google.com>
parent 779ed055

arch/x86/kvm/x86.c

+86 −1
Original line number Diff line number Diff line
@@ -136,6 +136,9 @@ static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); 
static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);



static DEFINE_MUTEX(vendor_module_lock);

static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);

static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);



struct kvm_x86_ops kvm_x86_ops __read_mostly;



#define KVM_X86_OP(func)					     \
@@ -3801,6 +3804,67 @@ static void record_steal_time(struct kvm_vcpu *vcpu) 
	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));

}



/*

 * Returns true if the MSR in question is managed via XSTATE, i.e. is context

 * switched with the rest of guest FPU state.  Note!  S_CET is _not_ context

 * switched via XSTATE even though it _is_ saved/restored via XSAVES/XRSTORS.

 * Because S_CET is loaded on VM-Enter and VM-Exit via dedicated VMCS fields,

 * the value saved/restored via XSTATE is always the host's value.  That detail

 * is _extremely_ important, as the guest's S_CET must _never_ be resident in

 * hardware while executing in the host.  Loading guest values for U_CET and

 * PL[0-3]_SSP while executing in the kernel is safe, as U_CET is specific to

 * userspace, and PL[0-3]_SSP are only consumed when transitioning to lower

 * privilege levels, i.e. are effectively only consumed by userspace as well.

 */

static bool is_xstate_managed_msr(struct kvm_vcpu *vcpu, u32 msr)

{

	if (!vcpu)

		return false;



	switch (msr) {

	case MSR_IA32_U_CET:

		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ||

		       guest_cpu_cap_has(vcpu, X86_FEATURE_IBT);

	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:

		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK);

	default:

		return false;

	}

}



/*

 * Lock (and if necessary, re-load) the guest FPU, i.e. XSTATE, and access an

 * MSR that is managed via XSTATE.  Note, the caller is responsible for doing

 * the initial FPU load, this helper only ensures that guest state is resident

 * in hardware (the kernel can load its FPU state in IRQ context).

 */

static __always_inline void kvm_access_xstate_msr(struct kvm_vcpu *vcpu,

						  struct msr_data *msr_info,

						  int access)

{

	BUILD_BUG_ON(access != MSR_TYPE_R && access != MSR_TYPE_W);



	KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm);

	KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm);



	kvm_fpu_get();

	if (access == MSR_TYPE_R)

		rdmsrq(msr_info->index, msr_info->data);

	else

		wrmsrq(msr_info->index, msr_info->data);

	kvm_fpu_put();

}



static __maybe_unused void kvm_set_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)

{

	kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_W);

}



static __maybe_unused void kvm_get_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)

{

	kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_R);

}



int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)

{

	u32 msr = msr_info->index;
@@ -4551,11 +4615,25 @@ static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, 
		    int (*do_msr)(struct kvm_vcpu *vcpu,

				  unsigned index, u64 *data))

{

	bool fpu_loaded = false;

	int i;



	for (i = 0; i < msrs->nmsrs; ++i)

	for (i = 0; i < msrs->nmsrs; ++i) {

		/*

		 * If userspace is accessing one or more XSTATE-managed MSRs,

		 * temporarily load the guest's FPU state so that the guest's

		 * MSR value(s) is resident in hardware and thus can be accessed

		 * via RDMSR/WRMSR.

		 */

		if (!fpu_loaded && is_xstate_managed_msr(vcpu, entries[i].index)) {

			kvm_load_guest_fpu(vcpu);

			fpu_loaded = true;

		}

		if (do_msr(vcpu, entries[i].index, &entries[i].data))

			break;

	}

	if (fpu_loaded)

		kvm_put_guest_fpu(vcpu);



	return i;

}
@@ -5965,6 +6043,7 @@ static int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, 
	struct kvm_one_reg one_reg;

	struct kvm_x86_reg_id *reg;

	u64 __user *user_val;

	bool load_fpu;

	int r;



	if (copy_from_user(&one_reg, argp, sizeof(one_reg)))
@@ -5991,12 +6070,18 @@ static int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, 


	guard(srcu)(&vcpu->kvm->srcu);



	load_fpu = is_xstate_managed_msr(vcpu, reg->index);

	if (load_fpu)

		kvm_load_guest_fpu(vcpu);



	user_val = u64_to_user_ptr(one_reg.addr);

	if (ioctl == KVM_GET_ONE_REG)

		r = kvm_get_one_msr(vcpu, reg->index, user_val);

	else

		r = kvm_set_one_msr(vcpu, reg->index, user_val);



	if (load_fpu)

		kvm_put_guest_fpu(vcpu);

	return r;

}