Commit 8c2899e7 authored by Oliver Upton's avatar Oliver Upton
Browse files

Merge branch kvm-arm64/nv-sve into kvmarm/next 



* kvm-arm64/nv-sve:
  : CPTR_EL2, FPSIMD/SVE support for nested
  :
  : This series brings support for honoring the guest hypervisor's CPTR_EL2
  : trap configuration when running a nested guest, along with support for
  : FPSIMD/SVE usage at L1 and L2.
  KVM: arm64: Allow the use of SVE+NV
  KVM: arm64: nv: Add additional trap setup for CPTR_EL2
  KVM: arm64: nv: Add trap description for CPTR_EL2
  KVM: arm64: nv: Add TCPAC/TTA to CPTR->CPACR conversion helper
  KVM: arm64: nv: Honor guest hypervisor's FP/SVE traps in CPTR_EL2
  KVM: arm64: nv: Load guest FP state for ZCR_EL2 trap
  KVM: arm64: nv: Handle CPACR_EL1 traps
  KVM: arm64: Spin off helper for programming CPTR traps
  KVM: arm64: nv: Ensure correct VL is loaded before saving SVE state
  KVM: arm64: nv: Use guest hypervisor's max VL when running nested guest
  KVM: arm64: nv: Save guest's ZCR_EL2 when in hyp context
  KVM: arm64: nv: Load guest hyp's ZCR into EL1 state
  KVM: arm64: nv: Handle ZCR_EL2 traps
  KVM: arm64: nv: Forward SVE traps to guest hypervisor
  KVM: arm64: nv: Forward FP/ASIMD traps to guest hypervisor

Signed-off-by: default avatarOliver Upton <oliver.upton@linux.dev>
parents 1270dad3 f1ee914f

arch/arm64/include/asm/kvm_emulate.h

+55 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#ifndef __ARM64_KVM_EMULATE_H__

#define __ARM64_KVM_EMULATE_H__



#include <linux/bitfield.h>

#include <linux/kvm_host.h>



#include <asm/debug-monitors.h>
@@ -55,6 +56,14 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); 
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);

int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);



static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)

{

	u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SVE) |

		  ESR_ELx_IL;



	kvm_inject_nested_sync(vcpu, esr);

}



#if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)

static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)

{
@@ -638,4 +647,50 @@ static __always_inline void kvm_reset_cptr_el2(struct kvm_vcpu *vcpu) 


	kvm_write_cptr_el2(val);

}



/*

 * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE

 * format if E2H isn't set.

 */

static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu)

{

	u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2);



	if (!vcpu_el2_e2h_is_set(vcpu))

		cptr = translate_cptr_el2_to_cpacr_el1(cptr);



	return cptr;

}



static inline bool ____cptr_xen_trap_enabled(const struct kvm_vcpu *vcpu,

					     unsigned int xen)

{

	switch (xen) {

	case 0b00:

	case 0b10:

		return true;

	case 0b01:

		return vcpu_el2_tge_is_set(vcpu) && !vcpu_is_el2(vcpu);

	case 0b11:

	default:

		return false;

	}

}



#define __guest_hyp_cptr_xen_trap_enabled(vcpu, xen)				\

	(!vcpu_has_nv(vcpu) ? false :						\

	 ____cptr_xen_trap_enabled(vcpu,					\

				   SYS_FIELD_GET(CPACR_ELx, xen,		\

						 vcpu_sanitised_cptr_el2(vcpu))))



static inline bool guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu *vcpu)

{

	return __guest_hyp_cptr_xen_trap_enabled(vcpu, FPEN);

}



static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu)

{

	return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN);

}



#endif /* __ARM64_KVM_EMULATE_H__ */

arch/arm64/include/asm/kvm_host.h

+6 −0
Original line number Diff line number Diff line
@@ -458,6 +458,7 @@ enum vcpu_sysreg { 
	MDCR_EL2,	/* Monitor Debug Configuration Register (EL2) */

	CPTR_EL2,	/* Architectural Feature Trap Register (EL2) */

	HACR_EL2,	/* Hypervisor Auxiliary Control Register */

	ZCR_EL2,	/* SVE Control Register (EL2) */

	TTBR0_EL2,	/* Translation Table Base Register 0 (EL2) */

	TTBR1_EL2,	/* Translation Table Base Register 1 (EL2) */

	TCR_EL2,	/* Translation Control Register (EL2) */
@@ -902,6 +903,9 @@ struct kvm_vcpu_arch { 


#define vcpu_sve_max_vq(vcpu)	sve_vq_from_vl((vcpu)->arch.sve_max_vl)



#define vcpu_sve_zcr_elx(vcpu)						\

	(unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1)



#define vcpu_sve_state_size(vcpu) ({					\

	size_t __size_ret;						\

	unsigned int __vcpu_vq;						\
@@ -1026,6 +1030,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) 
	case DACR32_EL2:	*val = read_sysreg_s(SYS_DACR32_EL2);	break;

	case IFSR32_EL2:	*val = read_sysreg_s(SYS_IFSR32_EL2);	break;

	case DBGVCR32_EL2:	*val = read_sysreg_s(SYS_DBGVCR32_EL2);	break;

	case ZCR_EL1:		*val = read_sysreg_s(SYS_ZCR_EL12);	break;

	default:		return false;

	}
@@ -1071,6 +1076,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) 
	case DACR32_EL2:	write_sysreg_s(val, SYS_DACR32_EL2);	break;

	case IFSR32_EL2:	write_sysreg_s(val, SYS_IFSR32_EL2);	break;

	case DBGVCR32_EL2:	write_sysreg_s(val, SYS_DBGVCR32_EL2);	break;

	case ZCR_EL1:		write_sysreg_s(val, SYS_ZCR_EL12);	break;

	default:		return false;

	}

arch/arm64/include/asm/kvm_nested.h

+3 −1
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) 


static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)

{

	u64 cpacr_el1 = 0;

	u64 cpacr_el1 = CPACR_ELx_RES1;



	if (cptr_el2 & CPTR_EL2_TTA)

		cpacr_el1 |= CPACR_ELx_TTA;
@@ -42,6 +42,8 @@ static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) 
	if (!(cptr_el2 & CPTR_EL2_TZ))

		cpacr_el1 |= CPACR_ELx_ZEN;



	cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);



	return cpacr_el1;

}

arch/arm64/kvm/arm.c

+0 −5
Original line number Diff line number Diff line
@@ -1458,11 +1458,6 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, 
	    test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))

		return -EINVAL;



	/* Disallow NV+SVE for the time being */

	if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features) &&

	    test_bit(KVM_ARM_VCPU_SVE, &features))

		return -EINVAL;



	if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))

		return 0;

arch/arm64/kvm/emulate-nested.c

+91 −0
Original line number Diff line number Diff line
@@ -79,6 +79,10 @@ enum cgt_group_id { 
	CGT_MDCR_E2TB,

	CGT_MDCR_TDCC,



	CGT_CPACR_E0POE,

	CGT_CPTR_TAM,

	CGT_CPTR_TCPAC,



	/*

	 * Anything after this point is a combination of coarse trap

	 * controls, which must all be evaluated to decide what to do.
@@ -106,6 +110,8 @@ enum cgt_group_id { 
	CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__,

	CGT_CNTHCTL_EL1PTEN,



	CGT_CPTR_TTA,



	/* Must be last */

	__NR_CGT_GROUP_IDS__

};
@@ -345,6 +351,24 @@ static const struct trap_bits coarse_trap_bits[] = { 
		.mask		= MDCR_EL2_TDCC,

		.behaviour	= BEHAVE_FORWARD_ANY,

	},

	[CGT_CPACR_E0POE] = {

		.index		= CPTR_EL2,

		.value		= CPACR_ELx_E0POE,

		.mask		= CPACR_ELx_E0POE,

		.behaviour	= BEHAVE_FORWARD_ANY,

	},

	[CGT_CPTR_TAM] = {

		.index		= CPTR_EL2,

		.value		= CPTR_EL2_TAM,

		.mask		= CPTR_EL2_TAM,

		.behaviour	= BEHAVE_FORWARD_ANY,

	},

	[CGT_CPTR_TCPAC] = {

		.index		= CPTR_EL2,

		.value		= CPTR_EL2_TCPAC,

		.mask		= CPTR_EL2_TCPAC,

		.behaviour	= BEHAVE_FORWARD_ANY,

	},

};



#define MCB(id, ...)						\
@@ -410,12 +434,26 @@ static enum trap_behaviour check_cnthctl_el1pten(struct kvm_vcpu *vcpu) 
	return BEHAVE_FORWARD_ANY;

}



static enum trap_behaviour check_cptr_tta(struct kvm_vcpu *vcpu)

{

	u64 val = __vcpu_sys_reg(vcpu, CPTR_EL2);



	if (!vcpu_el2_e2h_is_set(vcpu))

		val = translate_cptr_el2_to_cpacr_el1(val);



	if (val & CPACR_ELx_TTA)

		return BEHAVE_FORWARD_ANY;



	return BEHAVE_HANDLE_LOCALLY;

}



#define CCC(id, fn)				\

	[id - __COMPLEX_CONDITIONS__] = fn



static const complex_condition_check ccc[] = {

	CCC(CGT_CNTHCTL_EL1PCTEN, check_cnthctl_el1pcten),

	CCC(CGT_CNTHCTL_EL1PTEN, check_cnthctl_el1pten),

	CCC(CGT_CPTR_TTA, check_cptr_tta),

};



/*
@@ -1000,6 +1038,59 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = { 
	SR_TRAP(SYS_TRBPTR_EL1, 	CGT_MDCR_E2TB),

	SR_TRAP(SYS_TRBSR_EL1, 		CGT_MDCR_E2TB),

	SR_TRAP(SYS_TRBTRG_EL1,		CGT_MDCR_E2TB),

	SR_TRAP(SYS_CPACR_EL1,		CGT_CPTR_TCPAC),

	SR_TRAP(SYS_AMUSERENR_EL0,	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCFGR_EL0,		CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCGCR_EL0,		CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCNTENCLR0_EL0,	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCNTENCLR1_EL0,	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCNTENSET0_EL0,	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCNTENSET1_EL0,	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMCR_EL0,		CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR0_EL0(0),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR0_EL0(1),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR0_EL0(2),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR0_EL0(3),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(0),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(1),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(2),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(3),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(4),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(5),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(6),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(7),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(8),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(9),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(10),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(11),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(12),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(13),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(14),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVCNTR1_EL0(15),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER0_EL0(0),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER0_EL0(1),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER0_EL0(2),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER0_EL0(3),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(0),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(1),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(2),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(3),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(4),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(5),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(6),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(7),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(8),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(9),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(10),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(11),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(12),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(13),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(14),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(15),	CGT_CPTR_TAM),

	SR_TRAP(SYS_POR_EL0,		CGT_CPACR_E0POE),

	/* op0=2, op1=1, and CRn<0b1000 */

	SR_RANGE_TRAP(sys_reg(2, 1, 0, 0, 0),

		      sys_reg(2, 1, 7, 15, 7), CGT_CPTR_TTA),

	SR_TRAP(SYS_CNTP_TVAL_EL0,	CGT_CNTHCTL_EL1PTEN),

	SR_TRAP(SYS_CNTP_CVAL_EL0,	CGT_CNTHCTL_EL1PTEN),

	SR_TRAP(SYS_CNTP_CTL_EL0,	CGT_CNTHCTL_EL1PTEN),