Commit 3318e42b authored by Oliver Upton's avatar Oliver Upton
Browse files

Merge branch 'kvm-arm64/doublefault2' into kvmarm/next 



* kvm-arm64/doublefault2: (33 commits)
  : NV Support for FEAT_RAS + DoubleFault2
  :
  : Delegate the vSError context to the guest hypervisor when in a nested
  : state, including registers related to ESR propagation. Additionally,
  : catch up KVM's external abort infrastructure to the architecture,
  : implementing the effects of FEAT_DoubleFault2.
  :
  : This has some impact on non-nested guests, as SErrors deemed unmasked at
  : the time they're made pending are now immediately injected with an
  : emulated exception entry rather than using the VSE bit.
  KVM: arm64: Make RAS registers UNDEF when RAS isn't advertised
  KVM: arm64: Filter out HCR_EL2 bits when running in hypervisor context
  KVM: arm64: Check for SYSREGS_ON_CPU before accessing the CPU state
  KVM: arm64: Commit exceptions from KVM_SET_VCPU_EVENTS immediately
  KVM: arm64: selftests: Test ESR propagation for vSError injection
  KVM: arm64: Populate ESR_ELx.EC for emulated SError injection
  KVM: arm64: selftests: Catch up set_id_regs with the kernel
  KVM: arm64: selftests: Add SCTLR2_EL1 to get-reg-list
  KVM: arm64: selftests: Test SEAs are taken to SError vector when EASE=1
  KVM: arm64: selftests: Add basic SError injection test
  KVM: arm64: Don't retire MMIO instruction w/ pending (emulated) SError
  KVM: arm64: Advertise support for FEAT_DoubleFault2
  KVM: arm64: Advertise support for FEAT_SCTLR2
  KVM: arm64: nv: Enable vSErrors when HCRX_EL2.TMEA is set
  KVM: arm64: nv: Honor SError routing effects of SCTLR2_ELx.NMEA
  KVM: arm64: nv: Take "masked" aborts to EL2 when HCRX_EL2.TMEA is set
  KVM: arm64: Route SEAs to the SError vector when EASE is set
  KVM: arm64: nv: Ensure Address size faults affect correct ESR
  KVM: arm64: Factor out helper for selecting exception target EL
  KVM: arm64: Describe SCTLR2_ELx RESx masks
  ...

Signed-off-by: default avatarOliver Upton <oliver.upton@linux.dev>
parents c535d132 d9c5c232

arch/arm64/include/asm/kvm_emulate.h

+48 −3
Original line number Diff line number Diff line
@@ -45,16 +45,39 @@ bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); 
void kvm_skip_instr32(struct kvm_vcpu *vcpu);



void kvm_inject_undefined(struct kvm_vcpu *vcpu);

void kvm_inject_vabt(struct kvm_vcpu *vcpu);

void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);

void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);

int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);

int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);

void kvm_inject_size_fault(struct kvm_vcpu *vcpu);



static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr)

{

	return kvm_inject_sea(vcpu, false, addr);

}



static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr)

{

	return kvm_inject_sea(vcpu, true, addr);

}



static inline int kvm_inject_serror(struct kvm_vcpu *vcpu)

{

	/*

	 * ESR_ELx.ISV (later renamed to IDS) indicates whether or not

	 * ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information.

	 *

	 * Set the bit when injecting an SError w/o an ESR to indicate ISS

	 * does not follow the architected format.

	 */

	return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV);

}



void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);



void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);

int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);

int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);

int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);

int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr);



static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)

{
@@ -195,6 +218,11 @@ static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu) 
	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;

}



static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)

{

	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO;

}



static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)

{

	bool e2h, tge;
@@ -224,6 +252,20 @@ static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu) 
	return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu);

}



static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu)

{

	return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);

}



static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu)

{

	if (!is_nested_ctxt(vcpu))

		return false;



	return vcpu_el2_amo_is_set(vcpu) ||

	       (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);

}



/*

 * The layout of SPSR for an AArch32 state is different when observed from an

 * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
@@ -627,6 +669,9 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) 


		if (kvm_has_fpmr(kvm))

			vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM;



		if (kvm_has_sctlr2(kvm))

			vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En;

	}

}

#endif /* __ARM64_KVM_EMULATE_H__ */

arch/arm64/include/asm/kvm_host.h

+33 −3
Original line number Diff line number Diff line
@@ -523,6 +523,7 @@ enum vcpu_sysreg { 
	/* Anything from this can be RES0/RES1 sanitised */

	MARKER(__SANITISED_REG_START__),

	TCR2_EL2,	/* Extended Translation Control Register (EL2) */

	SCTLR2_EL2,	/* System Control Register 2 (EL2) */

	MDCR_EL2,	/* Monitor Debug Configuration Register (EL2) */

	CNTHCTL_EL2,	/* Counter-timer Hypervisor Control register */
@@ -537,6 +538,7 @@ enum vcpu_sysreg { 
	VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */

	VNCR(TCR_EL1),	/* Translation Control Register */

	VNCR(TCR2_EL1),	/* Extended Translation Control Register */

	VNCR(SCTLR2_EL1), /* System Control Register 2 */

	VNCR(ESR_EL1),	/* Exception Syndrome Register */

	VNCR(AFSR0_EL1),/* Auxiliary Fault Status Register 0 */

	VNCR(AFSR1_EL1),/* Auxiliary Fault Status Register 1 */
@@ -565,6 +567,10 @@ enum vcpu_sysreg { 


	VNCR(POR_EL1),	/* Permission Overlay Register 1 (EL1) */



	/* FEAT_RAS registers */

	VNCR(VDISR_EL2),

	VNCR(VSESR_EL2),



	VNCR(HFGRTR_EL2),

	VNCR(HFGWTR_EL2),

	VNCR(HFGITR_EL2),
@@ -817,7 +823,7 @@ struct kvm_vcpu_arch { 
	u8 iflags;



	/* State flags for kernel bookkeeping, unused by the hypervisor code */

	u8 sflags;

	u16 sflags;



	/*

	 * Don't run the guest (internal implementation need).
@@ -953,9 +959,21 @@ struct kvm_vcpu_arch { 
		__vcpu_flags_preempt_enable();			\

	} while (0)



#define __vcpu_test_and_clear_flag(v, flagset, f, m)		\

	({							\

		typeof(v->arch.flagset) set;			\

								\

		set = __vcpu_get_flag(v, flagset, f, m);	\

		__vcpu_clear_flag(v, flagset, f, m);		\

								\

		set;						\

	})



#define vcpu_get_flag(v, ...)	__vcpu_get_flag((v), __VA_ARGS__)

#define vcpu_set_flag(v, ...)	__vcpu_set_flag((v), __VA_ARGS__)

#define vcpu_clear_flag(v, ...)	__vcpu_clear_flag((v), __VA_ARGS__)

#define vcpu_test_and_clear_flag(v, ...)			\

	__vcpu_test_and_clear_flag((v), __VA_ARGS__)



/* KVM_ARM_VCPU_INIT completed */

#define VCPU_INITIALIZED	__vcpu_single_flag(cflags, BIT(0))
@@ -1015,6 +1033,8 @@ struct kvm_vcpu_arch { 
#define IN_WFI			__vcpu_single_flag(sflags, BIT(6))

/* KVM is currently emulating a nested ERET */

#define IN_NESTED_ERET		__vcpu_single_flag(sflags, BIT(7))

/* SError pending for nested guest */

#define NESTED_SERROR_PENDING	__vcpu_single_flag(sflags, BIT(8))





/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
@@ -1149,6 +1169,8 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) 
	 * System registers listed in the switch are not saved on every

	 * exit from the guest but are only saved on vcpu_put.

	 *

	 * SYSREGS_ON_CPU *MUST* be checked before using this helper.

	 *

	 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but

	 * should never be listed below, because the guest cannot modify its

	 * own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
@@ -1186,6 +1208,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) 
	case IFSR32_EL2:	*val = read_sysreg_s(SYS_IFSR32_EL2);	break;

	case DBGVCR32_EL2:	*val = read_sysreg_s(SYS_DBGVCR32_EL2);	break;

	case ZCR_EL1:		*val = read_sysreg_s(SYS_ZCR_EL12);	break;

	case SCTLR2_EL1:	*val = read_sysreg_s(SYS_SCTLR2_EL12);	break;

	default:		return false;

	}
@@ -1200,6 +1223,8 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) 
	 * System registers listed in the switch are not restored on every

	 * entry to the guest but are only restored on vcpu_load.

	 *

	 * SYSREGS_ON_CPU *MUST* be checked before using this helper.

	 *

	 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but

	 * should never be listed below, because the MPIDR should only be set

	 * once, before running the VCPU, and never changed later.
@@ -1236,6 +1261,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) 
	case IFSR32_EL2:	write_sysreg_s(val, SYS_IFSR32_EL2);	break;

	case DBGVCR32_EL2:	write_sysreg_s(val, SYS_DBGVCR32_EL2);	break;

	case ZCR_EL1:		write_sysreg_s(val, SYS_ZCR_EL12);	break;

	case SCTLR2_EL1:	write_sysreg_s(val, SYS_SCTLR2_EL12);	break;

	default:		return false;

	}
@@ -1387,8 +1413,6 @@ static inline bool kvm_arm_is_pvtime_enabled(struct kvm_vcpu_arch *vcpu_arch) 
	return (vcpu_arch->steal.base != INVALID_GPA);

}



void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);



struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);



DECLARE_KVM_HYP_PER_CPU(struct kvm_host_data, kvm_host_data);
@@ -1666,6 +1690,12 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val); 
#define kvm_has_s1poe(k)				\

	(kvm_has_feat((k), ID_AA64MMFR3_EL1, S1POE, IMP))



#define kvm_has_ras(k)					\

	(kvm_has_feat((k), ID_AA64PFR0_EL1, RAS, IMP))



#define kvm_has_sctlr2(k)				\

	(kvm_has_feat((k), ID_AA64MMFR3_EL1, SCTLRX, IMP))



static inline bool kvm_arch_has_irq_bypass(void)

{

	return true;

arch/arm64/include/asm/kvm_nested.h

+2 −0
Original line number Diff line number Diff line
@@ -80,6 +80,8 @@ extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); 
extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);



extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);

extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);

extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);



struct kvm_s2_trans {

	phys_addr_t output;

arch/arm64/include/asm/vncr_mapping.h

+2 −0
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ 
#define VNCR_SP_EL1             0x240

#define VNCR_VBAR_EL1           0x250

#define VNCR_TCR2_EL1		0x270

#define VNCR_SCTLR2_EL1		0x278

#define VNCR_PIRE0_EL1		0x290

#define VNCR_PIR_EL1		0x2A0

#define VNCR_POR_EL1		0x2A8
@@ -84,6 +85,7 @@ 
#define VNCR_ICH_HCR_EL2        0x4C0

#define VNCR_ICH_VMCR_EL2       0x4C8

#define VNCR_VDISR_EL2          0x500

#define VNCR_VSESR_EL2		0x508

#define VNCR_PMBLIMITR_EL1      0x800

#define VNCR_PMBPTR_EL1         0x810

#define VNCR_PMBSR_EL1          0x820

arch/arm64/kernel/cpufeature.c

+9 −0
Original line number Diff line number Diff line
@@ -303,6 +303,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { 
};



static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {

	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_DF2_SHIFT, 4, 0),

	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_GCS),

		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_GCS_SHIFT, 4, 0),

	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_frac_SHIFT, 4, 0),
@@ -500,6 +501,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = { 
	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_POE),

		       FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1POE_SHIFT, 4, 0),

	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),

	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_SCTLRX_SHIFT, 4, 0),

	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),

	ARM64_FTR_END,

};
@@ -3061,6 +3063,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { 
		.matches = has_pmuv3,

	},

#endif

	{

		.desc = "SCTLR2",

		.capability = ARM64_HAS_SCTLR2,

		.type = ARM64_CPUCAP_SYSTEM_FEATURE,

		.matches = has_cpuid_feature,

		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, SCTLRX, IMP)

	},

	{},

};