Commit 28654634 authored by Oliver Upton's avatar Oliver Upton
Browse files

Merge branch kvm-arm64/nv-s1pie-s1poe into kvmarm/next 



* kvm-arm64/nv-s1pie-s1poe: (36 commits)
  : NV support for S1PIE/S1POE, courtesy of Marc Zyngier
  :
  : Complete support for S1PIE/S1POE at vEL2, including:
  :
  :  - Save/restore of the vEL2 sysreg context
  :
  :  - Use the S1PIE/S1POE context for fast-path AT emulation
  :
  :  - Enlightening the software walker to the behavior of S1PIE/S1POE
  :
  :  - Like any other good NV series, some trap routing descriptions
  KVM: arm64: Handle WXN attribute
  KVM: arm64: Handle stage-1 permission overlays
  KVM: arm64: Make PAN conditions part of the S1 walk context
  KVM: arm64: Disable hierarchical permissions when POE is enabled
  KVM: arm64: Add POE save/restore for AT emulation fast-path
  KVM: arm64: Add save/restore support for POR_EL2
  KVM: arm64: Add basic support for POR_EL2
  KVM: arm64: Add kvm_has_s1poe() helper
  KVM: arm64: Subject S1PIE/S1POE registers to HCR_EL2.{TVM,TRVM}
  KVM: arm64: Drop bogus CPTR_EL2.E0POE trap routing
  arm64: Add encoding for POR_EL2
  KVM: arm64: Rely on visibility to let PIR*_ELx/TCR2_ELx UNDEF
  KVM: arm64: Hide S1PIE registers from userspace when disabled for guests
  KVM: arm64: Hide TCR2_EL1 from userspace when disabled for guests
  KVM: arm64: Define helper for EL2 registers with custom visibility
  KVM: arm64: Add a composite EL2 visibility helper
  KVM: arm64: Implement AT S1PIE support
  KVM: arm64: Disable hierarchical permissions when S1PIE is enabled
  KVM: arm64: Split S1 permission evaluation into direct and hierarchical parts
  KVM: arm64: Add AT fast-path support for S1PIE
  ...

Signed-off-by: default avatarOliver Upton <oliver.upton@linux.dev>
parents 81983758 1c6801d5

arch/arm64/include/asm/kvm_host.h

+34 −6
Original line number Diff line number Diff line
@@ -374,7 +374,7 @@ struct kvm_arch { 


	u64 ctr_el0;



	/* Masks for VNCR-baked sysregs */

	/* Masks for VNCR-backed and general EL2 sysregs */

	struct kvm_sysreg_masks	*sysreg_masks;



	/*
@@ -408,6 +408,9 @@ struct kvm_vcpu_fault_info { 
	r = __VNCR_START__ + ((VNCR_ ## r) / 8),	\

	__after_##r = __MAX__(__before_##r - 1, r)



#define MARKER(m)				\

	m, __after_##m = m - 1



enum vcpu_sysreg {

	__INVALID_SYSREG__,   /* 0 is reserved as an invalid value */

	MPIDR_EL1,	/* MultiProcessor Affinity Register */
@@ -475,6 +478,9 @@ enum vcpu_sysreg { 
	TTBR0_EL2,	/* Translation Table Base Register 0 (EL2) */

	TTBR1_EL2,	/* Translation Table Base Register 1 (EL2) */

	TCR_EL2,	/* Translation Control Register (EL2) */

	PIRE0_EL2,	/* Permission Indirection Register 0 (EL2) */

	PIR_EL2,	/* Permission Indirection Register 1 (EL2) */

	POR_EL2,	/* Permission Overlay Register 2 (EL2) */

	SPSR_EL2,	/* EL2 saved program status register */

	ELR_EL2,	/* EL2 exception link register */

	AFSR0_EL2,	/* Auxiliary Fault Status Register 0 (EL2) */
@@ -494,7 +500,12 @@ enum vcpu_sysreg { 
	CNTHV_CTL_EL2,

	CNTHV_CVAL_EL2,



	__VNCR_START__,	/* Any VNCR-capable reg goes after this point */

	/* Anything from this can be RES0/RES1 sanitised */

	MARKER(__SANITISED_REG_START__),

	TCR2_EL2,	/* Extended Translation Control Register (EL2) */



	/* Any VNCR-capable reg goes after this point */

	MARKER(__VNCR_START__),



	VNCR(SCTLR_EL1),/* System Control Register */

	VNCR(ACTLR_EL1),/* Auxiliary Control Register */
@@ -554,7 +565,7 @@ struct kvm_sysreg_masks { 
	struct {

		u64	res0;

		u64	res1;

	} mask[NR_SYS_REGS - __VNCR_START__];

	} mask[NR_SYS_REGS - __SANITISED_REG_START__];

};



struct kvm_cpu_context {
@@ -1002,13 +1013,13 @@ static inline u64 *___ctxt_sys_reg(const struct kvm_cpu_context *ctxt, int r) 


#define ctxt_sys_reg(c,r)	(*__ctxt_sys_reg(c,r))



u64 kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu *, enum vcpu_sysreg);

u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *, enum vcpu_sysreg, u64);

#define __vcpu_sys_reg(v,r)						\

	(*({								\

		const struct kvm_cpu_context *ctxt = &(v)->arch.ctxt;	\

		u64 *__r = __ctxt_sys_reg(ctxt, (r));			\

		if (vcpu_has_nv((v)) && (r) >= __VNCR_START__)		\

			*__r = kvm_vcpu_sanitise_vncr_reg((v), (r));	\

		if (vcpu_has_nv((v)) && (r) >= __SANITISED_REG_START__)	\

			*__r = kvm_vcpu_apply_reg_masks((v), (r), *__r);\

		__r;							\

	}))
@@ -1037,6 +1048,10 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) 
	case TTBR0_EL1:		*val = read_sysreg_s(SYS_TTBR0_EL12);	break;

	case TTBR1_EL1:		*val = read_sysreg_s(SYS_TTBR1_EL12);	break;

	case TCR_EL1:		*val = read_sysreg_s(SYS_TCR_EL12);	break;

	case TCR2_EL1:		*val = read_sysreg_s(SYS_TCR2_EL12);	break;

	case PIR_EL1:		*val = read_sysreg_s(SYS_PIR_EL12);	break;

	case PIRE0_EL1:		*val = read_sysreg_s(SYS_PIRE0_EL12);	break;

	case POR_EL1:		*val = read_sysreg_s(SYS_POR_EL12);	break;

	case ESR_EL1:		*val = read_sysreg_s(SYS_ESR_EL12);	break;

	case AFSR0_EL1:		*val = read_sysreg_s(SYS_AFSR0_EL12);	break;

	case AFSR1_EL1:		*val = read_sysreg_s(SYS_AFSR1_EL12);	break;
@@ -1083,6 +1098,10 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) 
	case TTBR0_EL1:		write_sysreg_s(val, SYS_TTBR0_EL12);	break;

	case TTBR1_EL1:		write_sysreg_s(val, SYS_TTBR1_EL12);	break;

	case TCR_EL1:		write_sysreg_s(val, SYS_TCR_EL12);	break;

	case TCR2_EL1:		write_sysreg_s(val, SYS_TCR2_EL12);	break;

	case PIR_EL1:		write_sysreg_s(val, SYS_PIR_EL12);	break;

	case PIRE0_EL1:		write_sysreg_s(val, SYS_PIRE0_EL12);	break;

	case POR_EL1:		write_sysreg_s(val, SYS_POR_EL12);	break;

	case ESR_EL1:		write_sysreg_s(val, SYS_ESR_EL12);	break;

	case AFSR0_EL1:		write_sysreg_s(val, SYS_AFSR0_EL12);	break;

	case AFSR1_EL1:		write_sysreg_s(val, SYS_AFSR1_EL12);	break;
@@ -1503,4 +1522,13 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val); 
	(system_supports_fpmr() &&			\

	 kvm_has_feat((k), ID_AA64PFR2_EL1, FPMR, IMP))



#define kvm_has_tcr2(k)				\

	(kvm_has_feat((k), ID_AA64MMFR3_EL1, TCRX, IMP))



#define kvm_has_s1pie(k)				\

	(kvm_has_feat((k), ID_AA64MMFR3_EL1, S1PIE, IMP))



#define kvm_has_s1poe(k)				\

	(kvm_has_feat((k), ID_AA64MMFR3_EL1, S1POE, IMP))



#endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/include/asm/vncr_mapping.h

+0 −1
Original line number Diff line number Diff line
@@ -50,7 +50,6 @@ 
#define VNCR_VBAR_EL1           0x250

#define VNCR_TCR2_EL1		0x270

#define VNCR_PIRE0_EL1		0x290

#define VNCR_PIRE0_EL2		0x298

#define VNCR_PIR_EL1		0x2A0

#define VNCR_POR_EL1		0x2A8

#define VNCR_ICH_LR0_EL2        0x400

arch/arm64/kvm/at.c

+404 −66
Original line number Diff line number Diff line
@@ -24,6 +24,9 @@ struct s1_walk_info { 
	unsigned int		txsz;

	int 	     		sl;

	bool	     		hpd;

	bool			e0poe;

	bool			poe;

	bool			pan;

	bool	     		be;

	bool	     		s2;

};
@@ -37,6 +40,16 @@ struct s1_walk_result { 
			u8	APTable;

			bool	UXNTable;

			bool	PXNTable;

			bool	uwxn;

			bool	uov;

			bool	ur;

			bool	uw;

			bool	ux;

			bool	pwxn;

			bool	pov;

			bool	pr;

			bool	pw;

			bool	px;

		};

		struct {

			u8	fst;
@@ -87,6 +100,51 @@ static enum trans_regime compute_translation_regime(struct kvm_vcpu *vcpu, u32 o 
	}

}



static bool s1pie_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime)

{

	if (!kvm_has_s1pie(vcpu->kvm))

		return false;



	switch (regime) {

	case TR_EL2:

	case TR_EL20:

		return vcpu_read_sys_reg(vcpu, TCR2_EL2) & TCR2_EL2_PIE;

	case TR_EL10:

		return  (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En) &&

			(__vcpu_sys_reg(vcpu, TCR2_EL1) & TCR2_EL1x_PIE);

	default:

		BUG();

	}

}



static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi)

{

	u64 val;



	if (!kvm_has_s1poe(vcpu->kvm)) {

		wi->poe = wi->e0poe = false;

		return;

	}



	switch (wi->regime) {

	case TR_EL2:

	case TR_EL20:

		val = vcpu_read_sys_reg(vcpu, TCR2_EL2);

		wi->poe = val & TCR2_EL2_POE;

		wi->e0poe = (wi->regime == TR_EL20) && (val & TCR2_EL2_E0POE);

		break;

	case TR_EL10:

		if (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En) {

			wi->poe = wi->e0poe = false;

			return;

		}



		val = __vcpu_sys_reg(vcpu, TCR2_EL1);

		wi->poe = val & TCR2_EL1x_POE;

		wi->e0poe = val & TCR2_EL1x_E0POE;

	}

}



static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi,

			 struct s1_walk_result *wr, u64 va)

{
@@ -98,6 +156,8 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, 


	wi->regime = compute_translation_regime(vcpu, op);

	as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W);

	wi->pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) &&

		  (*vcpu_cpsr(vcpu) & PSR_PAN_BIT);



	va55 = va & BIT(55);
@@ -180,6 +240,14 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, 
		    (va55 ?

		     FIELD_GET(TCR_HPD1, tcr) :

		     FIELD_GET(TCR_HPD0, tcr)));

	/* R_JHSVW */

	wi->hpd |= s1pie_enabled(vcpu, wi->regime);



	/* Do we have POE? */

	compute_s1poe(vcpu, wi);



	/* R_BVXDG */

	wi->hpd |= (wi->poe || wi->e0poe);



	/* Someone was silly enough to encode TG0/TG1 differently */

	if (va55) {
@@ -412,6 +480,11 @@ struct mmu_config { 
	u64	ttbr1;

	u64	tcr;

	u64	mair;

	u64	tcr2;

	u64	pir;

	u64	pire0;

	u64	por_el0;

	u64	por_el1;

	u64	sctlr;

	u64	vttbr;

	u64	vtcr;
@@ -424,6 +497,17 @@ static void __mmu_config_save(struct mmu_config *config) 
	config->ttbr1	= read_sysreg_el1(SYS_TTBR1);

	config->tcr	= read_sysreg_el1(SYS_TCR);

	config->mair	= read_sysreg_el1(SYS_MAIR);

	if (cpus_have_final_cap(ARM64_HAS_TCR2)) {

		config->tcr2	= read_sysreg_el1(SYS_TCR2);

		if (cpus_have_final_cap(ARM64_HAS_S1PIE)) {

			config->pir	= read_sysreg_el1(SYS_PIR);

			config->pire0	= read_sysreg_el1(SYS_PIRE0);

		}

		if (system_supports_poe()) {

			config->por_el1	= read_sysreg_el1(SYS_POR);

			config->por_el0	= read_sysreg_s(SYS_POR_EL0);

		}

	}

	config->sctlr	= read_sysreg_el1(SYS_SCTLR);

	config->vttbr	= read_sysreg(vttbr_el2);

	config->vtcr	= read_sysreg(vtcr_el2);
@@ -444,6 +528,17 @@ static void __mmu_config_restore(struct mmu_config *config) 
	write_sysreg_el1(config->ttbr1,	SYS_TTBR1);

	write_sysreg_el1(config->tcr,	SYS_TCR);

	write_sysreg_el1(config->mair,	SYS_MAIR);

	if (cpus_have_final_cap(ARM64_HAS_TCR2)) {

		write_sysreg_el1(config->tcr2, SYS_TCR2);

		if (cpus_have_final_cap(ARM64_HAS_S1PIE)) {

			write_sysreg_el1(config->pir, SYS_PIR);

			write_sysreg_el1(config->pire0, SYS_PIRE0);

		}

		if (system_supports_poe()) {

			write_sysreg_el1(config->por_el1, SYS_POR);

			write_sysreg_s(config->por_el0, SYS_POR_EL0);

		}

	}

	write_sysreg_el1(config->sctlr,	SYS_SCTLR);

	write_sysreg(config->vttbr,	vttbr_el2);

	write_sysreg(config->vtcr,	vtcr_el2);
@@ -739,6 +834,9 @@ static bool pan3_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 
	if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, PAN3))

		return false;



	if (s1pie_enabled(vcpu, regime))

		return true;



	if (regime == TR_EL10)

		sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);

	else
@@ -747,111 +845,343 @@ static bool pan3_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 
	return sctlr & SCTLR_EL1_EPAN;

}



static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)

static void compute_s1_direct_permissions(struct kvm_vcpu *vcpu,

					  struct s1_walk_info *wi,

					  struct s1_walk_result *wr)

{

	bool perm_fail, ur, uw, ux, pr, pw, px;

	struct s1_walk_result wr = {};

	struct s1_walk_info wi = {};

	int ret, idx;

	bool wxn;



	ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr);

	if (ret)

		goto compute_par;

	/* Non-hierarchical part of AArch64.S1DirectBasePermissions() */

	if (wi->regime != TR_EL2) {

		switch (FIELD_GET(PTE_USER | PTE_RDONLY, wr->desc)) {

		case 0b00:

			wr->pr = wr->pw = true;

			wr->ur = wr->uw = false;

			break;

		case 0b01:

			wr->pr = wr->pw = wr->ur = wr->uw = true;

			break;

		case 0b10:

			wr->pr = true;

			wr->pw = wr->ur = wr->uw = false;

			break;

		case 0b11:

			wr->pr = wr->ur = true;

			wr->pw = wr->uw = false;

			break;

		}



	if (wr.level == S1_MMU_DISABLED)

		goto compute_par;

		/* We don't use px for anything yet, but hey... */

		wr->px = !((wr->desc & PTE_PXN) || wr->uw);

		wr->ux = !(wr->desc & PTE_UXN);

	} else {

		wr->ur = wr->uw = wr->ux = false;



	idx = srcu_read_lock(&vcpu->kvm->srcu);

		if (!(wr->desc & PTE_RDONLY)) {

			wr->pr = wr->pw = true;

		} else {

			wr->pr = true;

			wr->pw = false;

		}



	ret = walk_s1(vcpu, &wi, &wr, vaddr);

		/* XN maps to UXN */

		wr->px = !(wr->desc & PTE_UXN);

	}



	srcu_read_unlock(&vcpu->kvm->srcu, idx);

	switch (wi->regime) {

	case TR_EL2:

	case TR_EL20:

		wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_WXN);

		break;

	case TR_EL10:

		wxn = (__vcpu_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_WXN);

		break;

	}



	if (ret)

		goto compute_par;

	wr->pwxn = wr->uwxn = wxn;

	wr->pov = wi->poe;

	wr->uov = wi->e0poe;

}



	/* FIXME: revisit when adding indirect permission support */

	/* AArch64.S1DirectBasePermissions() */

	if (wi.regime != TR_EL2) {

		switch (FIELD_GET(PTE_USER | PTE_RDONLY, wr.desc)) {

static void compute_s1_hierarchical_permissions(struct kvm_vcpu *vcpu,

						struct s1_walk_info *wi,

						struct s1_walk_result *wr)

{

	/* Hierarchical part of AArch64.S1DirectBasePermissions() */

	if (wi->regime != TR_EL2) {

		switch (wr->APTable) {

		case 0b00:

			pr = pw = true;

			ur = uw = false;

			break;

		case 0b01:

			pr = pw = ur = uw = true;

			wr->ur = wr->uw = false;

			break;

		case 0b10:

			pr = true;

			pw = ur = uw = false;

			wr->pw = wr->uw = false;

			break;

		case 0b11:

			pr = ur = true;

			pw = uw = false;

			wr->pw = wr->ur = wr->uw = false;

			break;

		}



		switch (wr.APTable) {

		case 0b00:

		wr->px &= !wr->PXNTable;

		wr->ux &= !wr->UXNTable;

	} else {

		if (wr->APTable & BIT(1))

			wr->pw = false;



		/* XN maps to UXN */

		wr->px &= !wr->UXNTable;

	}

}



#define perm_idx(v, r, i)	((vcpu_read_sys_reg((v), (r)) >> ((i) * 4)) & 0xf)



#define set_priv_perms(wr, r, w, x)	\

	do {				\

		(wr)->pr = (r);		\

		(wr)->pw = (w);		\

		(wr)->px = (x);		\

	} while (0)



#define set_unpriv_perms(wr, r, w, x)	\

	do {				\

		(wr)->ur = (r);		\

		(wr)->uw = (w);		\

		(wr)->ux = (x);		\

	} while (0)



#define set_priv_wxn(wr, v)		\

	do {				\

		(wr)->pwxn = (v);	\

	} while (0)



#define set_unpriv_wxn(wr, v)		\

	do {				\

		(wr)->uwxn = (v);	\

	} while (0)



/* Similar to AArch64.S1IndirectBasePermissions(), without GCS  */

#define set_perms(w, wr, ip)						\

	do {								\

		/* R_LLZDZ */						\

		switch ((ip)) {						\

		case 0b0000:						\

			set_ ## w ## _perms((wr), false, false, false);	\

			break;						\

		case 0b0001:						\

			set_ ## w ## _perms((wr), true , false, false);	\

			break;						\

		case 0b0010:						\

			set_ ## w ## _perms((wr), false, false, true );	\

			break;						\

		case 0b0011:						\

			set_ ## w ## _perms((wr), true , false, true );	\

			break;						\

		case 0b0100:						\

			set_ ## w ## _perms((wr), false, false, false);	\

			break;						\

		case 0b0101:						\

			set_ ## w ## _perms((wr), true , true , false);	\

			break;						\

		case 0b0110:						\

			set_ ## w ## _perms((wr), true , true , true );	\

			break;						\

		case 0b0111:						\

			set_ ## w ## _perms((wr), true , true , true );	\

			break;						\

		case 0b1000:						\

			set_ ## w ## _perms((wr), true , false, false);	\

			break;						\

		case 0b1001:						\

			set_ ## w ## _perms((wr), true , false, false);	\

			break;						\

		case 0b1010:						\

			set_ ## w ## _perms((wr), true , false, true );	\

			break;						\

		case 0b1011:						\

			set_ ## w ## _perms((wr), false, false, false);	\

			break;						\

		case 0b1100:						\

			set_ ## w ## _perms((wr), true , true , false);	\

			break;						\

		case 0b1101:						\

			set_ ## w ## _perms((wr), false, false, false);	\

			break;						\

		case 0b1110:						\

			set_ ## w ## _perms((wr), true , true , true );	\

			break;						\

		case 0b1111:						\

			set_ ## w ## _perms((wr), false, false, false);	\

			break;						\

		}							\

									\

		/* R_HJYGR */						\

		set_ ## w ## _wxn((wr), ((ip) == 0b0110));		\

									\

	} while (0)



static void compute_s1_indirect_permissions(struct kvm_vcpu *vcpu,

					    struct s1_walk_info *wi,

					    struct s1_walk_result *wr)

{

	u8 up, pp, idx;



	idx = pte_pi_index(wr->desc);



	switch (wi->regime) {

	case TR_EL10:

		pp = perm_idx(vcpu, PIR_EL1, idx);

		up = perm_idx(vcpu, PIRE0_EL1, idx);

		break;

		case 0b01:

			ur = uw = false;

	case TR_EL20:

		pp = perm_idx(vcpu, PIR_EL2, idx);

		up = perm_idx(vcpu, PIRE0_EL2, idx);

		break;

		case 0b10:

			pw = uw = false;

	case TR_EL2:

		pp = perm_idx(vcpu, PIR_EL2, idx);

		up = 0;

		break;

		case 0b11:

			pw = ur = uw = false;

	}



	set_perms(priv, wr, pp);



	if (wi->regime != TR_EL2)

		set_perms(unpriv, wr, up);

	else

		set_unpriv_perms(wr, false, false, false);



	wr->pov = wi->poe && !(pp & BIT(3));

	wr->uov = wi->e0poe && !(up & BIT(3));



	/* R_VFPJF */

	if (wr->px && wr->uw) {

		set_priv_perms(wr, false, false, false);

		set_unpriv_perms(wr, false, false, false);

	}

}



static void compute_s1_overlay_permissions(struct kvm_vcpu *vcpu,

					   struct s1_walk_info *wi,

					   struct s1_walk_result *wr)

{

	u8 idx, pov_perms, uov_perms;



	idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc);



	switch (wi->regime) {

	case TR_EL10:

		pov_perms = perm_idx(vcpu, POR_EL1, idx);

		uov_perms = perm_idx(vcpu, POR_EL0, idx);

		break;

	case TR_EL20:

		pov_perms = perm_idx(vcpu, POR_EL2, idx);

		uov_perms = perm_idx(vcpu, POR_EL0, idx);

		break;

	case TR_EL2:

		pov_perms = perm_idx(vcpu, POR_EL2, idx);

		uov_perms = 0;

		break;

	}



		/* We don't use px for anything yet, but hey... */

		px = !((wr.desc & PTE_PXN) || wr.PXNTable || uw);

		ux = !((wr.desc & PTE_UXN) || wr.UXNTable);

	if (pov_perms & ~POE_RXW)

		pov_perms = POE_NONE;



		if (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) {

			bool pan;

	if (wi->poe && wr->pov) {

		wr->pr &= pov_perms & POE_R;

		wr->px &= pov_perms & POE_X;

		wr->pw &= pov_perms & POE_W;

	}



			pan = *vcpu_cpsr(vcpu) & PSR_PAN_BIT;

			pan &= ur || uw || (pan3_enabled(vcpu, wi.regime) && ux);

			pw &= !pan;

			pr &= !pan;

	if (uov_perms & ~POE_RXW)

		uov_perms = POE_NONE;



	if (wi->e0poe && wr->uov) {

		wr->ur &= uov_perms & POE_R;

		wr->ux &= uov_perms & POE_X;

		wr->uw &= uov_perms & POE_W;

	}

}

	} else {

		ur = uw = ux = false;



		if (!(wr.desc & PTE_RDONLY)) {

			pr = pw = true;

		} else {

			pr = true;

			pw = false;

static void compute_s1_permissions(struct kvm_vcpu *vcpu,

				   struct s1_walk_info *wi,

				   struct s1_walk_result *wr)

{

	bool pan;



	if (!s1pie_enabled(vcpu, wi->regime))

		compute_s1_direct_permissions(vcpu, wi, wr);

	else

		compute_s1_indirect_permissions(vcpu, wi, wr);



	if (!wi->hpd)

		compute_s1_hierarchical_permissions(vcpu, wi, wr);



	if (wi->poe || wi->e0poe)

		compute_s1_overlay_permissions(vcpu, wi, wr);



	/* R_QXXPC */

	if (wr->pwxn) {

		if (!wr->pov && wr->pw)

			wr->px = false;

		if (wr->pov && wr->px)

			wr->pw = false;

	}



		if (wr.APTable & BIT(1))

			pw = false;

	/* R_NPBXC */

	if (wr->uwxn) {

		if (!wr->uov && wr->uw)

			wr->ux = false;

		if (wr->uov && wr->ux)

			wr->uw = false;

	}



		/* XN maps to UXN */

		px = !((wr.desc & PTE_UXN) || wr.UXNTable);

	pan = wi->pan && (wr->ur || wr->uw ||

			  (pan3_enabled(vcpu, wi->regime) && wr->ux));

	wr->pw &= !pan;

	wr->pr &= !pan;

}



	perm_fail = false;

static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)

{

	struct s1_walk_result wr = {};

	struct s1_walk_info wi = {};

	bool perm_fail = false;

	int ret, idx;



	ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr);

	if (ret)

		goto compute_par;



	if (wr.level == S1_MMU_DISABLED)

		goto compute_par;



	idx = srcu_read_lock(&vcpu->kvm->srcu);



	ret = walk_s1(vcpu, &wi, &wr, vaddr);



	srcu_read_unlock(&vcpu->kvm->srcu, idx);



	if (ret)

		goto compute_par;



	compute_s1_permissions(vcpu, &wi, &wr);



	switch (op) {

	case OP_AT_S1E1RP:

	case OP_AT_S1E1R:

	case OP_AT_S1E2R:

		perm_fail = !pr;

		perm_fail = !wr.pr;

		break;

	case OP_AT_S1E1WP:

	case OP_AT_S1E1W:

	case OP_AT_S1E2W:

		perm_fail = !pw;

		perm_fail = !wr.pw;

		break;

	case OP_AT_S1E0R:

		perm_fail = !ur;

		perm_fail = !wr.ur;

		break;

	case OP_AT_S1E0W:

		perm_fail = !uw;

		perm_fail = !wr.uw;

		break;

	case OP_AT_S1E1A:

	case OP_AT_S1E2A:
@@ -914,6 +1244,17 @@ static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 
	write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR1_EL1),	SYS_TTBR1);

	write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR_EL1),	SYS_TCR);

	write_sysreg_el1(vcpu_read_sys_reg(vcpu, MAIR_EL1),	SYS_MAIR);

	if (kvm_has_tcr2(vcpu->kvm)) {

		write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR2_EL1), SYS_TCR2);

		if (kvm_has_s1pie(vcpu->kvm)) {

			write_sysreg_el1(vcpu_read_sys_reg(vcpu, PIR_EL1), SYS_PIR);

			write_sysreg_el1(vcpu_read_sys_reg(vcpu, PIRE0_EL1), SYS_PIRE0);

		}

		if (kvm_has_s1poe(vcpu->kvm)) {

			write_sysreg_el1(vcpu_read_sys_reg(vcpu, POR_EL1), SYS_POR);

			write_sysreg_s(vcpu_read_sys_reg(vcpu, POR_EL0), SYS_POR_EL0);

		}

	}

	write_sysreg_el1(vcpu_read_sys_reg(vcpu, SCTLR_EL1),	SYS_SCTLR);

	__load_stage2(mmu, mmu->arch);
@@ -992,12 +1333,9 @@ void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 
	 * switching context behind everybody's back, disable interrupts...

	 */

	scoped_guard(write_lock_irqsave, &vcpu->kvm->mmu_lock) {

		struct kvm_s2_mmu *mmu;

		u64 val, hcr;

		bool fail;



		mmu = &vcpu->kvm->arch.mmu;



		val = hcr = read_sysreg(hcr_el2);

		val &= ~HCR_TGE;

		val |= HCR_VM;

arch/arm64/kvm/emulate-nested.c

+4 −8
Original line number Diff line number Diff line
@@ -79,7 +79,6 @@ enum cgt_group_id { 
	CGT_MDCR_E2TB,

	CGT_MDCR_TDCC,



	CGT_CPACR_E0POE,

	CGT_CPTR_TAM,

	CGT_CPTR_TCPAC,
@@ -362,12 +361,6 @@ static const struct trap_bits coarse_trap_bits[] = { 
		.mask		= MDCR_EL2_TDCC,

		.behaviour	= BEHAVE_FORWARD_ANY,

	},

	[CGT_CPACR_E0POE] = {

		.index		= CPTR_EL2,

		.value		= CPACR_ELx_E0POE,

		.mask		= CPACR_ELx_E0POE,

		.behaviour	= BEHAVE_FORWARD_ANY,

	},

	[CGT_CPTR_TAM] = {

		.index		= CPTR_EL2,

		.value		= CPTR_EL2_TAM,
@@ -711,6 +704,10 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = { 
	SR_TRAP(SYS_MAIR_EL1,		CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_AMAIR_EL1,		CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_CONTEXTIDR_EL1,	CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_PIR_EL1,		CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_PIRE0_EL1,		CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_POR_EL0,		CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_POR_EL1,		CGT_HCR_TVM_TRVM),

	SR_TRAP(SYS_TCR2_EL1,		CGT_HCR_TVM_TRVM_HCRX_TCR2En),

	SR_TRAP(SYS_DC_ZVA,		CGT_HCR_TDZ),

	SR_TRAP(SYS_DC_GVA,		CGT_HCR_TDZ),
@@ -1141,7 +1138,6 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = { 
	SR_TRAP(SYS_AMEVTYPER1_EL0(13),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(14),	CGT_CPTR_TAM),

	SR_TRAP(SYS_AMEVTYPER1_EL0(15),	CGT_CPTR_TAM),

	SR_TRAP(SYS_POR_EL0,		CGT_CPACR_E0POE),

	/* op0=2, op1=1, and CRn<0b1000 */

	SR_RANGE_TRAP(sys_reg(2, 1, 0, 0, 0),

		      sys_reg(2, 1, 7, 15, 7), CGT_CPTR_TTA),

arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h

+6 −5
Original line number Diff line number Diff line
@@ -58,7 +58,7 @@ static inline bool ctxt_has_s1pie(struct kvm_cpu_context *ctxt) 
		return false;



	vcpu = ctxt_to_vcpu(ctxt);

	return kvm_has_feat(kern_hyp_va(vcpu->kvm), ID_AA64MMFR3_EL1, S1PIE, IMP);

	return kvm_has_s1pie(kern_hyp_va(vcpu->kvm));

}



static inline bool ctxt_has_tcrx(struct kvm_cpu_context *ctxt)
@@ -69,7 +69,7 @@ static inline bool ctxt_has_tcrx(struct kvm_cpu_context *ctxt) 
		return false;



	vcpu = ctxt_to_vcpu(ctxt);

	return kvm_has_feat(kern_hyp_va(vcpu->kvm), ID_AA64MMFR3_EL1, TCRX, IMP);

	return kvm_has_tcr2(kern_hyp_va(vcpu->kvm));

}



static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt)
@@ -80,7 +80,7 @@ static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt) 
		return false;



	vcpu = ctxt_to_vcpu(ctxt);

	return kvm_has_feat(kern_hyp_va(vcpu->kvm), ID_AA64MMFR3_EL1, S1POE, IMP);

	return kvm_has_s1poe(kern_hyp_va(vcpu->kvm));

}



static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
@@ -152,9 +152,10 @@ static inline void __sysreg_restore_user_state(struct kvm_cpu_context *ctxt) 
	write_sysreg(ctxt_sys_reg(ctxt, TPIDRRO_EL0),	tpidrro_el0);

}



static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)

static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt,

					      u64 mpidr)

{

	write_sysreg(ctxt_sys_reg(ctxt, MPIDR_EL1),	vmpidr_el2);

	write_sysreg(mpidr,				vmpidr_el2);



	if (has_vhe() ||

	    !cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {