Commit 435a9f60 authored by Oliver Upton's avatar Oliver Upton
Browse files

Merge branch kvm-arm64/shadow-mmu into kvmarm/next 



* kvm-arm64/shadow-mmu:
  : Shadow stage-2 MMU support for NV, courtesy of Marc Zyngier
  :
  : Initial implementation of shadow stage-2 page tables to support a guest
  : hypervisor. In the author's words:
  :
  :   So here's the 10000m (approximately 30000ft for those of you stuck
  :   with the wrong units) view of what this is doing:
  :
  :     - for each {VMID,VTTBR,VTCR} tuple the guest uses, we use a
  :       separate shadow s2_mmu context. This context has its own "real"
  :       VMID and a set of page tables that are the combination of the
  :       guest's S2 and the host S2, built dynamically one fault at a time.
  :
  :     - these shadow S2 contexts are ephemeral, and behave exactly as
  :       TLBs. For all intent and purposes, they *are* TLBs, and we discard
  :       them pretty often.
  :
  :     - TLB invalidation takes three possible paths:
  :
  :       * either this is an EL2 S1 invalidation, and we directly emulate
  :         it as early as possible
  :
  :       * or this is an EL1 S1 invalidation, and we need to apply it to
  :         the shadow S2s (plural!) that match the VMID set by the L1 guest
  :
  :       * or finally, this is affecting S2, and we need to teardown the
  :         corresponding part of the shadow S2s, which invalidates the TLBs
  KVM: arm64: nv: Truely enable nXS TLBI operations
  KVM: arm64: nv: Add handling of NXS-flavoured TLBI operations
  KVM: arm64: nv: Add handling of range-based TLBI operations
  KVM: arm64: nv: Add handling of outer-shareable TLBI operations
  KVM: arm64: nv: Invalidate TLBs based on shadow S2 TTL-like information
  KVM: arm64: nv: Tag shadow S2 entries with guest's leaf S2 level
  KVM: arm64: nv: Handle FEAT_TTL hinted TLB operations
  KVM: arm64: nv: Handle TLBI IPAS2E1{,IS} operations
  KVM: arm64: nv: Handle TLBI ALLE1{,IS} operations
  KVM: arm64: nv: Handle TLBI VMALLS12E1{,IS} operations
  KVM: arm64: nv: Handle TLB invalidation targeting L2 stage-1
  KVM: arm64: nv: Handle EL2 Stage-1 TLB invalidation
  KVM: arm64: nv: Add Stage-1 EL2 invalidation primitives
  KVM: arm64: nv: Unmap/flush shadow stage 2 page tables
  KVM: arm64: nv: Handle shadow stage 2 page faults
  KVM: arm64: nv: Implement nested Stage-2 page table walk logic
  KVM: arm64: nv: Support multiple nested Stage-2 mmu structures

Signed-off-by: default avatarOliver Upton <oliver.upton@linux.dev>
parents a35d5b20 3cfde36d

arch/arm64/include/asm/esr.h

+1 −0
Original line number Diff line number Diff line
@@ -152,6 +152,7 @@ 
#define ESR_ELx_Xs_MASK		(GENMASK_ULL(4, 0))



/* ISS field definitions for exceptions taken in to Hyp */

#define ESR_ELx_FSC_ADDRSZ	(0x00)

#define ESR_ELx_CV		(UL(1) << 24)

#define ESR_ELx_COND_SHIFT	(20)

#define ESR_ELx_COND_MASK	(UL(0xF) << ESR_ELx_COND_SHIFT)

arch/arm64/include/asm/kvm_asm.h

+2 −0
Original line number Diff line number Diff line
@@ -232,6 +232,8 @@ extern void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, 
					phys_addr_t start, unsigned long pages);

extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);



extern int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding);



extern void __kvm_timer_set_cntvoff(u64 cntvoff);



extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);

arch/arm64/include/asm/kvm_host.h

+36 −0
Original line number Diff line number Diff line
@@ -189,6 +189,33 @@ struct kvm_s2_mmu { 
	uint64_t split_page_chunk_size;



	struct kvm_arch *arch;



	/*

	 * For a shadow stage-2 MMU, the virtual vttbr used by the

	 * host to parse the guest S2.

	 * This either contains:

	 * - the virtual VTTBR programmed by the guest hypervisor with

         *   CnP cleared

	 * - The value 1 (VMID=0, BADDR=0, CnP=1) if invalid

	 *

	 * We also cache the full VTCR which gets used for TLB invalidation,

	 * taking the ARM ARM's "Any of the bits in VTCR_EL2 are permitted

	 * to be cached in a TLB" to the letter.

	 */

	u64	tlb_vttbr;

	u64	tlb_vtcr;



	/*

	 * true when this represents a nested context where virtual

	 * HCR_EL2.VM == 1

	 */

	bool	nested_stage2_enabled;



	/*

	 *  0: Nobody is currently using this, check vttbr for validity

	 * >0: Somebody is actively using this.

	 */

	atomic_t refcnt;

};



struct kvm_arch_memory_slot {
@@ -256,6 +283,14 @@ struct kvm_arch { 
	 */

	u64 fgu[__NR_FGT_GROUP_IDS__];



	/*

	 * Stage 2 paging state for VMs with nested S2 using a virtual

	 * VMID.

	 */

	struct kvm_s2_mmu *nested_mmus;

	size_t nested_mmus_size;

	int nested_mmus_next;



	/* Interrupt controller */

	struct vgic_dist	vgic;
@@ -1306,6 +1341,7 @@ void kvm_vcpu_load_vhe(struct kvm_vcpu *vcpu); 
void kvm_vcpu_put_vhe(struct kvm_vcpu *vcpu);



int __init kvm_set_ipa_limit(void);

u32 kvm_get_pa_bits(struct kvm *kvm);



#define __KVM_HAVE_ARCH_VM_ALLOC

struct kvm *kvm_arch_alloc_vm(void);

arch/arm64/include/asm/kvm_mmu.h

+26 −0
Original line number Diff line number Diff line
@@ -98,6 +98,7 @@ alternative_cb_end 
#include <asm/mmu_context.h>

#include <asm/kvm_emulate.h>

#include <asm/kvm_host.h>

#include <asm/kvm_nested.h>



void kvm_update_va_mask(struct alt_instr *alt,

			__le32 *origptr, __le32 *updptr, int nr_inst);
@@ -165,6 +166,10 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, 
int create_hyp_stack(phys_addr_t phys_addr, unsigned long *haddr);

void __init free_hyp_pgds(void);



void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size);

void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end);

void kvm_stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end);



void stage2_unmap_vm(struct kvm *kvm);

int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type);

void kvm_uninit_stage2_mmu(struct kvm *kvm);
@@ -326,5 +331,26 @@ static inline struct kvm *kvm_s2_mmu_to_kvm(struct kvm_s2_mmu *mmu) 
{

	return container_of(mmu->arch, struct kvm, arch);

}



static inline u64 get_vmid(u64 vttbr)

{

	return (vttbr & VTTBR_VMID_MASK(kvm_get_vmid_bits())) >>

		VTTBR_VMID_SHIFT;

}



static inline bool kvm_s2_mmu_valid(struct kvm_s2_mmu *mmu)

{

	return !(mmu->tlb_vttbr & VTTBR_CNP_BIT);

}



static inline bool kvm_is_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)

{

	/*

	 * Be careful, mmu may not be fully initialised so do look at

	 * *any* of its fields.

	 */

	return &kvm->arch.mmu != mmu;

}



#endif /* __ASSEMBLY__ */

#endif /* __ARM64_KVM_MMU_H__ */

arch/arm64/include/asm/kvm_nested.h

+127 −0
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#include <linux/bitfield.h>

#include <linux/kvm_host.h>

#include <asm/kvm_emulate.h>

#include <asm/kvm_pgtable.h>



static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)

{
@@ -61,6 +62,125 @@ static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) 
}



extern bool forward_smc_trap(struct kvm_vcpu *vcpu);

extern void kvm_init_nested(struct kvm *kvm);

extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);

extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);

extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);



union tlbi_info;



extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,

				       const union tlbi_info *info,

				       void (*)(struct kvm_s2_mmu *,

						const union tlbi_info *));

extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);

extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);



struct kvm_s2_trans {

	phys_addr_t output;

	unsigned long block_size;

	bool writable;

	bool readable;

	int level;

	u32 esr;

	u64 upper_attr;

};



static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans)

{

	return trans->output;

}



static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans)

{

	return trans->block_size;

}



static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans)

{

	return trans->esr;

}



static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans)

{

	return trans->readable;

}



static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans)

{

	return trans->writable;

}



static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans)

{

	return !(trans->upper_attr & BIT(54));

}



extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,

			      struct kvm_s2_trans *result);

extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,

				    struct kvm_s2_trans *trans);

extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);

extern void kvm_nested_s2_wp(struct kvm *kvm);

extern void kvm_nested_s2_unmap(struct kvm *kvm);

extern void kvm_nested_s2_flush(struct kvm *kvm);



unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val);



static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr)

{

	struct kvm *kvm = vpcu->kvm;

	u8 CRm = sys_reg_CRm(instr);



	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&

	      sys_reg_Op1(instr) == TLBI_Op1_EL1))

		return false;



	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||

	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&

	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))

		return false;



	if (CRm == TLBI_CRm_nROS &&

	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))

		return false;



	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||

	     CRm == TLBI_CRm_RNS) &&

	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))

		return false;



	return true;

}



static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr)

{

	struct kvm *kvm = vpcu->kvm;

	u8 CRm = sys_reg_CRm(instr);



	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&

	      sys_reg_Op1(instr) == TLBI_Op1_EL2))

		return false;



	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||

	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&

	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))

		return false;



	if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS)

		return false;



	if (CRm == TLBI_CRm_nROS &&

	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))

		return false;



	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||

	     CRm == TLBI_CRm_RNS) &&

	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))

		return false;



	return true;

}



int kvm_init_nv_sysregs(struct kvm *kvm);
@@ -76,4 +196,11 @@ static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr) 
}

#endif



#define KVM_NV_GUEST_MAP_SZ	(KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0)



static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans)

{

	return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level);

}



#endif /* __ARM64_KVM_NESTED_H */