Commit 34c0d5a6 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/host_data into kvmarm-master/next 



* kvm-arm64/host_data:
  : .
  : Rationalise the host-specific data to live as part of the per-CPU state.
  :
  : From the cover letter:
  :
  : "It appears that over the years, we have accumulated a lot of cruft in
  : the kvm_vcpu_arch structure. Part of the gunk is data that is strictly
  : host CPU specific, and this result in two main problems:
  :
  : - the structure itself is stupidly large, over 8kB. With the
  :   arch-agnostic kvm_vcpu, we're above 10kB, which is insane. This has
  :   some ripple effects, as we need physically contiguous allocation to
  :   be able to map it at EL2 for !VHE. There is more to it though, as
  :   some data structures, although per-vcpu, could be allocated
  :   separately.
  :
  : - We lose track of the life-cycle of this data, because we're
  :   guaranteed that it will be around forever and we start relying on
  :   wrong assumptions. This is becoming a maintenance burden.
  :
  : This series rectifies some of these things, starting with the two main
  : offenders: debug and FP, a lot of which gets pushed out to the per-CPU
  : host structure. Indeed, their lifetime really isn't that of the vcpu,
  : but tied to the physical CPU the vpcu runs on.
  :
  : This results in a small reduction of the vcpu size, but mainly a much
  : clearer understanding of the life-cycle of these structures."
  : .
  KVM: arm64: Move management of __hyp_running_vcpu to load/put on VHE
  KVM: arm64: Exclude FP ownership from kvm_vcpu_arch
  KVM: arm64: Exclude host_fpsimd_state pointer from kvm_vcpu_arch
  KVM: arm64: Exclude mdcr_el2_host from kvm_vcpu_arch
  KVM: arm64: Exclude host_debug_data from vcpu_arch
  KVM: arm64: Add accessor for per-CPU state

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents fec50db7 9a393599

arch/arm64/include/asm/kvm_emulate.h

+2 −2
Original line number Diff line number Diff line
@@ -588,7 +588,7 @@ static __always_inline u64 kvm_get_reset_cptr_el2(struct kvm_vcpu *vcpu) 
		val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);



		if (!vcpu_has_sve(vcpu) ||

		    (vcpu->arch.fp_state != FP_STATE_GUEST_OWNED))

		    (*host_data_ptr(fp_owner) != FP_STATE_GUEST_OWNED))

			val |= CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN;

		if (cpus_have_final_cap(ARM64_SME))

			val |= CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN;
@@ -596,7 +596,7 @@ static __always_inline u64 kvm_get_reset_cptr_el2(struct kvm_vcpu *vcpu) 
		val = CPTR_NVHE_EL2_RES1;



		if (vcpu_has_sve(vcpu) &&

		    (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED))

		    (*host_data_ptr(fp_owner) == FP_STATE_GUEST_OWNED))

			val |= CPTR_EL2_TZ;

		if (cpus_have_final_cap(ARM64_SME))

			val &= ~CPTR_EL2_TSM;

arch/arm64/include/asm/kvm_host.h

+64 −25
Original line number Diff line number Diff line
@@ -530,8 +530,42 @@ struct kvm_cpu_context { 
	u64 *vncr_array;

};



/*

 * This structure is instantiated on a per-CPU basis, and contains

 * data that is:

 *

 * - tied to a single physical CPU, and

 * - either have a lifetime that does not extend past vcpu_put()

 * - or is an invariant for the lifetime of the system

 *

 * Use host_data_ptr(field) as a way to access a pointer to such a

 * field.

 */

struct kvm_host_data {

	struct kvm_cpu_context host_ctxt;

	struct user_fpsimd_state *fpsimd_state;	/* hyp VA */



	/* Ownership of the FP regs */

	enum {

		FP_STATE_FREE,

		FP_STATE_HOST_OWNED,

		FP_STATE_GUEST_OWNED,

	} fp_owner;



	/*

	 * host_debug_state contains the host registers which are

	 * saved and restored during world switches.

	 */

	 struct {

		/* {Break,watch}point registers */

		struct kvm_guest_debug_arch regs;

		/* Statistical profiling extension */

		u64 pmscr_el1;

		/* Self-hosted trace */

		u64 trfcr_el1;

		/* Values of trap registers for the host before guest entry. */

		u64 mdcr_el2;

	} host_debug_state;

};



struct kvm_host_psci_config {
@@ -592,19 +626,9 @@ struct kvm_vcpu_arch { 
	u64 mdcr_el2;

	u64 cptr_el2;



	/* Values of trap registers for the host before guest entry. */

	u64 mdcr_el2_host;



	/* Exception Information */

	struct kvm_vcpu_fault_info fault;



	/* Ownership of the FP regs */

	enum {

		FP_STATE_FREE,

		FP_STATE_HOST_OWNED,

		FP_STATE_GUEST_OWNED,

	} fp_state;



	/* Configuration flags, set once and for all before the vcpu can run */

	u8 cflags;
@@ -627,11 +651,10 @@ struct kvm_vcpu_arch { 
	 * We maintain more than a single set of debug registers to support

	 * debugging the guest from the host and to maintain separate host and

	 * guest state during world switches. vcpu_debug_state are the debug

	 * registers of the vcpu as the guest sees them.  host_debug_state are

	 * the host registers which are saved and restored during

	 * world switches. external_debug_state contains the debug

	 * values we want to debug the guest. This is set via the

	 * KVM_SET_GUEST_DEBUG ioctl.

	 * registers of the vcpu as the guest sees them.

	 *

	 * external_debug_state contains the debug values we want to debug the

	 * guest. This is set via the KVM_SET_GUEST_DEBUG ioctl.

	 *

	 * debug_ptr points to the set of debug registers that should be loaded

	 * onto the hardware when running the guest.
@@ -640,18 +663,8 @@ struct kvm_vcpu_arch { 
	struct kvm_guest_debug_arch vcpu_debug_state;

	struct kvm_guest_debug_arch external_debug_state;



	struct user_fpsimd_state *host_fpsimd_state;	/* hyp VA */

	struct task_struct *parent_task;



	struct {

		/* {Break,watch}point registers */

		struct kvm_guest_debug_arch regs;

		/* Statistical profiling extension */

		u64 pmscr_el1;

		/* Self-hosted trace */

		u64 trfcr_el1;

	} host_debug_state;



	/* VGIC state */

	struct vgic_cpu vgic_cpu;

	struct arch_timer_cpu timer_cpu;
@@ -1168,6 +1181,32 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); 


DECLARE_KVM_HYP_PER_CPU(struct kvm_host_data, kvm_host_data);



/*

 * How we access per-CPU host data depends on the where we access it from,

 * and the mode we're in:

 *

 * - VHE and nVHE hypervisor bits use their locally defined instance

 *

 * - the rest of the kernel use either the VHE or nVHE one, depending on

 *   the mode we're running in.

 *

 *   Unless we're in protected mode, fully deprivileged, and the nVHE

 *   per-CPU stuff is exclusively accessible to the protected EL2 code.

 *   In this case, the EL1 code uses the *VHE* data as its private state

 *   (which makes sense in a way as there shouldn't be any shared state

 *   between the host and the hypervisor).

 *

 * Yes, this is all totally trivial. Shoot me now.

 */

#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)

#define host_data_ptr(f)	(&this_cpu_ptr(&kvm_host_data)->f)

#else

#define host_data_ptr(f)						\

	(static_branch_unlikely(&kvm_protected_mode_initialized) ?	\

	 &this_cpu_ptr(&kvm_host_data)->f :				\

	 &this_cpu_ptr_hyp_sym(kvm_host_data)->f)

#endif



static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt)

{

	/* The host's MPIDR is immutable, so let's set it up at boot time */

arch/arm64/kvm/arm.c

+1 −7
Original line number Diff line number Diff line
@@ -378,12 +378,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 


	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;



	/*

	 * Default value for the FP state, will be overloaded at load

	 * time if we support FP (pretty likely)

	 */

	vcpu->arch.fp_state = FP_STATE_FREE;



	/* Set up the timer */

	kvm_timer_vcpu_init(vcpu);
@@ -1971,7 +1965,7 @@ static void cpu_set_hyp_vector(void) 


static void cpu_hyp_init_context(void)

{

	kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt);

	kvm_init_host_cpu_context(host_data_ptr(host_ctxt));



	if (!is_kernel_in_hyp_mode())

		cpu_init_hyp_mode();

arch/arm64/kvm/fpsimd.c

+6 −7
Original line number Diff line number Diff line
@@ -49,8 +49,6 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) 
	if (ret)

		return ret;



	vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);



	/*

	 * We need to keep current's task_struct pinned until its data has been

	 * unshared with the hypervisor to make sure it is not re-used by the
@@ -86,7 +84,8 @@ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) 
	 * guest in kvm_arch_vcpu_ctxflush_fp() and override this to

	 * FP_STATE_FREE if the flag set.

	 */

	vcpu->arch.fp_state = FP_STATE_HOST_OWNED;

	*host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;

	*host_data_ptr(fpsimd_state) = kern_hyp_va(&current->thread.uw.fpsimd_state);



	vcpu_clear_flag(vcpu, HOST_SVE_ENABLED);

	if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
@@ -110,7 +109,7 @@ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) 
		 * been saved, this is very unlikely to happen.

		 */

		if (read_sysreg_s(SYS_SVCR) & (SVCR_SM_MASK | SVCR_ZA_MASK)) {

			vcpu->arch.fp_state = FP_STATE_FREE;

			*host_data_ptr(fp_owner) = FP_STATE_FREE;

			fpsimd_save_and_flush_cpu_state();

		}

	}
@@ -126,7 +125,7 @@ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) 
void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)

{

	if (test_thread_flag(TIF_FOREIGN_FPSTATE))

		vcpu->arch.fp_state = FP_STATE_FREE;

		*host_data_ptr(fp_owner) = FP_STATE_FREE;

}



/*
@@ -142,7 +141,7 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) 


	WARN_ON_ONCE(!irqs_disabled());



	if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {

	if (*host_data_ptr(fp_owner) == FP_STATE_GUEST_OWNED) {



		/*

		 * Currently we do not support SME guests so SVCR is
@@ -196,7 +195,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) 
		isb();

	}



	if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {

	if (*host_data_ptr(fp_owner) == FP_STATE_GUEST_OWNED) {

		if (vcpu_has_sve(vcpu)) {

			__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);

arch/arm64/kvm/hyp/include/hyp/debug-sr.h

+4 −4
Original line number Diff line number Diff line
@@ -135,9 +135,9 @@ static inline void __debug_switch_to_guest_common(struct kvm_vcpu *vcpu) 
	if (!vcpu_get_flag(vcpu, DEBUG_DIRTY))

		return;



	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;

	host_ctxt = host_data_ptr(host_ctxt);

	guest_ctxt = &vcpu->arch.ctxt;

	host_dbg = &vcpu->arch.host_debug_state.regs;

	host_dbg = host_data_ptr(host_debug_state.regs);

	guest_dbg = kern_hyp_va(vcpu->arch.debug_ptr);



	__debug_save_state(host_dbg, host_ctxt);
@@ -154,9 +154,9 @@ static inline void __debug_switch_to_host_common(struct kvm_vcpu *vcpu) 
	if (!vcpu_get_flag(vcpu, DEBUG_DIRTY))

		return;



	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;

	host_ctxt = host_data_ptr(host_ctxt);

	guest_ctxt = &vcpu->arch.ctxt;

	host_dbg = &vcpu->arch.host_debug_state.regs;

	host_dbg = host_data_ptr(host_debug_state.regs);

	guest_dbg = kern_hyp_va(vcpu->arch.debug_ptr);



	__debug_save_state(guest_dbg, guest_ctxt);