Commit 4647c822 authored by Paolo Bonzini's avatar Paolo Bonzini
Browse files

Merge tag 'kvmarm-fixes-6.14-3' of... 

Merge tag 'kvmarm-fixes-6.14-3' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 fixes for 6.14, take #3

- Fix TCR_EL2 configuration to not use the ASID in TTBR1_EL2
  and not mess-up T1SZ/PS by using the HCR_EL2.E2H==0 layout.

- Bring back the VMID allocation to the vcpu_load phase, ensuring
  that we only setup VTTBR_EL2 once on VHE. This cures an ugly
  race that would lead to running with an unallocated VMID.
parents 0ad2507d fa808ed4

arch/arm64/include/asm/kvm_arm.h

+1 −1
Original line number Diff line number Diff line
@@ -119,7 +119,7 @@ 
#define TCR_EL2_IRGN0_MASK	TCR_IRGN0_MASK

#define TCR_EL2_T0SZ_MASK	0x3f

#define TCR_EL2_MASK	(TCR_EL2_TG0_MASK | TCR_EL2_SH0_MASK | \

			 TCR_EL2_ORGN0_MASK | TCR_EL2_IRGN0_MASK | TCR_EL2_T0SZ_MASK)

			 TCR_EL2_ORGN0_MASK | TCR_EL2_IRGN0_MASK)



/* VTCR_EL2 Registers bits */

#define VTCR_EL2_DS		TCR_EL2_DS

arch/arm64/include/asm/kvm_host.h

+1 −1
Original line number Diff line number Diff line
@@ -1259,7 +1259,7 @@ int kvm_arm_pvtime_has_attr(struct kvm_vcpu *vcpu, 
extern unsigned int __ro_after_init kvm_arm_vmid_bits;

int __init kvm_arm_vmid_alloc_init(void);

void __init kvm_arm_vmid_alloc_free(void);

bool kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid);

void kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid);

void kvm_arm_vmid_clear_active(void);



static inline void kvm_arm_pvtime_vcpu_init(struct kvm_vcpu_arch *vcpu_arch)

arch/arm64/kvm/arm.c

+17 −20
Original line number Diff line number Diff line
@@ -559,6 +559,16 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 
	mmu = vcpu->arch.hw_mmu;

	last_ran = this_cpu_ptr(mmu->last_vcpu_ran);



	/*

	 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2,

	 * which happens eagerly in VHE.

	 *

	 * Also, the VMID allocator only preserves VMIDs that are active at the

	 * time of rollover, so KVM might need to grab a new VMID for the MMU if

	 * this is called from kvm_sched_in().

	 */

	kvm_arm_vmid_update(&mmu->vmid);



	/*

	 * We guarantee that both TLBs and I-cache are private to each

	 * vcpu. If detecting that a vcpu from the same VM has
@@ -1138,18 +1148,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 
		 */

		preempt_disable();



		/*

		 * The VMID allocator only tracks active VMIDs per

		 * physical CPU, and therefore the VMID allocated may not be

		 * preserved on VMID roll-over if the task was preempted,

		 * making a thread's VMID inactive. So we need to call

		 * kvm_arm_vmid_update() in non-premptible context.

		 */

		if (kvm_arm_vmid_update(&vcpu->arch.hw_mmu->vmid) &&

		    has_vhe())

			__load_stage2(vcpu->arch.hw_mmu,

				      vcpu->arch.hw_mmu->arch);



		kvm_pmu_flush_hwstate(vcpu);



		local_irq_disable();
@@ -1980,7 +1978,7 @@ static int kvm_init_vector_slots(void) 
static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)

{

	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);

	unsigned long tcr, ips;

	unsigned long tcr;



	/*

	 * Calculate the raw per-cpu offset without a translation from the
@@ -1994,19 +1992,18 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) 
	params->mair_el2 = read_sysreg(mair_el1);



	tcr = read_sysreg(tcr_el1);

	ips = FIELD_GET(TCR_IPS_MASK, tcr);

	if (cpus_have_final_cap(ARM64_KVM_HVHE)) {

		tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);

		tcr |= TCR_EPD1_MASK;

	} else {

		unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);



		tcr &= TCR_EL2_MASK;

		tcr |= TCR_EL2_RES1;

	}

	tcr &= ~TCR_T0SZ_MASK;

	tcr |= TCR_T0SZ(hyp_va_bits);

	tcr &= ~TCR_EL2_PS_MASK;

	tcr |= FIELD_PREP(TCR_EL2_PS_MASK, ips);

		tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);

		if (lpa2_is_enabled())

			tcr |= TCR_EL2_DS;

	}

	tcr |= TCR_T0SZ(hyp_va_bits);

	params->tcr_el2 = tcr;



	params->pgd_pa = kvm_mmu_get_httbr();

arch/arm64/kvm/vmid.c

+3 −8
Original line number Diff line number Diff line
@@ -135,11 +135,10 @@ void kvm_arm_vmid_clear_active(void) 
	atomic64_set(this_cpu_ptr(&active_vmids), VMID_ACTIVE_INVALID);

}



bool kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid)

void kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid)

{

	unsigned long flags;

	u64 vmid, old_active_vmid;

	bool updated = false;



	vmid = atomic64_read(&kvm_vmid->id);
@@ -157,21 +156,17 @@ bool kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid) 
	if (old_active_vmid != 0 && vmid_gen_match(vmid) &&

	    0 != atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_vmids),

					  old_active_vmid, vmid))

		return false;

		return;



	raw_spin_lock_irqsave(&cpu_vmid_lock, flags);



	/* Check that our VMID belongs to the current generation. */

	vmid = atomic64_read(&kvm_vmid->id);

	if (!vmid_gen_match(vmid)) {

	if (!vmid_gen_match(vmid))

		vmid = new_vmid(kvm_vmid);

		updated = true;

	}



	atomic64_set(this_cpu_ptr(&active_vmids), vmid);

	raw_spin_unlock_irqrestore(&cpu_vmid_lock, flags);



	return updated;

}



/*