Commit 54b44ad2 authored by Oliver Upton's avatar Oliver Upton
Browse files

Merge branch kvm-arm64/sgi-injection into kvmarm/next 



* kvm-arm64/sgi-injection:
  : vSGI injection improvements + fixes, courtesy Marc Zyngier
  :
  : Avoid linearly searching for vSGI targets using a compressed MPIDR to
  : index a cache. While at it, fix some egregious bugs in KVM's mishandling
  : of vcpuid (user-controlled value) and vcpu_idx.
  KVM: arm64: Clarify the ordering requirements for vcpu/RD creation
  KVM: arm64: vgic-v3: Optimize affinity-based SGI injection
  KVM: arm64: Fast-track kvm_mpidr_to_vcpu() when mpidr_data is available
  KVM: arm64: Build MPIDR to vcpu index cache at runtime
  KVM: arm64: Simplify kvm_vcpu_get_mpidr_aff()
  KVM: arm64: Use vcpu_idx for invalidation tracking
  KVM: arm64: vgic: Use vcpu_idx for the debug information
  KVM: arm64: vgic-v2: Use cpuid from userspace as vcpu_id
  KVM: arm64: vgic-v3: Refactor GICv3 SGI generation
  KVM: arm64: vgic-its: Treat the collection target address as a vcpu_id
  KVM: arm64: vgic: Make kvm_vgic_inject_irq() take a vcpu pointer

Signed-off-by: default avatarOliver Upton <oliver.upton@linux.dev>
parents df26b779 f9940416

Documentation/virt/kvm/devices/arm-vgic-v3.rst

+7 −0
Original line number Diff line number Diff line
@@ -59,6 +59,13 @@ Groups: 
  It is invalid to mix calls with KVM_VGIC_V3_ADDR_TYPE_REDIST and

  KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION attributes.



  Note that to obtain reproducible results (the same VCPU being associated

  with the same redistributor across a save/restore operation), VCPU creation

  order, redistributor region creation order as well as the respective

  interleaves of VCPU and region creation MUST be preserved.  Any change in

  either ordering may result in a different vcpu_id/redistributor association,

  resulting in a VM that will fail to run at restore time.



  Errors:



    =======  =============================================================

arch/arm64/include/asm/kvm_emulate.h

+1 −1
Original line number Diff line number Diff line
@@ -470,7 +470,7 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) 


static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)

{

	return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;

	return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;

}



static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)

arch/arm64/include/asm/kvm_host.h

+28 −0
Original line number Diff line number Diff line
@@ -212,6 +212,31 @@ struct kvm_protected_vm { 
	struct kvm_hyp_memcache teardown_mc;

};



struct kvm_mpidr_data {

	u64			mpidr_mask;

	DECLARE_FLEX_ARRAY(u16, cmpidr_to_idx);

};



static inline u16 kvm_mpidr_index(struct kvm_mpidr_data *data, u64 mpidr)

{

	unsigned long mask = data->mpidr_mask;

	u64 aff = mpidr & MPIDR_HWID_BITMASK;

	int nbits, bit, bit_idx = 0;

	u16 index = 0;



	/*

	 * If this looks like RISC-V's BEXT or x86's PEXT

	 * instructions, it isn't by accident.

	 */

	nbits = fls(mask);

	for_each_set_bit(bit, &mask, nbits) {

		index |= (aff & BIT(bit)) >> (bit - bit_idx);

		bit_idx++;

	}



	return index;

}



struct kvm_arch {

	struct kvm_s2_mmu mmu;
@@ -253,6 +278,9 @@ struct kvm_arch { 
	/* VM-wide vCPU feature set */

	DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES);



	/* MPIDR to vcpu index mapping, optional */

	struct kvm_mpidr_data *mpidr_data;



	/*

	 * VM-wide PMU filter, implemented as a bitmap and big enough for

	 * up to 2^10 events (ARMv8.0) or 2^16 events (ARMv8.1+).

arch/arm64/kvm/arch_timer.c

+1 −1
Original line number Diff line number Diff line
@@ -458,7 +458,7 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, 
				   timer_ctx->irq.level);



	if (!userspace_irqchip(vcpu->kvm)) {

		ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,

		ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu,

					  timer_irq(timer_ctx),

					  timer_ctx->irq.level,

					  timer_ctx);

arch/arm64/kvm/arm.c

+76 −17
Original line number Diff line number Diff line
@@ -205,6 +205,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) 
	if (is_protected_kvm_enabled())

		pkvm_destroy_hyp_vm(kvm);



	kfree(kvm->arch.mpidr_data);

	kvm_destroy_vcpus(kvm);



	kvm_unshare_hyp(kvm, kvm + 1);
@@ -437,9 +438,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 
	 * We might get preempted before the vCPU actually runs, but

	 * over-invalidation doesn't affect correctness.

	 */

	if (*last_ran != vcpu->vcpu_id) {

	if (*last_ran != vcpu->vcpu_idx) {

		kvm_call_hyp(__kvm_flush_cpu_context, mmu);

		*last_ran = vcpu->vcpu_id;

		*last_ran = vcpu->vcpu_idx;

	}



	vcpu->cpu = cpu;
@@ -577,6 +578,57 @@ static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) 
	return vcpu_get_flag(vcpu, VCPU_INITIALIZED);

}



static void kvm_init_mpidr_data(struct kvm *kvm)

{

	struct kvm_mpidr_data *data = NULL;

	unsigned long c, mask, nr_entries;

	u64 aff_set = 0, aff_clr = ~0UL;

	struct kvm_vcpu *vcpu;



	mutex_lock(&kvm->arch.config_lock);



	if (kvm->arch.mpidr_data || atomic_read(&kvm->online_vcpus) == 1)

		goto out;



	kvm_for_each_vcpu(c, vcpu, kvm) {

		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);

		aff_set |= aff;

		aff_clr &= aff;

	}



	/*

	 * A significant bit can be either 0 or 1, and will only appear in

	 * aff_set. Use aff_clr to weed out the useless stuff.

	 */

	mask = aff_set ^ aff_clr;

	nr_entries = BIT_ULL(hweight_long(mask));



	/*

	 * Don't let userspace fool us. If we need more than a single page

	 * to describe the compressed MPIDR array, just fall back to the

	 * iterative method. Single vcpu VMs do not need this either.

	 */

	if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)

		data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),

			       GFP_KERNEL_ACCOUNT);



	if (!data)

		goto out;



	data->mpidr_mask = mask;



	kvm_for_each_vcpu(c, vcpu, kvm) {

		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);

		u16 index = kvm_mpidr_index(data, aff);



		data->cmpidr_to_idx[index] = c;

	}



	kvm->arch.mpidr_data = data;

out:

	mutex_unlock(&kvm->arch.config_lock);

}



/*

 * Handle both the initialisation that is being done when the vcpu is

 * run for the first time, as well as the updates that must be
@@ -600,6 +652,8 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 
	if (likely(vcpu_has_run_once(vcpu)))

		return 0;



	kvm_init_mpidr_data(kvm);



	kvm_arm_vcpu_init_debug(vcpu);



	if (likely(irqchip_in_kernel(kvm))) {
@@ -1136,27 +1190,23 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 
			  bool line_status)

{

	u32 irq = irq_level->irq;

	unsigned int irq_type, vcpu_idx, irq_num;

	int nrcpus = atomic_read(&kvm->online_vcpus);

	unsigned int irq_type, vcpu_id, irq_num;

	struct kvm_vcpu *vcpu = NULL;

	bool level = irq_level->level;



	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;

	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;

	vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);

	vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;

	vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);

	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;



	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);

	trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);



	switch (irq_type) {

	case KVM_ARM_IRQ_TYPE_CPU:

		if (irqchip_in_kernel(kvm))

			return -ENXIO;



		if (vcpu_idx >= nrcpus)

			return -EINVAL;



		vcpu = kvm_get_vcpu(kvm, vcpu_idx);

		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);

		if (!vcpu)

			return -EINVAL;
@@ -1168,17 +1218,14 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 
		if (!irqchip_in_kernel(kvm))

			return -ENXIO;



		if (vcpu_idx >= nrcpus)

			return -EINVAL;



		vcpu = kvm_get_vcpu(kvm, vcpu_idx);

		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);

		if (!vcpu)

			return -EINVAL;



		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)

			return -EINVAL;



		return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);

		return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);

	case KVM_ARM_IRQ_TYPE_SPI:

		if (!irqchip_in_kernel(kvm))

			return -ENXIO;
@@ -1186,7 +1233,7 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 
		if (irq_num < VGIC_NR_PRIVATE_IRQS)

			return -EINVAL;



		return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);

		return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);

	}



	return -EINVAL;
@@ -2378,6 +2425,18 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr) 
	unsigned long i;



	mpidr &= MPIDR_HWID_BITMASK;



	if (kvm->arch.mpidr_data) {

		u16 idx = kvm_mpidr_index(kvm->arch.mpidr_data, mpidr);



		vcpu = kvm_get_vcpu(kvm,

				    kvm->arch.mpidr_data->cmpidr_to_idx[idx]);

		if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))

			vcpu = NULL;



		return vcpu;

	}



	kvm_for_each_vcpu(i, vcpu, kvm) {

		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))

			return vcpu;