Commit 3d5689e0 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/lpi-xa-cache into kvmarm-master/next 



* kvm-arm64/lpi-xa-cache:
  : .
  : New and improved LPI translation cache from Oliver Upton.
  :
  : From the cover letter:
  :
  : "As discussed [*], here is the new take on the LPI translation cache,
  : migrating to an xarray indexed by (devid, eventid) per ITS.
  :
  : The end result is quite satisfying, as it becomes possible to rip out
  : other nasties such as the lpi_list_lock. To that end, patches 2-6 aren't
  : _directly_ related to the translation cache cleanup, but instead are
  : done to enable the cleanups at the end of the series.
  :
  : I changed out my test machine from the last time so the baseline has
  : moved a bit, but here are the results from the vgic_lpi_stress test:
  :
  : +----------------------------+------------+-------------------+
  : |       Configuration        |  v6.8-rc1  | v6.8-rc1 + series |
  : +----------------------------+------------+-------------------+
  : | -v 1 -d 1 -e 1 -i 1000000  | 2063296.81 |        1362602.35 |
  : | -v 16 -d 16 -e 16 -i 10000 |  610678.33 |        5200910.01 |
  : | -v 16 -d 16 -e 17 -i 10000 |  678361.53 |        5890675.51 |
  : | -v 32 -d 32 -e 1 -i 100000 |  580918.96 |        8304552.67 |
  : | -v 1 -d 1 -e 17 -i 1000    | 1512443.94 |         1425953.8 |
  : +----------------------------+------------+-------------------+
  :
  : Unlike last time, no dramatic regressions at any performance point. The
  : regression on a single interrupt stream is to be expected, as the
  : overheads of SRCU and two tree traversals (kvm_io_bus_get_dev(),
  : translation cache xarray) are likely greater than that of a linked-list
  : with a single node."
  : .
  KVM: selftests: Add stress test for LPI injection
  KVM: selftests: Use MPIDR_HWID_BITMASK from cputype.h
  KVM: selftests: Add helper for enabling LPIs on a redistributor
  KVM: selftests: Add a minimal library for interacting with an ITS
  KVM: selftests: Add quadword MMIO accessors
  KVM: selftests: Standardise layout of GIC frames
  KVM: selftests: Align with kernel's GIC definitions
  KVM: arm64: vgic-its: Get rid of the lpi_list_lock
  KVM: arm64: vgic-its: Rip out the global translation cache
  KVM: arm64: vgic-its: Use the per-ITS translation cache for injection
  KVM: arm64: vgic-its: Spin off helper for finding ITS by doorbell addr
  KVM: arm64: vgic-its: Maintain a translation cache per ITS
  KVM: arm64: vgic-its: Scope translation cache invalidations to an ITS
  KVM: arm64: vgic-its: Get rid of vgic_copy_lpi_list()
  KVM: arm64: vgic-debug: Use an xarray mark for debug iterator
  KVM: arm64: vgic-its: Walk LPI xarray in vgic_its_cmd_handle_movall()
  KVM: arm64: vgic-its: Walk LPI xarray in vgic_its_invall()
  KVM: arm64: vgic-its: Walk LPI xarray in its_sync_lpi_pending_table()
  KVM: Treat the device list as an rculist

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents 2d38f439 96d36ad9

arch/arm64/kvm/vgic/vgic-debug.c

+60 −22
Original line number Diff line number Diff line
@@ -28,27 +28,65 @@ struct vgic_state_iter { 
	int nr_lpis;

	int dist_id;

	int vcpu_id;

	int intid;

	unsigned long intid;

	int lpi_idx;

	u32 *lpi_array;

};



static void iter_next(struct vgic_state_iter *iter)

static void iter_next(struct kvm *kvm, struct vgic_state_iter *iter)

{

	struct vgic_dist *dist = &kvm->arch.vgic;



	if (iter->dist_id == 0) {

		iter->dist_id++;

		return;

	}



	/*

	 * Let the xarray drive the iterator after the last SPI, as the iterator

	 * has exhausted the sequentially-allocated INTID space.

	 */

	if (iter->intid >= (iter->nr_spis + VGIC_NR_PRIVATE_IRQS - 1)) {

		if (iter->lpi_idx < iter->nr_lpis)

			xa_find_after(&dist->lpi_xa, &iter->intid,

				      VGIC_LPI_MAX_INTID,

				      LPI_XA_MARK_DEBUG_ITER);

		iter->lpi_idx++;

		return;

	}



	iter->intid++;

	if (iter->intid == VGIC_NR_PRIVATE_IRQS &&

	    ++iter->vcpu_id < iter->nr_cpus)

		iter->intid = 0;

}



	if (iter->intid >= (iter->nr_spis + VGIC_NR_PRIVATE_IRQS)) {

		if (iter->lpi_idx < iter->nr_lpis)

			iter->intid = iter->lpi_array[iter->lpi_idx];

		iter->lpi_idx++;

static int iter_mark_lpis(struct kvm *kvm)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct vgic_irq *irq;

	unsigned long intid;

	int nr_lpis = 0;



	xa_for_each(&dist->lpi_xa, intid, irq) {

		if (!vgic_try_get_irq_kref(irq))

			continue;



		xa_set_mark(&dist->lpi_xa, intid, LPI_XA_MARK_DEBUG_ITER);

		nr_lpis++;

	}



	return nr_lpis;

}



static void iter_unmark_lpis(struct kvm *kvm)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct vgic_irq *irq;

	unsigned long intid;



	xa_for_each(&dist->lpi_xa, intid, irq) {

		xa_clear_mark(&dist->lpi_xa, intid, LPI_XA_MARK_DEBUG_ITER);

		vgic_put_irq(kvm, irq);

	}

}
@@ -61,15 +99,12 @@ static void iter_init(struct kvm *kvm, struct vgic_state_iter *iter, 


	iter->nr_cpus = nr_cpus;

	iter->nr_spis = kvm->arch.vgic.nr_spis;

	if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {

		iter->nr_lpis = vgic_copy_lpi_list(kvm, NULL, &iter->lpi_array);

		if (iter->nr_lpis < 0)

			iter->nr_lpis = 0;

	}

	if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)

		iter->nr_lpis = iter_mark_lpis(kvm);



	/* Fast forward to the right position if needed */

	while (pos--)

		iter_next(iter);

		iter_next(kvm, iter);

}



static bool end_of_vgic(struct vgic_state_iter *iter)
@@ -114,7 +149,7 @@ static void *vgic_debug_next(struct seq_file *s, void *v, loff_t *pos) 
	struct vgic_state_iter *iter = kvm->arch.vgic.iter;



	++*pos;

	iter_next(iter);

	iter_next(kvm, iter);

	if (end_of_vgic(iter))

		iter = NULL;

	return iter;
@@ -134,13 +169,14 @@ static void vgic_debug_stop(struct seq_file *s, void *v) 


	mutex_lock(&kvm->arch.config_lock);

	iter = kvm->arch.vgic.iter;

	kfree(iter->lpi_array);

	iter_unmark_lpis(kvm);

	kfree(iter);

	kvm->arch.vgic.iter = NULL;

	mutex_unlock(&kvm->arch.config_lock);

}



static void print_dist_state(struct seq_file *s, struct vgic_dist *dist)

static void print_dist_state(struct seq_file *s, struct vgic_dist *dist,

			     struct vgic_state_iter *iter)

{

	bool v3 = dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3;
@@ -149,7 +185,7 @@ static void print_dist_state(struct seq_file *s, struct vgic_dist *dist) 
	seq_printf(s, "vgic_model:\t%s\n", v3 ? "GICv3" : "GICv2");

	seq_printf(s, "nr_spis:\t%d\n", dist->nr_spis);

	if (v3)

		seq_printf(s, "nr_lpis:\t%d\n", atomic_read(&dist->lpi_count));

		seq_printf(s, "nr_lpis:\t%d\n", iter->nr_lpis);

	seq_printf(s, "enabled:\t%d\n", dist->enabled);

	seq_printf(s, "\n");
@@ -236,7 +272,7 @@ static int vgic_debug_show(struct seq_file *s, void *v) 
	unsigned long flags;



	if (iter->dist_id == 0) {

		print_dist_state(s, &kvm->arch.vgic);

		print_dist_state(s, &kvm->arch.vgic, iter);

		return 0;

	}
@@ -246,11 +282,13 @@ static int vgic_debug_show(struct seq_file *s, void *v) 
	if (iter->vcpu_id < iter->nr_cpus)

		vcpu = kvm_get_vcpu(kvm, iter->vcpu_id);



	/*

	 * Expect this to succeed, as iter_mark_lpis() takes a reference on

	 * every LPI to be visited.

	 */

	irq = vgic_get_irq(kvm, vcpu, iter->intid);

	if (!irq) {

		seq_printf(s, "       LPI %4d freed\n", iter->intid);

		return 0;

	}

	if (WARN_ON_ONCE(!irq))

		return -EINVAL;



	raw_spin_lock_irqsave(&irq->irq_lock, flags);

	print_irq_state(s, irq, vcpu);

arch/arm64/kvm/vgic/vgic-init.c

+0 −8
Original line number Diff line number Diff line
@@ -53,8 +53,6 @@ void kvm_vgic_early_init(struct kvm *kvm) 
{

	struct vgic_dist *dist = &kvm->arch.vgic;



	INIT_LIST_HEAD(&dist->lpi_translation_cache);

	raw_spin_lock_init(&dist->lpi_list_lock);

	xa_init_flags(&dist->lpi_xa, XA_FLAGS_LOCK_IRQ);

}
@@ -305,9 +303,6 @@ int vgic_init(struct kvm *kvm) 
		}

	}



	if (vgic_has_its(kvm))

		vgic_lpi_translation_cache_init(kvm);



	/*

	 * If we have GICv4.1 enabled, unconditionally request enable the

	 * v4 support so that we get HW-accelerated vSGIs. Otherwise, only
@@ -361,9 +356,6 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) 
		dist->vgic_cpu_base = VGIC_ADDR_UNDEF;

	}



	if (vgic_has_its(kvm))

		vgic_lpi_translation_cache_destroy(kvm);



	if (vgic_supports_direct_msis(kvm))

		vgic_v4_teardown(kvm);

arch/arm64/kvm/vgic/vgic-its.c

+104 −248
Original line number Diff line number Diff line
@@ -23,6 +23,8 @@ 
#include "vgic.h"

#include "vgic-mmio.h"



static struct kvm_device_ops kvm_arm_vgic_its_ops;



static int vgic_its_save_tables_v0(struct vgic_its *its);

static int vgic_its_restore_tables_v0(struct vgic_its *its);

static int vgic_its_commit_v0(struct vgic_its *its);
@@ -67,7 +69,7 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, 
	irq->target_vcpu = vcpu;

	irq->group = 1;



	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);

	xa_lock_irqsave(&dist->lpi_xa, flags);



	/*

	 * There could be a race with another vgic_add_lpi(), so we need to
@@ -82,17 +84,14 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, 
		goto out_unlock;

	}



	ret = xa_err(xa_store(&dist->lpi_xa, intid, irq, 0));

	ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0));

	if (ret) {

		xa_release(&dist->lpi_xa, intid);

		kfree(irq);

		goto out_unlock;

	}



	atomic_inc(&dist->lpi_count);



out_unlock:

	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);

	xa_unlock_irqrestore(&dist->lpi_xa, flags);



	if (ret)

		return ERR_PTR(ret);
@@ -150,14 +149,6 @@ struct its_ite { 
	u32 event_id;

};



struct vgic_translation_cache_entry {

	struct list_head	entry;

	phys_addr_t		db;

	u32			devid;

	u32			eventid;

	struct vgic_irq		*irq;

};



/**

 * struct vgic_its_abi - ITS abi ops and settings

 * @cte_esz: collection table entry size
@@ -253,7 +244,9 @@ static struct its_ite *find_ite(struct vgic_its *its, u32 device_id, 
#define GIC_LPI_OFFSET 8192



#define VITS_TYPER_IDBITS		16

#define VITS_MAX_EVENTID		(BIT(VITS_TYPER_IDBITS) - 1)

#define VITS_TYPER_DEVBITS		16

#define VITS_MAX_DEVID			(BIT(VITS_TYPER_DEVBITS) - 1)

#define VITS_DTE_MAX_DEVID_OFFSET	(BIT(14) - 1)

#define VITS_ITE_MAX_EVENTID_OFFSET	(BIT(16) - 1)
@@ -316,53 +309,6 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, 
	return 0;

}



#define GIC_LPI_MAX_INTID	((1 << INTERRUPT_ID_BITS_ITS) - 1)



/*

 * Create a snapshot of the current LPIs targeting @vcpu, so that we can

 * enumerate those LPIs without holding any lock.

 * Returns their number and puts the kmalloc'ed array into intid_ptr.

 */

int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	XA_STATE(xas, &dist->lpi_xa, GIC_LPI_OFFSET);

	struct vgic_irq *irq;

	unsigned long flags;

	u32 *intids;

	int irq_count, i = 0;



	/*

	 * There is an obvious race between allocating the array and LPIs

	 * being mapped/unmapped. If we ended up here as a result of a

	 * command, we're safe (locks are held, preventing another

	 * command). If coming from another path (such as enabling LPIs),

	 * we must be careful not to overrun the array.

	 */

	irq_count = atomic_read(&dist->lpi_count);

	intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL_ACCOUNT);

	if (!intids)

		return -ENOMEM;



	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);

	rcu_read_lock();



	xas_for_each(&xas, irq, GIC_LPI_MAX_INTID) {

		if (i == irq_count)

			break;

		/* We don't need to "get" the IRQ, as we hold the list lock. */

		if (vcpu && irq->target_vcpu != vcpu)

			continue;

		intids[i++] = irq->intid;

	}



	rcu_read_unlock();

	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);



	*intid_ptr = intids;

	return i;

}



static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)

{

	int ret = 0;
@@ -446,23 +392,18 @@ static u32 max_lpis_propbaser(u64 propbaser) 
static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)

{

	gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);

	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	unsigned long intid, flags;

	struct vgic_irq *irq;

	int last_byte_offset = -1;

	int ret = 0;

	u32 *intids;

	int nr_irqs, i;

	unsigned long flags;

	u8 pendmask;



	nr_irqs = vgic_copy_lpi_list(vcpu->kvm, vcpu, &intids);

	if (nr_irqs < 0)

		return nr_irqs;



	for (i = 0; i < nr_irqs; i++) {

	xa_for_each(&dist->lpi_xa, intid, irq) {

		int byte_offset, bit_nr;



		byte_offset = intids[i] / BITS_PER_BYTE;

		bit_nr = intids[i] % BITS_PER_BYTE;

		byte_offset = intid / BITS_PER_BYTE;

		bit_nr = intid % BITS_PER_BYTE;



		/*

		 * For contiguously allocated LPIs chances are we just read
@@ -472,25 +413,23 @@ static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu) 
			ret = kvm_read_guest_lock(vcpu->kvm,

						  pendbase + byte_offset,

						  &pendmask, 1);

			if (ret) {

				kfree(intids);

			if (ret)

				return ret;

			}



			last_byte_offset = byte_offset;

		}



		irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);

		irq = vgic_get_irq(vcpu->kvm, NULL, intid);

		if (!irq)

			continue;



		raw_spin_lock_irqsave(&irq->irq_lock, flags);

		if (irq->target_vcpu == vcpu)

			irq->pending_latch = pendmask & (1U << bit_nr);

		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);

		vgic_put_irq(vcpu->kvm, irq);

	}



	kfree(intids);



	return ret;

}
@@ -566,51 +505,52 @@ static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm, 
	return 0;

}



static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,

					       phys_addr_t db,

					       u32 devid, u32 eventid)

static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db)

{

	struct vgic_translation_cache_entry *cte;

	struct kvm_io_device *kvm_io_dev;

	struct vgic_io_device *iodev;



	list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {

		/*

		 * If we hit a NULL entry, there is nothing after this

		 * point.

		 */

		if (!cte->irq)

			break;

	kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db);

	if (!kvm_io_dev)

		return ERR_PTR(-EINVAL);



		if (cte->db != db || cte->devid != devid ||

		    cte->eventid != eventid)

			continue;

	if (kvm_io_dev->ops != &kvm_io_gic_ops)

		return ERR_PTR(-EINVAL);



		/*

		 * Move this entry to the head, as it is the most

		 * recently used.

		 */

		if (!list_is_first(&cte->entry, &dist->lpi_translation_cache))

			list_move(&cte->entry, &dist->lpi_translation_cache);

	iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);

	if (iodev->iodev_type != IODEV_ITS)

		return ERR_PTR(-EINVAL);



		return cte->irq;

	return iodev->its;

}



	return NULL;

static unsigned long vgic_its_cache_key(u32 devid, u32 eventid)

{

	return (((unsigned long)devid) << VITS_TYPER_IDBITS) | eventid;



}



static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db,

					     u32 devid, u32 eventid)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	unsigned long cache_key = vgic_its_cache_key(devid, eventid);

	struct vgic_its *its;

	struct vgic_irq *irq;

	unsigned long flags;



	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);

	if (devid > VITS_MAX_DEVID || eventid > VITS_MAX_EVENTID)

		return NULL;



	its = __vgic_doorbell_to_its(kvm, db);

	if (IS_ERR(its))

		return NULL;



	irq = __vgic_its_check_cache(dist, db, devid, eventid);

	rcu_read_lock();



	irq = xa_load(&its->translation_cache, cache_key);

	if (!vgic_try_get_irq_kref(irq))

		irq = NULL;



	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);

	rcu_read_unlock();



	return irq;

}
@@ -619,41 +559,13 @@ static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its, 
				       u32 devid, u32 eventid,

				       struct vgic_irq *irq)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct vgic_translation_cache_entry *cte;

	unsigned long flags;

	phys_addr_t db;

	unsigned long cache_key = vgic_its_cache_key(devid, eventid);

	struct vgic_irq *old;



	/* Do not cache a directly injected interrupt */

	if (irq->hw)

		return;



	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);



	if (unlikely(list_empty(&dist->lpi_translation_cache)))

		goto out;



	/*

	 * We could have raced with another CPU caching the same

	 * translation behind our back, so let's check it is not in

	 * already

	 */

	db = its->vgic_its_base + GITS_TRANSLATER;

	if (__vgic_its_check_cache(dist, db, devid, eventid))

		goto out;



	/* Always reuse the last entry (LRU policy) */

	cte = list_last_entry(&dist->lpi_translation_cache,

			      typeof(*cte), entry);



	/*

	 * Caching the translation implies having an extra reference

	 * to the interrupt, so drop the potential reference on what

	 * was in the cache, and increment it on the new interrupt.

	 */

	if (cte->irq)

		vgic_put_irq(kvm, cte->irq);



	/*

	 * The irq refcount is guaranteed to be nonzero while holding the

	 * its_lock, as the ITE (and the reference it holds) cannot be freed.
@@ -661,39 +573,44 @@ static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its, 
	lockdep_assert_held(&its->its_lock);

	vgic_get_irq_kref(irq);



	cte->db		= db;

	cte->devid	= devid;

	cte->eventid	= eventid;

	cte->irq	= irq;

	/*

	 * We could have raced with another CPU caching the same

	 * translation behind our back, ensure we don't leak a

	 * reference if that is the case.

	 */

	old = xa_store(&its->translation_cache, cache_key, irq, GFP_KERNEL_ACCOUNT);

	if (old)

		vgic_put_irq(kvm, old);

}



	/* Move the new translation to the head of the list */

	list_move(&cte->entry, &dist->lpi_translation_cache);

static void vgic_its_invalidate_cache(struct vgic_its *its)

{

	struct kvm *kvm = its->dev->kvm;

	struct vgic_irq *irq;

	unsigned long idx;



out:

	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);

	xa_for_each(&its->translation_cache, idx, irq) {

		xa_erase(&its->translation_cache, idx);

		vgic_put_irq(kvm, irq);

	}

}



void vgic_its_invalidate_cache(struct kvm *kvm)

void vgic_its_invalidate_all_caches(struct kvm *kvm)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct vgic_translation_cache_entry *cte;

	unsigned long flags;

	struct kvm_device *dev;

	struct vgic_its *its;



	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);

	rcu_read_lock();



	list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {

		/*

		 * If we hit a NULL entry, there is nothing after this

		 * point.

		 */

		if (!cte->irq)

			break;

	list_for_each_entry_rcu(dev, &kvm->devices, vm_node) {

		if (dev->ops != &kvm_arm_vgic_its_ops)

			continue;



		vgic_put_irq(kvm, cte->irq);

		cte->irq = NULL;

		its = dev->private;

		vgic_its_invalidate_cache(its);

	}



	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);

	rcu_read_unlock();

}



int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
@@ -725,8 +642,6 @@ int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its, 
struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)

{

	u64 address;

	struct kvm_io_device *kvm_io_dev;

	struct vgic_io_device *iodev;



	if (!vgic_has_its(kvm))

		return ERR_PTR(-ENODEV);
@@ -736,18 +651,7 @@ struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi) 


	address = (u64)msi->address_hi << 32 | msi->address_lo;



	kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);

	if (!kvm_io_dev)

		return ERR_PTR(-EINVAL);



	if (kvm_io_dev->ops != &kvm_io_gic_ops)

		return ERR_PTR(-EINVAL);



	iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);

	if (iodev->iodev_type != IODEV_ITS)

		return ERR_PTR(-EINVAL);



	return iodev->its;

	return __vgic_doorbell_to_its(kvm, address);

}



/*
@@ -883,7 +787,7 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its, 
		 * don't bother here since we clear the ITTE anyway and the

		 * pending state is a property of the ITTE struct.

		 */

		vgic_its_invalidate_cache(kvm);

		vgic_its_invalidate_cache(its);



		its_free_ite(kvm, ite);

		return 0;
@@ -920,7 +824,7 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, 
	ite->collection = collection;

	vcpu = collection_to_vcpu(kvm, collection);



	vgic_its_invalidate_cache(kvm);

	vgic_its_invalidate_cache(its);



	return update_affinity(ite->irq, vcpu);

}
@@ -955,7 +859,7 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id, 


	switch (type) {

	case GITS_BASER_TYPE_DEVICE:

		if (id >= BIT_ULL(VITS_TYPER_DEVBITS))

		if (id > VITS_MAX_DEVID)

			return false;

		break;

	case GITS_BASER_TYPE_COLLECTION:
@@ -1167,7 +1071,8 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, 
}



/* Requires the its_lock to be held. */

static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)

static void vgic_its_free_device(struct kvm *kvm, struct vgic_its *its,

				 struct its_device *device)

{

	struct its_ite *ite, *temp;
@@ -1179,7 +1084,7 @@ static void vgic_its_free_device(struct kvm *kvm, struct its_device *device) 
	list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)

		its_free_ite(kvm, ite);



	vgic_its_invalidate_cache(kvm);

	vgic_its_invalidate_cache(its);



	list_del(&device->dev_list);

	kfree(device);
@@ -1191,7 +1096,7 @@ static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its) 
	struct its_device *cur, *temp;



	list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)

		vgic_its_free_device(kvm, cur);

		vgic_its_free_device(kvm, its, cur);

}



/* its lock must be held */
@@ -1250,7 +1155,7 @@ static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its, 
	 * by removing the mapping and re-establishing it.

	 */

	if (device)

		vgic_its_free_device(kvm, device);

		vgic_its_free_device(kvm, its, device);



	/*

	 * The spec does not say whether unmapping a not-mapped device
@@ -1281,7 +1186,7 @@ static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its, 


	if (!valid) {

		vgic_its_free_collection(its, coll_id);

		vgic_its_invalidate_cache(kvm);

		vgic_its_invalidate_cache(its);

	} else {

		struct kvm_vcpu *vcpu;
@@ -1372,23 +1277,19 @@ static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its, 
int vgic_its_invall(struct kvm_vcpu *vcpu)

{

	struct kvm *kvm = vcpu->kvm;

	int irq_count, i = 0;

	u32 *intids;



	irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);

	if (irq_count < 0)

		return irq_count;

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct vgic_irq *irq;

	unsigned long intid;



	for (i = 0; i < irq_count; i++) {

		struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intids[i]);

	xa_for_each(&dist->lpi_xa, intid, irq) {

		irq = vgic_get_irq(kvm, NULL, intid);

		if (!irq)

			continue;



		update_lpi_config(kvm, irq, vcpu, false);

		vgic_put_irq(kvm, irq);

	}



	kfree(intids);



	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)

		its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
@@ -1431,10 +1332,10 @@ static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its, 
static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,

				      u64 *its_cmd)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct kvm_vcpu *vcpu1, *vcpu2;

	struct vgic_irq *irq;

	u32 *intids;

	int irq_count, i;

	unsigned long intid;



	/* We advertise GITS_TYPER.PTA==0, making the address the vcpu ID */

	vcpu1 = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
@@ -1446,12 +1347,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, 
	if (vcpu1 == vcpu2)

		return 0;



	irq_count = vgic_copy_lpi_list(kvm, vcpu1, &intids);

	if (irq_count < 0)

		return irq_count;



	for (i = 0; i < irq_count; i++) {

		irq = vgic_get_irq(kvm, NULL, intids[i]);

	xa_for_each(&dist->lpi_xa, intid, irq) {

		irq = vgic_get_irq(kvm, NULL, intid);

		if (!irq)

			continue;
@@ -1460,9 +1357,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, 
		vgic_put_irq(kvm, irq);

	}



	vgic_its_invalidate_cache(kvm);

	vgic_its_invalidate_cache(its);



	kfree(intids);

	return 0;

}
@@ -1813,7 +1709,7 @@ static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its, 


	its->enabled = !!(val & GITS_CTLR_ENABLE);

	if (!its->enabled)

		vgic_its_invalidate_cache(kvm);

		vgic_its_invalidate_cache(its);



	/*

	 * Try to process any pending commands. This function bails out early
@@ -1914,47 +1810,6 @@ static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its, 
	return ret;

}



/* Default is 16 cached LPIs per vcpu */

#define LPI_DEFAULT_PCPU_CACHE_SIZE	16



void vgic_lpi_translation_cache_init(struct kvm *kvm)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	unsigned int sz;

	int i;



	if (!list_empty(&dist->lpi_translation_cache))

		return;



	sz = atomic_read(&kvm->online_vcpus) * LPI_DEFAULT_PCPU_CACHE_SIZE;



	for (i = 0; i < sz; i++) {

		struct vgic_translation_cache_entry *cte;



		/* An allocation failure is not fatal */

		cte = kzalloc(sizeof(*cte), GFP_KERNEL_ACCOUNT);

		if (WARN_ON(!cte))

			break;



		INIT_LIST_HEAD(&cte->entry);

		list_add(&cte->entry, &dist->lpi_translation_cache);

	}

}



void vgic_lpi_translation_cache_destroy(struct kvm *kvm)

{

	struct vgic_dist *dist = &kvm->arch.vgic;

	struct vgic_translation_cache_entry *cte, *tmp;



	vgic_its_invalidate_cache(kvm);



	list_for_each_entry_safe(cte, tmp,

				 &dist->lpi_translation_cache, entry) {

		list_del(&cte->entry);

		kfree(cte);

	}

}



#define INITIAL_BASER_VALUE						  \

	(GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)		| \

	 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)		| \
@@ -1987,8 +1842,6 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) 
			kfree(its);

			return ret;

		}



		vgic_lpi_translation_cache_init(dev->kvm);

	}



	mutex_init(&its->its_lock);
@@ -2006,6 +1859,7 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) 


	INIT_LIST_HEAD(&its->device_list);

	INIT_LIST_HEAD(&its->collection_list);

	xa_init(&its->translation_cache);



	dev->kvm->arch.vgic.msis_require_devid = true;

	dev->kvm->arch.vgic.has_its = true;
@@ -2036,6 +1890,8 @@ static void vgic_its_destroy(struct kvm_device *kvm_dev) 


	vgic_its_free_device_list(kvm, its);

	vgic_its_free_collection_list(kvm, its);

	vgic_its_invalidate_cache(its);

	xa_destroy(&its->translation_cache);



	mutex_unlock(&its->its_lock);

	kfree(its);
@@ -2438,7 +2294,7 @@ static int vgic_its_restore_dte(struct vgic_its *its, u32 id, 


	ret = vgic_its_restore_itt(its, dev);

	if (ret) {

		vgic_its_free_device(its->dev->kvm, dev);

		vgic_its_free_device(its->dev->kvm, its, dev);

		return ret;

	}

arch/arm64/kvm/vgic/vgic-mmio-v3.c

+1 −1
Original line number Diff line number Diff line
@@ -277,7 +277,7 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, 
			return;



		vgic_flush_pending_lpis(vcpu);

		vgic_its_invalidate_cache(vcpu->kvm);

		vgic_its_invalidate_all_caches(vcpu->kvm);

		atomic_set_release(&vgic_cpu->ctlr, 0);

	} else {

		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,

arch/arm64/kvm/vgic/vgic.c

+2 −4
Original line number Diff line number Diff line
@@ -29,7 +29,6 @@ struct vgic_global kvm_vgic_global_state __ro_after_init = { 
 *       its->cmd_lock (mutex)

 *         its->its_lock (mutex)

 *           vgic_cpu->ap_list_lock		must be taken with IRQs disabled

 *             kvm->lpi_list_lock		must be taken with IRQs disabled

 *             vgic_dist->lpi_xa.xa_lock	must be taken with IRQs disabled

 *               vgic_irq->irq_lock		must be taken with IRQs disabled

 *
@@ -126,7 +125,6 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) 
	__xa_erase(&dist->lpi_xa, irq->intid);

	xa_unlock_irqrestore(&dist->lpi_xa, flags);



	atomic_dec(&dist->lpi_count);

	kfree_rcu(irq, rcu);

}