Merge tag 'vfio-v6.18-rc4' of https://github.com/awilliam/linux-vfio (39bcf0f7) · Commits · git / linux-nf

drivers/vfio/vfio_iommu_type1.c

+110 −63

Original line number	Diff line number	Diff line
		@@ -38,6 +38,7 @@
		#include <linux/workqueue.h>
		#include <linux/notifier.h>
		#include <linux/mm_inline.h>
		#include <linux/overflow.h>
		#include "vfio.h"

		#define DRIVER_VERSION "0.2"
		@@ -167,12 +168,14 @@ static struct vfio_dma vfio_find_dma(struct vfio_iommu iommu,
		{
		struct rb_node *node = iommu->dma_list.rb_node;

		WARN_ON(!size);

		while (node) {
		struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);

		if (start + size <= dma->iova)
		if (start + size - 1 < dma->iova)
		node = node->rb_left;
		else if (start >= dma->iova + dma->size)
		else if (start > dma->iova + dma->size - 1)
		node = node->rb_right;
		else
		return dma;
		@@ -182,16 +185,19 @@ static struct vfio_dma vfio_find_dma(struct vfio_iommu iommu,
		}

		static struct rb_node vfio_find_dma_first_node(struct vfio_iommu iommu,
		dma_addr_t start, u64 size)
		dma_addr_t start,
		dma_addr_t end)
		{
		struct rb_node *res = NULL;
		struct rb_node *node = iommu->dma_list.rb_node;
		struct vfio_dma *dma_res = NULL;

		WARN_ON(end < start);

		while (node) {
		struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);

		if (start < dma->iova + dma->size) {
		if (start <= dma->iova + dma->size - 1) {
		res = node;
		dma_res = dma;
		if (start >= dma->iova)
		@@ -201,7 +207,7 @@ static struct rb_node vfio_find_dma_first_node(struct vfio_iommu iommu,
		node = node->rb_right;
		}
		}
		if (res && size && dma_res->iova >= start + size)
		if (res && dma_res->iova > end)
		res = NULL;
		return res;
		}
		@@ -211,11 +217,13 @@ static void vfio_link_dma(struct vfio_iommu iommu, struct vfio_dma new)
		struct rb_node *link = &iommu->dma_list.rb_node, parent = NULL;
		struct vfio_dma *dma;

		WARN_ON(new->size != 0);

		while (*link) {
		parent = *link;
		dma = rb_entry(parent, struct vfio_dma, node);

		if (new->iova + new->size <= dma->iova)
		if (new->iova <= dma->iova)
		link = &(*link)->rb_left;
		else
		link = &(*link)->rb_right;
		@@ -895,14 +903,20 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
		unsigned long remote_vaddr;
		struct vfio_dma *dma;
		bool do_accounting;
		dma_addr_t iova_end;
		size_t iova_size;

		if (!iommu \|\| !pages)
		if (!iommu \|\| !pages \|\| npage <= 0)
		return -EINVAL;

		/* Supported for v2 version only */
		if (!iommu->v2)
		return -EACCES;

		if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) \|\|
		check_add_overflow(user_iova, iova_size - 1, &iova_end))
		return -EOVERFLOW;

		mutex_lock(&iommu->lock);

		if (WARN_ONCE(iommu->vaddr_invalid_count,
		@@ -1008,12 +1022,21 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data,
		{
		struct vfio_iommu *iommu = iommu_data;
		bool do_accounting;
		dma_addr_t iova_end;
		size_t iova_size;
		int i;

		/* Supported for v2 version only */
		if (WARN_ON(!iommu->v2))
		return;

		if (WARN_ON(npage <= 0))
		return;

		if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) \|\|
		check_add_overflow(user_iova, iova_size - 1, &iova_end)))
		return;

		mutex_lock(&iommu->lock);

		do_accounting = list_empty(&iommu->domain_list);
		@@ -1067,7 +1090,7 @@ static long vfio_sync_unpin(struct vfio_dma dma, struct vfio_domain domain,
		#define VFIO_IOMMU_TLB_SYNC_MAX 512

		static size_t unmap_unpin_fast(struct vfio_domain *domain,
		struct vfio_dma dma, dma_addr_t iova,
		struct vfio_dma *dma, dma_addr_t iova,
		size_t len, phys_addr_t phys, long *unlocked,
		struct list_head *unmapped_list,
		int *unmapped_cnt,
		@@ -1077,18 +1100,17 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain,
		struct vfio_regions entry = kzalloc(sizeof(entry), GFP_KERNEL);

		if (entry) {
		unmapped = iommu_unmap_fast(domain->domain, *iova, len,
		unmapped = iommu_unmap_fast(domain->domain, iova, len,
		iotlb_gather);

		if (!unmapped) {
		kfree(entry);
		} else {
		entry->iova = *iova;
		entry->iova = iova;
		entry->phys = phys;
		entry->len = unmapped;
		list_add_tail(&entry->list, unmapped_list);

		*iova += unmapped;
		(*unmapped_cnt)++;
		}
		}
		@@ -1107,18 +1129,17 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain,
		}

		static size_t unmap_unpin_slow(struct vfio_domain *domain,
		struct vfio_dma dma, dma_addr_t iova,
		struct vfio_dma *dma, dma_addr_t iova,
		size_t len, phys_addr_t phys,
		long *unlocked)
		{
		size_t unmapped = iommu_unmap(domain->domain, *iova, len);
		size_t unmapped = iommu_unmap(domain->domain, iova, len);

		if (unmapped) {
		unlocked += vfio_unpin_pages_remote(dma, iova,
		*unlocked += vfio_unpin_pages_remote(dma, iova,
		phys >> PAGE_SHIFT,
		unmapped >> PAGE_SHIFT,
		false);
		*iova += unmapped;
		cond_resched();
		}
		return unmapped;
		@@ -1127,12 +1148,12 @@ static size_t unmap_unpin_slow(struct vfio_domain *domain,
		static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		bool do_accounting)
		{
		dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
		struct vfio_domain domain, d;
		LIST_HEAD(unmapped_region_list);
		struct iommu_iotlb_gather iotlb_gather;
		int unmapped_region_cnt = 0;
		long unlocked = 0;
		size_t pos = 0;

		if (!dma->size)
		return 0;
		@@ -1156,13 +1177,14 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		}

		iommu_iotlb_gather_init(&iotlb_gather);
		while (iova < end) {
		while (pos < dma->size) {
		size_t unmapped, len;
		phys_addr_t phys, next;
		dma_addr_t iova = dma->iova + pos;

		phys = iommu_iova_to_phys(domain->domain, iova);
		if (WARN_ON(!phys)) {
		iova += PAGE_SIZE;
		pos += PAGE_SIZE;
		continue;
		}

		@@ -1171,7 +1193,7 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		* may require hardware cache flushing, try to find the
		* largest contiguous physical memory chunk to unmap.
		*/
		for (len = PAGE_SIZE; iova + len < end; len += PAGE_SIZE) {
		for (len = PAGE_SIZE; pos + len < dma->size; len += PAGE_SIZE) {
		next = iommu_iova_to_phys(domain->domain, iova + len);
		if (next != phys + len)
		break;
		@@ -1181,16 +1203,18 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		* First, try to use fast unmap/unpin. In case of failure,
		* switch to slow unmap/unpin path.
		*/
		unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys,
		unmapped = unmap_unpin_fast(domain, dma, iova, len, phys,
		&unlocked, &unmapped_region_list,
		&unmapped_region_cnt,
		&iotlb_gather);
		if (!unmapped) {
		unmapped = unmap_unpin_slow(domain, dma, &iova, len,
		unmapped = unmap_unpin_slow(domain, dma, iova, len,
		phys, &unlocked);
		if (WARN_ON(!unmapped))
		break;
		}

		pos += unmapped;
		}

		dma->iommu_mapped = false;
		@@ -1282,7 +1306,7 @@ static int update_user_bitmap(u64 __user bitmap, struct vfio_iommu iommu,
		}

		static int vfio_iova_dirty_bitmap(u64 __user bitmap, struct vfio_iommu iommu,
		dma_addr_t iova, size_t size, size_t pgsize)
		dma_addr_t iova, dma_addr_t iova_end, size_t pgsize)
		{
		struct vfio_dma *dma;
		struct rb_node *n;
		@@ -1299,8 +1323,8 @@ static int vfio_iova_dirty_bitmap(u64 __user bitmap, struct vfio_iommu iommu,
		if (dma && dma->iova != iova)
		return -EINVAL;

		dma = vfio_find_dma(iommu, iova + size - 1, 0);
		if (dma && dma->iova + dma->size != iova + size)
		dma = vfio_find_dma(iommu, iova_end, 1);
		if (dma && dma->iova + dma->size - 1 != iova_end)
		return -EINVAL;

		for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
		@@ -1309,7 +1333,7 @@ static int vfio_iova_dirty_bitmap(u64 __user bitmap, struct vfio_iommu iommu,
		if (dma->iova < iova)
		continue;

		if (dma->iova > iova + size - 1)
		if (dma->iova > iova_end)
		break;

		ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);
		@@ -1374,7 +1398,8 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
		int ret = -EINVAL, retries = 0;
		unsigned long pgshift;
		dma_addr_t iova = unmap->iova;
		u64 size = unmap->size;
		dma_addr_t iova_end;
		size_t size = unmap->size;
		bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL;
		bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR;
		struct rb_node n, first_n;
		@@ -1387,6 +1412,11 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
		goto unlock;
		}

		if (iova != unmap->iova \|\| size != unmap->size) {
		ret = -EOVERFLOW;
		goto unlock;
		}

		pgshift = __ffs(iommu->pgsize_bitmap);
		pgsize = (size_t)1 << pgshift;

		@@ -1396,11 +1426,16 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
		if (unmap_all) {
		if (iova \|\| size)
		goto unlock;
		size = U64_MAX;
		} else if (!size \|\| size & (pgsize - 1) \|\|
		iova + size - 1 < iova \|\| size > SIZE_MAX) {
		iova_end = ~(dma_addr_t)0;
		} else {
		if (!size \|\| size & (pgsize - 1))
		goto unlock;

		if (check_add_overflow(iova, size - 1, &iova_end)) {
		ret = -EOVERFLOW;
		goto unlock;
		}
		}

		/* When dirty tracking is enabled, allow only min supported pgsize */
		if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
		@@ -1446,17 +1481,17 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
		if (dma && dma->iova != iova)
		goto unlock;

		dma = vfio_find_dma(iommu, iova + size - 1, 0);
		if (dma && dma->iova + dma->size != iova + size)
		dma = vfio_find_dma(iommu, iova_end, 1);
		if (dma && dma->iova + dma->size - 1 != iova_end)
		goto unlock;
		}

		ret = 0;
		n = first_n = vfio_find_dma_first_node(iommu, iova, size);
		n = first_n = vfio_find_dma_first_node(iommu, iova, iova_end);

		while (n) {
		dma = rb_entry(n, struct vfio_dma, node);
		if (dma->iova >= iova + size)
		if (dma->iova > iova_end)
		break;

		if (!iommu->v2 && iova > dma->iova)
		@@ -1648,7 +1683,9 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
		{
		bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR;
		dma_addr_t iova = map->iova;
		dma_addr_t iova_end;
		unsigned long vaddr = map->vaddr;
		unsigned long vaddr_end;
		size_t size = map->size;
		int ret = 0, prot = 0;
		size_t pgsize;
		@@ -1656,8 +1693,15 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,

		/* Verify that none of our __u64 fields overflow */
		if (map->size != size \|\| map->vaddr != vaddr \|\| map->iova != iova)
		return -EOVERFLOW;

		if (!size)
		return -EINVAL;

		if (check_add_overflow(iova, size - 1, &iova_end) \|\|
		check_add_overflow(vaddr, size - 1, &vaddr_end))
		return -EOVERFLOW;

		/* READ/WRITE from device perspective */
		if (map->flags & VFIO_DMA_MAP_FLAG_WRITE)
		prot \|= IOMMU_WRITE;
		@@ -1673,13 +1717,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,

		WARN_ON((pgsize - 1) & PAGE_MASK);

		if (!size \|\| (size \| iova \| vaddr) & (pgsize - 1)) {
		ret = -EINVAL;
		goto out_unlock;
		}

		/* Don't allow IOVA or virtual address wrap */
		if (iova + size - 1 < iova \|\| vaddr + size - 1 < vaddr) {
		if ((size \| iova \| vaddr) & (pgsize - 1)) {
		ret = -EINVAL;
		goto out_unlock;
		}
		@@ -1710,7 +1748,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
		goto out_unlock;
		}

		if (!vfio_iommu_iova_dma_valid(iommu, iova, iova + size - 1)) {
		if (!vfio_iommu_iova_dma_valid(iommu, iova, iova_end)) {
		ret = -EINVAL;
		goto out_unlock;
		}
		@@ -1783,12 +1821,12 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,

		for (; n; n = rb_next(n)) {
		struct vfio_dma *dma;
		dma_addr_t iova;
		size_t pos = 0;

		dma = rb_entry(n, struct vfio_dma, node);
		iova = dma->iova;

		while (iova < dma->iova + dma->size) {
		while (pos < dma->size) {
		dma_addr_t iova = dma->iova + pos;
		phys_addr_t phys;
		size_t size;

		@@ -1804,14 +1842,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
		phys = iommu_iova_to_phys(d->domain, iova);

		if (WARN_ON(!phys)) {
		iova += PAGE_SIZE;
		pos += PAGE_SIZE;
		continue;
		}

		size = PAGE_SIZE;
		p = phys + size;
		i = iova + size;
		while (i < dma->iova + dma->size &&
		while (pos + size < dma->size &&
		p == iommu_iova_to_phys(d->domain, i)) {
		size += PAGE_SIZE;
		p += PAGE_SIZE;
		@@ -1819,9 +1857,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
		}
		} else {
		unsigned long pfn;
		unsigned long vaddr = dma->vaddr +
		(iova - dma->iova);
		size_t n = dma->iova + dma->size - iova;
		unsigned long vaddr = dma->vaddr + pos;
		size_t n = dma->size - pos;
		long npage;

		npage = vfio_pin_pages_remote(dma, vaddr,
		@@ -1852,7 +1889,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
		goto unwind;
		}

		iova += size;
		pos += size;
		}
		}

		@@ -1869,29 +1906,29 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
		unwind:
		for (; n; n = rb_prev(n)) {
		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
		dma_addr_t iova;
		size_t pos = 0;

		if (dma->iommu_mapped) {
		iommu_unmap(domain->domain, dma->iova, dma->size);
		continue;
		}

		iova = dma->iova;
		while (iova < dma->iova + dma->size) {
		while (pos < dma->size) {
		dma_addr_t iova = dma->iova + pos;
		phys_addr_t phys, p;
		size_t size;
		dma_addr_t i;

		phys = iommu_iova_to_phys(domain->domain, iova);
		if (!phys) {
		iova += PAGE_SIZE;
		pos += PAGE_SIZE;
		continue;
		}

		size = PAGE_SIZE;
		p = phys + size;
		i = iova + size;
		while (i < dma->iova + dma->size &&
		while (pos + size < dma->size &&
		p == iommu_iova_to_phys(domain->domain, i)) {
		size += PAGE_SIZE;
		p += PAGE_SIZE;
		@@ -2977,7 +3014,8 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,
		struct vfio_iommu_type1_dirty_bitmap_get range;
		unsigned long pgshift;
		size_t data_size = dirty.argsz - minsz;
		size_t iommu_pgsize;
		size_t size, iommu_pgsize;
		dma_addr_t iova, iova_end;

		if (!data_size \|\| data_size < sizeof(range))
		return -EINVAL;
		@@ -2986,14 +3024,24 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,
		sizeof(range)))
		return -EFAULT;

		if (range.iova + range.size < range.iova)
		iova = range.iova;
		size = range.size;

		if (iova != range.iova \|\| size != range.size)
		return -EOVERFLOW;

		if (!size)
		return -EINVAL;

		if (check_add_overflow(iova, size - 1, &iova_end))
		return -EOVERFLOW;

		if (!access_ok((void __user *)range.bitmap.data,
		range.bitmap.size))
		return -EINVAL;

		pgshift = __ffs(range.bitmap.pgsize);
		ret = verify_bitmap_size(range.size >> pgshift,
		ret = verify_bitmap_size(size >> pgshift,
		range.bitmap.size);
		if (ret)
		return ret;
		@@ -3007,19 +3055,18 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,
		ret = -EINVAL;
		goto out_unlock;
		}
		if (range.iova & (iommu_pgsize - 1)) {
		if (iova & (iommu_pgsize - 1)) {
		ret = -EINVAL;
		goto out_unlock;
		}
		if (!range.size \|\| range.size & (iommu_pgsize - 1)) {
		if (size & (iommu_pgsize - 1)) {
		ret = -EINVAL;
		goto out_unlock;
		}

		if (iommu->dirty_page_tracking)
		ret = vfio_iova_dirty_bitmap(range.bitmap.data,
		iommu, range.iova,
		range.size,
		iommu, iova, iova_end,
		range.bitmap.pgsize);
		else
		ret = -EINVAL;

tools/testing/selftests/vfio/lib/include/vfio_util.h

+23 −4

Original line number	Diff line number	Diff line
		@@ -206,10 +206,29 @@ struct vfio_pci_device vfio_pci_device_init(const char bdf, const char *iommu_
		void vfio_pci_device_cleanup(struct vfio_pci_device *device);
		void vfio_pci_device_reset(struct vfio_pci_device *device);

		void vfio_pci_dma_map(struct vfio_pci_device *device,
		struct vfio_dma_region *region);
		void vfio_pci_dma_unmap(struct vfio_pci_device *device,
		int __vfio_pci_dma_map(struct vfio_pci_device *device,
		struct vfio_dma_region *region);
		int __vfio_pci_dma_unmap(struct vfio_pci_device *device,
		struct vfio_dma_region *region,
		u64 *unmapped);
		int __vfio_pci_dma_unmap_all(struct vfio_pci_device device, u64 unmapped);

		static inline void vfio_pci_dma_map(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		{
		VFIO_ASSERT_EQ(__vfio_pci_dma_map(device, region), 0);
		}

		static inline void vfio_pci_dma_unmap(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		{
		VFIO_ASSERT_EQ(__vfio_pci_dma_unmap(device, region, NULL), 0);
		}

		static inline void vfio_pci_dma_unmap_all(struct vfio_pci_device *device)
		{
		VFIO_ASSERT_EQ(__vfio_pci_dma_unmap_all(device, NULL), 0);
		}

		void vfio_pci_config_access(struct vfio_pci_device *device, bool write,
		size_t config, size_t size, void *data);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

+81 −23

Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@
		#include <dirent.h>
		#include <fcntl.h>
		#include <libgen.h>
		#include <stdint.h>
		#include <stdlib.h>
		#include <string.h>
		#include <unistd.h>
		@@ -141,7 +142,7 @@ static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index,
		ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info);
		}

		static void vfio_iommu_dma_map(struct vfio_pci_device *device,
		static int vfio_iommu_dma_map(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		{
		struct vfio_iommu_type1_dma_map args = {
		@@ -152,10 +153,13 @@ static void vfio_iommu_dma_map(struct vfio_pci_device *device,
		.size = region->size,
		};

		ioctl_assert(device->container_fd, VFIO_IOMMU_MAP_DMA, &args);
		if (ioctl(device->container_fd, VFIO_IOMMU_MAP_DMA, &args))
		return -errno;

		return 0;
		}

		static void iommufd_dma_map(struct vfio_pci_device *device,
		static int iommufd_dma_map(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		{
		struct iommu_ioas_map args = {
		@@ -169,54 +173,108 @@ static void iommufd_dma_map(struct vfio_pci_device *device,
		.ioas_id = device->ioas_id,
		};

		ioctl_assert(device->iommufd, IOMMU_IOAS_MAP, &args);
		if (ioctl(device->iommufd, IOMMU_IOAS_MAP, &args))
		return -errno;

		return 0;
		}

		void vfio_pci_dma_map(struct vfio_pci_device *device,
		int __vfio_pci_dma_map(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		{
		int ret;

		if (device->iommufd)
		iommufd_dma_map(device, region);
		ret = iommufd_dma_map(device, region);
		else
		vfio_iommu_dma_map(device, region);
		ret = vfio_iommu_dma_map(device, region);

		if (ret)
		return ret;

		list_add(&region->link, &device->dma_regions);

		return 0;
		}

		static void vfio_iommu_dma_unmap(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		static int vfio_iommu_dma_unmap(int fd, u64 iova, u64 size, u32 flags,
		u64 *unmapped)
		{
		struct vfio_iommu_type1_dma_unmap args = {
		.argsz = sizeof(args),
		.iova = region->iova,
		.size = region->size,
		.iova = iova,
		.size = size,
		.flags = flags,
		};

		ioctl_assert(device->container_fd, VFIO_IOMMU_UNMAP_DMA, &args);
		if (ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &args))
		return -errno;

		if (unmapped)
		*unmapped = args.size;

		return 0;
		}

		static void iommufd_dma_unmap(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		static int iommufd_dma_unmap(int fd, u64 iova, u64 length, u32 ioas_id,
		u64 *unmapped)
		{
		struct iommu_ioas_unmap args = {
		.size = sizeof(args),
		.iova = region->iova,
		.length = region->size,
		.ioas_id = device->ioas_id,
		.iova = iova,
		.length = length,
		.ioas_id = ioas_id,
		};

		ioctl_assert(device->iommufd, IOMMU_IOAS_UNMAP, &args);
		if (ioctl(fd, IOMMU_IOAS_UNMAP, &args))
		return -errno;

		if (unmapped)
		*unmapped = args.length;

		return 0;
		}

		void vfio_pci_dma_unmap(struct vfio_pci_device *device,
		struct vfio_dma_region *region)
		int __vfio_pci_dma_unmap(struct vfio_pci_device *device,
		struct vfio_dma_region region, u64 unmapped)
		{
		int ret;

		if (device->iommufd)
		ret = iommufd_dma_unmap(device->iommufd, region->iova,
		region->size, device->ioas_id,
		unmapped);
		else
		ret = vfio_iommu_dma_unmap(device->container_fd, region->iova,
		region->size, 0, unmapped);

		if (ret)
		return ret;

		list_del_init(&region->link);

		return 0;
		}

		int __vfio_pci_dma_unmap_all(struct vfio_pci_device device, u64 unmapped)
		{
		int ret;
		struct vfio_dma_region curr, next;

		if (device->iommufd)
		iommufd_dma_unmap(device, region);
		ret = iommufd_dma_unmap(device->iommufd, 0, UINT64_MAX,
		device->ioas_id, unmapped);
		else
		vfio_iommu_dma_unmap(device, region);
		ret = vfio_iommu_dma_unmap(device->container_fd, 0, 0,
		VFIO_DMA_UNMAP_FLAG_ALL, unmapped);

		if (ret)
		return ret;

		list_for_each_entry_safe(curr, next, &device->dma_regions, link)
		list_del_init(&curr->link);

		list_del(&region->link);
		return 0;
		}

		static void vfio_pci_region_get(struct vfio_pci_device *device, int index,

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

+94 −1

Original line number	Diff line number	Diff line
		@@ -112,6 +112,8 @@ FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(anonymous, 0, 0);
		FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(anonymous_hugetlb_2mb, SZ_2M, MAP_HUGETLB \| MAP_HUGE_2MB);
		FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(anonymous_hugetlb_1gb, SZ_1G, MAP_HUGETLB \| MAP_HUGE_1GB);

		#undef FIXTURE_VARIANT_ADD_IOMMU_MODE

		FIXTURE_SETUP(vfio_dma_mapping_test)
		{
		self->device = vfio_pci_device_init(device_bdf, variant->iommu_mode);
		@@ -129,6 +131,7 @@ TEST_F(vfio_dma_mapping_test, dma_map_unmap)
		struct vfio_dma_region region;
		struct iommu_mapping mapping;
		u64 mapping_size = size;
		u64 unmapped;
		int rc;

		region.vaddr = mmap(NULL, size, PROT_READ \| PROT_WRITE, flags, -1, 0);
		@@ -184,7 +187,9 @@ TEST_F(vfio_dma_mapping_test, dma_map_unmap)
		}

		unmap:
		vfio_pci_dma_unmap(self->device, &region);
		rc = __vfio_pci_dma_unmap(self->device, &region, &unmapped);
		ASSERT_EQ(rc, 0);
		ASSERT_EQ(unmapped, region.size);
		printf("Unmapped IOVA 0x%lx\n", region.iova);
		ASSERT_EQ(INVALID_IOVA, __to_iova(self->device, region.vaddr));
		ASSERT_NE(0, iommu_mapping_get(device_bdf, region.iova, &mapping));
		@@ -192,6 +197,94 @@ TEST_F(vfio_dma_mapping_test, dma_map_unmap)
		ASSERT_TRUE(!munmap(region.vaddr, size));
		}

		FIXTURE(vfio_dma_map_limit_test) {
		struct vfio_pci_device *device;
		struct vfio_dma_region region;
		size_t mmap_size;
		};

		FIXTURE_VARIANT(vfio_dma_map_limit_test) {
		const char *iommu_mode;
		};

		#define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \
		FIXTURE_VARIANT_ADD(vfio_dma_map_limit_test, _iommu_mode) { \
		.iommu_mode = #_iommu_mode, \
		}

		FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES();

		#undef FIXTURE_VARIANT_ADD_IOMMU_MODE

		FIXTURE_SETUP(vfio_dma_map_limit_test)
		{
		struct vfio_dma_region *region = &self->region;
		u64 region_size = getpagesize();

		/*
		* Over-allocate mmap by double the size to provide enough backing vaddr
		* for overflow tests
		*/
		self->mmap_size = 2 * region_size;

		self->device = vfio_pci_device_init(device_bdf, variant->iommu_mode);
		region->vaddr = mmap(NULL, self->mmap_size, PROT_READ \| PROT_WRITE,
		MAP_ANONYMOUS \| MAP_PRIVATE, -1, 0);
		ASSERT_NE(region->vaddr, MAP_FAILED);

		/* One page prior to the end of address space */
		region->iova = ~(iova_t)0 & ~(region_size - 1);
		region->size = region_size;
		}

		FIXTURE_TEARDOWN(vfio_dma_map_limit_test)
		{
		vfio_pci_device_cleanup(self->device);
		ASSERT_EQ(munmap(self->region.vaddr, self->mmap_size), 0);
		}

		TEST_F(vfio_dma_map_limit_test, unmap_range)
		{
		struct vfio_dma_region *region = &self->region;
		u64 unmapped;
		int rc;

		vfio_pci_dma_map(self->device, region);
		ASSERT_EQ(region->iova, to_iova(self->device, region->vaddr));

		rc = __vfio_pci_dma_unmap(self->device, region, &unmapped);
		ASSERT_EQ(rc, 0);
		ASSERT_EQ(unmapped, region->size);
		}

		TEST_F(vfio_dma_map_limit_test, unmap_all)
		{
		struct vfio_dma_region *region = &self->region;
		u64 unmapped;
		int rc;

		vfio_pci_dma_map(self->device, region);
		ASSERT_EQ(region->iova, to_iova(self->device, region->vaddr));

		rc = __vfio_pci_dma_unmap_all(self->device, &unmapped);
		ASSERT_EQ(rc, 0);
		ASSERT_EQ(unmapped, region->size);
		}

		TEST_F(vfio_dma_map_limit_test, overflow)
		{
		struct vfio_dma_region *region = &self->region;
		int rc;

		region->size = self->mmap_size;

		rc = __vfio_pci_dma_map(self->device, region);
		ASSERT_EQ(rc, -EOVERFLOW);

		rc = __vfio_pci_dma_unmap(self->device, region, NULL);
		ASSERT_EQ(rc, -EOVERFLOW);
		}

		int main(int argc, char *argv[])
		{
		device_bdf = vfio_selftests_get_bdf(&argc, argv);