Commit 9163d835 authored by Bingbu Cao's avatar Bingbu Cao Committed by Hans Verkuil
Browse files

media: intel/ipu6: add IPU6 DMA mapping API and MMU table 



The Intel IPU6 has internal microcontrollers (scalar processor, SP) which
are used to execute the firmware. The SPs can access IPU internal memory
and system DRAM mapped to its an internal 32-bit virtual address space.

This patch adds a driver for the IPU MMU and a DMA mapping implementation
using the internal MMU. The system IOMMU may be used besides the IPU MMU.

Signed-off-by: default avatarBingbu Cao <bingbu.cao@intel.com>
Signed-off-by: default avatarSakari Ailus <sakari.ailus@linux.intel.com>
Signed-off-by: default avatarHans Verkuil <hverkuil-cisco@xs4all.nl>
parent fb26412f

drivers/media/pci/intel/ipu6/ipu6-dma.c

0 → 100644
+502 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (C) 2013--2024 Intel Corporation

 */



#include <linux/cacheflush.h>

#include <linux/dma-mapping.h>

#include <linux/iova.h>

#include <linux/list.h>

#include <linux/mm.h>

#include <linux/vmalloc.h>

#include <linux/scatterlist.h>

#include <linux/slab.h>

#include <linux/types.h>



#include "ipu6.h"

#include "ipu6-bus.h"

#include "ipu6-dma.h"

#include "ipu6-mmu.h"



struct vm_info {

	struct list_head list;

	struct page **pages;

	dma_addr_t ipu6_iova;

	void *vaddr;

	unsigned long size;

};



static struct vm_info *get_vm_info(struct ipu6_mmu *mmu, dma_addr_t iova)

{

	struct vm_info *info, *save;



	list_for_each_entry_safe(info, save, &mmu->vma_list, list) {

		if (iova >= info->ipu6_iova &&

		    iova < (info->ipu6_iova + info->size))

			return info;

	}



	return NULL;

}



static void __dma_clear_buffer(struct page *page, size_t size,

			       unsigned long attrs)

{

	void *ptr;



	if (!page)

		return;

	/*

	 * Ensure that the allocated pages are zeroed, and that any data

	 * lurking in the kernel direct-mapped region is invalidated.

	 */

	ptr = page_address(page);

	memset(ptr, 0, size);

	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)

		clflush_cache_range(ptr, size);

}



static struct page **__dma_alloc_buffer(struct device *dev, size_t size,

					gfp_t gfp, unsigned long attrs)

{

	int count = PHYS_PFN(size);

	int array_size = count * sizeof(struct page *);

	struct page **pages;

	int i = 0;



	pages = kvzalloc(array_size, GFP_KERNEL);

	if (!pages)

		return NULL;



	gfp |= __GFP_NOWARN;



	while (count) {

		int j, order = __fls(count);



		pages[i] = alloc_pages(gfp, order);

		while (!pages[i] && order)

			pages[i] = alloc_pages(gfp, --order);

		if (!pages[i])

			goto error;



		if (order) {

			split_page(pages[i], order);

			j = 1 << order;

			while (j--)

				pages[i + j] = pages[i] + j;

		}



		__dma_clear_buffer(pages[i], PAGE_SIZE << order, attrs);

		i += 1 << order;

		count -= 1 << order;

	}



	return pages;

error:

	while (i--)

		if (pages[i])

			__free_pages(pages[i], 0);

	kvfree(pages);

	return NULL;

}



static void __dma_free_buffer(struct device *dev, struct page **pages,

			      size_t size, unsigned long attrs)

{

	int count = PHYS_PFN(size);

	unsigned int i;



	for (i = 0; i < count && pages[i]; i++) {

		__dma_clear_buffer(pages[i], PAGE_SIZE, attrs);

		__free_pages(pages[i], 0);

	}



	kvfree(pages);

}



static void ipu6_dma_sync_single_for_cpu(struct device *dev,

					 dma_addr_t dma_handle,

					 size_t size,

					 enum dma_data_direction dir)

{

	void *vaddr;

	u32 offset;

	struct vm_info *info;

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;



	info = get_vm_info(mmu, dma_handle);

	if (WARN_ON(!info))

		return;



	offset = dma_handle - info->ipu6_iova;

	if (WARN_ON(size > (info->size - offset)))

		return;



	vaddr = info->vaddr + offset;

	clflush_cache_range(vaddr, size);

}



static void ipu6_dma_sync_sg_for_cpu(struct device *dev,

				     struct scatterlist *sglist,

				     int nents, enum dma_data_direction dir)

{

	struct scatterlist *sg;

	int i;



	for_each_sg(sglist, sg, nents, i)

		clflush_cache_range(page_to_virt(sg_page(sg)), sg->length);

}



static void *ipu6_dma_alloc(struct device *dev, size_t size,

			    dma_addr_t *dma_handle, gfp_t gfp,

			    unsigned long attrs)

{

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;

	struct pci_dev *pdev = to_ipu6_bus_device(dev)->isp->pdev;

	dma_addr_t pci_dma_addr, ipu6_iova;

	struct vm_info *info;

	unsigned long count;

	struct page **pages;

	struct iova *iova;

	unsigned int i;

	int ret;



	info = kzalloc(sizeof(*info), GFP_KERNEL);

	if (!info)

		return NULL;



	size = PAGE_ALIGN(size);

	count = PHYS_PFN(size);



	iova = alloc_iova(&mmu->dmap->iovad, count,

			  PHYS_PFN(dma_get_mask(dev)), 0);

	if (!iova)

		goto out_kfree;



	pages = __dma_alloc_buffer(dev, size, gfp, attrs);

	if (!pages)

		goto out_free_iova;



	dev_dbg(dev, "dma_alloc: size %zu iova low pfn %lu, high pfn %lu\n",

		size, iova->pfn_lo, iova->pfn_hi);

	for (i = 0; iova->pfn_lo + i <= iova->pfn_hi; i++) {

		pci_dma_addr = dma_map_page_attrs(&pdev->dev, pages[i], 0,

						  PAGE_SIZE, DMA_BIDIRECTIONAL,

						  attrs);

		dev_dbg(dev, "dma_alloc: mapped pci_dma_addr %pad\n",

			&pci_dma_addr);

		if (dma_mapping_error(&pdev->dev, pci_dma_addr)) {

			dev_err(dev, "pci_dma_mapping for page[%d] failed", i);

			goto out_unmap;

		}



		ret = ipu6_mmu_map(mmu->dmap->mmu_info,

				   PFN_PHYS(iova->pfn_lo + i), pci_dma_addr,

				   PAGE_SIZE);

		if (ret) {

			dev_err(dev, "ipu6_mmu_map for pci_dma[%d] %pad failed",

				i, &pci_dma_addr);

			dma_unmap_page_attrs(&pdev->dev, pci_dma_addr,

					     PAGE_SIZE, DMA_BIDIRECTIONAL,

					     attrs);

			goto out_unmap;

		}

	}



	info->vaddr = vmap(pages, count, VM_USERMAP, PAGE_KERNEL);

	if (!info->vaddr)

		goto out_unmap;



	*dma_handle = PFN_PHYS(iova->pfn_lo);



	info->pages = pages;

	info->ipu6_iova = *dma_handle;

	info->size = size;

	list_add(&info->list, &mmu->vma_list);



	return info->vaddr;



out_unmap:

	while (i--) {

		ipu6_iova = PFN_PHYS(iova->pfn_lo + i);

		pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,

						     ipu6_iova);

		dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,

				     DMA_BIDIRECTIONAL, attrs);



		ipu6_mmu_unmap(mmu->dmap->mmu_info, ipu6_iova, PAGE_SIZE);

	}



	__dma_free_buffer(dev, pages, size, attrs);



out_free_iova:

	__free_iova(&mmu->dmap->iovad, iova);

out_kfree:

	kfree(info);



	return NULL;

}



static void ipu6_dma_free(struct device *dev, size_t size, void *vaddr,

			  dma_addr_t dma_handle,

			  unsigned long attrs)

{

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;

	struct pci_dev *pdev = to_ipu6_bus_device(dev)->isp->pdev;

	struct iova *iova = find_iova(&mmu->dmap->iovad, PHYS_PFN(dma_handle));

	dma_addr_t pci_dma_addr, ipu6_iova;

	struct vm_info *info;

	struct page **pages;

	unsigned int i;



	if (WARN_ON(!iova))

		return;



	info = get_vm_info(mmu, dma_handle);

	if (WARN_ON(!info))

		return;



	if (WARN_ON(!info->vaddr))

		return;



	if (WARN_ON(!info->pages))

		return;



	list_del(&info->list);



	size = PAGE_ALIGN(size);



	pages = info->pages;



	vunmap(vaddr);



	for (i = 0; i < PHYS_PFN(size); i++) {

		ipu6_iova = PFN_PHYS(iova->pfn_lo + i);

		pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,

						     ipu6_iova);

		dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,

				     DMA_BIDIRECTIONAL, attrs);

	}



	ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),

		       PFN_PHYS(iova_size(iova)));



	__dma_free_buffer(dev, pages, size, attrs);



	mmu->tlb_invalidate(mmu);



	__free_iova(&mmu->dmap->iovad, iova);



	kfree(info);

}



static int ipu6_dma_mmap(struct device *dev, struct vm_area_struct *vma,

			 void *addr, dma_addr_t iova, size_t size,

			 unsigned long attrs)

{

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;

	size_t count = PHYS_PFN(PAGE_ALIGN(size));

	struct vm_info *info;

	size_t i;

	int ret;



	info = get_vm_info(mmu, iova);

	if (!info)

		return -EFAULT;



	if (!info->vaddr)

		return -EFAULT;



	if (vma->vm_start & ~PAGE_MASK)

		return -EINVAL;



	if (size > info->size)

		return -EFAULT;



	for (i = 0; i < count; i++) {

		ret = vm_insert_page(vma, vma->vm_start + PFN_PHYS(i),

				     info->pages[i]);

		if (ret < 0)

			return ret;

	}



	return 0;

}



static void ipu6_dma_unmap_sg(struct device *dev,

			      struct scatterlist *sglist,

			      int nents, enum dma_data_direction dir,

			      unsigned long attrs)

{

	struct pci_dev *pdev = to_ipu6_bus_device(dev)->isp->pdev;

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;

	struct iova *iova = find_iova(&mmu->dmap->iovad,

				      PHYS_PFN(sg_dma_address(sglist)));

	int i, npages, count;

	struct scatterlist *sg;

	dma_addr_t pci_dma_addr;



	if (!nents)

		return;



	if (WARN_ON(!iova))

		return;



	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)

		ipu6_dma_sync_sg_for_cpu(dev, sglist, nents, DMA_BIDIRECTIONAL);



	/* get the nents as orig_nents given by caller */

	count = 0;

	npages = iova_size(iova);

	for_each_sg(sglist, sg, nents, i) {

		if (sg_dma_len(sg) == 0 ||

		    sg_dma_address(sg) == DMA_MAPPING_ERROR)

			break;



		npages -= PHYS_PFN(PAGE_ALIGN(sg_dma_len(sg)));

		count++;

		if (npages <= 0)

			break;

	}



	/*

	 * Before IPU6 mmu unmap, return the pci dma address back to sg

	 * assume the nents is less than orig_nents as the least granule

	 * is 1 SZ_4K page

	 */

	dev_dbg(dev, "trying to unmap concatenated %u ents\n", count);

	for_each_sg(sglist, sg, count, i) {

		dev_dbg(dev, "ipu unmap sg[%d] %pad\n", i, &sg_dma_address(sg));

		pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,

						     sg_dma_address(sg));

		dev_dbg(dev, "return pci_dma_addr %pad back to sg[%d]\n",

			&pci_dma_addr, i);

		sg_dma_address(sg) = pci_dma_addr;

	}



	dev_dbg(dev, "ipu6_mmu_unmap low pfn %lu high pfn %lu\n",

		iova->pfn_lo, iova->pfn_hi);

	ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),

		       PFN_PHYS(iova_size(iova)));



	mmu->tlb_invalidate(mmu);



	dma_unmap_sg_attrs(&pdev->dev, sglist, nents, dir, attrs);



	__free_iova(&mmu->dmap->iovad, iova);

}



static int ipu6_dma_map_sg(struct device *dev, struct scatterlist *sglist,

			   int nents, enum dma_data_direction dir,

			   unsigned long attrs)

{

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;

	struct pci_dev *pdev = to_ipu6_bus_device(dev)->isp->pdev;

	struct scatterlist *sg;

	struct iova *iova;

	size_t npages = 0;

	unsigned long iova_addr;

	int i, count;



	for_each_sg(sglist, sg, nents, i) {

		if (sg->offset) {

			dev_err(dev, "Unsupported non-zero sg[%d].offset %x\n",

				i, sg->offset);

			return -EFAULT;

		}

	}



	dev_dbg(dev, "pci_dma_map_sg trying to map %d ents\n", nents);

	count  = dma_map_sg_attrs(&pdev->dev, sglist, nents, dir, attrs);

	if (count <= 0) {

		dev_err(dev, "pci_dma_map_sg %d ents failed\n", nents);

		return 0;

	}



	dev_dbg(dev, "pci_dma_map_sg %d ents mapped\n", count);



	for_each_sg(sglist, sg, count, i)

		npages += PHYS_PFN(PAGE_ALIGN(sg_dma_len(sg)));



	iova = alloc_iova(&mmu->dmap->iovad, npages,

			  PHYS_PFN(dma_get_mask(dev)), 0);

	if (!iova)

		return 0;



	dev_dbg(dev, "dmamap: iova low pfn %lu, high pfn %lu\n", iova->pfn_lo,

		iova->pfn_hi);



	iova_addr = iova->pfn_lo;

	for_each_sg(sglist, sg, count, i) {

		int ret;



		dev_dbg(dev, "mapping entry %d: iova 0x%llx phy %pad size %d\n",

			i, PFN_PHYS(iova_addr), &sg_dma_address(sg),

			sg_dma_len(sg));



		ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),

				   sg_dma_address(sg),

				   PAGE_ALIGN(sg_dma_len(sg)));

		if (ret)

			goto out_fail;



		sg_dma_address(sg) = PFN_PHYS(iova_addr);



		iova_addr += PHYS_PFN(PAGE_ALIGN(sg_dma_len(sg)));

	}



	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)

		ipu6_dma_sync_sg_for_cpu(dev, sglist, nents, DMA_BIDIRECTIONAL);



	return count;



out_fail:

	ipu6_dma_unmap_sg(dev, sglist, i, dir, attrs);



	return 0;

}



/*

 * Create scatter-list for the already allocated DMA buffer

 */

static int ipu6_dma_get_sgtable(struct device *dev, struct sg_table *sgt,

				void *cpu_addr, dma_addr_t handle, size_t size,

				unsigned long attrs)

{

	struct ipu6_mmu *mmu = to_ipu6_bus_device(dev)->mmu;

	struct vm_info *info;

	int n_pages;

	int ret = 0;



	info = get_vm_info(mmu, handle);

	if (!info)

		return -EFAULT;



	if (!info->vaddr)

		return -EFAULT;



	if (WARN_ON(!info->pages))

		return -ENOMEM;



	n_pages = PHYS_PFN(PAGE_ALIGN(size));



	ret = sg_alloc_table_from_pages(sgt, info->pages, n_pages, 0, size,

					GFP_KERNEL);

	if (ret)

		dev_warn(dev, "IPU6 get sgt table failed\n");



	return ret;

}



const struct dma_map_ops ipu6_dma_ops = {

	.alloc = ipu6_dma_alloc,

	.free = ipu6_dma_free,

	.mmap = ipu6_dma_mmap,

	.map_sg = ipu6_dma_map_sg,

	.unmap_sg = ipu6_dma_unmap_sg,

	.sync_single_for_cpu = ipu6_dma_sync_single_for_cpu,

	.sync_single_for_device = ipu6_dma_sync_single_for_cpu,

	.sync_sg_for_cpu = ipu6_dma_sync_sg_for_cpu,

	.sync_sg_for_device = ipu6_dma_sync_sg_for_cpu,

	.get_sgtable = ipu6_dma_get_sgtable,

};

drivers/media/pci/intel/ipu6/ipu6-dma.h

0 → 100644
+19 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/* Copyright (C) 2013--2024 Intel Corporation */



#ifndef IPU6_DMA_H

#define IPU6_DMA_H



#include <linux/dma-map-ops.h>

#include <linux/iova.h>



struct ipu6_mmu_info;



struct ipu6_dma_mapping {

	struct ipu6_mmu_info *mmu_info;

	struct iova_domain iovad;

};



extern const struct dma_map_ops ipu6_dma_ops;



#endif /* IPU6_DMA_H */

drivers/media/pci/intel/ipu6/ipu6-mmu.c

0 → 100644
+845 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/media/pci/intel/ipu6/ipu6-mmu.h

0 → 100644
+73 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/* Copyright (C) 2013--2024 Intel Corporation */



#ifndef IPU6_MMU_H

#define IPU6_MMU_H



#define ISYS_MMID 1

#define PSYS_MMID 0



#include <linux/list.h>

#include <linux/spinlock_types.h>

#include <linux/types.h>



struct device;

struct page;

struct ipu6_hw_variants;



struct ipu6_mmu_info {

	struct device *dev;



	u32 *l1_pt;

	u32 l1_pt_dma;

	u32 **l2_pts;



	u32 *dummy_l2_pt;

	u32 dummy_l2_pteval;

	void *dummy_page;

	u32 dummy_page_pteval;



	dma_addr_t aperture_start;

	dma_addr_t aperture_end;

	unsigned long pgsize_bitmap;



	spinlock_t lock;	/* Serialize access to users */

	struct ipu6_dma_mapping *dmap;

};



struct ipu6_mmu {

	struct list_head node;



	struct ipu6_mmu_hw *mmu_hw;

	unsigned int nr_mmus;

	unsigned int mmid;



	phys_addr_t pgtbl;

	struct device *dev;



	struct ipu6_dma_mapping *dmap;

	struct list_head vma_list;



	struct page *trash_page;

	dma_addr_t pci_trash_page; /* IOVA from PCI DMA services (parent) */

	dma_addr_t iova_trash_page; /* IOVA for IPU6 child nodes to use */



	bool ready;

	spinlock_t ready_lock;	/* Serialize access to bool ready */



	void (*tlb_invalidate)(struct ipu6_mmu *mmu);

};



struct ipu6_mmu *ipu6_mmu_init(struct device *dev,

			       void __iomem *base, int mmid,

			       const struct ipu6_hw_variants *hw);

void ipu6_mmu_cleanup(struct ipu6_mmu *mmu);

int ipu6_mmu_hw_init(struct ipu6_mmu *mmu);

void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu);

int ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,

		 phys_addr_t paddr, size_t size);

size_t ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,

		      size_t size);

phys_addr_t ipu6_mmu_iova_to_phys(struct ipu6_mmu_info *mmu_info,

				  dma_addr_t iova);

#endif