Commit a498d59c authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'dma-mapping-6.18-2025-09-30' of... 

Merge tag 'dma-mapping-6.18-2025-09-30' of git://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux

Pull dma-mapping updates from Marek Szyprowski:

 - Refactoring of DMA mapping API to physical addresses as the primary
   interface instead of page+offset parameters

   This gets much closer to Matthew Wilcox's long term wish for
   struct-pageless IO to cacheable DRAM and is supporting memdesc
   project which seeks to substantially transform how struct page works.

   An advantage of this approach is the possibility of introducing
   DMA_ATTR_MMIO, which covers existing 'dma_map_resource' flow in the
   common paths, what in turn lets to use recently introduced
   dma_iova_link() API to map PCI P2P MMIO without creating struct page

   Developped by Leon Romanovsky and Jason Gunthorpe

 - Minor clean-up by Petr Tesarik and Qianfeng Rong

* tag 'dma-mapping-6.18-2025-09-30' of git://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux:
  kmsan: fix missed kmsan_handle_dma() signature conversion
  mm/hmm: properly take MMIO path
  mm/hmm: migrate to physical address-based DMA mapping API
  dma-mapping: export new dma_*map_phys() interface
  xen: swiotlb: Open code map_resource callback
  dma-mapping: implement DMA_ATTR_MMIO for dma_(un)map_page_attrs()
  kmsan: convert kmsan_handle_dma to use physical addresses
  dma-mapping: convert dma_direct_*map_page to be phys_addr_t based
  iommu/dma: implement DMA_ATTR_MMIO for iommu_dma_(un)map_phys()
  iommu/dma: rename iommu_dma_*map_page to iommu_dma_*map_phys
  dma-mapping: rename trace_dma_*map_page to trace_dma_*map_phys
  dma-debug: refactor to use physical addresses for page mapping
  iommu/dma: implement DMA_ATTR_MMIO for dma_iova_link().
  dma-mapping: introduce new DMA attribute to indicate MMIO memory
  swiotlb: Remove redundant __GFP_NOWARN
  dma-direct: clean up the logic in __dma_direct_alloc_pages()
parents ee2fe81c ef3d979b

Documentation/core-api/dma-api.rst

+2 −2
Original line number Diff line number Diff line
@@ -761,7 +761,7 @@ example warning message may look like this:: 
	[<ffffffff80235177>] find_busiest_group+0x207/0x8a0

	[<ffffffff8064784f>] _spin_lock_irqsave+0x1f/0x50

	[<ffffffff803c7ea3>] check_unmap+0x203/0x490

	[<ffffffff803c8259>] debug_dma_unmap_page+0x49/0x50

	[<ffffffff803c8259>] debug_dma_unmap_phys+0x49/0x50

	[<ffffffff80485f26>] nv_tx_done_optimized+0xc6/0x2c0

	[<ffffffff80486c13>] nv_nic_irq_optimized+0x73/0x2b0

	[<ffffffff8026df84>] handle_IRQ_event+0x34/0x70
@@ -855,7 +855,7 @@ that a driver may be leaking mappings. 
dma-debug interface debug_dma_mapping_error() to debug drivers that fail

to check DMA mapping errors on addresses returned by dma_map_single() and

dma_map_page() interfaces. This interface clears a flag set by

debug_dma_map_page() to indicate that dma_mapping_error() has been called by

debug_dma_map_phys() to indicate that dma_mapping_error() has been called by

the driver. When driver does unmap, debug_dma_unmap() checks the flag and if

this flag is still set, prints warning message that includes call trace that

leads up to the unmap. This interface can be called from dma_mapping_error()

Documentation/core-api/dma-attributes.rst

+18 −0
Original line number Diff line number Diff line
@@ -130,3 +130,21 @@ accesses to DMA buffers in both privileged "supervisor" and unprivileged 
subsystem that the buffer is fully accessible at the elevated privilege

level (and ideally inaccessible or at least read-only at the

lesser-privileged levels).



DMA_ATTR_MMIO

-------------



This attribute indicates the physical address is not normal system

memory. It may not be used with kmap*()/phys_to_virt()/phys_to_page()

functions, it may not be cacheable, and access using CPU load/store

instructions may not be allowed.



Usually this will be used to describe MMIO addresses, or other non-cacheable

register addresses. When DMA mapping this sort of address we call

the operation Peer to Peer as a one device is DMA'ing to another device.

For PCI devices the p2pdma APIs must be used to determine if

DMA_ATTR_MMIO is appropriate.



For architectures that require cache flushing for DMA coherence

DMA_ATTR_MMIO will not perform any cache flushing. The address

provided must never be mapped cacheable into the CPU.

arch/powerpc/kernel/dma-iommu.c

+2 −2
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ 
#define can_map_direct(dev, addr) \

	((dev)->bus_dma_limit >= phys_to_dma((dev), (addr)))



bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr)

bool arch_dma_map_phys_direct(struct device *dev, phys_addr_t addr)

{

	if (likely(!dev->bus_dma_limit))

		return false;
@@ -24,7 +24,7 @@ bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr) 


#define is_direct_handle(dev, h) ((h) >= (dev)->archdata.dma_offset)



bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle)

bool arch_dma_unmap_phys_direct(struct device *dev, dma_addr_t dma_handle)

{

	if (likely(!dev->bus_dma_limit))

		return false;

drivers/iommu/dma-iommu.c

+31 −30
Original line number Diff line number Diff line
@@ -724,7 +724,12 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev 
static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,

		     unsigned long attrs)

{

	int prot = coherent ? IOMMU_CACHE : 0;

	int prot;



	if (attrs & DMA_ATTR_MMIO)

		prot = IOMMU_MMIO;

	else

		prot = coherent ? IOMMU_CACHE : 0;



	if (attrs & DMA_ATTR_PRIVILEGED)

		prot |= IOMMU_PRIV;
@@ -1190,11 +1195,9 @@ static inline size_t iova_unaligned(struct iova_domain *iovad, phys_addr_t phys, 
	return iova_offset(iovad, phys | size);

}



dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,

	      unsigned long offset, size_t size, enum dma_data_direction dir,

	      unsigned long attrs)

dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,

		enum dma_data_direction dir, unsigned long attrs)

{

	phys_addr_t phys = page_to_phys(page) + offset;

	bool coherent = dev_is_dma_coherent(dev);

	int prot = dma_info_to_prot(dir, coherent, attrs);

	struct iommu_domain *domain = iommu_get_dma_domain(dev);
@@ -1208,27 +1211,34 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 
	 */

	if (dev_use_swiotlb(dev, size, dir) &&

	    iova_unaligned(iovad, phys, size)) {

		if (attrs & DMA_ATTR_MMIO)

			return DMA_MAPPING_ERROR;



		phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs);

		if (phys == (phys_addr_t)DMA_MAPPING_ERROR)

			return DMA_MAPPING_ERROR;

	}



	if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))

	if (!coherent && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))

		arch_sync_dma_for_device(phys, size, dir);



	iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);

	if (iova == DMA_MAPPING_ERROR)

	if (iova == DMA_MAPPING_ERROR && !(attrs & DMA_ATTR_MMIO))

		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);

	return iova;

}



void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,

void iommu_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	struct iommu_domain *domain = iommu_get_dma_domain(dev);

	phys_addr_t phys;



	phys = iommu_iova_to_phys(domain, dma_handle);

	if (attrs & DMA_ATTR_MMIO) {

		__iommu_dma_unmap(dev, dma_handle, size);

		return;

	}



	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);

	if (WARN_ON(!phys))

		return;
@@ -1341,7 +1351,7 @@ static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *s 
	int i;



	for_each_sg(sg, s, nents, i)

		iommu_dma_unmap_page(dev, sg_dma_address(s),

		iommu_dma_unmap_phys(dev, sg_dma_address(s),

				sg_dma_len(s), dir, attrs);

}
@@ -1354,8 +1364,8 @@ static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg, 
	sg_dma_mark_swiotlb(sg);



	for_each_sg(sg, s, nents, i) {

		sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s),

				s->offset, s->length, dir, attrs);

		sg_dma_address(s) = iommu_dma_map_phys(dev, sg_phys(s),

				s->length, dir, attrs);

		if (sg_dma_address(s) == DMA_MAPPING_ERROR)

			goto out_unmap;

		sg_dma_len(s) = s->length;
@@ -1546,20 +1556,6 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 
		__iommu_dma_unmap(dev, start, end - start);

}



dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	return __iommu_dma_map(dev, phys, size,

			dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,

			dma_get_mask(dev));

}



void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	__iommu_dma_unmap(dev, handle, size);

}



static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)

{

	size_t alloc_size = PAGE_ALIGN(size);
@@ -1838,12 +1834,13 @@ static int __dma_iova_link(struct device *dev, dma_addr_t addr, 
		unsigned long attrs)

{

	bool coherent = dev_is_dma_coherent(dev);

	int prot = dma_info_to_prot(dir, coherent, attrs);



	if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))

	if (!coherent && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))

		arch_sync_dma_for_device(phys, size, dir);



	return iommu_map_nosync(iommu_get_dma_domain(dev), addr, phys, size,

			dma_info_to_prot(dir, coherent, attrs), GFP_ATOMIC);

			prot, GFP_ATOMIC);

}



static int iommu_dma_iova_bounce_and_link(struct device *dev, dma_addr_t addr,
@@ -1949,9 +1946,13 @@ int dma_iova_link(struct device *dev, struct dma_iova_state *state, 
		return -EIO;



	if (dev_use_swiotlb(dev, size, dir) &&

	    iova_unaligned(iovad, phys, size))

	    iova_unaligned(iovad, phys, size)) {

		if (attrs & DMA_ATTR_MMIO)

			return -EPERM;



		return iommu_dma_iova_link_swiotlb(dev, state, phys, offset,

				size, dir, attrs);

	}



	return __dma_iova_link(dev, state->addr + offset - iova_start_pad,

			phys - iova_start_pad,

drivers/virtio/virtio_ring.c

+2 −2
Original line number Diff line number Diff line
@@ -378,7 +378,7 @@ static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist 
		 * is initialized by the hardware. Explicitly check/unpoison it

		 * depending on the direction.

		 */

		kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction);

		kmsan_handle_dma(sg_phys(sg), sg->length, direction);

		*addr = (dma_addr_t)sg_phys(sg);

		return 0;

	}
@@ -3157,7 +3157,7 @@ dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr, 
	struct vring_virtqueue *vq = to_vvq(_vq);



	if (!vq->use_dma_api) {

		kmsan_handle_dma(virt_to_page(ptr), offset_in_page(ptr), size, dir);

		kmsan_handle_dma(virt_to_phys(ptr), size, dir);

		return (dma_addr_t)virt_to_phys(ptr);

	}