Commit afd81d91 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'dma-mapping-6.11-2024-07-19' of git://git.infradead.org/users/hch/dma-mapping 

Pull dma-mapping updates from Christoph Hellwig:

 - reduce duplicate swiotlb pool lookups (Michael Kelley)

 - minor small fixes (Yicong Yang, Yang Li)

* tag 'dma-mapping-6.11-2024-07-19' of git://git.infradead.org/users/hch/dma-mapping:
  swiotlb: fix kernel-doc description for swiotlb_del_transient
  swiotlb: reduce swiotlb pool lookups
  dma-mapping: benchmark: Don't starve others when doing the test
parents ebcfbf02 b69bdba5

drivers/iommu/dma-iommu.c

+4 −7
Original line number Diff line number Diff line
@@ -1078,7 +1078,6 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev, 
	if (!dev_is_dma_coherent(dev))

		arch_sync_dma_for_cpu(phys, size, dir);



	if (is_swiotlb_buffer(dev, phys))

	swiotlb_sync_single_for_cpu(dev, phys, size, dir);

}
@@ -1091,7 +1090,6 @@ static void iommu_dma_sync_single_for_device(struct device *dev, 
		return;



	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);

	if (is_swiotlb_buffer(dev, phys))

	swiotlb_sync_single_for_device(dev, phys, size, dir);



	if (!dev_is_dma_coherent(dev))
@@ -1186,7 +1184,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 
		arch_sync_dma_for_device(phys, size, dir);



	iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);

	if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))

	if (iova == DMA_MAPPING_ERROR)

		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);

	return iova;

}
@@ -1206,7 +1204,6 @@ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 


	__iommu_dma_unmap(dev, dma_handle, size);



	if (unlikely(is_swiotlb_buffer(dev, phys)))

	swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);

}

drivers/xen/swiotlb-xen.c

+20 −11
Original line number Diff line number Diff line
@@ -88,7 +88,8 @@ static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) 
	return 0;

}



static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)

static struct io_tlb_pool *xen_swiotlb_find_pool(struct device *dev,

						 dma_addr_t dma_addr)

{

	unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));

	unsigned long xen_pfn = bfn_to_local_pfn(bfn);
@@ -99,8 +100,8 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) 
	 * in our domain. Therefore _only_ check address within our domain.

	 */

	if (pfn_valid(PFN_DOWN(paddr)))

		return is_swiotlb_buffer(dev, paddr);

	return 0;

		return swiotlb_find_pool(dev, paddr);

	return NULL;

}



#ifdef CONFIG_X86
@@ -227,8 +228,9 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, 
	 * Ensure that the address returned is DMA'ble

	 */

	if (unlikely(!dma_capable(dev, dev_addr, size, true))) {

		swiotlb_tbl_unmap_single(dev, map, size, dir,

				attrs | DMA_ATTR_SKIP_CPU_SYNC);

		__swiotlb_tbl_unmap_single(dev, map, size, dir,

				attrs | DMA_ATTR_SKIP_CPU_SYNC,

				swiotlb_find_pool(dev, map));

		return DMA_MAPPING_ERROR;

	}
@@ -254,6 +256,7 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, 
		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr);

	struct io_tlb_pool *pool;



	BUG_ON(dir == DMA_NONE);
@@ -265,8 +268,10 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, 
	}



	/* NOTE: We use dev_addr here, not paddr! */

	if (is_xen_swiotlb_buffer(hwdev, dev_addr))

		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);

	pool = xen_swiotlb_find_pool(hwdev, dev_addr);

	if (pool)

		__swiotlb_tbl_unmap_single(hwdev, paddr, size, dir,

					   attrs, pool);

}



static void
@@ -274,6 +279,7 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, 
		size_t size, enum dma_data_direction dir)

{

	phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);

	struct io_tlb_pool *pool;



	if (!dev_is_dma_coherent(dev)) {

		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
@@ -282,8 +288,9 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, 
			xen_dma_sync_for_cpu(dev, dma_addr, size, dir);

	}



	if (is_xen_swiotlb_buffer(dev, dma_addr))

		swiotlb_sync_single_for_cpu(dev, paddr, size, dir);

	pool = xen_swiotlb_find_pool(dev, dma_addr);

	if (pool)

		__swiotlb_sync_single_for_cpu(dev, paddr, size, dir, pool);

}



static void
@@ -291,9 +298,11 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, 
		size_t size, enum dma_data_direction dir)

{

	phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);

	struct io_tlb_pool *pool;



	if (is_xen_swiotlb_buffer(dev, dma_addr))

		swiotlb_sync_single_for_device(dev, paddr, size, dir);

	pool = xen_swiotlb_find_pool(dev, dma_addr);

	if (pool)

		__swiotlb_sync_single_for_device(dev, paddr, size, dir, pool);



	if (!dev_is_dma_coherent(dev)) {

		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))

include/linux/scatterlist.h

+1 −1
Original line number Diff line number Diff line
@@ -332,7 +332,7 @@ static inline void sg_dma_unmark_bus_address(struct scatterlist *sg) 
 * Description:

 *   Returns true if the scatterlist was marked for SWIOTLB bouncing. Not all

 *   elements may have been bounced, so the caller would have to check

 *   individual SG entries with is_swiotlb_buffer().

 *   individual SG entries with swiotlb_find_pool().

 */

static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)

{

include/linux/swiotlb.h

+62 −43
Original line number Diff line number Diff line
@@ -42,24 +42,6 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, 
	int (*remap)(void *tlb, unsigned long nslabs));

extern void __init swiotlb_update_mem_attributes(void);



phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,

		size_t mapping_size,

		unsigned int alloc_aligned_mask, enum dma_data_direction dir,

		unsigned long attrs);



extern void swiotlb_tbl_unmap_single(struct device *hwdev,

				     phys_addr_t tlb_addr,

				     size_t mapping_size,

				     enum dma_data_direction dir,

				     unsigned long attrs);



void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,

		size_t size, enum dma_data_direction dir);

void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,

		size_t size, enum dma_data_direction dir);

dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,

		size_t size, enum dma_data_direction dir, unsigned long attrs);



#ifdef CONFIG_SWIOTLB



/**
@@ -143,37 +125,27 @@ struct io_tlb_mem { 
#endif

};



#ifdef CONFIG_SWIOTLB_DYNAMIC



struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);



#else



static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,

						    phys_addr_t paddr)

{

	return &dev->dma_io_tlb_mem->defpool;

}



#endif

struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr);



/**

 * is_swiotlb_buffer() - check if a physical address belongs to a swiotlb

 * swiotlb_find_pool() - find swiotlb pool to which a physical address belongs

 * @dev:        Device which has mapped the buffer.

 * @paddr:      Physical address within the DMA buffer.

 *

 * Check if @paddr points into a bounce buffer.

 * Find the swiotlb pool that @paddr points into.

 *

 * Return:

 * * %true if @paddr points into a bounce buffer

 * * %false otherwise

 * * pool address if @paddr points into a bounce buffer

 * * NULL if @paddr does not point into a bounce buffer. As such, this function

 *   can be used to determine if @paddr denotes a swiotlb bounce buffer.

 */

static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)

static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,

		phys_addr_t paddr)

{

	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;



	if (!mem)

		return false;

		return NULL;



#ifdef CONFIG_SWIOTLB_DYNAMIC

	/*
@@ -182,16 +154,19 @@ static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 
	 * If a SWIOTLB address is checked on another CPU, then it was

	 * presumably loaded by the device driver from an unspecified private

	 * data structure. Make sure that this load is ordered before reading

	 * dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool().

	 * dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool().

	 *

	 * This barrier pairs with smp_mb() in swiotlb_find_slots().

	 */

	smp_rmb();

	return READ_ONCE(dev->dma_uses_io_tlb) &&

		swiotlb_find_pool(dev, paddr);

	if (READ_ONCE(dev->dma_uses_io_tlb))

		return __swiotlb_find_pool(dev, paddr);

#else

	return paddr >= mem->defpool.start && paddr < mem->defpool.end;

	if (paddr >= mem->defpool.start && paddr < mem->defpool.end)

		return &mem->defpool;

#endif



	return NULL;

}



static inline bool is_swiotlb_force_bounce(struct device *dev)
@@ -219,9 +194,10 @@ static inline void swiotlb_dev_init(struct device *dev) 
{

}



static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)

static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,

		phys_addr_t paddr)

{

	return false;

	return NULL;

}

static inline bool is_swiotlb_force_bounce(struct device *dev)

{
@@ -260,6 +236,49 @@ static inline phys_addr_t default_swiotlb_limit(void) 
}

#endif /* CONFIG_SWIOTLB */



phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,

		size_t mapping_size, unsigned int alloc_aligned_mask,

		enum dma_data_direction dir, unsigned long attrs);

dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,

		size_t size, enum dma_data_direction dir, unsigned long attrs);



void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,

		size_t mapping_size, enum dma_data_direction dir,

		unsigned long attrs, struct io_tlb_pool *pool);

static inline void swiotlb_tbl_unmap_single(struct device *dev,

		phys_addr_t addr, size_t size, enum dma_data_direction dir,

		unsigned long attrs)

{

	struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);



	if (unlikely(pool))

		__swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool);

}



void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,

		size_t size, enum dma_data_direction dir,

		struct io_tlb_pool *pool);

static inline void swiotlb_sync_single_for_device(struct device *dev,

		phys_addr_t addr, size_t size, enum dma_data_direction dir)

{

	struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);



	if (unlikely(pool))

		__swiotlb_sync_single_for_device(dev, addr, size, dir, pool);

}



void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,

		size_t size, enum dma_data_direction dir,

		struct io_tlb_pool *pool);

static inline void swiotlb_sync_single_for_cpu(struct device *dev,

		phys_addr_t addr, size_t size, enum dma_data_direction dir)

{

	struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);



	if (unlikely(pool))

		__swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool);

}



extern void swiotlb_print_info(void);



#ifdef CONFIG_DMA_RESTRICTED_POOL

kernel/dma/direct.c

+3 −7
Original line number Diff line number Diff line
@@ -404,9 +404,7 @@ void dma_direct_sync_sg_for_device(struct device *dev, 
	for_each_sg(sgl, sg, nents, i) {

		phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));



		if (unlikely(is_swiotlb_buffer(dev, paddr)))

			swiotlb_sync_single_for_device(dev, paddr, sg->length,

						       dir);

		swiotlb_sync_single_for_device(dev, paddr, sg->length, dir);



		if (!dev_is_dma_coherent(dev))

			arch_sync_dma_for_device(paddr, sg->length,
@@ -430,9 +428,7 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, 
		if (!dev_is_dma_coherent(dev))

			arch_sync_dma_for_cpu(paddr, sg->length, dir);



		if (unlikely(is_swiotlb_buffer(dev, paddr)))

			swiotlb_sync_single_for_cpu(dev, paddr, sg->length,

						    dir);

		swiotlb_sync_single_for_cpu(dev, paddr, sg->length, dir);



		if (dir == DMA_FROM_DEVICE)

			arch_dma_mark_clean(paddr, sg->length);
@@ -640,7 +636,7 @@ size_t dma_direct_max_mapping_size(struct device *dev) 
bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr)

{

	return !dev_is_dma_coherent(dev) ||

	       is_swiotlb_buffer(dev, dma_to_phys(dev, dma_addr));

	       swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr));

}



/**