Commit 2c27c766 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'devmem-tcp-fixes' 

Mina Almasry says:

====================
devmem TCP fixes

Couple unrelated devmem TCP fixes bundled in a series for some
convenience.

- fix naming and provide page_pool_alloc_netmem for fragged
netmem.

- fix issues with dma-buf dma addresses being potentially
passed to dma_sync_for_* helpers.
====================

Link: https://patch.msgid.link/20241211212033.1684197-1-almasrymina@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 36317737 7dba339f

include/net/page_pool/helpers.h

+46 −14
Original line number Diff line number Diff line
@@ -115,22 +115,22 @@ static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, 
	return page_pool_alloc_frag(pool, offset, size, gfp);

}



static inline struct page *page_pool_alloc(struct page_pool *pool,

static inline netmem_ref page_pool_alloc_netmem(struct page_pool *pool,

						unsigned int *offset,

						unsigned int *size, gfp_t gfp)

{

	unsigned int max_size = PAGE_SIZE << pool->p.order;

	struct page *page;

	netmem_ref netmem;



	if ((*size << 1) > max_size) {

		*size = max_size;

		*offset = 0;

		return page_pool_alloc_pages(pool, gfp);

		return page_pool_alloc_netmems(pool, gfp);

	}



	page = page_pool_alloc_frag(pool, offset, *size, gfp);

	if (unlikely(!page))

		return NULL;

	netmem = page_pool_alloc_frag_netmem(pool, offset, *size, gfp);

	if (unlikely(!netmem))

		return 0;



	/* There is very likely not enough space for another fragment, so append

	 * the remaining size to the current fragment to avoid truesize
@@ -141,7 +141,14 @@ static inline struct page *page_pool_alloc(struct page_pool *pool, 
		pool->frag_offset = max_size;

	}



	return page;

	return netmem;

}



static inline struct page *page_pool_alloc(struct page_pool *pool,

					   unsigned int *offset,

					   unsigned int *size, gfp_t gfp)

{

	return netmem_to_page(page_pool_alloc_netmem(pool, offset, size, gfp));

}



/**
@@ -415,7 +422,21 @@ static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem) 
 */

static inline dma_addr_t page_pool_get_dma_addr(const struct page *page)

{

	return page_pool_get_dma_addr_netmem(page_to_netmem((struct page *)page));

	dma_addr_t ret = page->dma_addr;



	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)

		ret <<= PAGE_SHIFT;



	return ret;

}



static inline void __page_pool_dma_sync_for_cpu(const struct page_pool *pool,

						const dma_addr_t dma_addr,

						u32 offset, u32 dma_sync_size)

{

	dma_sync_single_range_for_cpu(pool->p.dev, dma_addr,

				      offset + pool->p.offset, dma_sync_size,

				      page_pool_get_dma_dir(pool));

}



/**
@@ -434,10 +455,21 @@ static inline void page_pool_dma_sync_for_cpu(const struct page_pool *pool, 
					      const struct page *page,

					      u32 offset, u32 dma_sync_size)

{

	dma_sync_single_range_for_cpu(pool->p.dev,

				      page_pool_get_dma_addr(page),

				      offset + pool->p.offset, dma_sync_size,

				      page_pool_get_dma_dir(pool));

	__page_pool_dma_sync_for_cpu(pool, page_pool_get_dma_addr(page), offset,

				     dma_sync_size);

}



static inline void

page_pool_dma_sync_netmem_for_cpu(const struct page_pool *pool,

				  const netmem_ref netmem, u32 offset,

				  u32 dma_sync_size)

{

	if (!pool->dma_sync_for_cpu)

		return;



	__page_pool_dma_sync_for_cpu(pool,

				     page_pool_get_dma_addr_netmem(netmem),

				     offset, dma_sync_size);

}



static inline bool page_pool_put(struct page_pool *pool)

include/net/page_pool/types.h

+3 −2
Original line number Diff line number Diff line
@@ -164,7 +164,8 @@ struct page_pool { 


	bool has_init_callback:1;	/* slow::init_callback is set */

	bool dma_map:1;			/* Perform DMA mapping */

	bool dma_sync:1;		/* Perform DMA sync */

	bool dma_sync:1;		/* Perform DMA sync for device */

	bool dma_sync_for_cpu:1;	/* Perform DMA sync for cpu */

#ifdef CONFIG_PAGE_POOL_STATS

	bool system:1;			/* This is a global percpu pool */

#endif
@@ -242,7 +243,7 @@ struct page_pool { 
};



struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);

netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp);

netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp);

struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,

				  unsigned int size, gfp_t gfp);

netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,

net/core/devmem.c

+5 −5
Original line number Diff line number Diff line
@@ -331,11 +331,11 @@ int mp_dmabuf_devmem_init(struct page_pool *pool) 
	if (!binding)

		return -EINVAL;



	if (!pool->dma_map)

		return -EOPNOTSUPP;



	if (pool->dma_sync)

		return -EOPNOTSUPP;

	/* dma-buf dma addresses do not need and should not be used with

	 * dma_sync_for_cpu/device. Force disable dma_sync.

	 */

	pool->dma_sync = false;

	pool->dma_sync_for_cpu = false;



	if (pool->p.order != 0)

		return -E2BIG;

net/core/page_pool.c

+8 −4
Original line number Diff line number Diff line
@@ -201,6 +201,7 @@ static int page_pool_init(struct page_pool *pool, 
	memcpy(&pool->slow, &params->slow, sizeof(pool->slow));



	pool->cpuid = cpuid;

	pool->dma_sync_for_cpu = true;



	/* Validate only known flags were used */

	if (pool->slow.flags & ~PP_FLAG_ALL)
@@ -287,6 +288,9 @@ static int page_pool_init(struct page_pool *pool, 
	}



	if (pool->mp_priv) {

		if (!pool->dma_map || !pool->dma_sync)

			return -EOPNOTSUPP;



		err = mp_dmabuf_devmem_init(pool);

		if (err) {

			pr_warn("%s() mem-provider init failed %d\n", __func__,
@@ -574,7 +578,7 @@ static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, 
/* For using page_pool replace: alloc_pages() API calls, but provide

 * synchronization guarantee for allocation side.

 */

netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp)

netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp)

{

	netmem_ref netmem;
@@ -590,11 +594,11 @@ netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp) 
		netmem = __page_pool_alloc_pages_slow(pool, gfp);

	return netmem;

}

EXPORT_SYMBOL(page_pool_alloc_netmem);

EXPORT_SYMBOL(page_pool_alloc_netmems);



struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)

{

	return netmem_to_page(page_pool_alloc_netmem(pool, gfp));

	return netmem_to_page(page_pool_alloc_netmems(pool, gfp));

}

EXPORT_SYMBOL(page_pool_alloc_pages);

ALLOW_ERROR_INJECTION(page_pool_alloc_pages, NULL);
@@ -992,7 +996,7 @@ netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, 
	}



	if (!netmem) {

		netmem = page_pool_alloc_netmem(pool, gfp);

		netmem = page_pool_alloc_netmems(pool, gfp);

		if (unlikely(!netmem)) {

			pool->frag_page = 0;

			return 0;