Commit 4dec64c5 authored by Mina Almasry's avatar Mina Almasry Committed by Jakub Kicinski
Browse files

page_pool: convert to use netmem 



Abstract the memory type from the page_pool so we can later add support
for new memory types. Convert the page_pool to use the new netmem type
abstraction, rather than use struct page directly.

As of this patch the netmem type is a no-op abstraction: it's always a
struct page underneath. All the page pool internals are converted to
use struct netmem instead of struct page, and the page pool now exports
2 APIs:

1. The existing struct page API.
2. The new struct netmem API.

Keeping the existing API is transitional; we do not want to refactor all
the current drivers using the page pool at once.

The netmem abstraction is currently a no-op. The page_pool uses
page_to_netmem() to convert allocated pages to netmem, and uses
netmem_to_page() to convert the netmem back to pages to pass to mm APIs,

Follow up patches to this series add non-paged netmem support to the
page_pool. This change is factored out on its own to limit the code
churn to this 1 patch, for ease of code review.

Signed-off-by: default avatarMina Almasry <almasrymina@google.com>
Reviewed-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/20240628003253.1694510-6-almasrymina@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent ac263276

include/linux/skbuff_ref.h

+2 −2
Original line number Diff line number Diff line
@@ -32,13 +32,13 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f) 
	__skb_frag_ref(&skb_shinfo(skb)->frags[f]);

}



bool napi_pp_put_page(struct page *page);

bool napi_pp_put_page(netmem_ref netmem);



static inline void

skb_page_unref(struct page *page, bool recycle)

{

#ifdef CONFIG_PAGE_POOL

	if (recycle && napi_pp_put_page(page))

	if (recycle && napi_pp_put_page(page_to_netmem(page)))

		return;

#endif

	put_page(page);

include/net/netmem.h

+15 −0
Original line number Diff line number Diff line
@@ -38,4 +38,19 @@ static inline netmem_ref page_to_netmem(struct page *page) 
	return (__force netmem_ref)page;

}



static inline int netmem_ref_count(netmem_ref netmem)

{

	return page_ref_count(netmem_to_page(netmem));

}



static inline unsigned long netmem_to_pfn(netmem_ref netmem)

{

	return page_to_pfn(netmem_to_page(netmem));

}



static inline netmem_ref netmem_compound_head(netmem_ref netmem)

{

	return page_to_netmem(compound_head(netmem_to_page(netmem)));

}



#endif /* _NET_NETMEM_H */

include/net/page_pool/helpers.h

+70 −21
Original line number Diff line number Diff line
@@ -55,6 +55,8 @@ 
#include <linux/dma-mapping.h>



#include <net/page_pool/types.h>

#include <net/net_debug.h>

#include <net/netmem.h>



#ifdef CONFIG_PAGE_POOL_STATS

/* Deprecated driver-facing API, use netlink instead */
@@ -212,6 +214,11 @@ page_pool_get_dma_dir(const struct page_pool *pool) 
	return pool->p.dma_dir;

}



static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr)

{

	atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr);

}



/**

 * page_pool_fragment_page() - split a fresh page into fragments

 * @page:	page to split
@@ -232,11 +239,12 @@ page_pool_get_dma_dir(const struct page_pool *pool) 
 */

static inline void page_pool_fragment_page(struct page *page, long nr)

{

	atomic_long_set(&page->pp_ref_count, nr);

	page_pool_fragment_netmem(page_to_netmem(page), nr);

}



static inline long page_pool_unref_page(struct page *page, long nr)

static inline long page_pool_unref_netmem(netmem_ref netmem, long nr)

{

	struct page *page = netmem_to_page(netmem);

	long ret;



	/* If nr == pp_ref_count then we have cleared all remaining
@@ -279,15 +287,41 @@ static inline long page_pool_unref_page(struct page *page, long nr) 
	return ret;

}



static inline long page_pool_unref_page(struct page *page, long nr)

{

	return page_pool_unref_netmem(page_to_netmem(page), nr);

}



static inline void page_pool_ref_netmem(netmem_ref netmem)

{

	atomic_long_inc(&netmem_to_page(netmem)->pp_ref_count);

}



static inline void page_pool_ref_page(struct page *page)

{

	atomic_long_inc(&page->pp_ref_count);

	page_pool_ref_netmem(page_to_netmem(page));

}



static inline bool page_pool_is_last_ref(struct page *page)

static inline bool page_pool_is_last_ref(netmem_ref netmem)

{

	/* If page_pool_unref_page() returns 0, we were the last user */

	return page_pool_unref_page(page, 1) == 0;

	return page_pool_unref_netmem(netmem, 1) == 0;

}



static inline void page_pool_put_netmem(struct page_pool *pool,

					netmem_ref netmem,

					unsigned int dma_sync_size,

					bool allow_direct)

{

	/* When page_pool isn't compiled-in, net/core/xdp.c doesn't

	 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.

	 */

#ifdef CONFIG_PAGE_POOL

	if (!page_pool_is_last_ref(netmem))

		return;



	page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct);

#endif

}



/**
@@ -308,15 +342,15 @@ static inline void page_pool_put_page(struct page_pool *pool, 
				      unsigned int dma_sync_size,

				      bool allow_direct)

{

	/* When page_pool isn't compiled-in, net/core/xdp.c doesn't

	 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.

	 */

#ifdef CONFIG_PAGE_POOL

	if (!page_pool_is_last_ref(page))

		return;

	page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size,

			     allow_direct);

}



	page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct);

#endif

static inline void page_pool_put_full_netmem(struct page_pool *pool,

					     netmem_ref netmem,

					     bool allow_direct)

{

	page_pool_put_netmem(pool, netmem, -1, allow_direct);

}



/**
@@ -331,7 +365,7 @@ static inline void page_pool_put_page(struct page_pool *pool, 
static inline void page_pool_put_full_page(struct page_pool *pool,

					   struct page *page, bool allow_direct)

{

	page_pool_put_page(pool, page, -1, allow_direct);

	page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct);

}



/**
@@ -365,6 +399,18 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, 
	page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct);

}



static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem)

{

	struct page *page = netmem_to_page(netmem);



	dma_addr_t ret = page->dma_addr;



	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)

		ret <<= PAGE_SHIFT;



	return ret;

}



/**

 * page_pool_get_dma_addr() - Retrieve the stored DMA address.

 * @page:	page allocated from a page pool
@@ -374,16 +420,14 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, 
 */

static inline dma_addr_t page_pool_get_dma_addr(const struct page *page)

{

	dma_addr_t ret = page->dma_addr;



	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)

		ret <<= PAGE_SHIFT;



	return ret;

	return page_pool_get_dma_addr_netmem(page_to_netmem((struct page *)page));

}



static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)

static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem,

						 dma_addr_t addr)

{

	struct page *page = netmem_to_page(netmem);



	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) {

		page->dma_addr = addr >> PAGE_SHIFT;
@@ -419,6 +463,11 @@ static inline void page_pool_dma_sync_for_cpu(const struct page_pool *pool, 
				      page_pool_get_dma_dir(pool));

}



static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)

{

	return page_pool_set_dma_addr_netmem(page_to_netmem(page), addr);

}



static inline bool page_pool_put(struct page_pool *pool)

{

	return refcount_dec_and_test(&pool->user_cnt);

include/net/page_pool/types.h

+11 −3
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#include <linux/dma-direction.h>

#include <linux/ptr_ring.h>

#include <linux/types.h>

#include <net/netmem.h>



#define PP_FLAG_DMA_MAP		BIT(0) /* Should page_pool do the DMA

					* map/unmap
@@ -40,7 +41,7 @@ 
#define PP_ALLOC_CACHE_REFILL	64

struct pp_alloc_cache {

	u32 count;

	struct page *cache[PP_ALLOC_CACHE_SIZE];

	netmem_ref cache[PP_ALLOC_CACHE_SIZE];

};



/**
@@ -73,7 +74,7 @@ struct page_pool_params { 
		struct net_device *netdev;

		unsigned int	flags;

/* private: used by test code only */

		void (*init_callback)(struct page *page, void *arg);

		void (*init_callback)(netmem_ref netmem, void *arg);

		void *init_arg;

	);

};
@@ -151,7 +152,7 @@ struct page_pool { 
	 */

	__cacheline_group_begin(frag) __aligned(4 * sizeof(long));

	long frag_users;

	struct page *frag_page;

	netmem_ref frag_page;

	unsigned int frag_offset;

	__cacheline_group_end(frag);
@@ -220,8 +221,12 @@ struct page_pool { 
};



struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);

netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp);

struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,

				  unsigned int size, gfp_t gfp);

netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,

				       unsigned int *offset, unsigned int size,

				       gfp_t gfp);

struct page_pool *page_pool_create(const struct page_pool_params *params);

struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,

					  int cpuid);
@@ -252,6 +257,9 @@ static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, 
}

#endif



void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,

				  unsigned int dma_sync_size,

				  bool allow_direct);

void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,

				unsigned int dma_sync_size,

				bool allow_direct);

include/trace/events/page_pool.h

+16 −14
Original line number Diff line number Diff line
@@ -42,51 +42,53 @@ TRACE_EVENT(page_pool_release, 
TRACE_EVENT(page_pool_state_release,



	TP_PROTO(const struct page_pool *pool,

		 const struct page *page, u32 release),

		 netmem_ref netmem, u32 release),



	TP_ARGS(pool, page, release),

	TP_ARGS(pool, netmem, release),



	TP_STRUCT__entry(

		__field(const struct page_pool *,	pool)

		__field(const struct page *,		page)

		__field(unsigned long,			netmem)

		__field(u32,				release)

		__field(unsigned long,			pfn)

	),



	TP_fast_assign(

		__entry->pool		= pool;

		__entry->page		= page;

		__entry->netmem		= (__force unsigned long)netmem;

		__entry->release	= release;

		__entry->pfn		= page_to_pfn(page);

		__entry->pfn		= netmem_to_pfn(netmem);

	),



	TP_printk("page_pool=%p page=%p pfn=0x%lx release=%u",

		  __entry->pool, __entry->page, __entry->pfn, __entry->release)

	TP_printk("page_pool=%p netmem=%p pfn=0x%lx release=%u",

		  __entry->pool, (void *)__entry->netmem,

		  __entry->pfn, __entry->release)

);



TRACE_EVENT(page_pool_state_hold,



	TP_PROTO(const struct page_pool *pool,

		 const struct page *page, u32 hold),

		 netmem_ref netmem, u32 hold),



	TP_ARGS(pool, page, hold),

	TP_ARGS(pool, netmem, hold),



	TP_STRUCT__entry(

		__field(const struct page_pool *,	pool)

		__field(const struct page *,		page)

		__field(unsigned long,			netmem)

		__field(u32,				hold)

		__field(unsigned long,			pfn)

	),



	TP_fast_assign(

		__entry->pool	= pool;

		__entry->page	= page;

		__entry->netmem	= (__force unsigned long)netmem;

		__entry->hold	= hold;

		__entry->pfn	= page_to_pfn(page);

		__entry->pfn	= netmem_to_pfn(netmem);

	),



	TP_printk("page_pool=%p page=%p pfn=0x%lx hold=%u",

		  __entry->pool, __entry->page, __entry->pfn, __entry->hold)

	TP_printk("page_pool=%p netmem=%p pfn=0x%lx hold=%u",

		  __entry->pool, (void *)__entry->netmem,

		  __entry->pfn, __entry->hold)

);



TRACE_EVENT(page_pool_update_nid,