Commit 2b0cfa6e authored by Lorenzo Bianconi's avatar Lorenzo Bianconi Committed by Jakub Kicinski
Browse files

net: add generic percpu page_pool allocator 



Introduce generic percpu page_pools allocator.
Moreover add page_pool_create_percpu() and cpuid filed in page_pool struct
in order to recycle the page in the page_pool "hot" cache if
napi_pp_put_page() is running on the same cpu.
This is a preliminary patch to add xdp multi-buff support for xdp running
in generic mode.

Acked-by: default avatarJesper Dangaard Brouer <hawk@kernel.org>
Reviewed-by: default avatarToke Hoiland-Jorgensen <toke@redhat.com>
Signed-off-by: default avatarLorenzo Bianconi <lorenzo@kernel.org>
Link: https://lore.kernel.org/r/80bc4285228b6f4220cd03de1999d86e46e3fcbd.1707729884.git.lorenzo@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 32e4a544

include/net/page_pool/types.h

+3 −0
Original line number Diff line number Diff line
@@ -128,6 +128,7 @@ struct page_pool_stats { 
struct page_pool {

	struct page_pool_params_fast p;



	int cpuid;

	bool has_init_callback;



	long frag_users;
@@ -203,6 +204,8 @@ struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); 
struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,

				  unsigned int size, gfp_t gfp);

struct page_pool *page_pool_create(const struct page_pool_params *params);

struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,

					  int cpuid);



struct xdp_mem_info;

net/core/dev.c

+45 −0
Original line number Diff line number Diff line
@@ -153,6 +153,8 @@ 
#include <linux/prandom.h>

#include <linux/once_lite.h>

#include <net/netdev_rx_queue.h>

#include <net/page_pool/types.h>

#include <net/page_pool/helpers.h>



#include "dev.h"

#include "net-sysfs.h"
@@ -450,6 +452,12 @@ static RAW_NOTIFIER_HEAD(netdev_chain); 
DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);

EXPORT_PER_CPU_SYMBOL(softnet_data);



/* Page_pool has a lockless array/stack to alloc/recycle pages.

 * PP consumers must pay attention to run APIs in the appropriate context

 * (e.g. NAPI context).

 */

static DEFINE_PER_CPU_ALIGNED(struct page_pool *, system_page_pool);



#ifdef CONFIG_LOCKDEP

/*

 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
@@ -11724,6 +11732,27 @@ static void __init net_dev_struct_check(void) 
 *

 */



/* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */

#define SYSTEM_PERCPU_PAGE_POOL_SIZE	((1 << 20) / PAGE_SIZE)



static int net_page_pool_create(int cpuid)

{

#if IS_ENABLED(CONFIG_PAGE_POOL)

	struct page_pool_params page_pool_params = {

		.pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE,

		.nid = NUMA_NO_NODE,

	};

	struct page_pool *pp_ptr;



	pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid);

	if (IS_ERR(pp_ptr))

		return -ENOMEM;



	per_cpu(system_page_pool, cpuid) = pp_ptr;

#endif

	return 0;

}



/*

 *       This is called single threaded during boot, so no need

 *       to take the rtnl semaphore.
@@ -11776,6 +11805,9 @@ static int __init net_dev_init(void) 
		init_gro_hash(&sd->backlog);

		sd->backlog.poll = process_backlog;

		sd->backlog.weight = weight_p;



		if (net_page_pool_create(i))

			goto out;

	}



	dev_boot_phase = 0;
@@ -11803,6 +11835,19 @@ static int __init net_dev_init(void) 
	WARN_ON(rc < 0);

	rc = 0;

out:

	if (rc < 0) {

		for_each_possible_cpu(i) {

			struct page_pool *pp_ptr;



			pp_ptr = per_cpu(system_page_pool, i);

			if (!pp_ptr)

				continue;



			page_pool_destroy(pp_ptr);

			per_cpu(system_page_pool, i) = NULL;

		}

	}



	return rc;

}

net/core/page_pool.c

+19 −4
Original line number Diff line number Diff line
@@ -171,13 +171,16 @@ static void page_pool_producer_unlock(struct page_pool *pool, 
}



static int page_pool_init(struct page_pool *pool,

			  const struct page_pool_params *params)

			  const struct page_pool_params *params,

			  int cpuid)

{

	unsigned int ring_qsize = 1024; /* Default */



	memcpy(&pool->p, &params->fast, sizeof(pool->p));

	memcpy(&pool->slow, &params->slow, sizeof(pool->slow));



	pool->cpuid = cpuid;



	/* Validate only known flags were used */

	if (pool->p.flags & ~(PP_FLAG_ALL))

		return -EINVAL;
@@ -253,10 +256,12 @@ static void page_pool_uninit(struct page_pool *pool) 
}



/**

 * page_pool_create() - create a page pool.

 * page_pool_create_percpu() - create a page pool for a given cpu.

 * @params: parameters, see struct page_pool_params

 * @cpuid: cpu identifier

 */

struct page_pool *page_pool_create(const struct page_pool_params *params)

struct page_pool *

page_pool_create_percpu(const struct page_pool_params *params, int cpuid)

{

	struct page_pool *pool;

	int err;
@@ -265,7 +270,7 @@ struct page_pool *page_pool_create(const struct page_pool_params *params) 
	if (!pool)

		return ERR_PTR(-ENOMEM);



	err = page_pool_init(pool, params);

	err = page_pool_init(pool, params, cpuid);

	if (err < 0)

		goto err_free;
@@ -282,6 +287,16 @@ struct page_pool *page_pool_create(const struct page_pool_params *params) 
	kfree(pool);

	return ERR_PTR(err);

}

EXPORT_SYMBOL(page_pool_create_percpu);



/**

 * page_pool_create() - create a page pool

 * @params: parameters, see struct page_pool_params

 */

struct page_pool *page_pool_create(const struct page_pool_params *params)

{

	return page_pool_create_percpu(params, -1);

}

EXPORT_SYMBOL(page_pool_create);



static void page_pool_return_page(struct page_pool *pool, struct page *page);

net/core/skbuff.c

+3 −2
Original line number Diff line number Diff line
@@ -923,9 +923,10 @@ bool napi_pp_put_page(struct page *page, bool napi_safe) 
	 */

	if (napi_safe || in_softirq()) {

		const struct napi_struct *napi = READ_ONCE(pp->p.napi);

		unsigned int cpuid = smp_processor_id();



		allow_direct = napi &&

			READ_ONCE(napi->list_owner) == smp_processor_id();

		allow_direct = napi && READ_ONCE(napi->list_owner) == cpuid;

		allow_direct |= (pp->cpuid == cpuid);

	}



	/* Driver set this to memory recycling info. Reset it on recycle.