Commit f77581bf authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'add-multi-buff-support-for-xdp-running-in-generic-mode' 

Lorenzo Bianconi says:

====================
add multi-buff support for xdp running in generic mode

Introduce multi-buffer support for xdp running in generic mode not always
linearizing the skb in netif_receive_generic_xdp routine.
Introduce generic percpu page_pools allocator.
====================

Link: https://lore.kernel.org/r/cover.1707729884.git.lorenzo@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 32e4a544 27accb3c

drivers/net/tun.c

+2 −2
Original line number Diff line number Diff line
@@ -1926,7 +1926,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
		rcu_read_lock();

		xdp_prog = rcu_dereference(tun->xdp_prog);

		if (xdp_prog) {

			ret = do_xdp_generic(xdp_prog, skb);

			ret = do_xdp_generic(xdp_prog, &skb);

			if (ret != XDP_PASS) {

				rcu_read_unlock();

				local_bh_enable();
@@ -2516,7 +2516,7 @@ static int tun_xdp_one(struct tun_struct *tun, 
	skb_record_rx_queue(skb, tfile->queue_index);



	if (skb_xdp) {

		ret = do_xdp_generic(xdp_prog, skb);

		ret = do_xdp_generic(xdp_prog, &skb);

		if (ret != XDP_PASS) {

			ret = 0;

			goto out;

drivers/net/veth.c

+2 −72
Original line number Diff line number Diff line
@@ -729,80 +729,10 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, 
	if (skb_shared(skb) || skb_head_is_locked(skb) ||

	    skb_shinfo(skb)->nr_frags ||

	    skb_headroom(skb) < XDP_PACKET_HEADROOM) {

		u32 size, len, max_head_size, off, truesize, page_offset;

		struct sk_buff *nskb;

		struct page *page;

		int i, head_off;

		void *va;



		/* We need a private copy of the skb and data buffers since

		 * the ebpf program can modify it. We segment the original skb

		 * into order-0 pages without linearize it.

		 *

		 * Make sure we have enough space for linear and paged area

		 */

		max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE -

						  VETH_XDP_HEADROOM);

		if (skb->len > PAGE_SIZE * MAX_SKB_FRAGS + max_head_size)

			goto drop;



		size = min_t(u32, skb->len, max_head_size);

		truesize = SKB_HEAD_ALIGN(size) + VETH_XDP_HEADROOM;



		/* Allocate skb head */

		va = page_pool_dev_alloc_va(rq->page_pool, &truesize);

		if (!va)

			goto drop;



		nskb = napi_build_skb(va, truesize);

		if (!nskb) {

			page_pool_free_va(rq->page_pool, va, true);

			goto drop;

		}



		skb_reserve(nskb, VETH_XDP_HEADROOM);

		skb_copy_header(nskb, skb);

		skb_mark_for_recycle(nskb);



		if (skb_copy_bits(skb, 0, nskb->data, size)) {

			consume_skb(nskb);

			goto drop;

		}

		skb_put(nskb, size);



		head_off = skb_headroom(nskb) - skb_headroom(skb);

		skb_headers_offset_update(nskb, head_off);



		/* Allocate paged area of new skb */

		off = size;

		len = skb->len - off;



		for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {

			size = min_t(u32, len, PAGE_SIZE);

			truesize = size;



			page = page_pool_dev_alloc(rq->page_pool, &page_offset,

						   &truesize);

			if (!page) {

				consume_skb(nskb);

				goto drop;

			}



			skb_add_rx_frag(nskb, i, page, page_offset, size,

					truesize);

			if (skb_copy_bits(skb, off,

					  page_address(page) + page_offset,

					  size)) {

				consume_skb(nskb);

		if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM))

			goto drop;

			}



			len -= size;

			off += size;

		}



		consume_skb(skb);

		skb = nskb;

		skb = *pskb;

	}



	/* SKB "head" area always have tailroom for skb_shared_info */

include/linux/netdevice.h

+1 −1
Original line number Diff line number Diff line
@@ -3958,7 +3958,7 @@ static inline void dev_consume_skb_any(struct sk_buff *skb) 
u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,

			     struct bpf_prog *xdp_prog);

void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);

int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);

int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb);

int netif_rx(struct sk_buff *skb);

int __netif_rx(struct sk_buff *skb);

include/linux/skbuff.h

+4 −0
Original line number Diff line number Diff line
@@ -3446,6 +3446,10 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f) 
	__skb_frag_ref(&skb_shinfo(skb)->frags[f]);

}



int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,

		    unsigned int headroom);

int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,

			 struct bpf_prog *prog);

bool napi_pp_put_page(struct page *page, bool napi_safe);



static inline void

include/net/page_pool/types.h

+3 −0
Original line number Diff line number Diff line
@@ -128,6 +128,7 @@ struct page_pool_stats { 
struct page_pool {

	struct page_pool_params_fast p;



	int cpuid;

	bool has_init_callback;



	long frag_users;
@@ -203,6 +204,8 @@ struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); 
struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,

				  unsigned int size, gfp_t gfp);

struct page_pool *page_pool_create(const struct page_pool_params *params);

struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,

					  int cpuid);



struct xdp_mem_info;