Commit b73e56f1 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'xdp-a-fistful-of-generic-changes-pt-iii' 

Alexander Lobakin says:

====================
xdp: a fistful of generic changes pt. III

XDP for idpf is currently 5.(6) chapters:
* convert Rx to libeth;
* convert Tx and stats to libeth;
* generic XDP and XSk code changes;
* generic XDP and XSk code additions pt. 1;
* generic XDP and XSk code additions pt. 2 (you are here);
* actual XDP for idpf via new libeth_xdp;
* XSk for idpf (via ^).

Part III.3 does the following:
* adds generic functions to build skbs from xdp_buffs (regular and
  XSk) and attach frags to xdp_buffs (regular and XSk);
* adds helper to optimize XSk xmit in drivers.

Everything is prereq for libeth_xdp, but will be useful standalone
as well: less code in drivers, faster XSk XDP_PASS, smaller object
code.
====================

Link: https://patch.msgid.link/20241218174435.1445282-1-aleksander.lobakin@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 29b54079 560d958c

drivers/net/ethernet/intel/i40e/i40e_xsk.c

+3 −27
Original line number Diff line number Diff line
@@ -395,32 +395,6 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, 
	WARN_ON_ONCE(1);

}



static int

i40e_add_xsk_frag(struct i40e_ring *rx_ring, struct xdp_buff *first,

		  struct xdp_buff *xdp, const unsigned int size)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first);



	if (!xdp_buff_has_frags(first)) {

		sinfo->nr_frags = 0;

		sinfo->xdp_frags_size = 0;

		xdp_buff_set_frags_flag(first);

	}



	if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) {

		xsk_buff_free(first);

		return -ENOMEM;

	}



	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++,

				   virt_to_page(xdp->data_hard_start),

				   XDP_PACKET_HEADROOM, size);

	sinfo->xdp_frags_size += size;

	xsk_buff_add_frag(xdp);



	return 0;

}



/**

 * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring

 * @rx_ring: Rx ring
@@ -486,8 +460,10 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) 


		if (!first)

			first = bi;

		else if (i40e_add_xsk_frag(rx_ring, first, bi, size))

		else if (!xsk_buff_add_frag(first, bi)) {

			xsk_buff_free(first);

			break;

		}



		if (++next_to_process == count)

			next_to_process = 0;

drivers/net/ethernet/intel/ice/ice_xsk.c

+2 −30
Original line number Diff line number Diff line
@@ -801,35 +801,6 @@ ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 
	return result;

}



static int

ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first,

		 struct xdp_buff *xdp, const unsigned int size)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first);



	if (!size)

		return 0;



	if (!xdp_buff_has_frags(first)) {

		sinfo->nr_frags = 0;

		sinfo->xdp_frags_size = 0;

		xdp_buff_set_frags_flag(first);

	}



	if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) {

		xsk_buff_free(first);

		return -ENOMEM;

	}



	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++,

				   virt_to_page(xdp->data_hard_start),

				   XDP_PACKET_HEADROOM, size);

	sinfo->xdp_frags_size += size;

	xsk_buff_add_frag(xdp);



	return 0;

}



/**

 * ice_clean_rx_irq_zc - consumes packets from the hardware ring

 * @rx_ring: AF_XDP Rx ring
@@ -895,7 +866,8 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, 


		if (!first) {

			first = xdp;

		} else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) {

		} else if (likely(size) && !xsk_buff_add_frag(first, xdp)) {

			xsk_buff_free(first);

			break;

		}

include/linux/skbuff.h

+12 −4
Original line number Diff line number Diff line
@@ -608,11 +608,19 @@ struct skb_shared_info { 
	 * Warning : all fields before dataref are cleared in __alloc_skb()

	 */

	atomic_t	dataref;

	unsigned int	xdp_frags_size;



	/* Intermediate layers must ensure that destructor_arg

	 * remains valid until skb destructor */

	union {

		struct {

			u32		xdp_frags_size;

			u32		xdp_frags_truesize;

		};



		/*

		 * Intermediate layers must ensure that destructor_arg

		 * remains valid until skb destructor.

		 */

		void		*destructor_arg;

	};



	/* must be last field, see pskb_expand_head() */

	skb_frag_t	frags[MAX_SKB_FRAGS];

include/net/page_pool/helpers.h

+9 −0
Original line number Diff line number Diff line
@@ -144,6 +144,15 @@ static inline netmem_ref page_pool_alloc_netmem(struct page_pool *pool, 
	return netmem;

}



static inline netmem_ref page_pool_dev_alloc_netmem(struct page_pool *pool,

						    unsigned int *offset,

						    unsigned int *size)

{

	gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;



	return page_pool_alloc_netmem(pool, offset, size, gfp);

}



static inline struct page *page_pool_alloc(struct page_pool *pool,

					   unsigned int *offset,

					   unsigned int *size, gfp_t gfp)

include/net/xdp.h

+97 −1
Original line number Diff line number Diff line
@@ -167,6 +167,93 @@ xdp_get_buff_len(const struct xdp_buff *xdp) 
	return len;

}



void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp);



/**

 * __xdp_buff_add_frag - attach frag to &xdp_buff

 * @xdp: XDP buffer to attach the frag to

 * @netmem: network memory containing the frag

 * @offset: offset at which the frag starts

 * @size: size of the frag

 * @truesize: total memory size occupied by the frag

 * @try_coalesce: whether to try coalescing the frags (not valid for XSk)

 *

 * Attach frag to the XDP buffer. If it currently has no frags attached,

 * initialize the related fields, otherwise check that the frag number

 * didn't reach the limit of ``MAX_SKB_FRAGS``. If possible, try coalescing

 * the frag with the previous one.

 * The function doesn't check/update the pfmemalloc bit. Please use the

 * non-underscored wrapper in drivers.

 *

 * Return: true on success, false if there's no space for the frag in

 * the shared info struct.

 */

static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,

				       u32 offset, u32 size, u32 truesize,

				       bool try_coalesce)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	skb_frag_t *prev;

	u32 nr_frags;



	if (!xdp_buff_has_frags(xdp)) {

		xdp_buff_set_frags_flag(xdp);



		nr_frags = 0;

		sinfo->xdp_frags_size = 0;

		sinfo->xdp_frags_truesize = 0;



		goto fill;

	}



	nr_frags = sinfo->nr_frags;

	prev = &sinfo->frags[nr_frags - 1];



	if (try_coalesce && netmem == skb_frag_netmem(prev) &&

	    offset == skb_frag_off(prev) + skb_frag_size(prev)) {

		skb_frag_size_add(prev, size);

		/* Guaranteed to only decrement the refcount */

		xdp_return_frag(netmem, xdp);

	} else if (unlikely(nr_frags == MAX_SKB_FRAGS)) {

		return false;

	} else {

fill:

		__skb_fill_netmem_desc_noacc(sinfo, nr_frags++, netmem,

					     offset, size);

	}



	sinfo->nr_frags = nr_frags;

	sinfo->xdp_frags_size += size;

	sinfo->xdp_frags_truesize += truesize;



	return true;

}



/**

 * xdp_buff_add_frag - attach frag to &xdp_buff

 * @xdp: XDP buffer to attach the frag to

 * @netmem: network memory containing the frag

 * @offset: offset at which the frag starts

 * @size: size of the frag

 * @truesize: total memory size occupied by the frag

 *

 * Version of __xdp_buff_add_frag() which takes care of the pfmemalloc bit.

 *

 * Return: true on success, false if there's no space for the frag in

 * the shared info struct.

 */

static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,

				     u32 offset, u32 size, u32 truesize)

{

	if (!__xdp_buff_add_frag(xdp, netmem, offset, size, truesize, true))

		return false;



	if (unlikely(netmem_is_pfmemalloc(netmem)))

		xdp_buff_set_frag_pfmemalloc(xdp);



	return true;

}



struct xdp_frame {

	void *data;

	u32 len;
@@ -230,7 +317,14 @@ xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags, 
			   unsigned int size, unsigned int truesize,

			   bool pfmemalloc)

{

	skb_shinfo(skb)->nr_frags = nr_frags;

	struct skb_shared_info *sinfo = skb_shinfo(skb);



	sinfo->nr_frags = nr_frags;

	/*

	 * ``destructor_arg`` is unionized with ``xdp_frags_{,true}size``,

	 * reset it after that these fields aren't used anymore.

	 */

	sinfo->destructor_arg = NULL;



	skb->len += size;

	skb->data_len += size;
@@ -242,6 +336,8 @@ xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags, 
void xdp_warn(const char *msg, const char *func, const int line);

#define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)



struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp);

struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp);

struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);

struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,

					   struct sk_buff *skb,