Commit 68ddc8ae authored by Alexander Lobakin's avatar Alexander Lobakin Committed by Jakub Kicinski
Browse files

xdp: add generic xdp_buff_add_frag() 



The code piece which would attach a frag to &xdp_buff is almost
identical across the drivers supporting XDP multi-buffer on Rx.
Make it a generic elegant "oneliner".
Also, I see lots of drivers calculating frags_truesize as
`xdp->frame_sz * nr_frags`. I can't say this is fully correct, since
frags might be backed by chunks of different sizes, especially with
stuff like the header split. Even page_pool_alloc() can give you two
different truesizes on two subsequent requests to allocate the same
buffer size. Add a field to &skb_shared_info (unionized as there's no
free slot currently on x86_64) to track the "true" truesize. It can
be used later when updating the skb.

Reviewed-by: default avatarMaciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: default avatarAlexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20241218174435.1445282-3-aleksander.lobakin@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent a19d0236

include/linux/skbuff.h

+12 −4
Original line number Diff line number Diff line
@@ -608,11 +608,19 @@ struct skb_shared_info { 
	 * Warning : all fields before dataref are cleared in __alloc_skb()

	 */

	atomic_t	dataref;

	unsigned int	xdp_frags_size;



	/* Intermediate layers must ensure that destructor_arg

	 * remains valid until skb destructor */

	union {

		struct {

			u32		xdp_frags_size;

			u32		xdp_frags_truesize;

		};



		/*

		 * Intermediate layers must ensure that destructor_arg

		 * remains valid until skb destructor.

		 */

		void		*destructor_arg;

	};



	/* must be last field, see pskb_expand_head() */

	skb_frag_t	frags[MAX_SKB_FRAGS];

include/net/xdp.h

+95 −1
Original line number Diff line number Diff line
@@ -167,6 +167,93 @@ xdp_get_buff_len(const struct xdp_buff *xdp) 
	return len;

}



void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp);



/**

 * __xdp_buff_add_frag - attach frag to &xdp_buff

 * @xdp: XDP buffer to attach the frag to

 * @netmem: network memory containing the frag

 * @offset: offset at which the frag starts

 * @size: size of the frag

 * @truesize: total memory size occupied by the frag

 * @try_coalesce: whether to try coalescing the frags (not valid for XSk)

 *

 * Attach frag to the XDP buffer. If it currently has no frags attached,

 * initialize the related fields, otherwise check that the frag number

 * didn't reach the limit of ``MAX_SKB_FRAGS``. If possible, try coalescing

 * the frag with the previous one.

 * The function doesn't check/update the pfmemalloc bit. Please use the

 * non-underscored wrapper in drivers.

 *

 * Return: true on success, false if there's no space for the frag in

 * the shared info struct.

 */

static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,

				       u32 offset, u32 size, u32 truesize,

				       bool try_coalesce)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	skb_frag_t *prev;

	u32 nr_frags;



	if (!xdp_buff_has_frags(xdp)) {

		xdp_buff_set_frags_flag(xdp);



		nr_frags = 0;

		sinfo->xdp_frags_size = 0;

		sinfo->xdp_frags_truesize = 0;



		goto fill;

	}



	nr_frags = sinfo->nr_frags;

	prev = &sinfo->frags[nr_frags - 1];



	if (try_coalesce && netmem == skb_frag_netmem(prev) &&

	    offset == skb_frag_off(prev) + skb_frag_size(prev)) {

		skb_frag_size_add(prev, size);

		/* Guaranteed to only decrement the refcount */

		xdp_return_frag(netmem, xdp);

	} else if (unlikely(nr_frags == MAX_SKB_FRAGS)) {

		return false;

	} else {

fill:

		__skb_fill_netmem_desc_noacc(sinfo, nr_frags++, netmem,

					     offset, size);

	}



	sinfo->nr_frags = nr_frags;

	sinfo->xdp_frags_size += size;

	sinfo->xdp_frags_truesize += truesize;



	return true;

}



/**

 * xdp_buff_add_frag - attach frag to &xdp_buff

 * @xdp: XDP buffer to attach the frag to

 * @netmem: network memory containing the frag

 * @offset: offset at which the frag starts

 * @size: size of the frag

 * @truesize: total memory size occupied by the frag

 *

 * Version of __xdp_buff_add_frag() which takes care of the pfmemalloc bit.

 *

 * Return: true on success, false if there's no space for the frag in

 * the shared info struct.

 */

static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,

				     u32 offset, u32 size, u32 truesize)

{

	if (!__xdp_buff_add_frag(xdp, netmem, offset, size, truesize, true))

		return false;



	if (unlikely(netmem_is_pfmemalloc(netmem)))

		xdp_buff_set_frag_pfmemalloc(xdp);



	return true;

}



struct xdp_frame {

	void *data;

	u32 len;
@@ -230,7 +317,14 @@ xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags, 
			   unsigned int size, unsigned int truesize,

			   bool pfmemalloc)

{

	skb_shinfo(skb)->nr_frags = nr_frags;

	struct skb_shared_info *sinfo = skb_shinfo(skb);



	sinfo->nr_frags = nr_frags;

	/*

	 * ``destructor_arg`` is unionized with ``xdp_frags_{,true}size``,

	 * reset it after that these fields aren't used anymore.

	 */

	sinfo->destructor_arg = NULL;



	skb->len += size;

	skb->data_len += size;

net/core/xdp.c

+11 −0
Original line number Diff line number Diff line
@@ -535,6 +535,17 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf, 
}

EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);



/**

 * xdp_return_frag -- free one XDP frag or decrement its refcount

 * @netmem: network memory reference to release

 * @xdp: &xdp_buff to release the frag for

 */

void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp)

{

	__xdp_return(netmem, xdp->rxq->mem.type, true, NULL);

}

EXPORT_SYMBOL_GPL(xdp_return_frag);



void xdp_return_buff(struct xdp_buff *xdp)

{

	struct skb_shared_info *sinfo;