Commit 36317737 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'xdp-a-fistful-of-generic-changes-pt-ii' 

Alexander Lobakin says:

====================
xdp: a fistful of generic changes pt. II (part)

XDP for idpf is currently 5.5 chapters:
* convert Rx to libeth;
* convert Tx and stats to libeth;
* generic XDP and XSk code changes;
* generic XDP and XSk code additions (you are here);
* actual XDP for idpf via new libeth_xdp;
* XSk for idpf (via ^).

Part III.2.1 does the following:
* allows mixing pages from several Page Pools within one XDP frame;
* optimizes &xdp_frame structure and removes no-more-used field;

Everything is prereq for libeth_xdp, but will be useful standalone
as well: faster xdp_return_frame_bulk() and xdp_frame fields access.
====================

Link: https://patch.msgid.link/20241211172649.761483-1-aleksander.lobakin@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents a42d71e3 0dffdb3b

drivers/net/ethernet/freescale/dpaa/dpaa_eth.c

+1 −1
Original line number Diff line number Diff line
@@ -2281,7 +2281,7 @@ static int dpaa_a050385_wa_xdpf(struct dpaa_priv *priv, 
	new_xdpf->len = xdpf->len;

	new_xdpf->headroom = priv->tx_headroom;

	new_xdpf->frame_sz = DPAA_BP_RAW_SIZE;

	new_xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;

	new_xdpf->mem_type = MEM_TYPE_PAGE_ORDER0;



	/* Release the initial buffer */

	xdp_return_frame_rx_napi(xdpf);

drivers/net/veth.c

+2 −2
Original line number Diff line number Diff line
@@ -634,7 +634,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 
			break;

		case XDP_TX:

			orig_frame = *frame;

			xdp->rxq->mem = frame->mem;

			xdp->rxq->mem.type = frame->mem_type;

			if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) {

				trace_xdp_exception(rq->dev, xdp_prog, act);

				frame = &orig_frame;
@@ -646,7 +646,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 
			goto xdp_xmit;

		case XDP_REDIRECT:

			orig_frame = *frame;

			xdp->rxq->mem = frame->mem;

			xdp->rxq->mem.type = frame->mem_type;

			if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) {

				frame = &orig_frame;

				stats->rx_drops++;

include/linux/skbuff.h

+26 −5
Original line number Diff line number Diff line
@@ -3674,7 +3674,7 @@ static inline void skb_frag_page_copy(skb_frag_t *fragto, 
bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);



/**

 * skb_frag_dma_map - maps a paged fragment via the DMA API

 * __skb_frag_dma_map - maps a paged fragment via the DMA API

 * @dev: the device to map the fragment to

 * @frag: the paged fragment to map

 * @offset: the offset within the fragment (starting at the
@@ -3684,7 +3684,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); 
 *

 * Maps the page associated with @frag to @device.

 */

static inline dma_addr_t skb_frag_dma_map(struct device *dev,

static inline dma_addr_t __skb_frag_dma_map(struct device *dev,

					    const skb_frag_t *frag,

					    size_t offset, size_t size,

					    enum dma_data_direction dir)
@@ -3693,6 +3693,27 @@ static inline dma_addr_t skb_frag_dma_map(struct device *dev, 
			    skb_frag_off(frag) + offset, size, dir);

}



#define skb_frag_dma_map(dev, frag, ...)				\

	CONCATENATE(_skb_frag_dma_map,					\

		    COUNT_ARGS(__VA_ARGS__))(dev, frag, ##__VA_ARGS__)



#define __skb_frag_dma_map1(dev, frag, offset, uf, uo) ({		\

	const skb_frag_t *uf = (frag);					\

	size_t uo = (offset);						\

									\

	__skb_frag_dma_map(dev, uf, uo, skb_frag_size(uf) - uo,		\

			   DMA_TO_DEVICE);				\

})

#define _skb_frag_dma_map1(dev, frag, offset)				\

	__skb_frag_dma_map1(dev, frag, offset, __UNIQUE_ID(frag_),	\

			    __UNIQUE_ID(offset_))

#define _skb_frag_dma_map0(dev, frag)					\

	_skb_frag_dma_map1(dev, frag, 0)

#define _skb_frag_dma_map2(dev, frag, offset, size)			\

	__skb_frag_dma_map(dev, frag, offset, size, DMA_TO_DEVICE)

#define _skb_frag_dma_map3(dev, frag, offset, size, dir)		\

	__skb_frag_dma_map(dev, frag, offset, size, dir)



static inline struct sk_buff *pskb_copy(struct sk_buff *skb,

					gfp_t gfp_mask)

{

include/net/page_pool/types.h

+2 −4
Original line number Diff line number Diff line
@@ -259,8 +259,7 @@ void page_pool_disable_direct_recycling(struct page_pool *pool); 
void page_pool_destroy(struct page_pool *pool);

void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),

			   const struct xdp_mem_info *mem);

void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data,

			       u32 count);

void page_pool_put_netmem_bulk(netmem_ref *data, u32 count);

#else

static inline void page_pool_destroy(struct page_pool *pool)

{
@@ -272,8 +271,7 @@ static inline void page_pool_use_xdp_mem(struct page_pool *pool, 
{

}



static inline void page_pool_put_netmem_bulk(struct page_pool *pool,

					     netmem_ref *data, u32 count)

static inline void page_pool_put_netmem_bulk(netmem_ref *data, u32 count)

{

}

#endif

include/net/xdp.h

+20 −12
Original line number Diff line number Diff line
@@ -11,6 +11,8 @@ 
#include <linux/netdevice.h>

#include <linux/skbuff.h> /* skb_shared_info */



#include <net/page_pool/types.h>



/**

 * DOC: XDP RX-queue information

 *
@@ -167,13 +169,13 @@ xdp_get_buff_len(const struct xdp_buff *xdp) 


struct xdp_frame {

	void *data;

	u16 len;

	u16 headroom;

	u32 len;

	u32 headroom;

	u32 metasize; /* uses lower 8-bits */

	/* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,

	 * while mem info is valid on remote CPU.

	 * while mem_type is valid on remote CPU.

	 */

	struct xdp_mem_info mem;

	enum xdp_mem_type mem_type:32;

	struct net_device *dev_rx; /* used by cpumap */

	u32 frame_sz;

	u32 flags; /* supported values defined in xdp_buff_flags */
@@ -193,14 +195,12 @@ xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame) 
#define XDP_BULK_QUEUE_SIZE	16

struct xdp_frame_bulk {

	int count;

	void *xa;

	netmem_ref q[XDP_BULK_QUEUE_SIZE];

};



static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq)

{

	/* bq->count will be zero'ed when bq->xa gets updated */

	bq->xa = NULL;

	bq->count = 0;

}



static inline struct skb_shared_info *
@@ -306,21 +306,29 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) 
	if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0))

		return NULL;



	/* rxq only valid until napi_schedule ends, convert to xdp_mem_info */

	xdp_frame->mem = xdp->rxq->mem;

	/* rxq only valid until napi_schedule ends, convert to xdp_mem_type */

	xdp_frame->mem_type = xdp->rxq->mem.type;



	return xdp_frame;

}



void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,

		  struct xdp_buff *xdp);

void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,

		  bool napi_direct, struct xdp_buff *xdp);

void xdp_return_frame(struct xdp_frame *xdpf);

void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);

void xdp_return_buff(struct xdp_buff *xdp);

void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq);

void xdp_return_frame_bulk(struct xdp_frame *xdpf,

			   struct xdp_frame_bulk *bq);



static inline void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq)

{

	if (unlikely(!bq->count))

		return;



	page_pool_put_netmem_bulk(bq->q, bq->count);

	bq->count = 0;

}



static __always_inline unsigned int

xdp_get_frame_len(const struct xdp_frame *xdpf)

{