Commit 207ff83c authored by Alexander Lobakin's avatar Alexander Lobakin Committed by Jakub Kicinski
Browse files

xdp: make __xdp_return() MP-agnostic 



Currently, __xdp_return() takes pointer to the virtual memory to free
a buffer. Apart from that this sometimes provokes redundant
data <--> page conversions, taking data pointer effectively prevents
lots of XDP code to support non-page-backed buffers, as there's no
mapping for the non-host memory (data is always NULL).
Just convert it to always take netmem reference. For
xdp_return_{buff,frame*}(), this chops off one page_address() per each
frag and adds one virt_to_netmem() (same as virt_to_page()) per header
buffer. For __xdp_return() itself, it removes one virt_to_page() for
MEM_TYPE_PAGE_POOL and another one for MEM_TYPE_PAGE_ORDER0, adding
one page_address() for [not really common nowadays]
MEM_TYPE_PAGE_SHARED, but the main effect is that the abovementioned
functions won't die or memleak anymore if the frame has non-host memory
attached and will correctly free those.

Signed-off-by: default avatarAlexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20241211172649.761483-4-aleksander.lobakin@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 56d95b0a

include/net/xdp.h

+2 −2
Original line number Diff line number Diff line
@@ -312,8 +312,8 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) 
	return xdp_frame;

}



void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,

		  struct xdp_buff *xdp);

void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,

		  bool napi_direct, struct xdp_buff *xdp);

void xdp_return_frame(struct xdp_frame *xdpf);

void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);

void xdp_return_buff(struct xdp_buff *xdp);

net/core/filter.c

+3 −6
Original line number Diff line number Diff line
@@ -4125,7 +4125,7 @@ static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, 


	if (release) {

		xsk_buff_del_tail(zc_frag);

		__xdp_return(NULL, mem_type, false, zc_frag);

		__xdp_return(0, mem_type, false, zc_frag);

	} else {

		zc_frag->data_end -= shrink;

	}
@@ -4142,11 +4142,8 @@ static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, 
		goto out;

	}



	if (release) {

		struct page *page = skb_frag_page(frag);



		__xdp_return(page_address(page), mem_type, false, NULL);

	}

	if (release)

		__xdp_return(skb_frag_netmem(frag), mem_type, false, NULL);



out:

	return release;

net/core/xdp.c

+19 −28
Original line number Diff line number Diff line
@@ -430,27 +430,25 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool); 
 * is used for those calls sites.  Thus, allowing for faster recycling

 * of xdp_frames/pages in those cases.

 */

void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,

		  struct xdp_buff *xdp)

void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,

		  bool napi_direct, struct xdp_buff *xdp)

{

	struct page *page;



	switch (mem_type) {

	case MEM_TYPE_PAGE_POOL:

		page = virt_to_head_page(data);

		netmem = netmem_compound_head(netmem);

		if (napi_direct && xdp_return_frame_no_direct())

			napi_direct = false;

		/* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)

		 * as mem->type knows this a page_pool page

		 */

		page_pool_put_full_page(page->pp, page, napi_direct);

		page_pool_put_full_netmem(netmem_get_pp(netmem), netmem,

					  napi_direct);

		break;

	case MEM_TYPE_PAGE_SHARED:

		page_frag_free(data);

		page_frag_free(__netmem_address(netmem));

		break;

	case MEM_TYPE_PAGE_ORDER0:

		page = virt_to_page(data); /* Assumes order0 page*/

		put_page(page);

		put_page(__netmem_to_page(netmem));

		break;

	case MEM_TYPE_XSK_BUFF_POOL:

		/* NB! Only valid from an xdp_buff! */
@@ -466,38 +464,34 @@ void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct, 
void xdp_return_frame(struct xdp_frame *xdpf)

{

	struct skb_shared_info *sinfo;

	int i;



	if (likely(!xdp_frame_has_frags(xdpf)))

		goto out;



	sinfo = xdp_get_shared_info_from_frame(xdpf);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);

	for (u32 i = 0; i < sinfo->nr_frags; i++)

		__xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,

			     false, NULL);



		__xdp_return(page_address(page), xdpf->mem_type, false, NULL);

	}

out:

	__xdp_return(xdpf->data, xdpf->mem_type, false, NULL);

	__xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, false, NULL);

}

EXPORT_SYMBOL_GPL(xdp_return_frame);



void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)

{

	struct skb_shared_info *sinfo;

	int i;



	if (likely(!xdp_frame_has_frags(xdpf)))

		goto out;



	sinfo = xdp_get_shared_info_from_frame(xdpf);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);

	for (u32 i = 0; i < sinfo->nr_frags; i++)

		__xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,

			     true, NULL);



		__xdp_return(page_address(page), xdpf->mem_type, true, NULL);

	}

out:

	__xdp_return(xdpf->data, xdpf->mem_type, true, NULL);

	__xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, true, NULL);

}

EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
@@ -544,20 +538,17 @@ EXPORT_SYMBOL_GPL(xdp_return_frame_bulk); 
void xdp_return_buff(struct xdp_buff *xdp)

{

	struct skb_shared_info *sinfo;

	int i;



	if (likely(!xdp_buff_has_frags(xdp)))

		goto out;



	sinfo = xdp_get_shared_info_from_buff(xdp);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);

	for (u32 i = 0; i < sinfo->nr_frags; i++)

		__xdp_return(skb_frag_netmem(&sinfo->frags[i]),

			     xdp->rxq->mem.type, true, xdp);



		__xdp_return(page_address(page), xdp->rxq->mem.type, true,

			     xdp);

	}

out:

	__xdp_return(xdp->data, xdp->rxq->mem.type, true, xdp);

	__xdp_return(virt_to_netmem(xdp->data), xdp->rxq->mem.type, true, xdp);

}

EXPORT_SYMBOL_GPL(xdp_return_buff);