Commit dea1526f authored by Amery Hung's avatar Amery Hung Committed by Martin KaFai Lau
Browse files

bpf: Allow bpf_xdp_shrink_data to shrink a frag from head and tail 



Move skb_frag_t adjustment into bpf_xdp_shrink_data() and extend its
functionality to be able to shrink an xdp fragment from both head and
tail. In a later patch, bpf_xdp_pull_data() will reuse it to shrink an
xdp fragment from head.

Additionally, in bpf_xdp_frags_shrink_tail(), breaking the loop when
bpf_xdp_shrink_data() returns false (i.e., not releasing the current
fragment) is not necessary as the loop condition, offset > 0, has the
same effect. Remove the else branch to simplify the code.

Signed-off-by: default avatarAmery Hung <ameryhung@gmail.com>
Signed-off-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
Reviewed-by: default avatarMaciej Fijalkowski <maciej.fijalkowski@intel.com>
Link: https://patch.msgid.link/20250922233356.3356453-3-ameryhung@gmail.com
parent 8f12d113

include/net/xdp_sock_drv.h

+18 −3
Original line number Diff line number Diff line
@@ -160,13 +160,23 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) 
	return ret;

}



static inline void xsk_buff_del_tail(struct xdp_buff *tail)

static inline void xsk_buff_del_frag(struct xdp_buff *xdp)

{

	struct xdp_buff_xsk *xskb = container_of(tail, struct xdp_buff_xsk, xdp);

	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);



	list_del(&xskb->list_node);

}



static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first)

{

	struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);

	struct xdp_buff_xsk *frag;



	frag = list_first_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk,

				list_node);

	return &frag->xdp;

}



static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)

{

	struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
@@ -389,8 +399,13 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) 
	return NULL;

}



static inline void xsk_buff_del_tail(struct xdp_buff *tail)

static inline void xsk_buff_del_frag(struct xdp_buff *xdp)

{

}



static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first)

{

	return NULL;

}



static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)

net/core/filter.c

+24 −17
Original line number Diff line number Diff line
@@ -4153,34 +4153,45 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) 
	return 0;

}



static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,

				   enum xdp_mem_type mem_type, bool release)

static struct xdp_buff *bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,

					       bool tail, bool release)

{

	struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp);

	struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(xdp) :

					  xsk_buff_get_head(xdp);



	if (release) {

		xsk_buff_del_tail(zc_frag);

		__xdp_return(0, mem_type, false, zc_frag);

		xsk_buff_del_frag(zc_frag);

	} else {

		if (tail)

			zc_frag->data_end -= shrink;

		else

			zc_frag->data += shrink;

	}



	return zc_frag;

}



static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,

				int shrink)

				int shrink, bool tail)

{

	enum xdp_mem_type mem_type = xdp->rxq->mem.type;

	bool release = skb_frag_size(frag) == shrink;

	netmem_ref netmem = skb_frag_netmem(frag);

	struct xdp_buff *zc_frag = NULL;



	if (mem_type == MEM_TYPE_XSK_BUFF_POOL) {

		bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release);

		goto out;

		netmem = 0;

		zc_frag = bpf_xdp_shrink_data_zc(xdp, shrink, tail, release);

	}



	if (release)

		__xdp_return(skb_frag_netmem(frag), mem_type, false, NULL);

	if (release) {

		__xdp_return(netmem, mem_type, false, zc_frag);

	} else {

		if (!tail)

			skb_frag_off_add(frag, shrink);

		skb_frag_size_sub(frag, shrink);

	}



out:

	return release;

}
@@ -4198,12 +4209,8 @@ static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset) 


		len_free += shrink;

		offset -= shrink;

		if (bpf_xdp_shrink_data(xdp, frag, shrink)) {

		if (bpf_xdp_shrink_data(xdp, frag, shrink, true))

			n_frags_free++;

		} else {

			skb_frag_size_sub(frag, shrink);

			break;

		}

	}

	sinfo->nr_frags -= n_frags_free;

	sinfo->xdp_frags_size -= len_free;