Commit d35c9659 authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'net-gro-cleanups-and-fast-path-refinement' 

Eric Dumazet says:

====================
net: gro: cleanups and fast path refinement

Current GRO stack has a 'fast path' for a subset of drivers,
users of napi_frags_skb().

With TCP zerocopy/direct uses, header split at receive is becoming
more important, and GRO fast path is disabled.

This series makes GRO (a bit) more efficient for almost all use cases.
====================

Link: https://lore.kernel.org/r/20240301193740.3436871-1-edumazet@google.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 9452c8b4 8f78010b

drivers/net/geneve.c

+1 −1
Original line number Diff line number Diff line
@@ -508,7 +508,7 @@ static struct sk_buff *geneve_gro_receive(struct sock *sk, 
	gh_len = geneve_hlen(gh);



	hlen = off_gnv + gh_len;

	if (skb_gro_header_hard(skb, hlen)) {

	if (!skb_gro_may_pull(skb, hlen)) {

		gh = skb_gro_header_slow(skb, hlen, off_gnv);

		if (unlikely(!gh))

			goto out;

include/net/gro.h

+16 −18
Original line number Diff line number Diff line
@@ -139,21 +139,16 @@ static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) 
	NAPI_GRO_CB(skb)->data_offset += len;

}



static inline void *skb_gro_header_fast(struct sk_buff *skb,

static inline void *skb_gro_header_fast(const struct sk_buff *skb,

					unsigned int offset)

{

	return NAPI_GRO_CB(skb)->frag0 + offset;

}



static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)

static inline bool skb_gro_may_pull(const struct sk_buff *skb,

				    unsigned int hlen)

{

	return NAPI_GRO_CB(skb)->frag0_len < hlen;

}



static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)

{

	NAPI_GRO_CB(skb)->frag0 = NULL;

	NAPI_GRO_CB(skb)->frag0_len = 0;

	return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);

}



static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
@@ -162,28 +157,30 @@ static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, 
	if (!pskb_may_pull(skb, hlen))

		return NULL;



	skb_gro_frag0_invalidate(skb);

	return skb->data + offset;

}



static inline void *skb_gro_header(struct sk_buff *skb,

					unsigned int hlen, unsigned int offset)

static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen,

				   unsigned int offset)

{

	void *ptr;



	ptr = skb_gro_header_fast(skb, offset);

	if (skb_gro_header_hard(skb, hlen))

	if (!skb_gro_may_pull(skb, hlen))

		ptr = skb_gro_header_slow(skb, hlen, offset);

	return ptr;

}



static inline void *skb_gro_network_header(struct sk_buff *skb)

static inline void *skb_gro_network_header(const struct sk_buff *skb)

{

	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +

	       skb_network_offset(skb);

	if (skb_gro_may_pull(skb, skb_gro_offset(skb)))

		return skb_gro_header_fast(skb, skb_network_offset(skb));



	return skb_network_header(skb);

}



static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto)

static inline __wsum inet_gro_compute_pseudo(const struct sk_buff *skb,

					     int proto)

{

	const struct iphdr *iph = skb_gro_network_header(skb);
@@ -421,7 +418,8 @@ static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) 
	return uh;

}



static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto)

static inline __wsum ip6_gro_compute_pseudo(const struct sk_buff *skb,

					    int proto)

{

	const struct ipv6hdr *iph = skb_gro_network_header(skb);

net/core/gro.c

+17 −8
Original line number Diff line number Diff line
@@ -369,15 +369,21 @@ static void gro_list_prepare(const struct list_head *head, 


static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)

{

	const struct skb_shared_info *pinfo = skb_shinfo(skb);

	const skb_frag_t *frag0 = &pinfo->frags[0];

	const struct skb_shared_info *pinfo;

	const skb_frag_t *frag0;

	unsigned int headlen;



	NAPI_GRO_CB(skb)->data_offset = 0;

	NAPI_GRO_CB(skb)->frag0 = NULL;

	NAPI_GRO_CB(skb)->frag0_len = 0;

	headlen = skb_headlen(skb);

	NAPI_GRO_CB(skb)->frag0 = skb->data;

	NAPI_GRO_CB(skb)->frag0_len = headlen;

	if (headlen)

		return;



	pinfo = skb_shinfo(skb);

	frag0 = &pinfo->frags[0];



	if (!skb_headlen(skb) && pinfo->nr_frags &&

	    !PageHighMem(skb_frag_page(frag0)) &&

	if (pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0)) &&

	    (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {

		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);

		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
@@ -700,7 +706,7 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 
	skb_reset_mac_header(skb);

	skb_gro_reset_offset(skb, hlen);



	if (unlikely(skb_gro_header_hard(skb, hlen))) {

	if (unlikely(!skb_gro_may_pull(skb, hlen))) {

		eth = skb_gro_header_slow(skb, hlen, 0);

		if (unlikely(!eth)) {

			net_warn_ratelimited("%s: dropping impossible skb from %s\n",
@@ -710,7 +716,10 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 
		}

	} else {

		eth = (const struct ethhdr *)skb->data;



		if (NAPI_GRO_CB(skb)->frag0 != skb->data)

			gro_pull_from_frag0(skb, hlen);



		NAPI_GRO_CB(skb)->frag0 += hlen;

		NAPI_GRO_CB(skb)->frag0_len -= hlen;

	}

net/ipv4/fou_core.c

+1 −1
Original line number Diff line number Diff line
@@ -351,7 +351,7 @@ static struct sk_buff *gue_gro_receive(struct sock *sk, 
	optlen = guehdr->hlen << 2;

	len += optlen;



	if (skb_gro_header_hard(skb, len)) {

	if (!skb_gro_may_pull(skb, len)) {

		guehdr = skb_gro_header_slow(skb, len, off);

		if (unlikely(!guehdr))

			goto out;

net/ipv4/gre_offload.c

+1 −1
Original line number Diff line number Diff line
@@ -174,7 +174,7 @@ static struct sk_buff *gre_gro_receive(struct list_head *head, 
		grehlen += GRE_HEADER_SECTION;



	hlen = off + grehlen;

	if (skb_gro_header_hard(skb, hlen)) {

	if (!skb_gro_may_pull(skb, hlen)) {

		greh = skb_gro_header_slow(skb, hlen, off);

		if (unlikely(!greh))

			goto out;