Commit c7583e9f authored by Eric Dumazet's avatar Eric Dumazet Committed by Paolo Abeni
Browse files

net: gro: enable fast path for more cases 



Currently the so-called GRO fast path is only enabled for
napi_frags_skb() callers.

After the prior patch, we no longer have to clear frag0 whenever
we pulled bytes to skb->head.

We therefore can initialize frag0 to skb->data so that GRO
fast path can be used in the following additional cases:

- Drivers using header split (populating skb->data with headers,
  and having payload in one or more page fragments).

- Drivers not using any page frag (entire packet is in skb->data)

Add a likely() in skb_gro_may_pull() to help the compiler
to generate better code if possible.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarPaolo Abeni <pabeni@redhat.com>
Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent bd56a29c

include/net/gro.h

+1 −1
Original line number Diff line number Diff line
@@ -148,7 +148,7 @@ static inline void *skb_gro_header_fast(const struct sk_buff *skb, 
static inline bool skb_gro_may_pull(const struct sk_buff *skb,

				    unsigned int hlen)

{

	return hlen <= NAPI_GRO_CB(skb)->frag0_len;

	return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);

}



static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,

net/core/gro.c

+16 −7
Original line number Diff line number Diff line
@@ -369,15 +369,21 @@ static void gro_list_prepare(const struct list_head *head, 


static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)

{

	const struct skb_shared_info *pinfo = skb_shinfo(skb);

	const skb_frag_t *frag0 = &pinfo->frags[0];

	const struct skb_shared_info *pinfo;

	const skb_frag_t *frag0;

	unsigned int headlen;



	NAPI_GRO_CB(skb)->data_offset = 0;

	NAPI_GRO_CB(skb)->frag0 = NULL;

	NAPI_GRO_CB(skb)->frag0_len = 0;

	headlen = skb_headlen(skb);

	NAPI_GRO_CB(skb)->frag0 = skb->data;

	NAPI_GRO_CB(skb)->frag0_len = headlen;

	if (headlen)

		return;



	pinfo = skb_shinfo(skb);

	frag0 = &pinfo->frags[0];



	if (!skb_headlen(skb) && pinfo->nr_frags &&

	    !PageHighMem(skb_frag_page(frag0)) &&

	if (pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0)) &&

	    (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {

		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);

		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
@@ -710,7 +716,10 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 
		}

	} else {

		eth = (const struct ethhdr *)skb->data;



		if (NAPI_GRO_CB(skb)->frag0 != skb->data)

			gro_pull_from_frag0(skb, hlen);



		NAPI_GRO_CB(skb)->frag0 += hlen;

		NAPI_GRO_CB(skb)->frag0_len -= hlen;

	}