Commit 0391ab57 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

net: add skbuff_clear() helper 



clang is unable to inline the memset() calls in net/core/skbuff.c
when initializing allocated sk_buff.

memset(skb, 0, offsetof(struct sk_buff, tail));

This is unfortunate, because:

1) calling external memset_orig() helper adds a call/ret and
   typical setup cost.

2) offsetof(struct sk_buff, tail) == 0xb8 = 0x80 + 0x38

   On x86_64, memset_orig() performs two 64 bytes clear,
   then has to loop 7 times to clear the final 56 bytes.

skbuff_clear() makes sure the minimal and optimal code
is generated.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20260109203836.1667441-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent d7161b07

net/core/skbuff.c

+22 −10
Original line number Diff line number Diff line
@@ -307,6 +307,23 @@ static struct sk_buff *napi_skb_cache_get(bool alloc) 
	return skb;

}



/*

 * Only clear those fields we need to clear, not those that we will

 * actually initialise later. Hence, don't put any more fields after

 * the tail pointer in struct sk_buff!

 */

static inline void skbuff_clear(struct sk_buff *skb)

{

	/* Replace memset(skb, 0, offsetof(struct sk_buff, tail))

	 * with two smaller memset(), with a barrier() between them.

	 * This forces the compiler to inline both calls.

	 */

	BUILD_BUG_ON(offsetof(struct sk_buff, tail) <= 128);

	memset(skb, 0, 128);

	barrier();

	memset((void *)skb + 128, 0, offsetof(struct sk_buff, tail) - 128);

}



/**

 * napi_skb_cache_get_bulk - obtain a number of zeroed skb heads from the cache

 * @skbs: pointer to an at least @n-sized array to fill with skb pointers
@@ -357,7 +374,7 @@ u32 napi_skb_cache_get_bulk(void **skbs, u32 n) 
		skbs[i] = nc->skb_cache[base + i];



		kasan_mempool_unpoison_object(skbs[i], skbuff_cache_size);

		memset(skbs[i], 0, offsetof(struct sk_buff, tail));

		skbuff_clear(skbs[i]);

	}



	nc->skb_count -= n;
@@ -424,7 +441,7 @@ struct sk_buff *slab_build_skb(void *data) 
	if (unlikely(!skb))

		return NULL;



	memset(skb, 0, offsetof(struct sk_buff, tail));

	skbuff_clear(skb);

	data = __slab_build_skb(data, &size);

	__finalize_skb_around(skb, data, size);
@@ -476,7 +493,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size) 
	if (unlikely(!skb))

		return NULL;



	memset(skb, 0, offsetof(struct sk_buff, tail));

	skbuff_clear(skb);

	__build_skb_around(skb, data, frag_size);



	return skb;
@@ -537,7 +554,7 @@ static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 
	if (unlikely(!skb))

		return NULL;



	memset(skb, 0, offsetof(struct sk_buff, tail));

	skbuff_clear(skb);

	__build_skb_around(skb, data, frag_size);



	return skb;
@@ -696,12 +713,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 
	 */

	prefetchw(data + SKB_WITH_OVERHEAD(size));



	/*

	 * Only clear those fields we need to clear, not those that we will

	 * actually initialise below. Hence, don't put any more fields after

	 * the tail pointer in struct sk_buff!

	 */

	memset(skb, 0, offsetof(struct sk_buff, tail));

	skbuff_clear(skb);

	__build_skb_around(skb, data, size);

	skb->pfmemalloc = pfmemalloc;