Commit 616d8604 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'net-expand-napi_skb_cache-use' 

Eric Dumazet says:

====================
net: expand napi_skb_cache use

This is a followup of commit e20dfbad ("net: fix napi_consume_skb()
with alien skbs").

Now the per-cpu napi_skb_cache is populated from TX completion path,
we can make use of this cache, especially for cpus not used
from a driver NAPI poll (primary user of napi_cache).

With this series, I consistently reach 130 Mpps on my UDP tx stress test
and reduce SLUB spinlock contention to smaller values.
====================

Link: https://patch.msgid.link/20251116202717.1542829-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents eb74ae2f 21664814

net/core/skbuff.c

+31 −17
Original line number Diff line number Diff line
@@ -280,13 +280,14 @@ EXPORT_SYMBOL(__netdev_alloc_frag_align); 
 */

static u32 skbuff_cache_size __read_mostly;



static struct sk_buff *napi_skb_cache_get(void)

static struct sk_buff *napi_skb_cache_get(bool alloc)

{

	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

	struct sk_buff *skb;



	local_lock_nested_bh(&napi_alloc_cache.bh_lock);

	if (unlikely(!nc->skb_count)) {

		if (alloc)

			nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,

						GFP_ATOMIC | __GFP_NOWARN,

						NAPI_SKB_CACHE_BULK,
@@ -530,7 +531,7 @@ static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 
{

	struct sk_buff *skb;



	skb = napi_skb_cache_get();

	skb = napi_skb_cache_get(true);

	if (unlikely(!skb))

		return NULL;
@@ -645,25 +646,38 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,

			    int flags, int node)

{

	struct sk_buff *skb = NULL;

	struct kmem_cache *cache;

	struct sk_buff *skb;

	bool pfmemalloc;

	u8 *data;



	cache = (flags & SKB_ALLOC_FCLONE)

		? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;



	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))

		gfp_mask |= __GFP_MEMALLOC;



	/* Get the HEAD */

	if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI &&

	    likely(node == NUMA_NO_NODE || node == numa_mem_id()))

		skb = napi_skb_cache_get();

	else

	if (flags & SKB_ALLOC_FCLONE) {

		cache = net_hotdata.skbuff_fclone_cache;

		goto fallback;

	}

	cache = net_hotdata.skbuff_cache;

	if (unlikely(node != NUMA_NO_NODE && node != numa_mem_id()))

		goto fallback;



	if (flags & SKB_ALLOC_NAPI) {

		skb = napi_skb_cache_get(true);

		if (unlikely(!skb))

			return NULL;

	} else if (!in_hardirq() && !irqs_disabled()) {

		local_bh_disable();

		skb = napi_skb_cache_get(false);

		local_bh_enable();

	}



	if (!skb) {

fallback:

		skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);

		if (unlikely(!skb))

			return NULL;

	}

	prefetchw(skb);



	/* We do our best to align skb_shared_info on a separate cache