Commit aa70d2d1 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

net: move skbuff_cache(s) to net_hotdata 



skbuff_cache, skbuff_fclone_cache and skb_small_head_cache
are used in rx/tx fast paths.

Move them to net_hotdata for better cache locality.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: default avatarDavid Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240306160031.874438-11-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 71c0de9b

include/linux/skbuff.h

+0 −1
Original line number Diff line number Diff line
@@ -1271,7 +1271,6 @@ static inline void consume_skb(struct sk_buff *skb) 


void __consume_stateless_skb(struct sk_buff *skb);

void  __kfree_skb(struct sk_buff *skb);

extern struct kmem_cache *skbuff_cache;



void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);

bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,

include/net/hotdata.h

+3 −0
Original line number Diff line number Diff line
@@ -16,6 +16,9 @@ struct net_hotdata { 
#endif

	struct list_head	offload_base;

	struct list_head	ptype_all;

	struct kmem_cache	*skbuff_cache;

	struct kmem_cache	*skbuff_fclone_cache;

	struct kmem_cache	*skb_small_head_cache;

	int			gro_normal_batch;

	int			netdev_budget;

	int			netdev_budget_usecs;

kernel/bpf/cpumap.c

+3 −1
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <linux/filter.h>

#include <linux/ptr_ring.h>

#include <net/xdp.h>

#include <net/hotdata.h>



#include <linux/sched.h>

#include <linux/workqueue.h>
@@ -326,7 +327,8 @@ static int cpu_map_kthread_run(void *data) 
		/* Support running another XDP prog on this CPU */

		nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list);

		if (nframes) {

			m = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, skbs);

			m = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,

						  gfp, nframes, skbs);

			if (unlikely(m == 0)) {

				for (i = 0; i < nframes; i++)

					skbs[i] = NULL; /* effect: xdp_return_frame */

net/bpf/test_run.c

+3 −1
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
#include <linux/rcupdate_trace.h>

#include <linux/sched/signal.h>

#include <net/bpf_sk_storage.h>

#include <net/hotdata.h>

#include <net/sock.h>

#include <net/tcp.h>

#include <net/net_namespace.h>
@@ -254,7 +255,8 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes, 
	int i, n;

	LIST_HEAD(list);



	n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs);

	n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,

				  (void **)skbs);

	if (unlikely(n == 0)) {

		for (i = 0; i < nframes; i++)

			xdp_return_frame(frames[i]);

net/core/skbuff.c

+20 −24
Original line number Diff line number Diff line
@@ -69,6 +69,7 @@ 
#include <net/sock.h>

#include <net/checksum.h>

#include <net/gso.h>

#include <net/hotdata.h>

#include <net/ip6_checksum.h>

#include <net/xfrm.h>

#include <net/mpls.h>
@@ -88,15 +89,10 @@ 
#include "dev.h"

#include "sock_destructor.h"



struct kmem_cache *skbuff_cache __ro_after_init;

static struct kmem_cache *skbuff_fclone_cache __ro_after_init;

#ifdef CONFIG_SKB_EXTENSIONS

static struct kmem_cache *skbuff_ext_cache __ro_after_init;

#endif





static struct kmem_cache *skb_small_head_cache __ro_after_init;



#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)



/* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
@@ -349,7 +345,7 @@ static struct sk_buff *napi_skb_cache_get(void) 
	struct sk_buff *skb;



	if (unlikely(!nc->skb_count)) {

		nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache,

		nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,

						      GFP_ATOMIC,

						      NAPI_SKB_CACHE_BULK,

						      nc->skb_cache);
@@ -358,7 +354,7 @@ static struct sk_buff *napi_skb_cache_get(void) 
	}



	skb = nc->skb_cache[--nc->skb_count];

	kasan_mempool_unpoison_object(skb, kmem_cache_size(skbuff_cache));

	kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache));



	return skb;

}
@@ -416,7 +412,7 @@ struct sk_buff *slab_build_skb(void *data) 
	struct sk_buff *skb;

	unsigned int size;



	skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);

	skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);

	if (unlikely(!skb))

		return NULL;
@@ -467,7 +463,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size) 
{

	struct sk_buff *skb;



	skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);

	skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);

	if (unlikely(!skb))

		return NULL;
@@ -578,7 +574,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 
	obj_size = SKB_HEAD_ALIGN(*size);

	if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&

	    !(flags & KMALLOC_NOT_NORMAL_BITS)) {

		obj = kmem_cache_alloc_node(skb_small_head_cache,

		obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,

				flags | __GFP_NOMEMALLOC | __GFP_NOWARN,

				node);

		*size = SKB_SMALL_HEAD_CACHE_SIZE;
@@ -586,7 +582,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 
			goto out;

		/* Try again but now we are using pfmemalloc reserves */

		ret_pfmemalloc = true;

		obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node);

		obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node);

		goto out;

	}
@@ -649,7 +645,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 
	u8 *data;



	cache = (flags & SKB_ALLOC_FCLONE)

		? skbuff_fclone_cache : skbuff_cache;

		? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;



	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))

		gfp_mask |= __GFP_MEMALLOC;
@@ -1095,7 +1091,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb) 
static void skb_kfree_head(void *head, unsigned int end_offset)

{

	if (end_offset == SKB_SMALL_HEAD_HEADROOM)

		kmem_cache_free(skb_small_head_cache, head);

		kmem_cache_free(net_hotdata.skb_small_head_cache, head);

	else

		kfree(head);

}
@@ -1162,7 +1158,7 @@ static void kfree_skbmem(struct sk_buff *skb) 


	switch (skb->fclone) {

	case SKB_FCLONE_UNAVAILABLE:

		kmem_cache_free(skbuff_cache, skb);

		kmem_cache_free(net_hotdata.skbuff_cache, skb);

		return;



	case SKB_FCLONE_ORIG:
@@ -1183,7 +1179,7 @@ static void kfree_skbmem(struct sk_buff *skb) 
	if (!refcount_dec_and_test(&fclones->fclone_ref))

		return;

fastpath:

	kmem_cache_free(skbuff_fclone_cache, fclones);

	kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);

}



void skb_release_head_state(struct sk_buff *skb)
@@ -1280,7 +1276,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb, 
	sa->skb_array[sa->skb_count++] = skb;



	if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {

		kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE,

		kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,

				     sa->skb_array);

		sa->skb_count = 0;

	}
@@ -1305,7 +1301,7 @@ kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) 
	}



	if (sa.skb_count)

		kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array);

		kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);

}

EXPORT_SYMBOL(kfree_skb_list_reason);
@@ -1467,9 +1463,9 @@ static void napi_skb_cache_put(struct sk_buff *skb) 
	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {

		for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)

			kasan_mempool_unpoison_object(nc->skb_cache[i],

						kmem_cache_size(skbuff_cache));

						kmem_cache_size(net_hotdata.skbuff_cache));



		kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF,

		kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF,

				     nc->skb_cache + NAPI_SKB_CACHE_HALF);

		nc->skb_count = NAPI_SKB_CACHE_HALF;

	}
@@ -2066,7 +2062,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 
		if (skb_pfmemalloc(skb))

			gfp_mask |= __GFP_MEMALLOC;



		n = kmem_cache_alloc(skbuff_cache, gfp_mask);

		n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);

		if (!n)

			return NULL;
@@ -5005,7 +5001,7 @@ static void skb_extensions_init(void) {} 


void __init skb_init(void)

{

	skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",

	net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",

					      sizeof(struct sk_buff),

					      0,

					      SLAB_HWCACHE_ALIGN|SLAB_PANIC|
@@ -5013,7 +5009,7 @@ void __init skb_init(void) 
					      offsetof(struct sk_buff, cb),

					      sizeof_field(struct sk_buff, cb),

					      NULL);

	skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",

	net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",

						sizeof(struct sk_buff_fclones),

						0,

						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
@@ -5022,7 +5018,7 @@ void __init skb_init(void) 
	 * struct skb_shared_info is located at the end of skb->head,

	 * and should not be copied to/from user.

	 */

	skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",

	net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",

						SKB_SMALL_HEAD_CACHE_SIZE,

						0,

						SLAB_HWCACHE_ALIGN | SLAB_PANIC,
@@ -5895,7 +5891,7 @@ void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 
{

	if (head_stolen) {

		skb_release_head_state(skb);

		kmem_cache_free(skbuff_cache, skb);

		kmem_cache_free(net_hotdata.skbuff_cache, skb);

	} else {

		__kfree_skb(skb);

	}