Commit c204fef9 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

net: move sysctl_mem_pcpu_rsv to net_hotdata 



sysctl_mem_pcpu_rsv is used in TCP fast path,
move it to net_hodata for better cache locality.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Reviewed-by: default avatarDavid Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240429134025.1233626-6-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent f3d93817

include/net/hotdata.h

+1 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ struct net_hotdata { 
	int			dev_rx_weight;

	int			sysctl_max_skb_frags;

	int			sysctl_skb_defer_max;

	int			sysctl_mem_pcpu_rsv;

};



#define inet_ehash_secret	net_hotdata.tcp_protocol.secret

include/net/proto_memory.h

+3 −3
Original line number Diff line number Diff line
@@ -3,10 +3,10 @@ 
#define _PROTO_MEMORY_H



#include <net/sock.h>

#include <net/hotdata.h>



/* 1 MB per cpu, in page units */

#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))

extern int sysctl_mem_pcpu_rsv;



static inline bool sk_has_memory_pressure(const struct sock *sk)

{
@@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val) 


	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);



	if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))

	if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))

		proto_memory_pcpu_drain(proto);

}
@@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val) 


	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);



	if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))

	if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))

		proto_memory_pcpu_drain(proto);

}

net/core/hotdata.c

+3 −2
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

#include <net/hotdata.h>

#include <linux/cache.h>

#include <linux/jiffies.h>

#include <linux/list.h>



#include <net/hotdata.h>

#include <net/proto_memory.h>



struct net_hotdata net_hotdata __cacheline_aligned = {

	.offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
@@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { 
	.dev_rx_weight = 64,

	.sysctl_max_skb_frags = MAX_SKB_FRAGS,

	.sysctl_skb_defer_max = 64,

	.sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE

};

EXPORT_SYMBOL(net_hotdata);

net/core/sock.c

+0 −1
Original line number Diff line number Diff line
@@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 
EXPORT_SYMBOL(sysctl_rmem_max);

__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;

__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;

int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE;



int sysctl_tstamp_allow_data __read_mostly = 1;

net/core/sysctl_net_core.c

+1 −1
Original line number Diff line number Diff line
@@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = { 
	},

	{

		.procname	= "mem_pcpu_rsv",

		.data		= &sysctl_mem_pcpu_rsv,

		.data		= &net_hotdata.sysctl_mem_pcpu_rsv,

		.maxlen		= sizeof(int),

		.mode		= 0644,

		.proc_handler	= proc_dointvec_minmax,