Commit 83081337 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

tcp: move tcp_memory_allocated into net_aligned_data 



____cacheline_aligned_in_smp attribute only makes sure to align
a field to a cache line. It does not prevent the linker to use
the remaining of the cache line for other variables, causing
potential false sharing.

Move tcp_memory_allocated into a dedicated cache line.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Reviewed-by: default avatarWillem de Bruijn <willemb@google.com>
Link: https://patch.msgid.link/20250630093540.3052835-4-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 998642e9

include/net/aligned_data.h

+3 −0
Original line number Diff line number Diff line
@@ -11,6 +11,9 @@ 
 */

struct net_aligned_data {

	atomic64_t	net_cookie ____cacheline_aligned_in_smp;

#if defined(CONFIG_INET)

	atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;

#endif

};



extern struct net_aligned_data net_aligned_data;

include/net/tcp.h

+0 −1
Original line number Diff line number Diff line
@@ -267,7 +267,6 @@ extern long sysctl_tcp_mem[3]; 
#define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */

#define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */



extern atomic_long_t tcp_memory_allocated;

DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);



extern struct percpu_counter tcp_sockets_allocated;

net/core/hotdata.c

+2 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include <linux/list.h>

#include <net/aligned_data.h>

#include <net/hotdata.h>

#include <net/ip.h>

#include <net/proto_memory.h>



struct net_hotdata net_hotdata __cacheline_aligned = {
@@ -25,3 +26,4 @@ struct net_hotdata net_hotdata __cacheline_aligned = { 
EXPORT_SYMBOL(net_hotdata);



struct net_aligned_data net_aligned_data;

EXPORT_IPV6_MOD(net_aligned_data);

net/ipv4/tcp.c

+0 −2
Original line number Diff line number Diff line
@@ -302,8 +302,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn); 
long sysctl_tcp_mem[3] __read_mostly;

EXPORT_IPV6_MOD(sysctl_tcp_mem);



atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;	/* Current allocated memory. */

EXPORT_IPV6_MOD(tcp_memory_allocated);

DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);

EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);

net/ipv4/tcp_ipv4.c

+2 −1
Original line number Diff line number Diff line
@@ -59,6 +59,7 @@ 
#include <linux/slab.h>

#include <linux/sched.h>



#include <net/aligned_data.h>

#include <net/net_namespace.h>

#include <net/icmp.h>

#include <net/inet_hashtables.h>
@@ -3390,7 +3391,7 @@ struct proto tcp_prot = { 
	.sockets_allocated	= &tcp_sockets_allocated,

	.orphan_count		= &tcp_orphan_count,



	.memory_allocated	= &tcp_memory_allocated,

	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,

	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,



	.memory_pressure	= &tcp_memory_pressure,