Commit 8077b9a9 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'net-introduce-net_aligned_data' 

Eric Dumazet says:

====================
net: introduce net_aligned_data

____cacheline_aligned_in_smp on small fields like
tcp_memory_allocated and udp_memory_allocated is not good enough.

It makes sure to put these fields at the start of a cache line,
but does not prevent the linker from using the cache line for other
fields, with potential performance impact.

nm -v vmlinux|egrep -5 "tcp_memory_allocated|udp_memory_allocated"

...
ffffffff83e35cc0 B tcp_memory_allocated
ffffffff83e35cc8 b __key.0
ffffffff83e35cc8 b __tcp_tx_delay_enabled.1
ffffffff83e35ce0 b tcp_orphan_timer
...
ffffffff849dddc0 B udp_memory_allocated
ffffffff849dddc8 B udp_encap_needed_key
ffffffff849dddd8 B udpv6_encap_needed_key
ffffffff849dddf0 b inetsw_lock

One solution is to move these sensitive fields to a structure,
so that the compiler is forced to add empty holes between each member.

nm -v vmlinux|egrep -2 "tcp_memory_allocated|udp_memory_allocated|net_aligned_data"

ffffffff885af970 b mem_id_init
ffffffff885af980 b __key.0
ffffffff885af9c0 B net_aligned_data
ffffffff885afa80 B page_pool_mem_providers
ffffffff885afa90 b __key.2

v1: https://lore.kernel.org/netdev/20250627200551.348096-1-edumazet@google.com/
====================

Link: https://patch.msgid.link/20250630093540.3052835-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents a8065af3 e3d48251

include/net/aligned_data.h

0 → 100644
+22 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-or-later */

#ifndef _NET_ALIGNED_DATA_H

#define _NET_ALIGNED_DATA_H



#include <linux/atomic.h>

#include <linux/types.h>



/* Structure holding cacheline aligned fields on SMP builds.

 * Each field or group should have an ____cacheline_aligned_in_smp

 * attribute to ensure no accidental false sharing can happen.

 */

struct net_aligned_data {

	atomic64_t	net_cookie ____cacheline_aligned_in_smp;

#if defined(CONFIG_INET)

	atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;

	atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp;

#endif

};



extern struct net_aligned_data net_aligned_data;



#endif /* _NET_ALIGNED_DATA_H */

include/net/tcp.h

+0 −1
Original line number Diff line number Diff line
@@ -267,7 +267,6 @@ extern long sysctl_tcp_mem[3]; 
#define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */

#define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */



extern atomic_long_t tcp_memory_allocated;

DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);



extern struct percpu_counter tcp_sockets_allocated;

include/net/udp.h

+0 −1
Original line number Diff line number Diff line
@@ -205,7 +205,6 @@ static inline void udp_hash4_dec(struct udp_hslot *hslot2) 


extern struct proto udp_prot;



extern atomic_long_t udp_memory_allocated;

DECLARE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);



/* sysctl variables for udp */

net/core/hotdata.c

+5 −0
Original line number Diff line number Diff line
@@ -2,7 +2,9 @@ 
#include <linux/cache.h>

#include <linux/jiffies.h>

#include <linux/list.h>

#include <net/aligned_data.h>

#include <net/hotdata.h>

#include <net/ip.h>

#include <net/proto_memory.h>



struct net_hotdata net_hotdata __cacheline_aligned = {
@@ -22,3 +24,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { 
	.sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE

};

EXPORT_SYMBOL(net_hotdata);



struct net_aligned_data net_aligned_data;

EXPORT_IPV6_MOD(net_aligned_data);

net/core/net_namespace.c

+2 −6
Original line number Diff line number Diff line
@@ -19,9 +19,9 @@ 
#include <linux/net_namespace.h>

#include <linux/sched/task.h>

#include <linux/uidgid.h>

#include <linux/cookie.h>

#include <linux/proc_fs.h>



#include <net/aligned_data.h>

#include <net/sock.h>

#include <net/netlink.h>

#include <net/net_namespace.h>
@@ -64,8 +64,6 @@ DECLARE_RWSEM(pernet_ops_rwsem); 


static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS;



DEFINE_COOKIE(net_cookie);



static struct net_generic *net_alloc_generic(void)

{

	unsigned int gen_ptrs = READ_ONCE(max_gen_ptrs);
@@ -434,9 +432,7 @@ static __net_init int setup_net(struct net *net) 
	LIST_HEAD(net_exit_list);

	int error = 0;



	preempt_disable();

	net->net_cookie = gen_cookie_next(&net_cookie);

	preempt_enable();

	net->net_cookie = atomic64_inc_return(&net_aligned_data.net_cookie);



	list_for_each_entry(ops, &pernet_list, list) {

		error = ops_init(ops, net);