Commit d4d8c6e6 authored by Kuniyuki Iwashima's avatar Kuniyuki Iwashima Committed by Jakub Kicinski
Browse files

tcp: Initialise ehash secrets during connect() and listen(). 



inet_ehashfn() and inet6_ehashfn() initialise random secrets
on the first call by net_get_random_once().

While the init part is patched out using static keys, with
CONFIG_STACKPROTECTOR_STRONG=y, this causes a compiler to
generate a stack canary due to an automatic variable,
unsigned long ___flags, in the DO_ONCE() macro being passed
to __do_once_start().

With FDO, this is visible in __inet_lookup_established() and
__inet6_lookup_established() too.

Let's initialise the secrets by get_random_sleepable_once()
in the slow paths: inet_hash() for listen(), and
inet_hash_connect() and inet6_hash_connect() for connect().

Note that IPv6 listener will initialise both IPv4 & IPv6 secrets
in inet_hash() for IPv4-mapped IPv6 address.

With the patch, the stack size is reduced by 16 bytes (___flags
 + a stack canary) and NOPs for the static key go away.

Before: __inet6_lookup_established()

       ...
       push   %rbx
       sub    $0x38,%rsp                # stack is 56 bytes
       mov    %edx,%ebx                 # sport
       mov    %gs:0x299419f(%rip),%rax  # load stack canary
       mov    %rax,0x30(%rsp)              and store it onto stack
       mov    0x440(%rdi),%r15          # net->ipv4.tcp_death_row.hashinfo
       nop
 32:   mov    %r8d,%ebp                 # hnum
       shl    $0x10,%ebp                # hnum << 16
       nop
 3d:   mov    0x70(%rsp),%r14d          # sdif
       or     %ebx,%ebp                 # INET_COMBINED_PORTS(sport, hnum)
       mov    0x11a8382(%rip),%eax      # inet6_ehashfn() ...

After: __inet6_lookup_established()

       ...
       push   %rbx
       sub    $0x28,%rsp                # stack is 40 bytes
       mov    0x60(%rsp),%ebp           # sdif
       mov    %r8d,%r14d                # hnum
       shl    $0x10,%r14d               # hnum << 16
       or     %edx,%r14d                # INET_COMBINED_PORTS(sport, hnum)
       mov    0x440(%rdi),%rax          # net->ipv4.tcp_death_row.hashinfo
       mov    0x1194f09(%rip),%r10d     # inet6_ehashfn() ...

Suggested-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarKuniyuki Iwashima <kuniyu@google.com>
Reviewed-by: default avatarEric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20260303235424.3877267-1-kuniyu@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent edad504c

include/linux/net.h

+2 −0
Original line number Diff line number Diff line
@@ -304,6 +304,8 @@ do { \ 


#define net_get_random_once(buf, nbytes)			\

	get_random_once((buf), (nbytes))

#define net_get_random_sleepable_once(buf, nbytes)		\

	get_random_sleepable_once((buf), (nbytes))



/*

 * E.g. XFS meta- & log-data is in slab pages, or bcache meta

include/net/inet6_hashtables.h

+2 −0
Original line number Diff line number Diff line
@@ -24,6 +24,8 @@ 


struct inet_hashinfo;



void inet6_init_ehash_secret(void);



static inline unsigned int __inet6_ehashfn(const u32 lhash,

				    const u16 lport,

				    const u32 fhash,

net/ipv4/inet_hashtables.c

+15 −2
Original line number Diff line number Diff line
@@ -30,12 +30,16 @@ 
#include <net/sock_reuseport.h>

#include <net/tcp.h>



static void inet_init_ehash_secret(void)

{

	net_get_random_sleepable_once(&inet_ehash_secret,

				      sizeof(inet_ehash_secret));

}



u32 inet_ehashfn(const struct net *net, const __be32 laddr,

		 const __u16 lport, const __be32 faddr,

		 const __be16 fport)

{

	net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));



	return lport + __inet_ehashfn(laddr, 0, faddr, fport,

				      inet_ehash_secret + net_hash_mix(net));

}
@@ -793,6 +797,13 @@ int inet_hash(struct sock *sk) 
		local_bh_enable();

		return 0;

	}



#if IS_ENABLED(CONFIG_IPV6)

	if (sk->sk_family == AF_INET6)

		inet6_init_ehash_secret();

#endif

	inet_init_ehash_secret();



	WARN_ON(!sk_unhashed(sk));

	ilb2 = inet_lhash2_bucket_sk(hashinfo, sk);
@@ -1239,6 +1250,8 @@ int inet_hash_connect(struct inet_timewait_death_row *death_row, 
	if (!inet_sk(sk)->inet_num)

		port_offset = inet_sk_port_offset(sk);



	inet_init_ehash_secret();



	hash_port0 = inet_ehashfn(net, inet->inet_rcv_saddr, 0,

				  inet->inet_daddr, inet->inet_dport);

net/ipv6/inet6_hashtables.c

+10 −3
Original line number Diff line number Diff line
@@ -23,15 +23,20 @@ 
#include <net/sock_reuseport.h>

#include <net/tcp.h>



void inet6_init_ehash_secret(void)

{

	net_get_random_sleepable_once(&inet6_ehash_secret,

				      sizeof(inet6_ehash_secret));

	net_get_random_sleepable_once(&tcp_ipv6_hash_secret,

				      sizeof(tcp_ipv6_hash_secret));

}



u32 inet6_ehashfn(const struct net *net,

		  const struct in6_addr *laddr, const u16 lport,

		  const struct in6_addr *faddr, const __be16 fport)

{

	u32 lhash, fhash;



	net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));

	net_get_random_once(&tcp_ipv6_hash_secret, sizeof(tcp_ipv6_hash_secret));



	lhash = (__force u32)laddr->s6_addr32[3];

	fhash = __ipv6_addr_jhash(faddr, tcp_ipv6_hash_secret);
@@ -363,6 +368,8 @@ int inet6_hash_connect(struct inet_timewait_death_row *death_row, 
	if (!inet_sk(sk)->inet_num)

		port_offset = inet6_sk_port_offset(sk);



	inet6_init_ehash_secret();



	hash_port0 = inet6_ehashfn(net, daddr, 0, saddr, inet->inet_dport);



	return __inet_hash_connect(death_row, sk, port_offset, hash_port0,