Commit 6a0f72c5 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'udp_tunnel-gro-optimizations' 

Paolo Abeni says:

====================
udp_tunnel: GRO optimizations

The UDP tunnel GRO stage is source of measurable overhead for workload
based on UDP-encapsulated traffic: each incoming packets requires a full
UDP socket lookup and an indirect call.

In the most common setups a single UDP tunnel device is used. In such
case we can optimize both the lookup and the indirect call.

Patch 1 tracks per netns the active UDP tunnels and replaces the socket
lookup with a single destination port comparison when possible.

Patch 2 tracks the different types of UDP tunnels and replaces the
indirect call with a static one when there is a single UDP tunnel type
active.

I measure ~10% performance improvement in TCP over UDP tunnel stream
tests on top of this series.

v4: https://lore.kernel.org/cover.1741718157.git.pabeni@redhat.com
v3: https://lore.kernel.org/cover.1741632298.git.pabeni@redhat.com
v2: https://lore.kernel.org/cover.1741338765.git.pabeni@redhat.com
v1: https://lore.kernel.org/cover.1741275846.git.pabeni@redhat.com
====================

Link: https://patch.msgid.link/cover.1744040675.git.pabeni@redhat.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 5ac40e6b 5d7f5b2f

include/linux/udp.h

+16 −0
Original line number Diff line number Diff line
@@ -101,6 +101,13 @@ struct udp_sock { 


	/* Cache friendly copy of sk->sk_peek_off >= 0 */

	bool		peeking_with_offset;



	/*

	 * Accounting for the tunnel GRO fastpath.

	 * Unprotected by compilers guard, as it uses space available in

	 * the last UDP socket cacheline.

	 */

	struct hlist_node	tunnel_list;

};



#define udp_test_bit(nr, sk)			\
@@ -219,4 +226,13 @@ static inline void udp_allow_gso(struct sock *sk) 


#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)



static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)

{

#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

	return rcu_dereference(net->ipv4.udp_tunnel_gro[is_ipv6].sk);

#else

	return NULL;

#endif

}



#endif	/* _LINUX_UDP_H */

include/net/netns/ipv4.h

+11 −0
Original line number Diff line number Diff line
@@ -47,6 +47,11 @@ struct sysctl_fib_multipath_hash_seed { 
};

#endif



struct udp_tunnel_gro {

	struct sock __rcu *sk;

	struct hlist_head list;

};



struct netns_ipv4 {

	/* Cacheline organization can be found documented in

	 * Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst.
@@ -85,6 +90,11 @@ struct netns_ipv4 { 
	struct inet_timewait_death_row tcp_death_row;

	struct udp_table *udp_table;



#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

	/* Not in a pernet subsys because need to be available at GRO stage */

	struct udp_tunnel_gro udp_tunnel_gro[2];

#endif



#ifdef CONFIG_SYSCTL

	struct ctl_table_header	*forw_hdr;

	struct ctl_table_header	*frags_hdr;
@@ -277,4 +287,5 @@ struct netns_ipv4 { 
	struct hlist_head	*inet_addr_lst;

	struct delayed_work	addr_chk_work;

};



#endif

include/net/udp.h

+1 −0
Original line number Diff line number Diff line
@@ -290,6 +290,7 @@ static inline void udp_lib_init_sock(struct sock *sk) 
	struct udp_sock *up = udp_sk(sk);



	skb_queue_head_init(&up->reader_queue);

	INIT_HLIST_NODE(&up->tunnel_list);

	up->forward_threshold = sk->sk_rcvbuf >> 2;

	set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);

}

include/net/udp_tunnel.h

+16 −0
Original line number Diff line number Diff line
@@ -191,6 +191,21 @@ static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum) 
}

#endif



#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add);

void udp_tunnel_update_gro_rcv(struct sock *sk, bool add);

#else

static inline void udp_tunnel_update_gro_lookup(struct net *net,

						struct sock *sk, bool add) {}

static inline void udp_tunnel_update_gro_rcv(struct sock *sk, bool add) {}

#endif



static inline void udp_tunnel_cleanup_gro(struct sock *sk)

{

	udp_tunnel_update_gro_rcv(sk, false);

	udp_tunnel_update_gro_lookup(sock_net(sk), sk, false);

}



static inline void udp_tunnel_encap_enable(struct sock *sk)

{

	if (udp_test_and_set_bit(ENCAP_ENABLED, sk))
@@ -200,6 +215,7 @@ static inline void udp_tunnel_encap_enable(struct sock *sk) 
	if (READ_ONCE(sk->sk_family) == PF_INET6)

		ipv6_stub->udpv6_encap_enable();

#endif

	udp_tunnel_update_gro_rcv(sk, true);

	udp_encap_enable();

}

net/ipv4/udp.c

+12 −1
Original line number Diff line number Diff line
@@ -2897,8 +2897,10 @@ void udp_destroy_sock(struct sock *sk) 
			if (encap_destroy)

				encap_destroy(sk);

		}

		if (udp_test_bit(ENCAP_ENABLED, sk))

		if (udp_test_bit(ENCAP_ENABLED, sk)) {

			static_branch_dec(&udp_encap_needed_key);

			udp_tunnel_cleanup_gro(sk);

		}

	}

}
@@ -3810,6 +3812,15 @@ static void __net_init udp_set_table(struct net *net) 


static int __net_init udp_pernet_init(struct net *net)

{

#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

	int i;



	/* No tunnel is configured */

	for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {

		INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);

		RCU_INIT_POINTER(net->ipv4.udp_tunnel_gro[i].sk, NULL);

	}

#endif

	udp_sysctl_init(net);

	udp_set_table(net);