Commit a36283e2 authored by Paolo Abeni's avatar Paolo Abeni Committed by Jakub Kicinski
Browse files

udp_tunnel: create a fastpath GRO lookup. 



Most UDP tunnels bind a socket to a local port, with ANY address, no
peer and no interface index specified.
Additionally it's quite common to have a single tunnel device per
namespace.

Track in each namespace the UDP tunnel socket respecting the above.
When only a single one is present, store a reference in the netns.

When such reference is not NULL, UDP tunnel GRO lookup just need to
match the incoming packet destination port vs the socket local port.

The tunnel socket never sets the reuse[port] flag[s]. When bound to no
address and interface, no other socket can exist in the same netns
matching the specified local port.

Matching packets with non-local destination addresses will be
aggregated, and eventually segmented as needed - no behavior changes
intended.

Restrict the optimization to kernel sockets only: it covers all the
relevant use-cases, and user-space owned sockets could be disconnected
and rebound after setup_udp_tunnel_sock(), breaking the uniqueness
assumption

Note that the UDP tunnel socket reference is stored into struct
netns_ipv4 for both IPv4 and IPv6 tunnels. That is intentional to keep
all the fastpath-related netns fields in the same struct and allow
cacheline-based optimization. Currently both the IPv4 and IPv6 socket
pointer share the same cacheline as the `udp_table` field.

Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
Reviewed-by: default avatarWillem de Bruijn <willemb@google.com>
Link: https://patch.msgid.link/41d16bc8d1257d567f9344c445b4ae0b4a91ede4.1744040675.git.pabeni@redhat.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 5ac40e6b

include/linux/udp.h

+16 −0
Original line number Diff line number Diff line
@@ -101,6 +101,13 @@ struct udp_sock { 


	/* Cache friendly copy of sk->sk_peek_off >= 0 */

	bool		peeking_with_offset;



	/*

	 * Accounting for the tunnel GRO fastpath.

	 * Unprotected by compilers guard, as it uses space available in

	 * the last UDP socket cacheline.

	 */

	struct hlist_node	tunnel_list;

};



#define udp_test_bit(nr, sk)			\
@@ -219,4 +226,13 @@ static inline void udp_allow_gso(struct sock *sk) 


#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)



static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)

{

#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

	return rcu_dereference(net->ipv4.udp_tunnel_gro[is_ipv6].sk);

#else

	return NULL;

#endif

}



#endif	/* _LINUX_UDP_H */

include/net/netns/ipv4.h

+11 −0
Original line number Diff line number Diff line
@@ -47,6 +47,11 @@ struct sysctl_fib_multipath_hash_seed { 
};

#endif



struct udp_tunnel_gro {

	struct sock __rcu *sk;

	struct hlist_head list;

};



struct netns_ipv4 {

	/* Cacheline organization can be found documented in

	 * Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst.
@@ -85,6 +90,11 @@ struct netns_ipv4 { 
	struct inet_timewait_death_row tcp_death_row;

	struct udp_table *udp_table;



#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

	/* Not in a pernet subsys because need to be available at GRO stage */

	struct udp_tunnel_gro udp_tunnel_gro[2];

#endif



#ifdef CONFIG_SYSCTL

	struct ctl_table_header	*forw_hdr;

	struct ctl_table_header	*frags_hdr;
@@ -277,4 +287,5 @@ struct netns_ipv4 { 
	struct hlist_head	*inet_addr_lst;

	struct delayed_work	addr_chk_work;

};



#endif

include/net/udp.h

+1 −0
Original line number Diff line number Diff line
@@ -290,6 +290,7 @@ static inline void udp_lib_init_sock(struct sock *sk) 
	struct udp_sock *up = udp_sk(sk);



	skb_queue_head_init(&up->reader_queue);

	INIT_HLIST_NODE(&up->tunnel_list);

	up->forward_threshold = sk->sk_rcvbuf >> 2;

	set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);

}

include/net/udp_tunnel.h

+12 −0
Original line number Diff line number Diff line
@@ -191,6 +191,18 @@ static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum) 
}

#endif



#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add);

#else

static inline void udp_tunnel_update_gro_lookup(struct net *net,

						struct sock *sk, bool add) {}

#endif



static inline void udp_tunnel_cleanup_gro(struct sock *sk)

{

	udp_tunnel_update_gro_lookup(sock_net(sk), sk, false);

}



static inline void udp_tunnel_encap_enable(struct sock *sk)

{

	if (udp_test_and_set_bit(ENCAP_ENABLED, sk))

net/ipv4/udp.c

+12 −1
Original line number Diff line number Diff line
@@ -2897,8 +2897,10 @@ void udp_destroy_sock(struct sock *sk) 
			if (encap_destroy)

				encap_destroy(sk);

		}

		if (udp_test_bit(ENCAP_ENABLED, sk))

		if (udp_test_bit(ENCAP_ENABLED, sk)) {

			static_branch_dec(&udp_encap_needed_key);

			udp_tunnel_cleanup_gro(sk);

		}

	}

}
@@ -3810,6 +3812,15 @@ static void __net_init udp_set_table(struct net *net) 


static int __net_init udp_pernet_init(struct net *net)

{

#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

	int i;



	/* No tunnel is configured */

	for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {

		INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);

		RCU_INIT_POINTER(net->ipv4.udp_tunnel_gro[i].sk, NULL);

	}

#endif

	udp_sysctl_init(net);

	udp_set_table(net);