Commit 70f1fbee authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'ipv6-tcp-no-longer-rebuild-fl6-at-each-transmit' 

Eric Dumazet says:

====================
ipv6: tcp: no longer rebuild fl6 at each transmit

TCP v6 spends a good amount of time rebuilding a fresh fl6 at each
transmit in inet6_csk_xmit()/inet6_csk_route_socket().

TCP v4 caches the information in inet->cork.fl.u.ip4 instead.

This series changes TCP v6 to behave the same, saving cpu cycles
and reducing cache line misses and stack use.
====================

Link: https://patch.msgid.link/20260206173426.1638518-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 792aaea9 97d7ae6e

include/linux/ipv6.h

+4 −0
Original line number Diff line number Diff line
@@ -230,6 +230,10 @@ struct ipv6_fl_socklist; 
struct ipv6_pinfo {

	/* Used in tx path (inet6_csk_route_socket(), ip6_xmit()) */

	struct in6_addr 	saddr;

	union {

		struct in6_addr daddr;

		struct in6_addr final;

	};

	__be32			flow_label;

	u32			dst_cookie;

	struct ipv6_txoptions __rcu	*opt;

include/net/inet6_connection_sock.h

+3 −1
Original line number Diff line number Diff line
@@ -18,7 +18,9 @@ struct sk_buff; 
struct sock;

struct sockaddr;



struct dst_entry *inet6_csk_route_req(const struct sock *sk, struct flowi6 *fl6,

struct dst_entry *inet6_csk_route_req(const struct sock *sk,

				      struct dst_entry *dst,

				      struct flowi6 *fl6,

				      const struct request_sock *req, u8 proto);



int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);

net/ipv6/af_inet6.c

+2 −2
Original line number Diff line number Diff line
@@ -825,7 +825,7 @@ int inet6_sk_rebuild_header(struct sock *sk) 
{

	struct ipv6_pinfo *np = inet6_sk(sk);

	struct inet_sock *inet = inet_sk(sk);

	struct in6_addr *final_p, final;

	struct in6_addr *final_p;

	struct dst_entry *dst;

	struct flowi6 *fl6;
@@ -847,7 +847,7 @@ int inet6_sk_rebuild_header(struct sock *sk) 
	security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));



	rcu_read_lock();

	final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &final);

	final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &np->final);

	rcu_read_unlock();



	dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);

net/ipv6/datagram.c

+11 −10
Original line number Diff line number Diff line
@@ -72,12 +72,12 @@ static void ip6_datagram_flow_key_init(struct flowi6 *fl6, 
int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr)

{

	struct ip6_flowlabel *flowlabel = NULL;

	struct in6_addr *final_p, final;

	struct ipv6_txoptions *opt;

	struct dst_entry *dst;

	struct inet_sock *inet = inet_sk(sk);

	struct ipv6_pinfo *np = inet6_sk(sk);

	struct flowi6 fl6;

	struct ipv6_txoptions *opt;

	struct in6_addr *final_p;

	struct dst_entry *dst;

	struct flowi6 *fl6;

	int err = 0;



	if (inet6_test_bit(SNDFLOW, sk) &&
@@ -86,14 +86,15 @@ int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr) 
		if (IS_ERR(flowlabel))

			return -EINVAL;

	}

	ip6_datagram_flow_key_init(&fl6, sk);

	fl6 = &inet_sk(sk)->cork.fl.u.ip6;

	ip6_datagram_flow_key_init(fl6, sk);



	rcu_read_lock();

	opt = flowlabel ? flowlabel->opt : rcu_dereference(np->opt);

	final_p = fl6_update_dst(&fl6, opt, &final);

	final_p = fl6_update_dst(fl6, opt, &np->final);

	rcu_read_unlock();



	dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl6, final_p);

	dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);

	if (IS_ERR(dst)) {

		err = PTR_ERR(dst);

		goto out;
@@ -101,17 +102,17 @@ int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr) 


	if (fix_sk_saddr) {

		if (ipv6_addr_any(&np->saddr))

			np->saddr = fl6.saddr;

			np->saddr = fl6->saddr;



		if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {

			sk->sk_v6_rcv_saddr = fl6.saddr;

			sk->sk_v6_rcv_saddr = fl6->saddr;

			inet->inet_rcv_saddr = LOOPBACK4_IPV6;

			if (sk->sk_prot->rehash)

				sk->sk_prot->rehash(sk);

		}

	}



	ip6_sk_dst_store_flow(sk, dst, &fl6);

	ip6_sk_dst_store_flow(sk, dst, fl6);



out:

	fl6_sock_release(flowlabel);

net/ipv6/inet6_connection_sock.c

+30 −27
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <net/sock_reuseport.h>



struct dst_entry *inet6_csk_route_req(const struct sock *sk,

				      struct dst_entry *dst,

				      struct flowi6 *fl6,

				      const struct request_sock *req,

				      u8 proto)
@@ -32,7 +33,6 @@ struct dst_entry *inet6_csk_route_req(const struct sock *sk, 
	const struct inet_request_sock *ireq = inet_rsk(req);

	const struct ipv6_pinfo *np = inet6_sk(sk);

	struct in6_addr *final_p, final;

	struct dst_entry *dst;



	memset(fl6, 0, sizeof(*fl6));

	fl6->flowi6_proto = proto;
@@ -48,10 +48,11 @@ struct dst_entry *inet6_csk_route_req(const struct sock *sk, 
	fl6->flowi6_uid = sk_uid(sk);

	security_req_classify_flow(req, flowi6_to_flowi_common(fl6));



	if (!dst) {

		dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);

		if (IS_ERR(dst))

			return NULL;



	}

	return dst;

}
@@ -60,7 +61,7 @@ static struct dst_entry *inet6_csk_route_socket(struct sock *sk, 
{

	struct inet_sock *inet = inet_sk(sk);

	struct ipv6_pinfo *np = inet6_sk(sk);

	struct in6_addr *final_p, final;

	struct in6_addr *final_p;

	struct dst_entry *dst;



	memset(fl6, 0, sizeof(*fl6));
@@ -77,41 +78,41 @@ static struct dst_entry *inet6_csk_route_socket(struct sock *sk, 
	security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));



	rcu_read_lock();

	final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &final);

	final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &np->final);

	rcu_read_unlock();



	dst = __sk_dst_check(sk, np->dst_cookie);

	if (!dst) {

	dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);



	if (!IS_ERR(dst))

		ip6_dst_store(sk, dst, false, false);

	}



	return dst;

}



int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl_unused)

{

	struct flowi6 *fl6 = &inet_sk(sk)->cork.fl.u.ip6;

	struct ipv6_pinfo *np = inet6_sk(sk);

	struct flowi6 fl6;

	struct dst_entry *dst;

	int res;



	dst = inet6_csk_route_socket(sk, &fl6);

	dst = __sk_dst_check(sk, np->dst_cookie);

	if (unlikely(!dst)) {

		dst = inet6_csk_route_socket(sk, fl6);

		if (IS_ERR(dst)) {

			WRITE_ONCE(sk->sk_err_soft, -PTR_ERR(dst));

			sk->sk_route_caps = 0;

			kfree_skb(skb);

			return PTR_ERR(dst);

		}

		/* Restore final destination back after routing done */

		fl6->daddr = sk->sk_v6_daddr;

	}



	rcu_read_lock();

	skb_dst_set_noref(skb, dst);



	/* Restore final destination back after routing done */

	fl6.daddr = sk->sk_v6_daddr;



	res = ip6_xmit(sk, skb, &fl6, sk->sk_mark, rcu_dereference(np->opt),

	res = ip6_xmit(sk, skb, fl6, sk->sk_mark, rcu_dereference(np->opt),

		       np->tclass, READ_ONCE(sk->sk_priority));

	rcu_read_unlock();

	return res;
@@ -120,13 +121,15 @@ EXPORT_SYMBOL_GPL(inet6_csk_xmit); 


struct dst_entry *inet6_csk_update_pmtu(struct sock *sk, u32 mtu)

{

	struct flowi6 fl6;

	struct dst_entry *dst = inet6_csk_route_socket(sk, &fl6);

	struct flowi6 *fl6 = &inet_sk(sk)->cork.fl.u.ip6;

	struct dst_entry *dst;



	dst = inet6_csk_route_socket(sk, fl6);



	if (IS_ERR(dst))

		return NULL;

	dst->ops->update_pmtu(dst, sk, NULL, mtu, true);



	dst = inet6_csk_route_socket(sk, &fl6);

	dst = inet6_csk_route_socket(sk, fl6);

	return IS_ERR(dst) ? NULL : dst;

}