Commit 2f651683 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'tcp-add-fast-path-in-timer-handlers' 

Eric Dumazet says:

====================
tcp: add fast path in timer handlers

As mentioned in Netconf 2024:

TCP retransmit and delack timers are not stopped from
inet_csk_clear_xmit_timer() because we do not define
INET_CSK_CLEAR_TIMERS.

Enabling INET_CSK_CLEAR_TIMERS leads to lower performance,
mainly because del_timer() and mod_timer() happen from
different cpus quite often.

What we can do instead is to add fast paths to tcp_write_timer()
and tcp_delack_timer() to avoid socket spinlock acquisition.
====================

Link: https://patch.msgid.link/20241002173042.917928-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents d454184b 81df4fa9

include/net/inet_connection_sock.h

+5 −4
Original line number Diff line number Diff line
@@ -197,12 +197,12 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what) 
	struct inet_connection_sock *icsk = inet_csk(sk);



	if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0) {

		icsk->icsk_pending = 0;

		smp_store_release(&icsk->icsk_pending, 0);

#ifdef INET_CSK_CLEAR_TIMERS

		sk_stop_timer(sk, &icsk->icsk_retransmit_timer);

#endif

	} else if (what == ICSK_TIME_DACK) {

		icsk->icsk_ack.pending = 0;

		smp_store_release(&icsk->icsk_ack.pending, 0);

		icsk->icsk_ack.retry = 0;

#ifdef INET_CSK_CLEAR_TIMERS

		sk_stop_timer(sk, &icsk->icsk_delack_timer);
@@ -229,11 +229,12 @@ static inline void inet_csk_reset_xmit_timer(struct sock *sk, const int what, 


	if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0 ||

	    what == ICSK_TIME_LOSS_PROBE || what == ICSK_TIME_REO_TIMEOUT) {

		icsk->icsk_pending = what;

		smp_store_release(&icsk->icsk_pending, what);

		icsk->icsk_timeout = jiffies + when;

		sk_reset_timer(sk, &icsk->icsk_retransmit_timer, icsk->icsk_timeout);

	} else if (what == ICSK_TIME_DACK) {

		icsk->icsk_ack.pending |= ICSK_ACK_TIMER;

		smp_store_release(&icsk->icsk_ack.pending,

				  icsk->icsk_ack.pending | ICSK_ACK_TIMER);

		icsk->icsk_ack.timeout = jiffies + when;

		sk_reset_timer(sk, &icsk->icsk_delack_timer, icsk->icsk_ack.timeout);

	} else {

net/ipv4/inet_connection_sock.c

+4 −2
Original line number Diff line number Diff line
@@ -775,7 +775,8 @@ void inet_csk_clear_xmit_timers(struct sock *sk) 
{

	struct inet_connection_sock *icsk = inet_csk(sk);



	icsk->icsk_pending = icsk->icsk_ack.pending = 0;

	smp_store_release(&icsk->icsk_pending, 0);

	smp_store_release(&icsk->icsk_ack.pending, 0);



	sk_stop_timer(sk, &icsk->icsk_retransmit_timer);

	sk_stop_timer(sk, &icsk->icsk_delack_timer);
@@ -790,7 +791,8 @@ void inet_csk_clear_xmit_timers_sync(struct sock *sk) 
	/* ongoing timer handlers need to acquire socket lock. */

	sock_not_owned_by_me(sk);



	icsk->icsk_pending = icsk->icsk_ack.pending = 0;

	smp_store_release(&icsk->icsk_pending, 0);

	smp_store_release(&icsk->icsk_ack.pending, 0);



	sk_stop_timer_sync(sk, &icsk->icsk_retransmit_timer);

	sk_stop_timer_sync(sk, &icsk->icsk_delack_timer);

net/ipv4/inet_diag.c

+6 −4
Original line number Diff line number Diff line
@@ -247,6 +247,7 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk, 
	struct nlmsghdr  *nlh;

	struct nlattr *attr;

	void *info = NULL;

	u8 icsk_pending;

	int protocol;



	cb_data = cb->data;
@@ -307,14 +308,15 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk, 
		goto out;

	}



	if (icsk->icsk_pending == ICSK_TIME_RETRANS ||

	    icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||

	    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {

	icsk_pending = smp_load_acquire(&icsk->icsk_pending);

	if (icsk_pending == ICSK_TIME_RETRANS ||

	    icsk_pending == ICSK_TIME_REO_TIMEOUT ||

	    icsk_pending == ICSK_TIME_LOSS_PROBE) {

		r->idiag_timer = 1;

		r->idiag_retrans = icsk->icsk_retransmits;

		r->idiag_expires =

			jiffies_delta_to_msecs(icsk->icsk_timeout - jiffies);

	} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {

	} else if (icsk_pending == ICSK_TIME_PROBE0) {

		r->idiag_timer = 4;

		r->idiag_retrans = icsk->icsk_probes_out;

		r->idiag_expires =

net/ipv4/tcp_ipv4.c

+6 −4
Original line number Diff line number Diff line
@@ -2900,15 +2900,17 @@ static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i) 
	__be32 src = inet->inet_rcv_saddr;

	__u16 destp = ntohs(inet->inet_dport);

	__u16 srcp = ntohs(inet->inet_sport);

	u8 icsk_pending;

	int rx_queue;

	int state;



	if (icsk->icsk_pending == ICSK_TIME_RETRANS ||

	    icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||

	    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {

	icsk_pending = smp_load_acquire(&icsk->icsk_pending);

	if (icsk_pending == ICSK_TIME_RETRANS ||

	    icsk_pending == ICSK_TIME_REO_TIMEOUT ||

	    icsk_pending == ICSK_TIME_LOSS_PROBE) {

		timer_active	= 1;

		timer_expires	= icsk->icsk_timeout;

	} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {

	} else if (icsk_pending == ICSK_TIME_PROBE0) {

		timer_active	= 4;

		timer_expires	= icsk->icsk_timeout;

	} else if (timer_pending(&sk->sk_timer)) {

net/ipv4/tcp_output.c

+4 −3
Original line number Diff line number Diff line
@@ -2960,7 +2960,7 @@ void tcp_send_loss_probe(struct sock *sk) 
		WARN_ONCE(tp->packets_out,

			  "invalid inflight: %u state %u cwnd %u mss %d\n",

			  tp->packets_out, sk->sk_state, tcp_snd_cwnd(tp), mss);

		inet_csk(sk)->icsk_pending = 0;

		smp_store_release(&inet_csk(sk)->icsk_pending, 0);

		return;

	}
@@ -2993,7 +2993,7 @@ void tcp_send_loss_probe(struct sock *sk) 


	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);

	/* Reset s.t. tcp_rearm_rto will restart timer from now */

	inet_csk(sk)->icsk_pending = 0;

	smp_store_release(&inet_csk(sk)->icsk_pending, 0);

rearm_timer:

	tcp_rearm_rto(sk);

}
@@ -4224,7 +4224,8 @@ void tcp_send_delayed_ack(struct sock *sk) 
		if (!time_before(timeout, icsk->icsk_ack.timeout))

			timeout = icsk->icsk_ack.timeout;

	}

	icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;

	smp_store_release(&icsk->icsk_ack.pending,

			  icsk->icsk_ack.pending | ICSK_ACK_SCHED | ICSK_ACK_TIMER);

	icsk->icsk_ack.timeout = timeout;

	sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);

}