Commit 81df4fa9 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

tcp: add a fast path in tcp_delack_timer() 



delack timer is not stopped from inet_csk_clear_xmit_timer()
because we do not define INET_CSK_CLEAR_TIMERS.

This is a conscious choice : inet_csk_clear_xmit_timer()
is often called from another cpu. Calling del_timer()
would cause false sharing and lock contention.

This means that very often, tcp_delack_timer() is called
at the timer expiration, while there is no ACK to transmit.

This can be detected very early, avoiding the socket spinlock.

Notes:
- test about tp->compressed_ack is racy,
  but in the unlikely case there is a race, the dedicated
  compressed_ack_timer hrtimer would close it.

- Even if the fast path is not taken, reading
  icsk->icsk_ack.pending and tp->compressed_ack
  before acquiring the socket spinlock reduces
  acquisition time and chances of contention.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20241002173042.917928-4-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 3b784293

include/net/inet_connection_sock.h

+3 −2
Original line number Diff line number Diff line
@@ -202,7 +202,7 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what) 
		sk_stop_timer(sk, &icsk->icsk_retransmit_timer);

#endif

	} else if (what == ICSK_TIME_DACK) {

		icsk->icsk_ack.pending = 0;

		smp_store_release(&icsk->icsk_ack.pending, 0);

		icsk->icsk_ack.retry = 0;

#ifdef INET_CSK_CLEAR_TIMERS

		sk_stop_timer(sk, &icsk->icsk_delack_timer);
@@ -233,7 +233,8 @@ static inline void inet_csk_reset_xmit_timer(struct sock *sk, const int what, 
		icsk->icsk_timeout = jiffies + when;

		sk_reset_timer(sk, &icsk->icsk_retransmit_timer, icsk->icsk_timeout);

	} else if (what == ICSK_TIME_DACK) {

		icsk->icsk_ack.pending |= ICSK_ACK_TIMER;

		smp_store_release(&icsk->icsk_ack.pending,

				  icsk->icsk_ack.pending | ICSK_ACK_TIMER);

		icsk->icsk_ack.timeout = jiffies + when;

		sk_reset_timer(sk, &icsk->icsk_delack_timer, icsk->icsk_ack.timeout);

	} else {

net/ipv4/inet_connection_sock.c

+2 −2
Original line number Diff line number Diff line
@@ -776,7 +776,7 @@ void inet_csk_clear_xmit_timers(struct sock *sk) 
	struct inet_connection_sock *icsk = inet_csk(sk);



	smp_store_release(&icsk->icsk_pending, 0);

	icsk->icsk_ack.pending = 0;

	smp_store_release(&icsk->icsk_ack.pending, 0);



	sk_stop_timer(sk, &icsk->icsk_retransmit_timer);

	sk_stop_timer(sk, &icsk->icsk_delack_timer);
@@ -792,7 +792,7 @@ void inet_csk_clear_xmit_timers_sync(struct sock *sk) 
	sock_not_owned_by_me(sk);



	smp_store_release(&icsk->icsk_pending, 0);

	icsk->icsk_ack.pending = 0;

	smp_store_release(&icsk->icsk_ack.pending, 0);



	sk_stop_timer_sync(sk, &icsk->icsk_retransmit_timer);

	sk_stop_timer_sync(sk, &icsk->icsk_delack_timer);

net/ipv4/tcp_output.c

+2 −1
Original line number Diff line number Diff line
@@ -4224,7 +4224,8 @@ void tcp_send_delayed_ack(struct sock *sk) 
		if (!time_before(timeout, icsk->icsk_ack.timeout))

			timeout = icsk->icsk_ack.timeout;

	}

	icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;

	smp_store_release(&icsk->icsk_ack.pending,

			  icsk->icsk_ack.pending | ICSK_ACK_SCHED | ICSK_ACK_TIMER);

	icsk->icsk_ack.timeout = timeout;

	sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);

}

net/ipv4/tcp_timer.c

+9 −0
Original line number Diff line number Diff line
@@ -361,6 +361,14 @@ static void tcp_delack_timer(struct timer_list *t) 
			from_timer(icsk, t, icsk_delack_timer);

	struct sock *sk = &icsk->icsk_inet.sk;



	/* Avoid taking socket spinlock if there is no ACK to send.

	 * The compressed_ack check is racy, but a separate hrtimer

	 * will take care of it eventually.

	 */

	if (!(smp_load_acquire(&icsk->icsk_ack.pending) & ICSK_ACK_TIMER) &&

	    !READ_ONCE(tcp_sk(sk)->compressed_ack))

		goto out;



	bh_lock_sock(sk);

	if (!sock_owned_by_user(sk)) {

		tcp_delack_timer_handler(sk);
@@ -371,6 +379,7 @@ static void tcp_delack_timer(struct timer_list *t) 
			sock_hold(sk);

	}

	bh_unlock_sock(sk);

out:

	sock_put(sk);

}

net/mptcp/protocol.c

+2 −1
Original line number Diff line number Diff line
@@ -3504,7 +3504,8 @@ static void schedule_3rdack_retransmission(struct sock *ssk) 
	timeout += jiffies;



	WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER);

	icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;

	smp_store_release(&icsk->icsk_ack.pending,

			  icsk->icsk_ack.pending | ICSK_ACK_SCHED | ICSK_ACK_TIMER);

	icsk->icsk_ack.timeout = timeout;

	sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout);

}