Commit ae9d5b19 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

tcp: use RCU in __inet{6}_check_established() 



When __inet_hash_connect() has to try many 4-tuples before
finding an available one, we see a high spinlock cost from
__inet_check_established() and/or __inet6_check_established().

This patch adds an RCU lookup to avoid the spinlock
acquisition when the 4-tuple is found in the hash table.

Note that there are still spin_lock_bh() calls in
__inet_hash_connect() to protect inet_bind_hashbucket,
this will be fixed later in this series.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Reviewed-by: default avatarJason Xing <kerneljasonxing@gmail.com>
Tested-by: default avatarJason Xing <kerneljasonxing@gmail.com>
Reviewed-by: default avatarKuniyuki Iwashima <kuniyu@amazon.com>
Link: https://patch.msgid.link/20250302124237.3913746-2-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 7ff1c88f

net/ipv4/inet_hashtables.c

+16 −3
Original line number Diff line number Diff line
@@ -551,11 +551,24 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row, 
	unsigned int hash = inet_ehashfn(net, daddr, lport,

					 saddr, inet->inet_dport);

	struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);

	spinlock_t *lock = inet_ehash_lockp(hinfo, hash);

	struct sock *sk2;

	const struct hlist_nulls_node *node;

	struct inet_timewait_sock *tw = NULL;

	const struct hlist_nulls_node *node;

	struct sock *sk2;

	spinlock_t *lock;



	rcu_read_lock();

	sk_nulls_for_each(sk2, node, &head->chain) {

		if (sk2->sk_hash != hash ||

		    !inet_match(net, sk2, acookie, ports, dif, sdif))

			continue;

		if (sk2->sk_state == TCP_TIME_WAIT)

			break;

		rcu_read_unlock();

		return -EADDRNOTAVAIL;

	}

	rcu_read_unlock();



	lock = inet_ehash_lockp(hinfo, hash);

	spin_lock(lock);



	sk_nulls_for_each(sk2, node, &head->chain) {

net/ipv6/inet6_hashtables.c

+16 −3
Original line number Diff line number Diff line
@@ -276,11 +276,24 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row, 
	const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,

						inet->inet_dport);

	struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);

	spinlock_t *lock = inet_ehash_lockp(hinfo, hash);

	struct sock *sk2;

	const struct hlist_nulls_node *node;

	struct inet_timewait_sock *tw = NULL;

	const struct hlist_nulls_node *node;

	struct sock *sk2;

	spinlock_t *lock;



	rcu_read_lock();

	sk_nulls_for_each(sk2, node, &head->chain) {

		if (sk2->sk_hash != hash ||

		    !inet6_match(net, sk2, saddr, daddr, ports, dif, sdif))

			continue;

		if (sk2->sk_state == TCP_TIME_WAIT)

			break;

		rcu_read_unlock();

		return -EADDRNOTAVAIL;

	}

	rcu_read_unlock();



	lock = inet_ehash_lockp(hinfo, hash);

	spin_lock(lock);



	sk_nulls_for_each(sk2, node, &head->chain) {