mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-02-25 23:20:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

tcp: use RCU in __inet{6}_check_established()

Browse Source 
When __inet_hash_connect() has to try many 4-tuples before
finding an available one, we see a high spinlock cost from
__inet_check_established() and/or __inet6_check_established().

This patch adds an RCU lookup to avoid the spinlock
acquisition when the 4-tuple is found in the hash table.

Note that there are still spin_lock_bh() calls in
__inet_hash_connect() to protect inet_bind_hashbucket,
this will be fixed later in this series.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Tested-by: Jason Xing <kerneljasonxing@gmail.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://patch.msgid.link/20250302124237.3913746-2-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet
2025-03-02 12:42:34 +00:00
committed by Jakub Kicinski
parent 7ff1c88fc8
commit ae9d5b19b3
2 changed files with 32 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
net/ipv4/inet_hashtables.c
View File

@@ -551,11 +551,24 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,
unsigned int hash = inet_ehashfn(net, daddr, lport,
saddr, inet->inet_dport);
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
struct sock *sk2;
const struct hlist_nulls_node *node;
struct inet_timewait_sock *tw = NULL;
const struct hlist_nulls_node *node;
struct sock *sk2;
spinlock_t *lock;
rcu_read_lock();
sk_nulls_for_each(sk2, node, &head->chain) {
if (sk2->sk_hash != hash ||
!inet_match(net, sk2, acookie, ports, dif, sdif))
continue;
if (sk2->sk_state == TCP_TIME_WAIT)
break;
rcu_read_unlock();
return -EADDRNOTAVAIL;
}
rcu_read_unlock();
lock = inet_ehash_lockp(hinfo, hash);
spin_lock(lock);
sk_nulls_for_each(sk2, node, &head->chain) {

19
net/ipv6/inet6_hashtables.c
View File

@@ -276,11 +276,24 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
inet->inet_dport);
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
struct sock *sk2;
const struct hlist_nulls_node *node;
struct inet_timewait_sock *tw = NULL;
const struct hlist_nulls_node *node;
struct sock *sk2;
spinlock_t *lock;
rcu_read_lock();
sk_nulls_for_each(sk2, node, &head->chain) {
if (sk2->sk_hash != hash ||
!inet6_match(net, sk2, saddr, daddr, ports, dif, sdif))
continue;
if (sk2->sk_state == TCP_TIME_WAIT)
break;
rcu_read_unlock();
return -EADDRNOTAVAIL;
}
rcu_read_unlock();
lock = inet_ehash_lockp(hinfo, hash);
spin_lock(lock);
sk_nulls_for_each(sk2, node, &head->chain) {