Commit f796feab authored by Paolo Abeni's avatar Paolo Abeni Committed by Jakub Kicinski
Browse files

udp: add local "peek offset enabled" flag 



We want to re-organize the struct sock layout. The sk_peek_off
field location is problematic, as most protocols want it in the
RX read area, while UDP wants it on a cacheline different from
sk_receive_queue.

Create a local (inside udp_sock) copy of the 'peek offset is enabled'
flag and place it inside the same cacheline of reader_queue.

Check such flag before reading sk_peek_off. This will save potential
false sharing and cache misses in the fast-path.

Tested under UDP flood with small packets. The struct sock layout
update causes a 4% performance drop, and this patch restores completely
the original tput.

Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
Reviewed-by: default avatarEric Dumazet <edumazet@google.com>
Reviewed-by: default avatarWillem de Bruijn <willemb@google.com>
Link: https://lore.kernel.org/r/67ab679c15fbf49fa05b3ffe05d91c47ab84f147.1708426665.git.pabeni@redhat.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 26b2a265

include/linux/udp.h

+10 −0
Original line number Diff line number Diff line
@@ -92,6 +92,9 @@ struct udp_sock { 


	/* This fields follows rcvbuf value, and is touched by udp_recvmsg */

	int		forward_threshold;



	/* Cache friendly copy of sk->sk_peek_off >= 0 */

	bool		peeking_with_offset;

};



#define udp_test_bit(nr, sk)			\
@@ -109,6 +112,13 @@ struct udp_sock { 


#define udp_sk(ptr) container_of_const(ptr, struct udp_sock, inet.sk)



static inline int udp_set_peek_off(struct sock *sk, int val)

{

	sk_set_peek_off(sk, val);

	WRITE_ONCE(udp_sk(sk)->peeking_with_offset, val >= 0);

	return 0;

}



static inline void udp_set_no_check6_tx(struct sock *sk, bool val)

{

	udp_assign_bit(NO_CHECK6_TX, sk, val);

net/ipv4/af_inet.c

+1 −1
Original line number Diff line number Diff line
@@ -1103,7 +1103,7 @@ const struct proto_ops inet_dgram_ops = { 
	.recvmsg	   = inet_recvmsg,

	.mmap		   = sock_no_mmap,

	.splice_eof	   = inet_splice_eof,

	.set_peek_off	   = sk_set_peek_off,

	.set_peek_off	   = udp_set_peek_off,

#ifdef CONFIG_COMPAT

	.compat_ioctl	   = inet_compat_ioctl,

#endif

net/ipv4/udp.c

+1 −1
Original line number Diff line number Diff line
@@ -1589,7 +1589,7 @@ int udp_init_sock(struct sock *sk) 


void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)

{

	if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {

	if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset))) {

		bool slow = lock_sock_fast(sk);



		sk_peek_offset_bwd(sk, len);

net/ipv6/af_inet6.c

+1 −1
Original line number Diff line number Diff line
@@ -736,7 +736,7 @@ const struct proto_ops inet6_dgram_ops = { 
	.recvmsg	   = inet6_recvmsg,		/* retpoline's sake */

	.read_skb	   = udp_read_skb,

	.mmap		   = sock_no_mmap,

	.set_peek_off	   = sk_set_peek_off,

	.set_peek_off	   = udp_set_peek_off,

#ifdef CONFIG_COMPAT

	.compat_ioctl	   = inet6_compat_ioctl,

#endif