Commit e7b1b078 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'net-three-additions-to-net_hotdata' 

Eric Dumazet says:

====================
net: three additions to net_hotdata

This series moves three fast path sysctls to net_hotdata.

To avoid <net/hotdata.h> inclusion from <net/sock.h>,
create <net/proto_memory.h> to hold proto memory definitions.
====================

Link: https://lore.kernel.org/r/20240429134025.1233626-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 05d6d492 c204fef9

include/linux/skbuff.h

+0 −2
Original line number Diff line number Diff line
@@ -353,8 +353,6 @@ struct sk_buff; 


#define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS



extern int sysctl_max_skb_frags;



/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to

 * segment using its current segmentation instead.

 */

include/net/hotdata.h

+3 −0
Original line number Diff line number Diff line
@@ -38,6 +38,9 @@ struct net_hotdata { 
	int			max_backlog;

	int			dev_tx_weight;

	int			dev_rx_weight;

	int			sysctl_max_skb_frags;

	int			sysctl_skb_defer_max;

	int			sysctl_mem_pcpu_rsv;

};



#define inet_ehash_secret	net_hotdata.tcp_protocol.secret

include/net/proto_memory.h

0 → 100644
+83 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-or-later */

#ifndef _PROTO_MEMORY_H

#define _PROTO_MEMORY_H



#include <net/sock.h>

#include <net/hotdata.h>



/* 1 MB per cpu, in page units */

#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))



static inline bool sk_has_memory_pressure(const struct sock *sk)

{

	return sk->sk_prot->memory_pressure != NULL;

}



static inline bool

proto_memory_pressure(const struct proto *prot)

{

	if (!prot->memory_pressure)

		return false;

	return !!READ_ONCE(*prot->memory_pressure);

}



static inline bool sk_under_global_memory_pressure(const struct sock *sk)

{

	return proto_memory_pressure(sk->sk_prot);

}



static inline bool sk_under_memory_pressure(const struct sock *sk)

{

	if (!sk->sk_prot->memory_pressure)

		return false;



	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&

	    mem_cgroup_under_socket_pressure(sk->sk_memcg))

		return true;



	return !!READ_ONCE(*sk->sk_prot->memory_pressure);

}



static inline long

proto_memory_allocated(const struct proto *prot)

{

	return max(0L, atomic_long_read(prot->memory_allocated));

}



static inline long

sk_memory_allocated(const struct sock *sk)

{

	return proto_memory_allocated(sk->sk_prot);

}



static inline void proto_memory_pcpu_drain(struct proto *proto)

{

	int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);



	if (val)

		atomic_long_add(val, proto->memory_allocated);

}



static inline void

sk_memory_allocated_add(const struct sock *sk, int val)

{

	struct proto *proto = sk->sk_prot;



	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);



	if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))

		proto_memory_pcpu_drain(proto);

}



static inline void

sk_memory_allocated_sub(const struct sock *sk, int val)

{

	struct proto *proto = sk->sk_prot;



	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);



	if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))

		proto_memory_pcpu_drain(proto);

}



#endif /* _PROTO_MEMORY_H */

include/net/sock.h

+0 −78
Original line number Diff line number Diff line
@@ -1371,75 +1371,6 @@ static inline int sk_under_cgroup_hierarchy(struct sock *sk, 
#endif

}



static inline bool sk_has_memory_pressure(const struct sock *sk)

{

	return sk->sk_prot->memory_pressure != NULL;

}



static inline bool sk_under_global_memory_pressure(const struct sock *sk)

{

	return sk->sk_prot->memory_pressure &&

		!!READ_ONCE(*sk->sk_prot->memory_pressure);

}



static inline bool sk_under_memory_pressure(const struct sock *sk)

{

	if (!sk->sk_prot->memory_pressure)

		return false;



	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&

	    mem_cgroup_under_socket_pressure(sk->sk_memcg))

		return true;



	return !!READ_ONCE(*sk->sk_prot->memory_pressure);

}



static inline long

proto_memory_allocated(const struct proto *prot)

{

	return max(0L, atomic_long_read(prot->memory_allocated));

}



static inline long

sk_memory_allocated(const struct sock *sk)

{

	return proto_memory_allocated(sk->sk_prot);

}



/* 1 MB per cpu, in page units */

#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))

extern int sysctl_mem_pcpu_rsv;



static inline void proto_memory_pcpu_drain(struct proto *proto)

{

	int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);



	if (val)

		atomic_long_add(val, proto->memory_allocated);

}



static inline void

sk_memory_allocated_add(const struct sock *sk, int val)

{

	struct proto *proto = sk->sk_prot;



	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);



	if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))

		proto_memory_pcpu_drain(proto);

}



static inline void

sk_memory_allocated_sub(const struct sock *sk, int val)

{

	struct proto *proto = sk->sk_prot;



	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);



	if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))

		proto_memory_pcpu_drain(proto);

}



#define SK_ALLOC_PERCPU_COUNTER_BATCH 16



static inline void sk_sockets_allocated_dec(struct sock *sk)
@@ -1466,15 +1397,6 @@ proto_sockets_allocated_sum_positive(struct proto *prot) 
	return percpu_counter_sum_positive(prot->sockets_allocated);

}



static inline bool

proto_memory_pressure(struct proto *prot)

{

	if (!prot->memory_pressure)

		return false;

	return !!READ_ONCE(*prot->memory_pressure);

}





#ifdef CONFIG_PROC_FS

#define PROTO_INUSE_NR	64	/* should be enough for the first time */

struct prot_inuse {

include/net/tcp.h

+1 −9
Original line number Diff line number Diff line
@@ -296,14 +296,6 @@ static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 
	return seq3 - seq2 >= seq1 - seq2;

}



static inline bool tcp_out_of_memory(struct sock *sk)

{

	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&

	    sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))

		return true;

	return false;

}



static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)

{

	sk_wmem_queued_add(sk, -skb->truesize);
@@ -316,7 +308,7 @@ static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 


void sk_forced_mem_schedule(struct sock *sk, int size);



bool tcp_check_oom(struct sock *sk, int shift);

bool tcp_check_oom(const struct sock *sk, int shift);





extern struct proto tcp_prot;