Commit 489cee4c authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'rtnetlink-per-netns-rtnl' 

Kuniyuki Iwashima says:

====================
rtnetlink: Per-netns RTNL.

rtnl_lock() is a "Big Kernel Lock" in the networking slow path and
serialised all rtnetlink requests until 4.13.

Since RTNL_FLAG_DOIT_UNLOCKED and RTNL_FLAG_DUMP_UNLOCKED have been
introduced in 4.14 and 6.9, respectively, rtnetlink message handlers
are ready to be converted to RTNL-less/free.

15 out of 44 dumpit()s have been converted to RCU so far, and the
progress is pretty good.  We can now dump various major network
resources without RTNL.

12 out of 87 doit()s have been converted, but most of the converted
doit()s are also on the reader side of RTNL; their message types are
RTM_GET*.

So, most of RTM_(NEW|DEL|SET)* operations are still serialised by RTNL.

For example, one of our services creates 2K netns and a small number
of network interfaces in each netns that require too many writer-side
rtnetlink requests, and setting up a single host takes 10+ minutes.

RTNL is still a huge pain for network configuration paths, and we need
more granular locking, given converting all doit()s would be unfeasible.

Actually, most RTNL users do not need to freeze multiple netns, and such
users can be protected by per-netns RTNL mutex.  The exceptions would be
RTM_NEWLINK, RTM_DELLINK, and RTM_SETLINK.  (See [0] and [1])

This series is the first step of the per-netns RTNL conversion that
gradually replaces rtnl_lock() with rtnl_net_lock(net) under
CONFIG_DEBUG_NET_SMALL_RTNL.

[0]: https://netdev.bots.linux.dev/netconf/2024/index.html
[1]: https://lpc.events/event/18/contributions/1959/
====================

Link: https://patch.msgid.link/20241004221031.77743-1-kuniyu@amazon.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents f178812d 03fa5348

include/linux/rtnetlink.h

+62 −7
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ 
#include <linux/netdevice.h>

#include <linux/wait.h>

#include <linux/refcount.h>

#include <linux/cleanup.h>

#include <uapi/linux/rtnetlink.h>



extern int rtnetlink_send(struct sk_buff *skb, struct net *net, u32 pid, u32 group, int echo);
@@ -47,13 +46,15 @@ extern int rtnl_is_locked(void); 
extern int rtnl_lock_killable(void);

extern bool refcount_dec_and_rtnl_lock(refcount_t *r);



DEFINE_LOCK_GUARD_0(rtnl, rtnl_lock(), rtnl_unlock())



extern wait_queue_head_t netdev_unregistering_wq;

extern atomic_t dev_unreg_count;

extern struct rw_semaphore pernet_ops_rwsem;

extern struct rw_semaphore net_rwsem;



#define ASSERT_RTNL() \

	WARN_ONCE(!rtnl_is_locked(), \

		  "RTNL: assertion failed at %s (%d)\n", __FILE__,  __LINE__)



#ifdef CONFIG_PROVE_LOCKING

extern bool lockdep_rtnl_is_held(void);

#else
@@ -95,6 +96,64 @@ static inline bool lockdep_rtnl_is_held(void) 
#define rcu_replace_pointer_rtnl(rp, p)			\

	rcu_replace_pointer(rp, p, lockdep_rtnl_is_held())



#ifdef CONFIG_DEBUG_NET_SMALL_RTNL

void __rtnl_net_lock(struct net *net);

void __rtnl_net_unlock(struct net *net);

void rtnl_net_lock(struct net *net);

void rtnl_net_unlock(struct net *net);

int rtnl_net_lock_cmp_fn(const struct lockdep_map *a, const struct lockdep_map *b);



bool rtnl_net_is_locked(struct net *net);



#define ASSERT_RTNL_NET(net)						\

	WARN_ONCE(!rtnl_net_is_locked(net),				\

		  "RTNL_NET: assertion failed at %s (%d)\n",		\

		  __FILE__,  __LINE__)



bool lockdep_rtnl_net_is_held(struct net *net);



#define rcu_dereference_rtnl_net(net, p)				\

	rcu_dereference_check(p, lockdep_rtnl_net_is_held(net))

#define rtnl_net_dereference(net, p)					\

	rcu_dereference_protected(p, lockdep_rtnl_net_is_held(net))

#define rcu_replace_pointer_rtnl_net(net, rp, p)			\

	rcu_replace_pointer(rp, p, lockdep_rtnl_net_is_held(net))

#else

static inline void __rtnl_net_lock(struct net *net) {}

static inline void __rtnl_net_unlock(struct net *net) {}



static inline void rtnl_net_lock(struct net *net)

{

	rtnl_lock();

}



static inline void rtnl_net_unlock(struct net *net)

{

	rtnl_unlock();

}



static inline void ASSERT_RTNL_NET(struct net *net)

{

	ASSERT_RTNL();

}



static inline void *rcu_dereference_rtnl_net(struct net *net, void *p)

{

	return rcu_dereference_rtnl(p);

}



static inline void *rtnl_net_dereference(struct net *net, void *p)

{

	return rtnl_dereference(p);

}



static inline void *rcu_replace_pointer_rtnl_net(struct net *net,

						 void *rp, void *p)

{

	return rcu_replace_pointer_rtnl(rp, p);

}

#endif



static inline struct netdev_queue *dev_ingress_queue(struct net_device *dev)

{

	return rtnl_dereference(dev->ingress_queue);
@@ -122,10 +181,6 @@ void rtnetlink_init(void); 
void __rtnl_unlock(void);

void rtnl_kfree_skbs(struct sk_buff *head, struct sk_buff *tail);



#define ASSERT_RTNL() \

	WARN_ONCE(!rtnl_is_locked(), \

		  "RTNL: assertion failed at %s (%d)\n", __FILE__,  __LINE__)



extern int ndo_dflt_fdb_dump(struct sk_buff *skb,

			     struct netlink_callback *cb,

			     struct net_device *dev,

include/net/net_namespace.h

+4 −0
Original line number Diff line number Diff line
@@ -188,6 +188,10 @@ struct net { 
#if IS_ENABLED(CONFIG_SMC)

	struct netns_smc	smc;

#endif

#ifdef CONFIG_DEBUG_NET_SMALL_RTNL

	/* Move to a better place when the config guard is removed. */

	struct mutex		rtnl_mutex;

#endif

} __randomize_layout;



#include <linux/seq_file_net.h>

net/Kconfig.debug

+15 −0
Original line number Diff line number Diff line
@@ -24,3 +24,18 @@ config DEBUG_NET 
	help

	  Enable extra sanity checks in networking.

	  This is mostly used by fuzzers, but is safe to select.



config DEBUG_NET_SMALL_RTNL

	bool "Add extra per-netns mutex inside RTNL"

	depends on DEBUG_KERNEL && NET && LOCK_DEBUGGING_SUPPORT

	select PROVE_LOCKING

	default n

	help

	  rtnl_lock() is being replaced with rtnl_net_lock() that

	  acquires the global RTNL and a small per-netns RTNL mutex.



	  During the conversion, rtnl_net_lock() just adds an extra

	  mutex in every RTNL scope and slows down the operations.



	  Once the conversion completes, rtnl_lock() will be removed

	  and rtnetlink will gain per-netns scalability.

net/core/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -45,3 +45,4 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o 
obj-$(CONFIG_OF)	+= of_net.o

obj-$(CONFIG_NET_TEST) += net_test.o

obj-$(CONFIG_NET_DEVMEM) += devmem.o

obj-$(CONFIG_DEBUG_NET_SMALL_RTNL) += rtnl_net_debug.o

net/core/net_namespace.c

+6 −0
Original line number Diff line number Diff line
@@ -334,6 +334,12 @@ static __net_init void preinit_net(struct net *net, struct user_namespace *user_ 
	idr_init(&net->netns_ids);

	spin_lock_init(&net->nsid_lock);

	mutex_init(&net->ipv4.ra_mutex);



#ifdef CONFIG_DEBUG_NET_SMALL_RTNL

	mutex_init(&net->rtnl_mutex);

	lock_set_cmp_fn(&net->rtnl_mutex, rtnl_net_lock_cmp_fn, NULL);

#endif



	preinit_net_sysctl(net);

}