Commit 4c032725 authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'net-cover-more-per-cpu-storage-with-local-nested-bh-locking' 

Sebastian Andrzej Siewior says:

====================
net: Cover more per-CPU storage with local nested BH locking

I was looking at the build-time defined per-CPU variables in net/ and
added the needed local-BH-locks in order to be able to remove the
current per-CPU lock in local_bh_disable() on PREMPT_RT.

The work is not yet complete, I just wanted to post what I have so far
instead of sitting on it.

v3: https://lore.kernel.org/all/20250430124758.1159480-1-bigeasy@linutronix.de/
v2: https://lore.kernel.org/all/20250414160754.503321-1-bigeasy@linutronix.de
v1: https://lore.kernel.org/all/20250309144653.825351-1-bigeasy@linutronix.de
====================

Link: https://patch.msgid.link/20250512092736.229935-1-bigeasy@linutronix.de


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 67fa7564 c50d295c

include/linux/netdevice.h

+6 −1
Original line number Diff line number Diff line
@@ -3503,7 +3503,12 @@ struct softnet_data { 
};



DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);

DECLARE_PER_CPU(struct page_pool *, system_page_pool);



struct page_pool_bh {

	struct page_pool *pool;

	local_lock_t bh_lock;

};

DECLARE_PER_CPU(struct page_pool_bh, system_page_pool);



#ifndef CONFIG_PREEMPT_RT

static inline int dev_recursion_level(void)

include/linux/netdevice_xmit.h

+3 −0
Original line number Diff line number Diff line
@@ -8,6 +8,9 @@ struct netdev_xmit { 
#ifdef CONFIG_NET_EGRESS

	u8  skip_txqueue;

#endif

#if IS_ENABLED(CONFIG_NET_ACT_MIRRED)

	u8 sched_mirred_nest;

#endif

};



#endif

net/core/dev.c

+10 −5
Original line number Diff line number Diff line
@@ -462,7 +462,9 @@ EXPORT_PER_CPU_SYMBOL(softnet_data); 
 * PP consumers must pay attention to run APIs in the appropriate context

 * (e.g. NAPI context).

 */

DEFINE_PER_CPU(struct page_pool *, system_page_pool);

DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = {

	.bh_lock = INIT_LOCAL_LOCK(bh_lock),

};



#ifdef CONFIG_LOCKDEP

/*
@@ -5322,7 +5324,10 @@ netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog) 
	struct sk_buff *skb = *pskb;

	int err, hroom, troom;



	if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog))

	local_lock_nested_bh(&system_page_pool.bh_lock);

	err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog);

	local_unlock_nested_bh(&system_page_pool.bh_lock);

	if (!err)

		return 0;



	/* In case we have to go down the path and also linearize,
@@ -12712,7 +12717,7 @@ static int net_page_pool_create(int cpuid) 
		return err;

	}



	per_cpu(system_page_pool, cpuid) = pp_ptr;

	per_cpu(system_page_pool.pool, cpuid) = pp_ptr;

#endif

	return 0;

}
@@ -12842,13 +12847,13 @@ static int __init net_dev_init(void) 
		for_each_possible_cpu(i) {

			struct page_pool *pp_ptr;



			pp_ptr = per_cpu(system_page_pool, i);

			pp_ptr = per_cpu(system_page_pool.pool, i);

			if (!pp_ptr)

				continue;



			xdp_unreg_page_pool(pp_ptr);

			page_pool_destroy(pp_ptr);

			per_cpu(system_page_pool, i) = NULL;

			per_cpu(system_page_pool.pool, i) = NULL;

		}

	}

net/core/dst_cache.c

+27 −3
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
struct dst_cache_pcpu {

	unsigned long refresh_ts;

	struct dst_entry *dst;

	local_lock_t bh_lock;

	u32 cookie;

	union {

		struct in_addr in_saddr;
@@ -65,10 +66,15 @@ static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, 


struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)

{

	struct dst_entry *dst;



	if (!dst_cache->cache)

		return NULL;



	return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));

	local_lock_nested_bh(&dst_cache->cache->bh_lock);

	dst = dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));

	local_unlock_nested_bh(&dst_cache->cache->bh_lock);

	return dst;

}

EXPORT_SYMBOL_GPL(dst_cache_get);
@@ -80,12 +86,16 @@ struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) 
	if (!dst_cache->cache)

		return NULL;



	local_lock_nested_bh(&dst_cache->cache->bh_lock);

	idst = this_cpu_ptr(dst_cache->cache);

	dst = dst_cache_per_cpu_get(dst_cache, idst);

	if (!dst)

	if (!dst) {

		local_unlock_nested_bh(&dst_cache->cache->bh_lock);

		return NULL;

	}



	*saddr = idst->in_saddr.s_addr;

	local_unlock_nested_bh(&dst_cache->cache->bh_lock);

	return dst_rtable(dst);

}

EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
@@ -98,9 +108,11 @@ void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, 
	if (!dst_cache->cache)

		return;



	local_lock_nested_bh(&dst_cache->cache->bh_lock);

	idst = this_cpu_ptr(dst_cache->cache);

	dst_cache_per_cpu_dst_set(idst, dst, 0);

	idst->in_saddr.s_addr = saddr;

	local_unlock_nested_bh(&dst_cache->cache->bh_lock);

}

EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
@@ -113,10 +125,13 @@ void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, 
	if (!dst_cache->cache)

		return;



	local_lock_nested_bh(&dst_cache->cache->bh_lock);



	idst = this_cpu_ptr(dst_cache->cache);

	dst_cache_per_cpu_dst_set(idst, dst,

				  rt6_get_cookie(dst_rt6_info(dst)));

	idst->in6_saddr = *saddr;

	local_unlock_nested_bh(&dst_cache->cache->bh_lock);

}

EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
@@ -129,12 +144,17 @@ struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, 
	if (!dst_cache->cache)

		return NULL;



	local_lock_nested_bh(&dst_cache->cache->bh_lock);



	idst = this_cpu_ptr(dst_cache->cache);

	dst = dst_cache_per_cpu_get(dst_cache, idst);

	if (!dst)

	if (!dst) {

		local_unlock_nested_bh(&dst_cache->cache->bh_lock);

		return NULL;

	}



	*saddr = idst->in6_saddr;

	local_unlock_nested_bh(&dst_cache->cache->bh_lock);

	return dst;

}

EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
@@ -142,10 +162,14 @@ EXPORT_SYMBOL_GPL(dst_cache_get_ip6); 


int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)

{

	unsigned int i;



	dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,

					    gfp | __GFP_ZERO);

	if (!dst_cache->cache)

		return -ENOMEM;

	for_each_possible_cpu(i)

		local_lock_init(&per_cpu_ptr(dst_cache->cache, i)->bh_lock);



	dst_cache_reset(dst_cache);

	return 0;

net/core/page_pool.c

+4 −0
Original line number Diff line number Diff line
@@ -839,6 +839,10 @@ static bool page_pool_napi_local(const struct page_pool *pool) 
	const struct napi_struct *napi;

	u32 cpuid;



	/* On PREEMPT_RT the softirq can be preempted by the consumer */

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		return false;



	if (unlikely(!in_softirq()))

		return false;