Commit b9eef339 authored by Sebastian Andrzej Siewior's avatar Sebastian Andrzej Siewior Committed by Paolo Abeni
Browse files

xdp: Use nested-BH locking for system_page_pool 



system_page_pool is a per-CPU variable and relies on disabled BH for its
locking. Without per-CPU locking in local_bh_disable() on PREEMPT_RT
this data structure requires explicit locking.

Make a struct with a page_pool member (original system_page_pool) and a
local_lock_t and use local_lock_nested_bh() for locking. This change
adds only lockdep coverage and does not alter the functional behaviour
for !PREEMPT_RT.

Cc: Andrew Lunn <andrew+netdev@lunn.ch>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Jesper Dangaard Brouer <hawk@kernel.org>
Cc: John Fastabend <john.fastabend@gmail.com>
Reviewed-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: https://patch.msgid.link/20250512092736.229935-6-bigeasy@linutronix.de


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent bc57eda6

include/linux/netdevice.h

+6 −1
Original line number Diff line number Diff line
@@ -3503,7 +3503,12 @@ struct softnet_data { 
};



DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);

DECLARE_PER_CPU(struct page_pool *, system_page_pool);



struct page_pool_bh {

	struct page_pool *pool;

	local_lock_t bh_lock;

};

DECLARE_PER_CPU(struct page_pool_bh, system_page_pool);



#ifndef CONFIG_PREEMPT_RT

static inline int dev_recursion_level(void)

net/core/dev.c

+10 −5
Original line number Diff line number Diff line
@@ -462,7 +462,9 @@ EXPORT_PER_CPU_SYMBOL(softnet_data); 
 * PP consumers must pay attention to run APIs in the appropriate context

 * (e.g. NAPI context).

 */

DEFINE_PER_CPU(struct page_pool *, system_page_pool);

DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = {

	.bh_lock = INIT_LOCAL_LOCK(bh_lock),

};



#ifdef CONFIG_LOCKDEP

/*
@@ -5322,7 +5324,10 @@ netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog) 
	struct sk_buff *skb = *pskb;

	int err, hroom, troom;



	if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog))

	local_lock_nested_bh(&system_page_pool.bh_lock);

	err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog);

	local_unlock_nested_bh(&system_page_pool.bh_lock);

	if (!err)

		return 0;



	/* In case we have to go down the path and also linearize,
@@ -12712,7 +12717,7 @@ static int net_page_pool_create(int cpuid) 
		return err;

	}



	per_cpu(system_page_pool, cpuid) = pp_ptr;

	per_cpu(system_page_pool.pool, cpuid) = pp_ptr;

#endif

	return 0;

}
@@ -12842,13 +12847,13 @@ static int __init net_dev_init(void) 
		for_each_possible_cpu(i) {

			struct page_pool *pp_ptr;



			pp_ptr = per_cpu(system_page_pool, i);

			pp_ptr = per_cpu(system_page_pool.pool, i);

			if (!pp_ptr)

				continue;



			xdp_unreg_page_pool(pp_ptr);

			page_pool_destroy(pp_ptr);

			per_cpu(system_page_pool, i) = NULL;

			per_cpu(system_page_pool.pool, i) = NULL;

		}

	}

net/core/xdp.c

+10 −5
Original line number Diff line number Diff line
@@ -739,25 +739,27 @@ static noinline bool xdp_copy_frags_from_zc(struct sk_buff *skb, 
 */

struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp)

{

	struct page_pool *pp = this_cpu_read(system_page_pool);

	const struct xdp_rxq_info *rxq = xdp->rxq;

	u32 len = xdp->data_end - xdp->data_meta;

	u32 truesize = xdp->frame_sz;

	struct sk_buff *skb;

	struct sk_buff *skb = NULL;

	struct page_pool *pp;

	int metalen;

	void *data;



	if (!IS_ENABLED(CONFIG_PAGE_POOL))

		return NULL;



	local_lock_nested_bh(&system_page_pool.bh_lock);

	pp = this_cpu_read(system_page_pool.pool);

	data = page_pool_dev_alloc_va(pp, &truesize);

	if (unlikely(!data))

		return NULL;

		goto out;



	skb = napi_build_skb(data, truesize);

	if (unlikely(!skb)) {

		page_pool_free_va(pp, data, true);

		return NULL;

		goto out;

	}



	skb_mark_for_recycle(skb);
@@ -776,13 +778,16 @@ struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp) 
	if (unlikely(xdp_buff_has_frags(xdp)) &&

	    unlikely(!xdp_copy_frags_from_zc(skb, xdp, pp))) {

		napi_consume_skb(skb, true);

		return NULL;

		skb = NULL;

		goto out;

	}



	xsk_buff_free(xdp);



	skb->protocol = eth_type_trans(skb, rxq->dev);



out:

	local_unlock_nested_bh(&system_page_pool.bh_lock);

	return skb;

}

EXPORT_SYMBOL_GPL(xdp_build_skb_from_zc);