Commit 255d75ef authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'xsk-minor-optimizations-around-locks' 

Jason Xing says:

====================
xsk: minor optimizations around locks

Two optimizations regarding xsk_tx_list_lock and cq_lock can yield a
performance increase because of avoiding disabling and enabling
interrupts frequently.
====================

Link: https://patch.msgid.link/20251030000646.18859-1-kerneljasonxing@gmail.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 27cb3de7 30ed05ad

include/net/xsk_buff_pool.h

+9 −4
Original line number Diff line number Diff line
@@ -85,11 +85,16 @@ struct xsk_buff_pool { 
	bool unaligned;

	bool tx_sw_csum;

	void *addrs;

	/* Mutual exclusion of the completion ring in the SKB mode. Two cases to protect:

	 * NAPI TX thread and sendmsg error paths in the SKB destructor callback and when

	 * sockets share a single cq when the same netdev and queue id is shared.

	/* Mutual exclusion of the completion ring in the SKB mode.

	 * Protect: NAPI TX thread and sendmsg error paths in the SKB

	 * destructor callback.

	 */

	spinlock_t cq_lock;

	spinlock_t cq_prod_lock;

	/* Mutual exclusion of the completion ring in the SKB mode.

	 * Protect: when sockets share a single cq when the same netdev

	 * and queue id is shared.

	 */

	spinlock_t cq_cached_prod_lock;

	struct xdp_buff_xsk *free_heads[];

};

net/xdp/xsk.c

+6 −9
Original line number Diff line number Diff line
@@ -548,12 +548,11 @@ static int xsk_wakeup(struct xdp_sock *xs, u8 flags) 


static int xsk_cq_reserve_locked(struct xsk_buff_pool *pool)

{

	unsigned long flags;

	int ret;



	spin_lock_irqsave(&pool->cq_lock, flags);

	spin_lock(&pool->cq_cached_prod_lock);

	ret = xskq_prod_reserve(pool->cq);

	spin_unlock_irqrestore(&pool->cq_lock, flags);

	spin_unlock(&pool->cq_cached_prod_lock);



	return ret;

}
@@ -566,7 +565,7 @@ static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool, 
	unsigned long flags;

	u32 idx;



	spin_lock_irqsave(&pool->cq_lock, flags);

	spin_lock_irqsave(&pool->cq_prod_lock, flags);

	idx = xskq_get_prod(pool->cq);



	xskq_prod_write_addr(pool->cq, idx,
@@ -583,16 +582,14 @@ static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool, 
		}

	}

	xskq_prod_submit_n(pool->cq, descs_processed);

	spin_unlock_irqrestore(&pool->cq_lock, flags);

	spin_unlock_irqrestore(&pool->cq_prod_lock, flags);

}



static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n)

{

	unsigned long flags;



	spin_lock_irqsave(&pool->cq_lock, flags);

	spin_lock(&pool->cq_cached_prod_lock);

	xskq_prod_cancel_n(pool->cq, n);

	spin_unlock_irqrestore(&pool->cq_lock, flags);

	spin_unlock(&pool->cq_cached_prod_lock);

}



static void xsk_inc_num_desc(struct sk_buff *skb)

net/xdp/xsk_buff_pool.c

+6 −9
Original line number Diff line number Diff line
@@ -12,26 +12,22 @@ 


void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs)

{

	unsigned long flags;



	if (!xs->tx)

		return;



	spin_lock_irqsave(&pool->xsk_tx_list_lock, flags);

	spin_lock(&pool->xsk_tx_list_lock);

	list_add_rcu(&xs->tx_list, &pool->xsk_tx_list);

	spin_unlock_irqrestore(&pool->xsk_tx_list_lock, flags);

	spin_unlock(&pool->xsk_tx_list_lock);

}



void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs)

{

	unsigned long flags;



	if (!xs->tx)

		return;



	spin_lock_irqsave(&pool->xsk_tx_list_lock, flags);

	spin_lock(&pool->xsk_tx_list_lock);

	list_del_rcu(&xs->tx_list);

	spin_unlock_irqrestore(&pool->xsk_tx_list_lock, flags);

	spin_unlock(&pool->xsk_tx_list_lock);

}



void xp_destroy(struct xsk_buff_pool *pool)
@@ -94,7 +90,8 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, 
	INIT_LIST_HEAD(&pool->xskb_list);

	INIT_LIST_HEAD(&pool->xsk_tx_list);

	spin_lock_init(&pool->xsk_tx_list_lock);

	spin_lock_init(&pool->cq_lock);

	spin_lock_init(&pool->cq_prod_lock);

	spin_lock_init(&pool->cq_cached_prod_lock);

	refcount_set(&pool->users, 1);



	pool->fq = xs->fq_tmp;