Commit 1200097f authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

net: call skb_defer_free_flush() from __napi_busy_loop() 



skb_defer_free_flush() is currently called from net_rx_action()
and napi_threaded_poll().

We should also call it from __napi_busy_loop() otherwise
there is the risk the percpu queue can grow until an IPI
is forced from skb_attempt_defer_free() adding a latency spike.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Cc: Samiullah Khawaja <skhawaja@google.com>
Acked-by: default avatarStanislav Fomichev <sdf@google.com>
Reviewed-by: default avatarJiri Pirko <jiri@nvidia.com>
Link: https://lore.kernel.org/r/20240227210105.3815474-1-edumazet@google.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 99123622

net/core/dev.c

+22 −21
Original line number Diff line number Diff line
@@ -6173,6 +6173,27 @@ struct napi_struct *napi_by_id(unsigned int napi_id) 
	return NULL;

}



static void skb_defer_free_flush(struct softnet_data *sd)

{

	struct sk_buff *skb, *next;



	/* Paired with WRITE_ONCE() in skb_attempt_defer_free() */

	if (!READ_ONCE(sd->defer_list))

		return;



	spin_lock(&sd->defer_lock);

	skb = sd->defer_list;

	sd->defer_list = NULL;

	sd->defer_count = 0;

	spin_unlock(&sd->defer_lock);



	while (skb != NULL) {

		next = skb->next;

		napi_consume_skb(skb, 1);

		skb = next;

	}

}



#if defined(CONFIG_NET_RX_BUSY_POLL)



static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
@@ -6297,6 +6318,7 @@ static void __napi_busy_loop(unsigned int napi_id, 
		if (work > 0)

			__NET_ADD_STATS(dev_net(napi->dev),

					LINUX_MIB_BUSYPOLLRXPACKETS, work);

		skb_defer_free_flush(this_cpu_ptr(&softnet_data));

		local_bh_enable();



		if (!loop_end || loop_end(loop_end_arg, start_time))
@@ -6726,27 +6748,6 @@ static int napi_thread_wait(struct napi_struct *napi) 
	return -1;

}



static void skb_defer_free_flush(struct softnet_data *sd)

{

	struct sk_buff *skb, *next;



	/* Paired with WRITE_ONCE() in skb_attempt_defer_free() */

	if (!READ_ONCE(sd->defer_list))

		return;



	spin_lock(&sd->defer_lock);

	skb = sd->defer_list;

	sd->defer_list = NULL;

	sd->defer_count = 0;

	spin_unlock(&sd->defer_lock);



	while (skb != NULL) {

		next = skb->next;

		napi_consume_skb(skb, 1);

		skb = next;

	}

}



static int napi_threaded_poll(void *data)

{

	struct napi_struct *napi = data;