Commit b4e8ae5c authored by Stefan Roesch's avatar Stefan Roesch Committed by Jakub Kicinski
Browse files

net: add napi_busy_loop_rcu() 



This adds the napi_busy_loop_rcu() function. This function assumes that
the calling function is already holding the rcu read lock and
napi_busy_loop() does not need to take the rcu read lock. Add a
NAPI_F_NO_SCHED flag, which tells __napi_busy_loop() to abort if we
need to reschedule rather than drop the RCU read lock and reschedule.

Signed-off-by: default avatarStefan Roesch <shr@devkernel.io>
Link: https://lore.kernel.org/r/20230608163839.2891748-3-shr@devkernel.io


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 13d381b4

include/net/busy_poll.h

+4 −0
Original line number Diff line number Diff line
@@ -48,6 +48,10 @@ void napi_busy_loop(unsigned int napi_id, 
		    bool (*loop_end)(void *, unsigned long),

		    void *loop_end_arg, bool prefer_busy_poll, u16 budget);



void napi_busy_loop_rcu(unsigned int napi_id,

			bool (*loop_end)(void *, unsigned long),

			void *loop_end_arg, bool prefer_busy_poll, u16 budget);



#else /* CONFIG_NET_RX_BUSY_POLL */

static inline unsigned long net_busy_loop_on(void)

{

net/core/dev.c

+15 −0
Original line number Diff line number Diff line
@@ -6179,6 +6179,7 @@ static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule) 


enum {

	NAPI_F_PREFER_BUSY_POLL	= 1,

	NAPI_F_END_ON_RESCHED	= 2,

};



static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
@@ -6285,6 +6286,8 @@ static void __napi_busy_loop(unsigned int napi_id, 
			break;



		if (unlikely(need_resched())) {

			if (flags & NAPI_F_END_ON_RESCHED)

				break;

			if (napi_poll)

				busy_poll_stop(napi, have_poll_lock, flags, budget);

			if (!IS_ENABLED(CONFIG_PREEMPT_RT))
@@ -6304,6 +6307,18 @@ static void __napi_busy_loop(unsigned int napi_id, 
		preempt_enable();

}



void napi_busy_loop_rcu(unsigned int napi_id,

			bool (*loop_end)(void *, unsigned long),

			void *loop_end_arg, bool prefer_busy_poll, u16 budget)

{

	unsigned flags = NAPI_F_END_ON_RESCHED;



	if (prefer_busy_poll)

		flags |= NAPI_F_PREFER_BUSY_POLL;



	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);

}



void napi_busy_loop(unsigned int napi_id,

		    bool (*loop_end)(void *, unsigned long),

		    void *loop_end_arg, bool prefer_busy_poll, u16 budget)