Commit adaad279 authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge branch 'for-io_uring-add-napi-busy-polling-support' of... 

Merge branch 'for-io_uring-add-napi-busy-polling-support' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux into for-6.9/io_uring

Pull netdev side of the io_uring napi support.

* 'for-io_uring-add-napi-busy-polling-support' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux:
  net: add napi_busy_loop_rcu()
  net: split off __napi_busy_poll from napi_busy_poll
parents b4bb1900 b4e8ae5c

include/net/busy_poll.h

+4 −0
Original line number Diff line number Diff line
@@ -48,6 +48,10 @@ void napi_busy_loop(unsigned int napi_id, 
		    bool (*loop_end)(void *, unsigned long),

		    void *loop_end_arg, bool prefer_busy_poll, u16 budget);



void napi_busy_loop_rcu(unsigned int napi_id,

			bool (*loop_end)(void *, unsigned long),

			void *loop_end_arg, bool prefer_busy_poll, u16 budget);



#else /* CONFIG_NET_RX_BUSY_POLL */

static inline unsigned long net_busy_loop_on(void)

{

net/core/dev.c

+43 −14
Original line number Diff line number Diff line
@@ -6177,8 +6177,13 @@ static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule) 
	clear_bit(NAPI_STATE_SCHED, &napi->state);

}



static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,

			   u16 budget)

enum {

	NAPI_F_PREFER_BUSY_POLL	= 1,

	NAPI_F_END_ON_RESCHED	= 2,

};



static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,

			   unsigned flags, u16 budget)

{

	bool skip_schedule = false;

	unsigned long timeout;
@@ -6198,7 +6203,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool 


	local_bh_disable();



	if (prefer_busy_poll) {

	if (flags & NAPI_F_PREFER_BUSY_POLL) {

		napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);

		timeout = READ_ONCE(napi->dev->gro_flush_timeout);

		if (napi->defer_hard_irqs_count && timeout) {
@@ -6222,23 +6227,23 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool 
	local_bh_enable();

}



void napi_busy_loop(unsigned int napi_id,

static void __napi_busy_loop(unsigned int napi_id,

		      bool (*loop_end)(void *, unsigned long),

		    void *loop_end_arg, bool prefer_busy_poll, u16 budget)

		      void *loop_end_arg, unsigned flags, u16 budget)

{

	unsigned long start_time = loop_end ? busy_loop_current_time() : 0;

	int (*napi_poll)(struct napi_struct *napi, int budget);

	void *have_poll_lock = NULL;

	struct napi_struct *napi;



	WARN_ON_ONCE(!rcu_read_lock_held());



restart:

	napi_poll = NULL;



	rcu_read_lock();



	napi = napi_by_id(napi_id);

	if (!napi)

		goto out;

		return;



	if (!IS_ENABLED(CONFIG_PREEMPT_RT))

		preempt_disable();
@@ -6254,14 +6259,14 @@ void napi_busy_loop(unsigned int napi_id, 
			 */

			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |

				   NAPIF_STATE_IN_BUSY_POLL)) {

				if (prefer_busy_poll)

				if (flags & NAPI_F_PREFER_BUSY_POLL)

					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);

				goto count;

			}

			if (cmpxchg(&napi->state, val,

				    val | NAPIF_STATE_IN_BUSY_POLL |

					  NAPIF_STATE_SCHED) != val) {

				if (prefer_busy_poll)

				if (flags & NAPI_F_PREFER_BUSY_POLL)

					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);

				goto count;

			}
@@ -6281,12 +6286,15 @@ void napi_busy_loop(unsigned int napi_id, 
			break;



		if (unlikely(need_resched())) {

			if (flags & NAPI_F_END_ON_RESCHED)

				break;

			if (napi_poll)

				busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);

				busy_poll_stop(napi, have_poll_lock, flags, budget);

			if (!IS_ENABLED(CONFIG_PREEMPT_RT))

				preempt_enable();

			rcu_read_unlock();

			cond_resched();

			rcu_read_lock();

			if (loop_end(loop_end_arg, start_time))

				return;

			goto restart;
@@ -6294,10 +6302,31 @@ void napi_busy_loop(unsigned int napi_id, 
		cpu_relax();

	}

	if (napi_poll)

		busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);

		busy_poll_stop(napi, have_poll_lock, flags, budget);

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))

		preempt_enable();

out:

}



void napi_busy_loop_rcu(unsigned int napi_id,

			bool (*loop_end)(void *, unsigned long),

			void *loop_end_arg, bool prefer_busy_poll, u16 budget)

{

	unsigned flags = NAPI_F_END_ON_RESCHED;



	if (prefer_busy_poll)

		flags |= NAPI_F_PREFER_BUSY_POLL;



	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);

}



void napi_busy_loop(unsigned int napi_id,

		    bool (*loop_end)(void *, unsigned long),

		    void *loop_end_arg, bool prefer_busy_poll, u16 budget)

{

	unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;



	rcu_read_lock();

	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);

	rcu_read_unlock();

}

EXPORT_SYMBOL(napi_busy_loop);