Commit ff183d42 authored by Stefan Roesch's avatar Stefan Roesch Committed by Jens Axboe
Browse files

io-uring: add sqpoll support for napi busy poll 



This adds the sqpoll support to the io-uring napi.

Signed-off-by: default avatarStefan Roesch <shr@devkernel.io>
Suggested-by: default avatarOlivier Langlois <olivier@trillion01.com>
Link: https://lore.kernel.org/r/20230608163839.2891748-6-shr@devkernel.io


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8d0c12a8

io_uring/napi.c

+24 −0
Original line number Diff line number Diff line
@@ -252,4 +252,28 @@ void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) 
		io_napi_blocking_busy_loop(ctx, iowq);

}



/*

 * io_napi_sqpoll_busy_poll() - busy poll loop for sqpoll

 * @ctx: pointer to io-uring context structure

 *

 * Splice of the napi list and execute the napi busy poll loop.

 */

int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx)

{

	LIST_HEAD(napi_list);

	bool is_stale = false;



	if (!READ_ONCE(ctx->napi_busy_poll_to))

		return 0;

	if (list_empty_careful(&ctx->napi_list))

		return 0;



	rcu_read_lock();

	is_stale = __io_napi_do_busy_loop(ctx, NULL);

	rcu_read_unlock();



	io_napi_remove_stale(ctx, is_stale);

	return 1;

}



#endif

io_uring/napi.h

+5 −1
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock); 
void __io_napi_adjust_timeout(struct io_ring_ctx *ctx,

		struct io_wait_queue *iowq, struct timespec64 *ts);

void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq);

int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx);



static inline bool io_napi(struct io_ring_ctx *ctx)

{
@@ -83,7 +84,10 @@ static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, 
				     struct io_wait_queue *iowq)

{

}



static inline int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx)

{

	return 0;

}

#endif /* CONFIG_NET_RX_BUSY_POLL */



#endif

io_uring/sqpoll.c

+4 −0
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
#include <uapi/linux/io_uring.h>



#include "io_uring.h"

#include "napi.h"

#include "sqpoll.h"



#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
@@ -194,6 +195,9 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) 
			ret = io_submit_sqes(ctx, to_submit);

		mutex_unlock(&ctx->uring_lock);



		if (io_napi(ctx))

			ret += io_napi_sqpoll_busy_poll(ctx);



		if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))

			wake_up(&ctx->sqo_sq_wait);

		if (creds)