Commit 428f1382 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring/napi: ensure napi polling is aborted when work is available 



While testing io_uring NAPI with DEFER_TASKRUN, I ran into slowdowns and
stalls in packet delivery. Turns out that while
io_napi_busy_loop_should_end() aborts appropriately on regular
task_work, it does not abort if we have local task_work pending.

Move io_has_work() into the private io_uring.h header, and gate whether
we should continue polling on that as well. This makes NAPI polling on
send/receive work as designed with IORING_SETUP_DEFER_TASKRUN as well.

Fixes: 8d0c12a8 ("io-uring: add napi busy poll support")
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3fb1764c

io_uring/io_uring.c

+0 −11
Original line number Diff line number Diff line
@@ -122,11 +122,6 @@ 
#define IO_COMPL_BATCH			32

#define IO_REQ_ALLOC_BATCH		8



enum {

	IO_CHECK_CQ_OVERFLOW_BIT,

	IO_CHECK_CQ_DROPPED_BIT,

};



struct io_defer_entry {

	struct list_head	list;

	struct io_kiocb		*req;
@@ -2479,12 +2474,6 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) 
	return ret;

}



static inline bool io_has_work(struct io_ring_ctx *ctx)

{

	return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||

	       !llist_empty(&ctx->work_llist);

}



static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,

			    int wake_flags, void *key)

{

io_uring/io_uring.h

+11 −0
Original line number Diff line number Diff line
@@ -448,4 +448,15 @@ static inline bool io_file_can_poll(struct io_kiocb *req) 
	}

	return false;

}



enum {

	IO_CHECK_CQ_OVERFLOW_BIT,

	IO_CHECK_CQ_DROPPED_BIT,

};



static inline bool io_has_work(struct io_ring_ctx *ctx)

{

	return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||

	       !llist_empty(&ctx->work_llist);

}

#endif

io_uring/napi.c

+1 −1
Original line number Diff line number Diff line
@@ -122,7 +122,7 @@ static bool io_napi_busy_loop_should_end(void *data, 


	if (signal_pending(current))

		return true;

	if (io_should_wake(iowq))

	if (io_should_wake(iowq) || io_has_work(iowq->ctx))

		return true;

	if (io_napi_busy_loop_timeout(start_time, iowq->napi_busy_poll_to))

		return true;