Commit dac6a0ea authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: ensure iopoll runs local task work as well 



Combine the two checks we have for task_work running and whether or not
we need to shuffle the mutex into one, so we unify how task_work is run
in the iopoll loop. This helps ensure that local task_work is run when
needed, and also optimizes that path to avoid a mutex shuffle if it's
not needed.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8ac5d85a

io_uring/io_uring.c

+20 −19
Original line number Diff line number Diff line
@@ -1428,12 +1428,18 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) 
		 * forever, while the workqueue is stuck trying to acquire the

		 * very same mutex.

		 */

		if (wq_list_empty(&ctx->iopoll_list)) {

		if (wq_list_empty(&ctx->iopoll_list) ||

		    io_task_work_pending(ctx)) {

			if (!llist_empty(&ctx->work_llist))

				__io_run_local_work(ctx, true);

			if (task_work_pending(current) ||

			    wq_list_empty(&ctx->iopoll_list)) {

				u32 tail = ctx->cached_cq_tail;



				mutex_unlock(&ctx->uring_lock);

			ret = io_run_task_work_ctx(ctx);

				ret = io_run_task_work();

				mutex_lock(&ctx->uring_lock);



				if (ret < 0)

					break;
@@ -1442,11 +1448,6 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) 
				    wq_list_empty(&ctx->iopoll_list))

					break;

			}



		if (task_work_pending(current)) {

			mutex_unlock(&ctx->uring_lock);

			io_run_task_work();

			mutex_lock(&ctx->uring_lock);

		}

		ret = io_do_iopoll(ctx, !min);

		if (ret < 0)

io_uring/io_uring.h

+6 −0
Original line number Diff line number Diff line
@@ -236,6 +236,12 @@ static inline int io_run_task_work(void) 
	return 0;

}



static inline bool io_task_work_pending(struct io_ring_ctx *ctx)

{

	return test_thread_flag(TIF_NOTIFY_SIGNAL) ||

		!wq_list_empty(&ctx->work_llist);

}



static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)

{

	int ret = 0;