Commit 0667db14 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: refactor io_req_complete_post() 



Make io_req_complete_post() to push all IORING_SETUP_IOPOLL requests
to task_work, it's much cleaner and should normally happen. We couldn't
do it before because there was a possibility of looping in

complete_post() -> tw -> complete_post() -> ...

Also, unexport the function and inline __io_req_complete_post().

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Tested-by: default avatarMing Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/ea19c032ace3e0dd96ac4d991a063b0188037014.1710799188.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 23fbdde6

io_uring/io_uring.c

+11 −18
Original line number Diff line number Diff line
@@ -920,11 +920,21 @@ bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags) 
	return posted;

}



static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)

static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)

{

	struct io_ring_ctx *ctx = req->ctx;

	struct io_rsrc_node *rsrc_node = NULL;



	/*

	 * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires

	 * the submitter task context, IOPOLL protects with uring_lock.

	 */

	if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL)) {

		req->io_task_work.func = io_req_task_complete;

		io_req_task_work_add(req);

		return;

	}



	io_cq_lock(ctx);

	if (!(req->flags & REQ_F_CQE_SKIP)) {

		if (!io_fill_cqe_req(ctx, req))
@@ -968,23 +978,6 @@ static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) 
	}

}



void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)

{

	struct io_ring_ctx *ctx = req->ctx;



	if (ctx->task_complete) {

		req->io_task_work.func = io_req_task_complete;

		io_req_task_work_add(req);

	} else if (!(issue_flags & IO_URING_F_UNLOCKED) ||

		   !(ctx->flags & IORING_SETUP_IOPOLL)) {

		__io_req_complete_post(req, issue_flags);

	} else {

		mutex_lock(&ctx->uring_lock);

		__io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED);

		mutex_unlock(&ctx->uring_lock);

	}

}



void io_req_defer_failed(struct io_kiocb *req, s32 res)

	__must_hold(&ctx->uring_lock)

{

io_uring/io_uring.h

+0 −1
Original line number Diff line number Diff line
@@ -65,7 +65,6 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow); 
void io_req_cqe_overflow(struct io_kiocb *req);

int io_run_task_work_sig(struct io_ring_ctx *ctx);

void io_req_defer_failed(struct io_kiocb *req, s32 res);

void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags);

bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);

bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags);

void __io_commit_cqring_flush(struct io_ring_ctx *ctx);