Commit 7be20254 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: unify task_work cancelation checks 



Rather than do per-tw checking, which needs to dip into the task_struct
for checking flags, do it upfront before running task_work. This places
a 'cancel' member in io_tw_token_t, which is assigned before running
task_work for that given ctx.

This is both more efficient in doing it upfront rather than for every
task_work, and it means that io_should_terminate_tw() can be made
private in io_uring.c rather than need to be called by various
callbacks of task_work.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 211ddde0

include/linux/io_uring_types.h

+1 −0
Original line number Diff line number Diff line
@@ -474,6 +474,7 @@ struct io_ring_ctx { 
 * ONLY core io_uring.c should instantiate this struct.

 */

struct io_tw_state {

	bool cancel;

};

/* Alias to use in code that doesn't instantiate struct io_tw_state */

typedef struct io_tw_state io_tw_token_t;

io_uring/io_uring.c

+20 −7
Original line number Diff line number Diff line
@@ -265,6 +265,20 @@ static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref) 
	complete(&ctx->ref_comp);

}



/*

 * Terminate the request if either of these conditions are true:

 *

 * 1) It's being executed by the original task, but that task is marked

 *    with PF_EXITING as it's exiting.

 * 2) PF_KTHREAD is set, in which case the invoker of the task_work is

 *    our fallback task_work.

 * 3) The ring has been closed and is going away.

 */

static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)

{

	return (current->flags & (PF_EXITING | PF_KTHREAD)) || percpu_ref_is_dying(&ctx->refs);

}



static __cold void io_fallback_req_func(struct work_struct *work)

{

	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
@@ -275,8 +289,10 @@ static __cold void io_fallback_req_func(struct work_struct *work) 


	percpu_ref_get(&ctx->refs);

	mutex_lock(&ctx->uring_lock);

	llist_for_each_entry_safe(req, tmp, node, io_task_work.node)

	llist_for_each_entry_safe(req, tmp, node, io_task_work.node) {

		ts.cancel = io_should_terminate_tw(req->ctx);

		req->io_task_work.func(req, ts);

	}

	io_submit_flush_completions(ctx);

	mutex_unlock(&ctx->uring_lock);

	percpu_ref_put(&ctx->refs);
@@ -1147,6 +1163,7 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, 
			ctx = req->ctx;

			mutex_lock(&ctx->uring_lock);

			percpu_ref_get(&ctx->refs);

			ts.cancel = io_should_terminate_tw(ctx);

		}

		INDIRECT_CALL_2(req->io_task_work.func,

				io_poll_task_func, io_req_rw_complete,
@@ -1205,11 +1222,6 @@ struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, 
{

	struct llist_node *node;



	if (unlikely(current->flags & PF_EXITING)) {

		io_fallback_tw(tctx, true);

		return NULL;

	}



	node = llist_del_all(&tctx->task_list);

	if (node) {

		node = llist_reverse_order(node);
@@ -1399,6 +1411,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw, 
	if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)

		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);

again:

	tw.cancel = io_should_terminate_tw(ctx);

	min_events -= ret;

	ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);

	if (ctx->retry_llist.first)
@@ -1458,7 +1471,7 @@ void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw) 
	struct io_ring_ctx *ctx = req->ctx;



	io_tw_lock(ctx, tw);

	if (unlikely(io_should_terminate_tw(ctx)))

	if (unlikely(tw.cancel))

		io_req_defer_failed(req, -EFAULT);

	else if (req->flags & REQ_F_FORCE_ASYNC)

		io_queue_iowq(req);

io_uring/io_uring.h

+0 −13
Original line number Diff line number Diff line
@@ -558,19 +558,6 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) 
		      ctx->submitter_task == current);

}



/*

 * Terminate the request if either of these conditions are true:

 *

 * 1) It's being executed by the original task, but that task is marked

 *    with PF_EXITING as it's exiting.

 * 2) PF_KTHREAD is set, in which case the invoker of the task_work is

 *    our fallback task_work.

 */

static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)

{

	return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs);

}



static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res)

{

	io_req_set_res(req, res, 0);

io_uring/poll.c

+1 −1
Original line number Diff line number Diff line
@@ -224,7 +224,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) 
{

	int v;



	if (unlikely(io_should_terminate_tw(req->ctx)))

	if (unlikely(tw.cancel))

		return -ECANCELED;



	do {

io_uring/timeout.c

+1 −1
Original line number Diff line number Diff line
@@ -324,7 +324,7 @@ static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw) 
	int ret;



	if (prev) {

		if (!io_should_terminate_tw(req->ctx)) {

		if (!tw.cancel) {

			struct io_cancel_data cd = {

				.ctx		= req->ctx,

				.data		= prev->cqe.user_data,