Commit 8e5b3b89 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: remove struct io_tw_state::locked 



ctx is always locked for task_work now, so get rid of struct
io_tw_state::locked. Note I'm stopping one step before removing
io_tw_state altogether, which is not empty, because it still serves the
purpose of indicating which function is a tw callback and forcing users
not to invoke them carelessly out of a wrong context. The removal can
always be done later.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Tested-by: default avatarMing Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/e95e1ea116d0bfa54b656076e6a977bc221392a4.1710799188.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 92219afb

include/linux/io_uring_types.h

+0 −2
Original line number Diff line number Diff line
@@ -438,8 +438,6 @@ struct io_ring_ctx { 
};



struct io_tw_state {

	/* ->uring_lock is taken, callbacks can use io_tw_lock to lock it */

	bool locked;

};



enum {

io_uring/io_uring.c

+8 −23
Original line number Diff line number Diff line
@@ -247,14 +247,12 @@ static __cold void io_fallback_req_func(struct work_struct *work) 
						fallback_work.work);

	struct llist_node *node = llist_del_all(&ctx->fallback_llist);

	struct io_kiocb *req, *tmp;

	struct io_tw_state ts = { .locked = true, };

	struct io_tw_state ts = {};



	percpu_ref_get(&ctx->refs);

	mutex_lock(&ctx->uring_lock);

	llist_for_each_entry_safe(req, tmp, node, io_task_work.node)

		req->io_task_work.func(req, &ts);

	if (WARN_ON_ONCE(!ts.locked))

		return;

	io_submit_flush_completions(ctx);

	mutex_unlock(&ctx->uring_lock);

	percpu_ref_put(&ctx->refs);
@@ -1157,11 +1155,9 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts) 
		return;

	if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)

		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);

	if (ts->locked) {



	io_submit_flush_completions(ctx);

	mutex_unlock(&ctx->uring_lock);

		ts->locked = false;

	}

	percpu_ref_put(&ctx->refs);

}
@@ -1185,8 +1181,6 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, 
		if (req->ctx != ctx) {

			ctx_flush_and_put(ctx, &ts);

			ctx = req->ctx;



			ts.locked = true;

			mutex_lock(&ctx->uring_lock);

			percpu_ref_get(&ctx->refs);

		}
@@ -1459,22 +1453,16 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, 
static inline int io_run_local_work_locked(struct io_ring_ctx *ctx,

					   int min_events)

{

	struct io_tw_state ts = { .locked = true, };

	int ret;

	struct io_tw_state ts = {};



	if (llist_empty(&ctx->work_llist))

		return 0;



	ret = __io_run_local_work(ctx, &ts, min_events);

	/* shouldn't happen! */

	if (WARN_ON_ONCE(!ts.locked))

		mutex_lock(&ctx->uring_lock);

	return ret;

	return __io_run_local_work(ctx, &ts, min_events);

}



static int io_run_local_work(struct io_ring_ctx *ctx, int min_events)

{

	struct io_tw_state ts = { .locked = true };

	struct io_tw_state ts = {};

	int ret;



	mutex_lock(&ctx->uring_lock);
@@ -1702,10 +1690,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) 


void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)

{

	if (ts->locked)

	io_req_complete_defer(req);

	else

		io_req_complete_post(req, IO_URING_F_UNLOCKED);

}



/*

io_uring/io_uring.h

+1 −4
Original line number Diff line number Diff line
@@ -351,10 +351,7 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx) 


static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts)

{

	if (!ts->locked) {

		mutex_lock(&ctx->uring_lock);

		ts->locked = true;

	}

	lockdep_assert_held(&ctx->uring_lock);

}



/*

io_uring/poll.c

+1 −1
Original line number Diff line number Diff line
@@ -322,7 +322,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts) 
			__poll_t mask = mangle_poll(req->cqe.res &

						    req->apoll_events);



			if (!io_fill_cqe_req_aux(req, ts->locked, mask,

			if (!io_fill_cqe_req_aux(req, true, mask,

						 IORING_CQE_F_MORE)) {

				io_req_set_res(req, mask, 0);

				return IOU_POLL_REMOVE_POLL_USE_RES;

io_uring/rw.c

+2 −4
Original line number Diff line number Diff line
@@ -305,11 +305,9 @@ void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts) 


	io_req_io_end(req);



	if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) {

		unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED;

	if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))

		req->cqe.flags |= io_put_kbuf(req, 0);



		req->cqe.flags |= io_put_kbuf(req, issue_flags);

	}

	io_req_task_complete(req, ts);

}