Commit bcf8a029 authored by Caleb Sander Mateos's avatar Caleb Sander Mateos Committed by Jens Axboe
Browse files

io_uring: introduce type alias for io_tw_state 



In preparation for changing how io_tw_state is passed, introduce a type
alias io_tw_token_t for struct io_tw_state *. This allows for changing
the representation in one place, without having to update the many
functions that just forward their struct io_tw_state * argument.

Also add a comment to struct io_tw_state to explain its purpose.

Signed-off-by: default avatarCaleb Sander Mateos <csander@purestorage.com>
Link: https://lore.kernel.org/r/20250217022511.1150145-1-csander@purestorage.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 496f56bf

include/linux/io_uring_types.h

+8 −1
Original line number Diff line number Diff line
@@ -436,8 +436,15 @@ struct io_ring_ctx { 
	struct io_mapped_region		param_region;

};



/*

 * Token indicating function is called in task work context:

 * ctx->uring_lock is held and any completions generated will be flushed.

 * ONLY core io_uring.c should instantiate this struct.

 */

struct io_tw_state {

};

/* Alias to use in code that doesn't instantiate struct io_tw_state */

typedef struct io_tw_state *io_tw_token_t;



enum {

	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
@@ -563,7 +570,7 @@ enum { 
	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),

};



typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);

typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);



struct io_task_work {

	struct llist_node		node;

io_uring/futex.c

+8 −8
Original line number Diff line number Diff line
@@ -44,30 +44,30 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) 
	io_alloc_cache_free(&ctx->futex_cache, kfree);

}



static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	req->async_data = NULL;

	hlist_del_init(&req->hash_node);

	io_req_task_complete(req, ts);

	io_req_task_complete(req, tw);

}



static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	struct io_futex_data *ifd = req->async_data;

	struct io_ring_ctx *ctx = req->ctx;



	io_tw_lock(ctx, ts);

	io_tw_lock(ctx, tw);

	if (!io_alloc_cache_put(&ctx->futex_cache, ifd))

		kfree(ifd);

	__io_futex_complete(req, ts);

	__io_futex_complete(req, tw);

}



static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);

	struct futex_vector *futexv = req->async_data;



	io_tw_lock(req->ctx, ts);

	io_tw_lock(req->ctx, tw);



	if (!iof->futexv_unqueued) {

		int res;
@@ -79,7 +79,7 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts) 


	kfree(req->async_data);

	req->flags &= ~REQ_F_ASYNC_DATA;

	__io_futex_complete(req, ts);

	__io_futex_complete(req, tw);

}



static bool io_futexv_claim(struct io_futex *iof)

io_uring/io_uring.c

+14 −14
Original line number Diff line number Diff line
@@ -543,7 +543,7 @@ static void io_queue_iowq(struct io_kiocb *req) 
		io_queue_linked_timeout(link);

}



static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts)

static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)

{

	io_queue_iowq(req);

}
@@ -1022,7 +1022,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) 
	return nxt;

}



static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)

static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw)

{

	if (!ctx)

		return;
@@ -1277,7 +1277,7 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, 
}



static int __io_run_local_work_loop(struct llist_node **node,

				    struct io_tw_state *ts,

				    io_tw_token_t tw,

				    int events)

{

	int ret = 0;
@@ -1288,7 +1288,7 @@ static int __io_run_local_work_loop(struct llist_node **node, 
						    io_task_work.node);

		INDIRECT_CALL_2(req->io_task_work.func,

				io_poll_task_func, io_req_rw_complete,

				req, ts);

				req, tw);

		*node = next;

		if (++ret >= events)

			break;
@@ -1297,7 +1297,7 @@ static int __io_run_local_work_loop(struct llist_node **node, 
	return ret;

}



static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,

static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,

			       int min_events, int max_events)

{

	struct llist_node *node;
@@ -1310,7 +1310,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, 
		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);

again:

	min_events -= ret;

	ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events);

	ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);

	if (ctx->retry_llist.first)

		goto retry_done;
@@ -1319,7 +1319,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, 
	 * running the pending items.

	 */

	node = llist_reverse_order(llist_del_all(&ctx->work_llist));

	ret += __io_run_local_work_loop(&node, ts, max_events - ret);

	ret += __io_run_local_work_loop(&node, tw, max_events - ret);

	ctx->retry_llist.first = node;

	loops++;
@@ -1357,15 +1357,15 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events, 
	return ret;

}



static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts)

static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)

{

	io_tw_lock(req->ctx, ts);

	io_tw_lock(req->ctx, tw);

	io_req_defer_failed(req, req->cqe.res);

}



void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts)

void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)

{

	io_tw_lock(req->ctx, ts);

	io_tw_lock(req->ctx, tw);

	if (unlikely(io_should_terminate_tw()))

		io_req_defer_failed(req, -EFAULT);

	else if (req->flags & REQ_F_FORCE_ASYNC)
@@ -1583,7 +1583,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) 
	return 0;

}



void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)

void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	io_req_complete_defer(req);

}
@@ -1763,9 +1763,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) 
	return ret;

}



int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)

int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)

{

	io_tw_lock(req->ctx, ts);

	io_tw_lock(req->ctx, tw);

	return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|

				 IO_URING_F_COMPLETE_DEFER);

}

io_uring/io_uring.h

+4 −4
Original line number Diff line number Diff line
@@ -90,9 +90,9 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx, 
				 unsigned flags);

bool io_alloc_async_data(struct io_kiocb *req);

void io_req_task_queue(struct io_kiocb *req);

void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts);

void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);

void io_req_task_queue_fail(struct io_kiocb *req, int ret);

void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts);

void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);

struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);

struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);

void tctx_task_work(struct callback_head *cb);
@@ -104,7 +104,7 @@ int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, 
				     int start, int end);

void io_req_queue_iowq(struct io_kiocb *req);



int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);

int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);

int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);

int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);

void __io_submit_flush_completions(struct io_ring_ctx *ctx);
@@ -376,7 +376,7 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx) 
	return task_work_pending(current) || io_local_work_pending(ctx);

}



static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts)

static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw)

{

	lockdep_assert_held(&ctx->uring_lock);

}

io_uring/msg_ring.c

+1 −1
Original line number Diff line number Diff line
@@ -71,7 +71,7 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) 
	return target_ctx->task_complete;

}



static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	struct io_ring_ctx *ctx = req->ctx;