Commit c33e779a authored by Caleb Sander Mateos's avatar Caleb Sander Mateos Committed by Jens Axboe
Browse files

io_uring: add wrapper type for io_req_tw_func_t arg 



In preparation for uring_cmd implementations to implement functions
with the io_req_tw_func_t signature, introduce a wrapper struct
io_tw_req to hide the struct io_kiocb * argument. The intention is for
only the io_uring core to access the inner struct io_kiocb *. uring_cmd
implementations should instead call a helper from io_uring/cmd.h to
convert struct io_tw_req to struct io_uring_cmd *.

Signed-off-by: default avatarCaleb Sander Mateos <csander@purestorage.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 4531d165

include/linux/io_uring_types.h

+5 −1
Original line number Diff line number Diff line
@@ -615,7 +615,11 @@ enum { 
	REQ_F_SQE_COPIED	= IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT),

};



typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);

struct io_tw_req {

	struct io_kiocb *req;

};



typedef void (*io_req_tw_func_t)(struct io_tw_req tw_req, io_tw_token_t tw);



struct io_task_work {

	struct llist_node		node;

io_uring/futex.c

+9 −7
Original line number Diff line number Diff line
@@ -41,24 +41,26 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) 
	io_alloc_cache_free(&ctx->futex_cache, kfree);

}



static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)

static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw)

{

	hlist_del_init(&req->hash_node);

	io_req_task_complete(req, tw);

	hlist_del_init(&tw_req.req->hash_node);

	io_req_task_complete(tw_req, tw);

}



static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)

static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_kiocb *req = tw_req.req;

	struct io_ring_ctx *ctx = req->ctx;



	io_tw_lock(ctx, tw);

	io_cache_free(&ctx->futex_cache, req->async_data);

	io_req_async_data_clear(req, 0);

	__io_futex_complete(req, tw);

	__io_futex_complete(tw_req, tw);

}



static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)

static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_kiocb *req = tw_req.req;

	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);

	struct futex_vector *futexv = req->async_data;
@@ -73,7 +75,7 @@ static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw) 
	}



	io_req_async_data_free(req);

	__io_futex_complete(req, tw);

	__io_futex_complete(tw_req, tw);

}



static bool io_futexv_claim(struct io_futex *iof)

io_uring/io_uring.c

+12 −9
Original line number Diff line number Diff line
@@ -291,7 +291,7 @@ static __cold void io_fallback_req_func(struct work_struct *work) 
	mutex_lock(&ctx->uring_lock);

	ts.cancel = io_should_terminate_tw(ctx);

	llist_for_each_entry_safe(req, tmp, node, io_task_work.node)

		req->io_task_work.func(req, ts);

		req->io_task_work.func((struct io_tw_req){req}, ts);

	io_submit_flush_completions(ctx);

	mutex_unlock(&ctx->uring_lock);

	percpu_ref_put(&ctx->refs);
@@ -539,9 +539,9 @@ static void io_queue_iowq(struct io_kiocb *req) 
	io_wq_enqueue(tctx->io_wq, &req->work);

}



static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)

static void io_req_queue_iowq_tw(struct io_tw_req tw_req, io_tw_token_t tw)

{

	io_queue_iowq(req);

	io_queue_iowq(tw_req.req);

}



void io_req_queue_iowq(struct io_kiocb *req)
@@ -1166,7 +1166,7 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, 
		}

		INDIRECT_CALL_2(req->io_task_work.func,

				io_poll_task_func, io_req_rw_complete,

				req, ts);

				(struct io_tw_req){req}, ts);

		node = next;

		(*count)++;

		if (unlikely(need_resched())) {
@@ -1389,7 +1389,7 @@ static int __io_run_local_work_loop(struct llist_node **node, 
						    io_task_work.node);

		INDIRECT_CALL_2(req->io_task_work.func,

				io_poll_task_func, io_req_rw_complete,

				req, tw);

				(struct io_tw_req){req}, tw);

		*node = next;

		if (++ret >= events)

			break;
@@ -1459,14 +1459,17 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events, 
	return ret;

}



static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)

static void io_req_task_cancel(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_kiocb *req = tw_req.req;



	io_tw_lock(req->ctx, tw);

	io_req_defer_failed(req, req->cqe.res);

}



void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)

void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_kiocb *req = tw_req.req;

	struct io_ring_ctx *ctx = req->ctx;



	io_tw_lock(ctx, tw);
@@ -1702,9 +1705,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events) 
	return 0;

}



void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)

void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw)

{

	io_req_complete_defer(req);

	io_req_complete_defer(tw_req.req);

}



/*

io_uring/io_uring.h

+2 −2
Original line number Diff line number Diff line
@@ -149,9 +149,9 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 
void __io_req_task_work_add(struct io_kiocb *req, unsigned flags);

void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags);

void io_req_task_queue(struct io_kiocb *req);

void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);

void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw);

void io_req_task_queue_fail(struct io_kiocb *req, int ret);

void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);

void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw);

struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);

struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);

void tctx_task_work(struct callback_head *cb);

io_uring/msg_ring.c

+2 −1
Original line number Diff line number Diff line
@@ -70,8 +70,9 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) 
	return target_ctx->task_complete;

}



static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)

static void io_msg_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_kiocb *req = tw_req.req;

	struct io_ring_ctx *ctx = req->ctx;



	io_add_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags);