Commit e0e4ab52 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: refactor DEFER_TASKRUN multishot checks 



We disallow DEFER_TASKRUN multishots from running by io-wq, which is
checked by individual opcodes in the issue path. We can consolidate all
it in io_wq_submit_work() at the same time moving the checks out of the
hot path.

Suggested-by: default avatarJens Axboe <axboe@kernel.dk>
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/e492f0f11588bb5aa11d7d24e6f53b7c7628afdb.1709905727.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3a96378e

io_uring/io_uring.c

+20 −0
Original line number Diff line number Diff line
@@ -944,6 +944,8 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags) 
	u64 user_data = req->cqe.user_data;

	struct io_uring_cqe *cqe;



	lockdep_assert(!io_wq_current_is_worker());



	if (!defer)

		return __io_post_aux_cqe(ctx, user_data, res, cflags, false);
@@ -1968,6 +1970,24 @@ void io_wq_submit_work(struct io_wq_work *work) 
		goto fail;

	}



	/*

	 * If DEFER_TASKRUN is set, it's only allowed to post CQEs from the

	 * submitter task context. Final request completions are handed to the

	 * right context, however this is not the case of auxiliary CQEs,

	 * which is the main mean of operation for multishot requests.

	 * Don't allow any multishot execution from io-wq. It's more restrictive

	 * than necessary and also cleaner.

	 */

	if (req->flags & REQ_F_APOLL_MULTISHOT) {

		err = -EBADFD;

		if (!io_file_can_poll(req))

			goto fail;

		err = -ECANCELED;

		if (io_arm_poll_handler(req, issue_flags) != IO_APOLL_OK)

			goto fail;

		return;

	}



	if (req->flags & REQ_F_FORCE_ASYNC) {

		bool opcode_poll = def->pollin || def->pollout;

io_uring/net.c

+0 −21
Original line number Diff line number Diff line
@@ -78,19 +78,6 @@ struct io_sr_msg { 
 */

#define MULTISHOT_MAX_RETRY	32



static inline bool io_check_multishot(struct io_kiocb *req,

				      unsigned int issue_flags)

{

	/*

	 * When ->locked_cq is set we only allow to post CQEs from the original

	 * task context. Usual request completions will be handled in other

	 * generic paths but multipoll may decide to post extra cqes.

	 */

	return !(issue_flags & IO_URING_F_IOWQ) ||

		!(req->flags & REQ_F_APOLL_MULTISHOT) ||

		!req->ctx->task_complete;

}



int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
@@ -862,9 +849,6 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 
	    (sr->flags & IORING_RECVSEND_POLL_FIRST))

		return io_setup_async_msg(req, kmsg, issue_flags);



	if (!io_check_multishot(req, issue_flags))

		return io_setup_async_msg(req, kmsg, issue_flags);



	flags = sr->msg_flags;

	if (force_nonblock)

		flags |= MSG_DONTWAIT;
@@ -956,9 +940,6 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) 
	    (sr->flags & IORING_RECVSEND_POLL_FIRST))

		return -EAGAIN;



	if (!io_check_multishot(req, issue_flags))

		return -EAGAIN;



	sock = sock_from_file(req->file);

	if (unlikely(!sock))

		return -ENOTSOCK;
@@ -1408,8 +1389,6 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) 
	struct file *file;

	int ret, fd;



	if (!io_check_multishot(req, issue_flags))

		return -EAGAIN;

retry:

	if (!fixed) {

		fd = __get_unused_fd_flags(accept->flags, accept->nofile);

io_uring/rw.c

+0 −2
Original line number Diff line number Diff line
@@ -933,8 +933,6 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) 
	 */

	if (!io_file_can_poll(req))

		return -EBADFD;

	if (issue_flags & IO_URING_F_IOWQ)

		return -EAGAIN;



	ret = __io_read(req, issue_flags);