Commit 6e7248ad authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: refactor io_cqring_wake() 



Instead of smp_mb() + __io_cqring_wake() in __io_cq_unlock_post_flush()
use equivalent io_cqring_wake(). With that we can clean it up further
and remove __io_cqring_wake().

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/662ee5d898168ac206be06038525e97b64072a46.1680782017.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent d73a572d

io_uring/io_uring.c

+2 −4
Original line number Diff line number Diff line
@@ -640,10 +640,8 @@ static inline void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx) 
	 * it will re-check the wakeup conditions once we return we can safely

	 * skip waking it up.

	 */

	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) {

		smp_mb();

		__io_cqring_wake(ctx);

	}

	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))

		io_cqring_wake(ctx);

}



void io_cq_unlock_post(struct io_ring_ctx *ctx)

io_uring/io_uring.h

+2 −9
Original line number Diff line number Diff line
@@ -228,8 +228,7 @@ static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) 
				poll_to_key(EPOLL_URING_WAKE | EPOLLIN));

}



/* requires smb_mb() prior, see wq_has_sleeper() */

static inline void __io_cqring_wake(struct io_ring_ctx *ctx)

static inline void io_cqring_wake(struct io_ring_ctx *ctx)

{

	/*

	 * Trigger waitqueue handler on all waiters on our waitqueue. This
@@ -241,17 +240,11 @@ static inline void __io_cqring_wake(struct io_ring_ctx *ctx) 
	 * waitqueue handlers, we know we have a dependency between eventfd or

	 * epoll and should terminate multishot poll at that point.

	 */

	if (waitqueue_active(&ctx->cq_wait))

	if (wq_has_sleeper(&ctx->cq_wait))

		__wake_up(&ctx->cq_wait, TASK_NORMAL, 0,

				poll_to_key(EPOLL_URING_WAKE | EPOLLIN));

}



static inline void io_cqring_wake(struct io_ring_ctx *ctx)

{

	smp_mb();

	__io_cqring_wake(ctx);

}



static inline bool io_sqring_full(struct io_ring_ctx *ctx)

{

	struct io_rings *r = ctx->rings;