Commit 95041b93 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: add io_file_can_poll() helper 



This adds a flag to avoid dipping dereferencing file and then f_op to
figure out if the file has a poll handler defined or not. We generally
call this at least twice for networked workloads, and if using ring
provided buffers, we do it on every buffer selection. Particularly the
latter is troublesome, as it's otherwise a very fast operation.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 521223d7

include/linux/io_uring_types.h

+3 −0
Original line number Diff line number Diff line
@@ -464,6 +464,7 @@ enum { 
	REQ_F_ISREG_BIT,

	REQ_F_POLL_NO_LAZY_BIT,

	REQ_F_CANCEL_SEQ_BIT,

	REQ_F_CAN_POLL_BIT,



	/* not a real bit, just to check we're not overflowing the space */

	__REQ_F_LAST_BIT,
@@ -538,6 +539,8 @@ enum { 
	REQ_F_POLL_NO_LAZY	= IO_REQ_FLAG(REQ_F_POLL_NO_LAZY_BIT),

	/* cancel sequence is set and valid */

	REQ_F_CANCEL_SEQ	= IO_REQ_FLAG(REQ_F_CANCEL_SEQ_BIT),

	/* file is pollable */

	REQ_F_CAN_POLL		= IO_REQ_FLAG(REQ_F_CAN_POLL_BIT),

};



typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);

io_uring/io_uring.c

+1 −1
Original line number Diff line number Diff line
@@ -1968,7 +1968,7 @@ void io_wq_submit_work(struct io_wq_work *work) 
	if (req->flags & REQ_F_FORCE_ASYNC) {

		bool opcode_poll = def->pollin || def->pollout;



		if (opcode_poll && file_can_poll(req->file)) {

		if (opcode_poll && io_file_can_poll(req)) {

			needs_poll = true;

			issue_flags |= IO_URING_F_NONBLOCK;

		}

io_uring/io_uring.h

+12 −0
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#include <linux/lockdep.h>

#include <linux/resume_user_mode.h>

#include <linux/kasan.h>

#include <linux/poll.h>

#include <linux/io_uring_types.h>

#include <uapi/linux/eventpoll.h>

#include "io-wq.h"
@@ -398,4 +399,15 @@ static inline size_t uring_sqe_size(struct io_ring_ctx *ctx) 
		return 2 * sizeof(struct io_uring_sqe);

	return sizeof(struct io_uring_sqe);

}



static inline bool io_file_can_poll(struct io_kiocb *req)

{

	if (req->flags & REQ_F_CAN_POLL)

		return true;

	if (file_can_poll(req->file)) {

		req->flags |= REQ_F_CAN_POLL;

		return true;

	}

	return false;

}

#endif

io_uring/kbuf.c

+1 −1
Original line number Diff line number Diff line
@@ -180,7 +180,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, 
	req->buf_list = bl;

	req->buf_index = buf->bid;



	if (issue_flags & IO_URING_F_UNLOCKED || !file_can_poll(req->file)) {

	if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) {

		/*

		 * If we came in unlocked, we have no choice but to consume the

		 * buffer here, otherwise nothing ensures that the buffer won't

io_uring/poll.c

+1 −1
Original line number Diff line number Diff line
@@ -724,7 +724,7 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) 


	if (!def->pollin && !def->pollout)

		return IO_APOLL_ABORTED;

	if (!file_can_poll(req->file))

	if (!io_file_can_poll(req))

		return IO_APOLL_ABORTED;

	if (!(req->flags & REQ_F_APOLL_MULTISHOT))

		mask |= EPOLLONESHOT;