Commit 35645ac3 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: combine REQ_F_NOWAIT_{READ,WRITE} flags 



Merge REQ_F_NOWAIT_READ and REQ_F_NOWAIT_WRITE into one flag, i.e.
REQ_F_SUPPORT_NOWAIT. First it gets rid of dependence on CONFIG_64BIT
but also simplifies the code.

One thing to consider is when we don't have ->{read,write}_iter and go
through loop_rw_iter(). Just fail it with -EAGAIN if we expect nowait
behaviour but not sure whether it supports it.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/f832a20e5186c2e79c6519280c238f559a1d2bbc.1634425438.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e74ead13

fs/io_uring.c

+21 −40
Original line number Diff line number Diff line
@@ -732,8 +732,7 @@ enum { 
	REQ_F_ARM_LTIMEOUT_BIT,

	REQ_F_ASYNC_DATA_BIT,

	/* keep async read/write and isreg together and in order */

	REQ_F_NOWAIT_READ_BIT,

	REQ_F_NOWAIT_WRITE_BIT,

	REQ_F_SUPPORT_NOWAIT_BIT,

	REQ_F_ISREG_BIT,



	/* not a real bit, just to check we're not overflowing the space */
@@ -774,10 +773,8 @@ enum { 
	REQ_F_COMPLETE_INLINE	= BIT(REQ_F_COMPLETE_INLINE_BIT),

	/* caller should reissue async */

	REQ_F_REISSUE		= BIT(REQ_F_REISSUE_BIT),

	/* supports async reads */

	REQ_F_NOWAIT_READ	= BIT(REQ_F_NOWAIT_READ_BIT),

	/* supports async writes */

	REQ_F_NOWAIT_WRITE	= BIT(REQ_F_NOWAIT_WRITE_BIT),

	/* supports async reads/writes */

	REQ_F_SUPPORT_NOWAIT	= BIT(REQ_F_SUPPORT_NOWAIT_BIT),

	/* regular file */

	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),

	/* has creds assigned */
@@ -1390,18 +1387,13 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) 
	return false;

}



#define FFS_ASYNC_READ		0x1UL

#define FFS_ASYNC_WRITE		0x2UL

#ifdef CONFIG_64BIT

#define FFS_ISREG		0x4UL

#else

#define FFS_ISREG		0x0UL

#endif

#define FFS_MASK		~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)

#define FFS_NOWAIT		0x1UL

#define FFS_ISREG		0x2UL

#define FFS_MASK		~(FFS_NOWAIT|FFS_ISREG)



static inline bool io_req_ffs_set(struct io_kiocb *req)

{

	return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE);

	return req->flags & REQ_F_FIXED_FILE;

}



static inline void io_req_track_inflight(struct io_kiocb *req)
@@ -2772,7 +2764,7 @@ static bool io_bdev_nowait(struct block_device *bdev) 
 * any file. For now, just ensure that anything potentially problematic is done

 * inline.

 */

static bool __io_file_supports_nowait(struct file *file, int rw)

static bool __io_file_supports_nowait(struct file *file)

{

	umode_t mode = file_inode(file)->i_mode;
@@ -2795,24 +2787,14 @@ static bool __io_file_supports_nowait(struct file *file, int rw) 
	/* any ->read/write should understand O_NONBLOCK */

	if (file->f_flags & O_NONBLOCK)

		return true;



	if (!(file->f_mode & FMODE_NOWAIT))

		return false;



	if (rw == READ)

		return file->f_op->read_iter != NULL;



	return file->f_op->write_iter != NULL;

	return file->f_mode & FMODE_NOWAIT;

}



static bool io_file_supports_nowait(struct io_kiocb *req, int rw)

static inline bool io_file_supports_nowait(struct io_kiocb *req)

{

	if (rw == READ && (req->flags & REQ_F_NOWAIT_READ))

	if (likely(req->flags & REQ_F_SUPPORT_NOWAIT))

		return true;

	else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE))

		return true;



	return __io_file_supports_nowait(req->file, rw);

	return __io_file_supports_nowait(req->file);

}



static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
@@ -2844,7 +2826,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, 
	 * reliably. If not, or it IOCB_NOWAIT is set, don't retry.

	 */

	if ((kiocb->ki_flags & IOCB_NOWAIT) ||

	    ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw)))

	    ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))

		req->flags |= REQ_F_NOWAIT;



	ioprio = READ_ONCE(sqe->ioprio);
@@ -3235,7 +3217,8 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) 
	 */

	if (kiocb->ki_flags & IOCB_HIPRI)

		return -EOPNOTSUPP;

	if (kiocb->ki_flags & IOCB_NOWAIT)

	if ((kiocb->ki_flags & IOCB_NOWAIT) &&

	    !(kiocb->ki_filp->f_flags & O_NONBLOCK))

		return -EAGAIN;



	while (iov_iter_count(iter)) {
@@ -3475,7 +3458,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) 


	if (force_nonblock) {

		/* If the file doesn't support async, just async punt */

		if (unlikely(!io_file_supports_nowait(req, READ))) {

		if (unlikely(!io_file_supports_nowait(req))) {

			ret = io_setup_async_rw(req, iovec, s, true);

			return ret ?: -EAGAIN;

		}
@@ -3599,7 +3582,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) 


	if (force_nonblock) {

		/* If the file doesn't support async, just async punt */

		if (unlikely(!io_file_supports_nowait(req, WRITE)))

		if (unlikely(!io_file_supports_nowait(req)))

			goto copy_iov;



		/* file path doesn't support NOWAIT for non-direct_IO */
@@ -3631,7 +3614,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) 
	}

	kiocb->ki_flags |= IOCB_WRITE;



	if (req->file->f_op->write_iter)

	if (likely(req->file->f_op->write_iter))

		ret2 = call_write_iter(req->file, kiocb, &s->iter);

	else if (req->file->f_op->write)

		ret2 = loop_rw_iter(WRITE, req, &s->iter);
@@ -6778,10 +6761,8 @@ static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file 
{

	unsigned long file_ptr = (unsigned long) file;



	if (__io_file_supports_nowait(file, READ))

		file_ptr |= FFS_ASYNC_READ;

	if (__io_file_supports_nowait(file, WRITE))

		file_ptr |= FFS_ASYNC_WRITE;

	if (__io_file_supports_nowait(file))

		file_ptr |= FFS_NOWAIT;

	if (S_ISREG(file_inode(file)->i_mode))

		file_ptr |= FFS_ISREG;

	file_slot->file_ptr = file_ptr;
@@ -6800,7 +6781,7 @@ static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx, 
	file = (struct file *) (file_ptr & FFS_MASK);

	file_ptr &= ~FFS_MASK;

	/* mask in overlapping REQ_F and FFS bits */

	req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT);

	req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);

	io_req_set_rsrc_node(req, ctx);

	return file;

}