Commit 99f15d8d authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: move uring_cmd handling to its own file 



Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent cd40cae2

io_uring/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -4,5 +4,5 @@ 


obj-$(CONFIG_IO_URING)		+= io_uring.o xattr.o nop.o fs.o splice.o \

					sync.o advise.o filetable.o \

					openclose.o

					openclose.o uring_cmd.o

obj-$(CONFIG_IO_WQ)		+= io-wq.o

io_uring/io_uring.c

+4 −123
Original line number Diff line number Diff line
@@ -99,6 +99,7 @@ 
#include "sync.h"

#include "advise.h"

#include "openclose.h"

#include "uring_cmd.h"



#define IORING_MAX_ENTRIES	32768

#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
@@ -473,14 +474,6 @@ struct io_cancel_data { 
	int seq;

};



/*

 * The URING_CMD payload starts at 'cmd' in the first sqe, and continues into

 * the following sqe if SQE128 is used.

 */

#define uring_cmd_pdu_size(is_sqe128)				\

	((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) -	\

		offsetof(struct io_uring_sqe, cmd))



struct io_op_def {

	/* needs req->file assigned */

	unsigned		needs_file : 1;
@@ -988,11 +981,6 @@ static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, 
	return matched;

}



static inline bool req_has_async_data(struct io_kiocb *req)

{

	return req->flags & REQ_F_ASYNC_DATA;

}



static inline void req_fail_link_node(struct io_kiocb *req, int res)

{

	req_set_fail(req);
@@ -1743,7 +1731,7 @@ static void io_req_complete_post(struct io_kiocb *req) 
	io_cqring_ev_posted(ctx);

}



static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)

inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)

{

	if (issue_flags & IO_URING_F_COMPLETE_DEFER)

		req->flags |= REQ_F_COMPLETE_INLINE;
@@ -2151,7 +2139,7 @@ static void __io_req_task_work_add(struct io_kiocb *req, 
	}

}



static void io_req_task_work_add(struct io_kiocb *req)

void io_req_task_work_add(struct io_kiocb *req)

{

	struct io_uring_task *tctx = req->task->io_uring;
@@ -3268,7 +3256,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, 
	}

}



static inline bool io_alloc_async_data(struct io_kiocb *req)

bool io_alloc_async_data(struct io_kiocb *req)

{

	WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);

	req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
@@ -3714,111 +3702,6 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) 
	return ret;

}



static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);



	ioucmd->task_work_cb(ioucmd);

}



void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,

			void (*task_work_cb)(struct io_uring_cmd *))

{

	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);



	ioucmd->task_work_cb = task_work_cb;

	req->io_task_work.func = io_uring_cmd_work;

	io_req_task_work_add(req);

}

EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);



static inline void io_req_set_cqe32_extra(struct io_kiocb *req,

					  u64 extra1, u64 extra2)

{

	req->extra1 = extra1;

	req->extra2 = extra2;

	req->flags |= REQ_F_CQE32_INIT;

}



/*

 * Called by consumers of io_uring_cmd, if they originally returned

 * -EIOCBQUEUED upon receiving the command.

 */

void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)

{

	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);



	if (ret < 0)

		req_set_fail(req);



	io_req_set_res(req, 0, ret);

	if (req->ctx->flags & IORING_SETUP_CQE32)

		io_req_set_cqe32_extra(req, res2, 0);

	__io_req_complete(req, 0);

}

EXPORT_SYMBOL_GPL(io_uring_cmd_done);



static int io_uring_cmd_prep_async(struct io_kiocb *req)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);

	size_t cmd_size;



	cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);



	memcpy(req->async_data, ioucmd->cmd, cmd_size);

	return 0;

}



static int io_uring_cmd_prep(struct io_kiocb *req,

			     const struct io_uring_sqe *sqe)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);



	if (sqe->rw_flags || sqe->__pad1)

		return -EINVAL;

	ioucmd->cmd = sqe->cmd;

	ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);

	return 0;

}



static int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);

	struct io_ring_ctx *ctx = req->ctx;

	struct file *file = req->file;

	int ret;



	if (!req->file->f_op->uring_cmd)

		return -EOPNOTSUPP;



	if (ctx->flags & IORING_SETUP_SQE128)

		issue_flags |= IO_URING_F_SQE128;

	if (ctx->flags & IORING_SETUP_CQE32)

		issue_flags |= IO_URING_F_CQE32;

	if (ctx->flags & IORING_SETUP_IOPOLL)

		issue_flags |= IO_URING_F_IOPOLL;



	if (req_has_async_data(req))

		ioucmd->cmd = req->async_data;



	ret = file->f_op->uring_cmd(ioucmd, issue_flags);

	if (ret == -EAGAIN) {

		if (!req_has_async_data(req)) {

			if (io_alloc_async_data(req))

				return -ENOMEM;

			io_uring_cmd_prep_async(req);

		}

		return -EAGAIN;

	}



	if (ret != -EIOCBQUEUED) {

		io_uring_cmd_done(ioucmd, ret, 0);

		return IOU_OK;

	}



	return IOU_ISSUE_SKIP_COMPLETE;

}



static int io_msg_ring_prep(struct io_kiocb *req,

			    const struct io_uring_sqe *sqe)

{
@@ -11533,8 +11416,6 @@ static int __init io_uring_init(void) 


	BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));



	BUILD_BUG_ON(sizeof(struct io_uring_cmd) > 64);



	for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {

		BUG_ON(!io_op_defs[i].prep);

		BUG_ON(!io_op_defs[i].issue);

io_uring/io_uring.h

+9 −0
Original line number Diff line number Diff line
@@ -25,6 +25,11 @@ static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) 
	req->cqe.flags = cflags;

}



static inline bool req_has_async_data(struct io_kiocb *req)

{

	return req->flags & REQ_F_ASYNC_DATA;

}



static inline void io_put_file(struct file *file)

{

	if (file)
@@ -53,6 +58,8 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx, 
	lockdep_assert_held(&ctx->uring_lock);

}



void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);



struct file *io_file_get_normal(struct io_kiocb *req, int fd);

struct file *io_file_get_fixed(struct io_kiocb *req, int fd,

			       unsigned issue_flags);
@@ -65,5 +72,7 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, 
void io_rsrc_node_switch(struct io_ring_ctx *ctx,

			 struct io_rsrc_data *data_to_kill);

bool io_is_uring_fops(struct file *file);

bool io_alloc_async_data(struct io_kiocb *req);

void io_req_task_work_add(struct io_kiocb *req);



#endif

io_uring/uring_cmd.c

0 → 100644
+115 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

#include <linux/kernel.h>

#include <linux/errno.h>

#include <linux/file.h>

#include <linux/io_uring.h>



#include <uapi/linux/io_uring.h>



#include "io_uring_types.h"

#include "io_uring.h"

#include "uring_cmd.h"



static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);



	ioucmd->task_work_cb(ioucmd);

}



void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,

			void (*task_work_cb)(struct io_uring_cmd *))

{

	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);



	ioucmd->task_work_cb = task_work_cb;

	req->io_task_work.func = io_uring_cmd_work;

	io_req_task_work_add(req);

}

EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);



static inline void io_req_set_cqe32_extra(struct io_kiocb *req,

					  u64 extra1, u64 extra2)

{

	req->extra1 = extra1;

	req->extra2 = extra2;

	req->flags |= REQ_F_CQE32_INIT;

}



/*

 * Called by consumers of io_uring_cmd, if they originally returned

 * -EIOCBQUEUED upon receiving the command.

 */

void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)

{

	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);



	if (ret < 0)

		req_set_fail(req);



	io_req_set_res(req, 0, ret);

	if (req->ctx->flags & IORING_SETUP_CQE32)

		io_req_set_cqe32_extra(req, res2, 0);

	__io_req_complete(req, 0);

}

EXPORT_SYMBOL_GPL(io_uring_cmd_done);



int io_uring_cmd_prep_async(struct io_kiocb *req)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);

	size_t cmd_size;



	cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);



	memcpy(req->async_data, ioucmd->cmd, cmd_size);

	return 0;

}



int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);



	if (sqe->rw_flags || sqe->__pad1)

		return -EINVAL;

	ioucmd->cmd = sqe->cmd;

	ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);

	return 0;

}



int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)

{

	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);

	struct io_ring_ctx *ctx = req->ctx;

	struct file *file = req->file;

	int ret;



	if (!req->file->f_op->uring_cmd)

		return -EOPNOTSUPP;



	if (ctx->flags & IORING_SETUP_SQE128)

		issue_flags |= IO_URING_F_SQE128;

	if (ctx->flags & IORING_SETUP_CQE32)

		issue_flags |= IO_URING_F_CQE32;

	if (ctx->flags & IORING_SETUP_IOPOLL)

		issue_flags |= IO_URING_F_IOPOLL;



	if (req_has_async_data(req))

		ioucmd->cmd = req->async_data;



	ret = file->f_op->uring_cmd(ioucmd, issue_flags);

	if (ret == -EAGAIN) {

		if (!req_has_async_data(req)) {

			if (io_alloc_async_data(req))

				return -ENOMEM;

			io_uring_cmd_prep_async(req);

		}

		return -EAGAIN;

	}



	if (ret != -EIOCBQUEUED) {

		io_uring_cmd_done(ioucmd, ret, 0);

		return IOU_OK;

	}



	return IOU_ISSUE_SKIP_COMPLETE;

}

io_uring/uring_cmd.h

0 → 100644
+13 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0



int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);

int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);

int io_uring_cmd_prep_async(struct io_kiocb *req);



/*

 * The URING_CMD payload starts at 'cmd' in the first sqe, and continues into

 * the following sqe if SQE128 is used.

 */

#define uring_cmd_pdu_size(is_sqe128)				\

	((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) -	\

		offsetof(struct io_uring_sqe, cmd))