Commit 620a50c9 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

io_uring: uring_cmd: add multishot support 

Add UAPI flag IORING_URING_CMD_MULTISHOT for supporting multishot
uring_cmd operations with provided buffer.

This enables drivers to post multiple completion events from a single
uring_cmd submission, which is useful for:

- Notifying userspace of device events (e.g., interrupt handling)
- Supporting devices with multiple event sources (e.g., multi-queue devices)
- Avoiding the need for device poll() support when events originate
  from multiple sources device-wide

The implementation adds two new APIs:
- io_uring_cmd_select_buffer(): selects a buffer from the provided
  buffer group for multishot uring_cmd
- io_uring_mshot_cmd_post_cqe(): posts a CQE after event data is
  pushed to the provided buffer

Multishot uring_cmd must be used with buffer select (IOSQE_BUFFER_SELECT)
and is mutually exclusive with IORING_URING_CMD_FIXED for now.

The ublk driver will be the first user of this functionality:

	https://github.com/ming1/linux/commits/ublk-devel/



Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250821040210.1152145-3-ming.lei@redhat.com


[axboe: fold in fix for !CONFIG_IO_URING]
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent d589bcdd

include/linux/io_uring/cmd.h

+26 −0
Original line number Diff line number Diff line
@@ -70,6 +70,21 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, 
/* Execute the request from a blocking context */

void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd);



/*

 * Select a buffer from the provided buffer group for multishot uring_cmd.

 * Returns the selected buffer address and size.

 */

struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,

					    unsigned buf_group, size_t *len,

					    unsigned int issue_flags);



/*

 * Complete a multishot uring_cmd event. This will post a CQE to the completion

 * queue and update the provided buffer.

 */

bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,

				 struct io_br_sel *sel, unsigned int issue_flags);



#else

static inline int

io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
@@ -102,6 +117,17 @@ static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, 
static inline void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)

{

}

static inline struct io_br_sel

io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, unsigned buf_group,

			   size_t *len, unsigned int issue_flags)

{

	return (struct io_br_sel) { .val = -EOPNOTSUPP };

}

static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,

				ssize_t ret, unsigned int issue_flags)

{

	return true;

}

#endif



/*

include/uapi/linux/io_uring.h

+5 −1
Original line number Diff line number Diff line
@@ -298,9 +298,13 @@ enum io_uring_op { 
 * sqe->uring_cmd_flags		top 8bits aren't available for userspace

 * IORING_URING_CMD_FIXED	use registered buffer; pass this flag

 *				along with setting sqe->buf_index.

 * IORING_URING_CMD_MULTISHOT	must be used with buffer select, like other

 *				multishot commands. Not compatible with

 *				IORING_URING_CMD_FIXED, for now.

 */

#define IORING_URING_CMD_FIXED	(1U << 0)

#define IORING_URING_CMD_MASK	IORING_URING_CMD_FIXED

#define IORING_URING_CMD_MULTISHOT	(1U << 1)

#define IORING_URING_CMD_MASK	(IORING_URING_CMD_FIXED | IORING_URING_CMD_MULTISHOT)





/*

io_uring/opdef.c

+1 −0
Original line number Diff line number Diff line
@@ -413,6 +413,7 @@ const struct io_issue_def io_issue_defs[] = { 
#endif

	},

	[IORING_OP_URING_CMD] = {

		.buffer_select		= 1,

		.needs_file		= 1,

		.plug			= 1,

		.iopoll			= 1,

io_uring/uring_cmd.c

+70 −1
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#include "io_uring.h"

#include "alloc_cache.h"

#include "rsrc.h"

#include "kbuf.h"

#include "uring_cmd.h"

#include "poll.h"
@@ -194,8 +195,21 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 
	if (ioucmd->flags & ~IORING_URING_CMD_MASK)

		return -EINVAL;



	if (ioucmd->flags & IORING_URING_CMD_FIXED)

	if (ioucmd->flags & IORING_URING_CMD_FIXED) {

		if (ioucmd->flags & IORING_URING_CMD_MULTISHOT)

			return -EINVAL;

		req->buf_index = READ_ONCE(sqe->buf_index);

	}



	if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) {

		if (ioucmd->flags & IORING_URING_CMD_FIXED)

			return -EINVAL;

		if (!(req->flags & REQ_F_BUFFER_SELECT))

			return -EINVAL;

	} else {

		if (req->flags & REQ_F_BUFFER_SELECT)

			return -EINVAL;

	}



	ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
@@ -251,6 +265,10 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) 
	}



	ret = file->f_op->uring_cmd(ioucmd, issue_flags);

	if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) {

		if (ret >= 0)

			return IOU_ISSUE_SKIP_COMPLETE;

	}

	if (ret == -EAGAIN) {

		ioucmd->flags |= IORING_URING_CMD_REISSUE;

		return ret;
@@ -333,3 +351,54 @@ bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd, 
		return false;

	return io_req_post_cqe32(req, cqe);

}



/*

 * Work with io_uring_mshot_cmd_post_cqe() together for committing the

 * provided buffer upfront

 */

struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,

					    unsigned buf_group, size_t *len,

					    unsigned int issue_flags)

{

	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);



	if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))

		return (struct io_br_sel) { .val = -EINVAL };



	if (WARN_ON_ONCE(!io_do_buffer_select(req)))

		return (struct io_br_sel) { .val = -EINVAL };



	return io_buffer_select(req, len, buf_group, issue_flags);

}

EXPORT_SYMBOL_GPL(io_uring_cmd_buffer_select);



/*

 * Return true if this multishot uring_cmd needs to be completed, otherwise

 * the event CQE is posted successfully.

 *

 * This function must use `struct io_br_sel` returned from

 * io_uring_cmd_buffer_select() for committing the buffer in the same

 * uring_cmd submission context.

 */

bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,

				 struct io_br_sel *sel, unsigned int issue_flags)

{

	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);

	unsigned int cflags = 0;



	if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))

		return true;



	if (sel->val > 0) {

		cflags = io_put_kbuf(req, sel->val, sel->buf_list);

		if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE))

			return false;

	}



	io_kbuf_recycle(req, sel->buf_list, issue_flags);

	if (sel->val < 0)

		req_set_fail(req);

	io_req_set_res(req, sel->val, cflags);

	return true;

}

EXPORT_SYMBOL_GPL(io_uring_mshot_cmd_post_cqe);