Commit 2b61bb1d authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/kbuf: unify legacy buf provision and removal 



Combine IORING_OP_PROVIDE_BUFFERS and IORING_OP_REMOVE_BUFFERS
->issue(), so that we can deduplicate ring locking and list lookups.
This way we further reduce code for legacy provided buffers. Locking is
also separated from buffer related handling, which makes it a bit
simpler with label jumps.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/f61af131622ad4337c2fb9f7c453d5b0102c7b90.1747150490.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c724e801

io_uring/kbuf.c

+28 −45
Original line number Diff line number Diff line
@@ -450,30 +450,6 @@ int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 
	return 0;

}



int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)

{

	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);

	struct io_ring_ctx *ctx = req->ctx;

	struct io_buffer_list *bl;

	int ret = 0;



	io_ring_submit_lock(ctx, issue_flags);



	ret = -ENOENT;

	bl = io_buffer_get_list(ctx, p->bgid);

	if (bl) {

		ret = -EINVAL;

		/* can't use provide/remove buffers command on mapped buffers */

		if (!(bl->flags & IOBL_BUF_RING))

			ret = io_remove_buffers_legacy(ctx, bl, p->nbufs);

	}

	io_ring_submit_unlock(ctx, issue_flags);

	if (ret < 0)

		req_set_fail(req);

	io_req_set_res(req, ret, 0);

	return IOU_OK;

}



int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

	unsigned long size, tmp_check;
@@ -535,37 +511,44 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, 
	return i ? 0 : -ENOMEM;

}



int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)

static int __io_manage_buffers_legacy(struct io_kiocb *req,

					struct io_buffer_list *bl)

{

	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);

	struct io_ring_ctx *ctx = req->ctx;

	struct io_buffer_list *bl;

	int ret = 0;



	io_ring_submit_lock(ctx, issue_flags);

	int ret;



	bl = io_buffer_get_list(ctx, p->bgid);

	if (unlikely(!bl)) {

		bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);

	if (!bl) {

			ret = -ENOMEM;

			goto err;

		}

		if (req->opcode != IORING_OP_PROVIDE_BUFFERS)

			return -ENOENT;

		bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);

		if (!bl)

			return -ENOMEM;



		INIT_LIST_HEAD(&bl->buf_list);

		ret = io_buffer_add_list(ctx, bl, p->bgid);

		ret = io_buffer_add_list(req->ctx, bl, p->bgid);

		if (ret) {

			kfree(bl);

			goto err;

			return ret;

		}

	}

	/* can't add buffers via this command for a mapped buffer ring */

	if (bl->flags & IOBL_BUF_RING) {

		ret = -EINVAL;

		goto err;

	/* can't use provide/remove buffers command on mapped buffers */

	if (bl->flags & IOBL_BUF_RING)

		return -EINVAL;

	if (req->opcode == IORING_OP_PROVIDE_BUFFERS)

		return io_add_buffers(req->ctx, p, bl);

	return io_remove_buffers_legacy(req->ctx, bl, p->nbufs);

}



	ret = io_add_buffers(ctx, p, bl);

err:

int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags)

{

	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);

	struct io_ring_ctx *ctx = req->ctx;

	struct io_buffer_list *bl;

	int ret;



	io_ring_submit_lock(ctx, issue_flags);

	bl = io_buffer_get_list(ctx, p->bgid);

	ret = __io_manage_buffers_legacy(req, bl);

	io_ring_submit_unlock(ctx, issue_flags);



	if (ret < 0)

io_uring/kbuf.h

+1 −3
Original line number Diff line number Diff line
@@ -66,10 +66,8 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg); 
void io_destroy_buffers(struct io_ring_ctx *ctx);



int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);

int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);



int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);

int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);

int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags);



int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);

int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);

io_uring/opdef.c

+2 −2
Original line number Diff line number Diff line
@@ -333,13 +333,13 @@ const struct io_issue_def io_issue_defs[] = { 
		.audit_skip		= 1,

		.iopoll			= 1,

		.prep			= io_provide_buffers_prep,

		.issue			= io_provide_buffers,

		.issue			= io_manage_buffers_legacy,

	},

	[IORING_OP_REMOVE_BUFFERS] = {

		.audit_skip		= 1,

		.iopoll			= 1,

		.prep			= io_remove_buffers_prep,

		.issue			= io_remove_buffers,

		.issue			= io_manage_buffers_legacy,

	},

	[IORING_OP_TEE] = {

		.needs_file		= 1,