Commit bd32923e authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: don't store bgid in req->buf_index 



Pass buffer group id into the rest of helpers via struct buf_sel_arg
and remove all reassignments of req->buf_index back to bgid. Now, it
only stores buffer indexes, and the group is provided by callers.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3ea9fa08113ecb4d9224b943e7806e80a324bdf9.1743437358.git.asml.silence@gmail.com
Link: https://lore.kernel.org/io-uring/0c01d76ff12986c2f48614db8610caff8f78c869.1743500909.git.asml.silence@gmail.com/


[axboe: fold in patch from second link]
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c0e96505

include/linux/io_uring_types.h

+1 −2
Original line number Diff line number Diff line
@@ -653,8 +653,7 @@ struct io_kiocb { 
	u8				iopoll_completed;

	/*

	 * Can be either a fixed buffer index, or used with provided buffers.

	 * For the latter, before issue it points to the buffer group ID,

	 * and after selection it points to the buffer ID itself.

	 * For the latter, it points to the selected buffer ID.

	 */

	u16				buf_index;

io_uring/kbuf.c

+4 −7
Original line number Diff line number Diff line
@@ -92,7 +92,6 @@ void io_kbuf_drop_legacy(struct io_kiocb *req) 
{

	if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED)))

		return;

	req->buf_index = req->kbuf->bgid;

	req->flags &= ~REQ_F_BUFFER_SELECTED;

	kfree(req->kbuf);

	req->kbuf = NULL;
@@ -110,7 +109,6 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) 
	bl = io_buffer_get_list(ctx, buf->bgid);

	list_add(&buf->list, &bl->buf_list);

	req->flags &= ~REQ_F_BUFFER_SELECTED;

	req->buf_index = buf->bgid;



	io_ring_submit_unlock(ctx, issue_flags);

	return true;
@@ -302,7 +300,7 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, 
	int ret = -ENOENT;



	io_ring_submit_lock(ctx, issue_flags);

	bl = io_buffer_get_list(ctx, req->buf_index);

	bl = io_buffer_get_list(ctx, arg->buf_group);

	if (unlikely(!bl))

		goto out_unlock;
@@ -335,7 +333,7 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg) 


	lockdep_assert_held(&ctx->uring_lock);



	bl = io_buffer_get_list(ctx, req->buf_index);

	bl = io_buffer_get_list(ctx, arg->buf_group);

	if (unlikely(!bl))

		return -ENOENT;
@@ -355,10 +353,9 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr) 
	struct io_buffer_list *bl = req->buf_list;

	bool ret = true;



	if (bl) {

	if (bl)

		ret = io_kbuf_commit(req, bl, len, nr);

		req->buf_index = bl->bgid;

	}



	req->flags &= ~REQ_F_BUFFER_RING;

	return ret;

}

io_uring/kbuf.h

+1 −1
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ struct buf_sel_arg { 
	size_t max_len;

	unsigned short nr_iovs;

	unsigned short mode;

	unsigned buf_group;

};



void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
@@ -94,7 +95,6 @@ static inline bool io_kbuf_recycle_ring(struct io_kiocb *req) 
	 * to monopolize the buffer.

	 */

	if (req->buf_list) {

		req->buf_index = req->buf_list->bgid;

		req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT);

		return true;

	}

io_uring/net.c

+2 −1
Original line number Diff line number Diff line
@@ -190,7 +190,6 @@ static inline void io_mshot_prep_retry(struct io_kiocb *req, 
	sr->done_io = 0;

	sr->retry = false;

	sr->len = 0; /* get from the provided buffer */

	req->buf_index = sr->buf_group;

}



static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg,
@@ -568,6 +567,7 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, 
		.iovs = &kmsg->fast_iov,

		.max_len = min_not_zero(sr->len, INT_MAX),

		.nr_iovs = 1,

		.buf_group = sr->buf_group,

	};



	if (kmsg->vec.iovec) {
@@ -1056,6 +1056,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg 
			.iovs = &kmsg->fast_iov,

			.nr_iovs = 1,

			.mode = KBUF_MODE_EXPAND,

			.buf_group = sr->buf_group,

		};



		if (kmsg->vec.iovec) {