Commit 1b5add75 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring/kbuf: pass in struct io_buffer_list to commit/recycle helpers 

Rather than have this implied being in the io_kiocb, pass it in directly
so it's immediately obvious where these users of ->buf_list are coming
from.

Link: https://lore.kernel.org/r/20250821020750.598432-6-axboe@kernel.dk


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent b22743f2

io_uring/io_uring.c

+3 −3
Original line number Diff line number Diff line
@@ -1007,7 +1007,7 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) 
	lockdep_assert_held(&req->ctx->uring_lock);



	req_set_fail(req);

	io_req_set_res(req, res, io_put_kbuf(req, res));

	io_req_set_res(req, res, io_put_kbuf(req, res, req->buf_list));

	if (def->fail)

		def->fail(req);

	io_req_complete_defer(req);
@@ -2025,11 +2025,11 @@ static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int r 


	switch (io_arm_poll_handler(req, 0)) {

	case IO_APOLL_READY:

		io_kbuf_recycle(req, 0);

		io_kbuf_recycle(req, req->buf_list, 0);

		io_req_task_queue(req);

		break;

	case IO_APOLL_ABORTED:

		io_kbuf_recycle(req, 0);

		io_kbuf_recycle(req, req->buf_list, 0);

		io_queue_iowq(req);

		break;

	case IO_APOLL_OK:

io_uring/kbuf.c

+5 −4
Original line number Diff line number Diff line
@@ -354,9 +354,9 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg) 
	return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs);

}



static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)

static inline bool __io_put_kbuf_ring(struct io_kiocb *req,

				      struct io_buffer_list *bl, int len, int nr)

{

	struct io_buffer_list *bl = req->buf_list;

	bool ret = true;



	if (bl)
@@ -366,7 +366,8 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr) 
	return ret;

}



unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs)

unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl,

			    int len, int nbufs)

{

	unsigned int ret;
@@ -377,7 +378,7 @@ unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs) 
		return ret;

	}



	if (!__io_put_kbuf_ring(req, len, nbufs))

	if (!__io_put_kbuf_ring(req, bl, len, nbufs))

		ret |= IORING_CQE_F_BUF_MORE;

	return ret;

}

io_uring/kbuf.h

+13 −9
Original line number Diff line number Diff line
@@ -80,14 +80,16 @@ int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg); 
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);

void io_kbuf_drop_legacy(struct io_kiocb *req);



unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs);

unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl,

			    int len, int nbufs);

bool io_kbuf_commit(struct io_kiocb *req,

		    struct io_buffer_list *bl, int len, int nr);



struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,

					    unsigned int bgid);



static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)

static inline bool io_kbuf_recycle_ring(struct io_kiocb *req,

					struct io_buffer_list *bl)

{

	/*

	 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
@@ -96,7 +98,7 @@ static inline bool io_kbuf_recycle_ring(struct io_kiocb *req) 
	 * The exception is partial io, that case we should increment bl->head

	 * to monopolize the buffer.

	 */

	if (req->buf_list) {

	if (bl) {

		req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT);

		return true;

	}
@@ -110,29 +112,31 @@ static inline bool io_do_buffer_select(struct io_kiocb *req) 
	return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));

}



static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)

static inline bool io_kbuf_recycle(struct io_kiocb *req, struct io_buffer_list *bl,

				   unsigned issue_flags)

{

	if (req->flags & REQ_F_BL_NO_RECYCLE)

		return false;

	if (req->flags & REQ_F_BUFFER_SELECTED)

		return io_kbuf_recycle_legacy(req, issue_flags);

	if (req->flags & REQ_F_BUFFER_RING)

		return io_kbuf_recycle_ring(req);

		return io_kbuf_recycle_ring(req, bl);

	return false;

}



static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len)

static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len,

				       struct io_buffer_list *bl)

{

	if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))

		return 0;

	return __io_put_kbufs(req, len, 1);

	return __io_put_kbufs(req, bl, len, 1);

}



static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len,

					int nbufs)

					struct io_buffer_list *bl, int nbufs)

{

	if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))

		return 0;

	return __io_put_kbufs(req, len, nbufs);

	return __io_put_kbufs(req, bl, len, nbufs);

}

#endif

io_uring/net.c

+14 −14
Original line number Diff line number Diff line
@@ -494,12 +494,12 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) 
	return nbufs;

}



static int io_net_kbuf_recyle(struct io_kiocb *req,

static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,

			      struct io_async_msghdr *kmsg, int len)

{

	req->flags |= REQ_F_BL_NO_RECYCLE;

	if (req->flags & REQ_F_BUFFERS_COMMIT)

		io_kbuf_commit(req, req->buf_list, len, io_bundle_nbufs(kmsg, len));

		io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));

	return IOU_RETRY;

}
@@ -511,11 +511,11 @@ static inline bool io_send_finish(struct io_kiocb *req, int *ret, 
	unsigned int cflags;



	if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {

		cflags = io_put_kbuf(req, *ret);

		cflags = io_put_kbuf(req, *ret, req->buf_list);

		goto finish;

	}



	cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret));

	cflags = io_put_kbufs(req, *ret, req->buf_list, io_bundle_nbufs(kmsg, *ret));



	if (bundle_finished || req->flags & REQ_F_BL_EMPTY)

		goto finish;
@@ -681,7 +681,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) 
			sr->len -= ret;

			sr->buf += ret;

			sr->done_io += ret;

			return io_net_kbuf_recyle(req, kmsg, ret);

			return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret);

		}

		if (ret == -ERESTARTSYS)

			ret = -EINTR;
@@ -871,7 +871,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, 
	if (sr->flags & IORING_RECVSEND_BUNDLE) {

		size_t this_ret = *ret - sr->done_io;



		cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret));

		cflags |= io_put_kbufs(req, this_ret, req->buf_list, io_bundle_nbufs(kmsg, this_ret));

		if (sr->flags & IORING_RECV_RETRY)

			cflags = req->cqe.flags | (cflags & CQE_F_MASK);

		if (sr->mshot_len && *ret >= sr->mshot_len)
@@ -893,7 +893,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, 
			return false;

		}

	} else {

		cflags |= io_put_kbuf(req, *ret);

		cflags |= io_put_kbuf(req, *ret, req->buf_list);

	}



	/*
@@ -1045,7 +1045,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 
		if (req->flags & REQ_F_APOLL_MULTISHOT) {

			ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);

			if (ret) {

				io_kbuf_recycle(req, issue_flags);

				io_kbuf_recycle(req, req->buf_list, issue_flags);

				return ret;

			}

		}
@@ -1070,13 +1070,13 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 
	if (ret < min_ret) {

		if (ret == -EAGAIN && force_nonblock) {

			if (issue_flags & IO_URING_F_MULTISHOT)

				io_kbuf_recycle(req, issue_flags);

				io_kbuf_recycle(req, req->buf_list, issue_flags);



			return IOU_RETRY;

		}

		if (ret > 0 && io_net_retry(sock, flags)) {

			sr->done_io += ret;

			return io_net_kbuf_recyle(req, kmsg, ret);

			return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret);

		}

		if (ret == -ERESTARTSYS)

			ret = -EINTR;
@@ -1090,7 +1090,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 
	else if (sr->done_io)

		ret = sr->done_io;

	else

		io_kbuf_recycle(req, issue_flags);

		io_kbuf_recycle(req, req->buf_list, issue_flags);



	if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))

		goto retry_multishot;
@@ -1214,7 +1214,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) 
	if (ret < min_ret) {

		if (ret == -EAGAIN && force_nonblock) {

			if (issue_flags & IO_URING_F_MULTISHOT)

				io_kbuf_recycle(req, issue_flags);

				io_kbuf_recycle(req, req->buf_list, issue_flags);



			return IOU_RETRY;

		}
@@ -1222,7 +1222,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) 
			sr->len -= ret;

			sr->buf += ret;

			sr->done_io += ret;

			return io_net_kbuf_recyle(req, kmsg, ret);

			return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret);

		}

		if (ret == -ERESTARTSYS)

			ret = -EINTR;
@@ -1238,7 +1238,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) 
	else if (sr->done_io)

		ret = sr->done_io;

	else

		io_kbuf_recycle(req, issue_flags);

		io_kbuf_recycle(req, req->buf_list, issue_flags);



	if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))

		goto retry_multishot;

io_uring/poll.c

+3 −3
Original line number Diff line number Diff line
@@ -316,10 +316,10 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) 


	ret = io_poll_check_events(req, tw);

	if (ret == IOU_POLL_NO_ACTION) {

		io_kbuf_recycle(req, 0);

		io_kbuf_recycle(req, req->buf_list, 0);

		return;

	} else if (ret == IOU_POLL_REQUEUE) {

		io_kbuf_recycle(req, 0);

		io_kbuf_recycle(req, req->buf_list, 0);

		__io_poll_execute(req, 0);

		return;

	}
@@ -686,7 +686,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) 
	req->flags |= REQ_F_POLLED;

	ipt.pt._qproc = io_async_queue_proc;



	io_kbuf_recycle(req, issue_flags);

	io_kbuf_recycle(req, req->buf_list, issue_flags);



	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);

	if (ret)