Commit 13ee854e authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/kbuf: remove legacy kbuf caching 



Remove all struct io_buffer caches. It makes it a fair bit simpler.
Apart from from killing a bunch of lines and juggling between lists,
__io_put_kbuf_list() doesn't need ->completion_lock locking now.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/18287217466ee2576ea0b1e72daccf7b22c7e856.1738724373.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent dc39fb10

include/linux/io_uring_types.h

+0 −3
Original line number Diff line number Diff line
@@ -360,7 +360,6 @@ struct io_ring_ctx { 


	spinlock_t		completion_lock;



	struct list_head	io_buffers_comp;

	struct list_head	cq_overflow_list;



	struct hlist_head	waitid_list;
@@ -379,8 +378,6 @@ struct io_ring_ctx { 
	unsigned int		file_alloc_start;

	unsigned int		file_alloc_end;



	struct list_head	io_buffers_cache;



	/* Keep this last, we don't need it for the fast path */

	struct wait_queue_head		poll_wq;

	struct io_restriction		restrictions;

io_uring/io_uring.c

+0 −2
Original line number Diff line number Diff line
@@ -323,7 +323,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	init_waitqueue_head(&ctx->sqo_sq_wait);

	INIT_LIST_HEAD(&ctx->sqd_list);

	INIT_LIST_HEAD(&ctx->cq_overflow_list);

	INIT_LIST_HEAD(&ctx->io_buffers_cache);

	ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,

			    sizeof(struct async_poll), 0);

	ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
@@ -348,7 +347,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	spin_lock_init(&ctx->completion_lock);

	raw_spin_lock_init(&ctx->timeout_lock);

	INIT_WQ_LIST(&ctx->iopoll_list);

	INIT_LIST_HEAD(&ctx->io_buffers_comp);

	INIT_LIST_HEAD(&ctx->defer_list);

	INIT_LIST_HEAD(&ctx->timeout_list);

	INIT_LIST_HEAD(&ctx->ltimeout_list);

io_uring/kbuf.c

+7 −50
Original line number Diff line number Diff line
@@ -70,9 +70,7 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) 


void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags)

{

	spin_lock(&req->ctx->completion_lock);

	__io_put_kbuf_list(req, len);

	spin_unlock(&req->ctx->completion_lock);

}



static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
@@ -345,7 +343,9 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, 
		struct io_buffer *nxt;



		nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);

		list_move(&nxt->list, &ctx->io_buffers_cache);

		list_del(&nxt->list);

		kfree(nxt);



		if (++i == nbufs)

			return i;

		cond_resched();
@@ -363,8 +363,6 @@ static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) 
void io_destroy_buffers(struct io_ring_ctx *ctx)

{

	struct io_buffer_list *bl;

	struct list_head *item, *tmp;

	struct io_buffer *buf;



	while (1) {

		unsigned long index = 0;
@@ -378,19 +376,6 @@ void io_destroy_buffers(struct io_ring_ctx *ctx) 
			break;

		io_put_bl(ctx, bl);

	}



	/*

	 * Move deferred locked entries to cache before pruning

	 */

	spin_lock(&ctx->completion_lock);

	if (!list_empty(&ctx->io_buffers_comp))

		list_splice_init(&ctx->io_buffers_comp, &ctx->io_buffers_cache);

	spin_unlock(&ctx->completion_lock);



	list_for_each_safe(item, tmp, &ctx->io_buffers_cache) {

		buf = list_entry(item, struct io_buffer, list);

		kfree(buf);

	}

}



static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
@@ -479,33 +464,6 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe 
	return 0;

}



static int io_refill_buffer_cache(struct io_ring_ctx *ctx)

{

	struct io_buffer *buf;



	/*

	 * Completions that don't happen inline (eg not under uring_lock) will

	 * add to ->io_buffers_comp. If we don't have any free buffers, check

	 * the completion list and splice those entries first.

	 */

	if (!list_empty_careful(&ctx->io_buffers_comp)) {

		spin_lock(&ctx->completion_lock);

		if (!list_empty(&ctx->io_buffers_comp)) {

			list_splice_init(&ctx->io_buffers_comp,

						&ctx->io_buffers_cache);

			spin_unlock(&ctx->completion_lock);

			return 0;

		}

		spin_unlock(&ctx->completion_lock);

	}



	buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);

	if (!buf)

		return -ENOMEM;

	list_add_tail(&buf->list, &ctx->io_buffers_cache);

	return 0;

}



static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,

			  struct io_buffer_list *bl)

{
@@ -514,12 +472,11 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, 
	int i, bid = pbuf->bid;



	for (i = 0; i < pbuf->nbufs; i++) {

		if (list_empty(&ctx->io_buffers_cache) &&

		    io_refill_buffer_cache(ctx))

		buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);

		if (!buf)

			break;

		buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,

					list);

		list_move_tail(&buf->list, &bl->buf_list);



		list_add_tail(&buf->list, &bl->buf_list);

		buf->addr = addr;

		buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);

		buf->bid = bid;

io_uring/kbuf.h

+2 −3
Original line number Diff line number Diff line
@@ -166,8 +166,9 @@ static inline void __io_put_kbuf_list(struct io_kiocb *req, int len) 
		__io_put_kbuf_ring(req, len, 1);

	} else {

		req->buf_index = req->kbuf->bgid;

		list_add(&req->kbuf->list, &req->ctx->io_buffers_comp);

		req->flags &= ~REQ_F_BUFFER_SELECTED;

		kfree(req->kbuf);

		req->kbuf = NULL;

	}

}
@@ -176,10 +177,8 @@ static inline void io_kbuf_drop(struct io_kiocb *req) 
	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))

		return;



	spin_lock(&req->ctx->completion_lock);

	/* len == 0 is fine here, non-ring will always drop all of it */

	__io_put_kbuf_list(req, 0);

	spin_unlock(&req->ctx->completion_lock);

}



static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,