Commit 528407b1 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/rsrc: consolidate node caching 



We store one pre-allocated rsrc node in ->rsrc_backup_node, merge it
with ->rsrc_node_cache.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/6d5410e51ccd29be7a716be045b51d6b371baef6.1681210788.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 786788a8

include/linux/io_uring_types.h

+0 −1
Original line number Diff line number Diff line
@@ -326,7 +326,6 @@ struct io_ring_ctx { 
	struct io_restriction		restrictions;



	/* slow path rsrc auxilary data, used by update/register */

	struct io_rsrc_node		*rsrc_backup_node;

	struct io_mapped_ubuf		*dummy_ubuf;

	struct io_rsrc_data		*file_data;

	struct io_rsrc_data		*buf_data;

io_uring/alloc_cache.h

+5 −0
Original line number Diff line number Diff line
@@ -23,6 +23,11 @@ static inline bool io_alloc_cache_put(struct io_alloc_cache *cache, 
	return false;

}



static inline bool io_alloc_cache_empty(struct io_alloc_cache *cache)

{

	return !cache->list.next;

}



static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache)

{

	if (cache->list.next) {

io_uring/io_uring.c

+0 −2
Original line number Diff line number Diff line
@@ -2852,8 +2852,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 
	/* there are no registered resources left, nobody uses it */

	if (ctx->rsrc_node)

		io_rsrc_node_destroy(ctx, ctx->rsrc_node);

	if (ctx->rsrc_backup_node)

		io_rsrc_node_destroy(ctx, ctx->rsrc_backup_node);



	WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));

io_uring/rsrc.c

+11 −9
Original line number Diff line number Diff line
@@ -230,7 +230,7 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx, 
			 struct io_rsrc_data *data_to_kill)

	__must_hold(&ctx->uring_lock)

{

	WARN_ON_ONCE(!ctx->rsrc_backup_node);

	WARN_ON_ONCE(io_alloc_cache_empty(&ctx->rsrc_node_cache));

	WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);



	if (data_to_kill) {
@@ -245,18 +245,20 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx, 
		ctx->rsrc_node = NULL;

	}



	if (!ctx->rsrc_node) {

		ctx->rsrc_node = ctx->rsrc_backup_node;

		ctx->rsrc_backup_node = NULL;

	}

	if (!ctx->rsrc_node)

		ctx->rsrc_node = io_rsrc_node_alloc(ctx);

}



int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)

{

	if (ctx->rsrc_backup_node)

	if (io_alloc_cache_empty(&ctx->rsrc_node_cache)) {

		struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);



		if (!node)

			return -ENOMEM;

		io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);

	}

	return 0;

	ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);

	return ctx->rsrc_backup_node ? 0 : -ENOMEM;

}



__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,