Commit 69bbc6ad authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/rsrc: add custom limit for node caching 



The number of entries in the rsrc node cache is limited to 512, which
still seems unnecessarily large. Add per cache thresholds and set to
to 32 for the rsrc node cache.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d0cd538b944dac0bf878e276fc0199f21e6bccea.1680576071.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 757ef468

include/linux/io_uring_types.h

+1 −0
Original line number Diff line number Diff line
@@ -190,6 +190,7 @@ struct io_ev_fd { 
struct io_alloc_cache {

	struct io_wq_work_node	list;

	unsigned int		nr_cached;

	unsigned int		max_cached;

	size_t			elem_size;

};

io_uring/alloc_cache.h

+4 −2
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ struct io_cache_entry { 
static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,

				      struct io_cache_entry *entry)

{

	if (cache->nr_cached < IO_ALLOC_CACHE_MAX) {

	if (cache->nr_cached < cache->max_cached) {

		cache->nr_cached++;

		wq_stack_add_head(&entry->node, &cache->list);

		/* KASAN poisons object */
@@ -38,10 +38,12 @@ static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *c 
	return NULL;

}



static inline void io_alloc_cache_init(struct io_alloc_cache *cache, size_t size)

static inline void io_alloc_cache_init(struct io_alloc_cache *cache,

				       unsigned max_nr, size_t size)

{

	cache->list.next = NULL;

	cache->nr_cached = 0;

	cache->max_cached = max_nr;

	cache->elem_size = size;

}

io_uring/io_uring.c

+6 −3
Original line number Diff line number Diff line
@@ -310,9 +310,12 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	INIT_LIST_HEAD(&ctx->sqd_list);

	INIT_LIST_HEAD(&ctx->cq_overflow_list);

	INIT_LIST_HEAD(&ctx->io_buffers_cache);

	io_alloc_cache_init(&ctx->rsrc_node_cache, sizeof(struct io_rsrc_node));

	io_alloc_cache_init(&ctx->apoll_cache, sizeof(struct async_poll));

	io_alloc_cache_init(&ctx->netmsg_cache, sizeof(struct io_async_msghdr));

	io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,

			    sizeof(struct io_rsrc_node));

	io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,

			    sizeof(struct async_poll));

	io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,

			    sizeof(struct io_async_msghdr));

	init_completion(&ctx->ref_comp);

	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);

	mutex_init(&ctx->uring_lock);

io_uring/rsrc.h

+2 −0
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ 


#include "alloc_cache.h"



#define IO_NODE_ALLOC_CACHE_MAX 32



#define IO_RSRC_TAG_TABLE_SHIFT	(PAGE_SHIFT - 3)

#define IO_RSRC_TAG_TABLE_MAX	(1U << IO_RSRC_TAG_TABLE_SHIFT)

#define IO_RSRC_TAG_TABLE_MASK	(IO_RSRC_TAG_TABLE_MAX - 1)