Commit 414d0f45 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring/alloc_cache: switch to array based caching 



Currently lists are being used to manage this, but best practice is
usually to have these in an array instead as that it cheaper to manage.

Outside of that detail, games are also played with KASAN as the list
is inside the cached entry itself.

Finally, all users of this need a struct io_cache_entry embedded in
their struct, which is union'ized with something else in there that
isn't used across the free -> realloc cycle.

Get rid of all of that, and simply have it be an array. This will not
change the memory used, as we're just trading an 8-byte member entry
for the per-elem array size.

This reduces the overhead of the recycled allocations, and it reduces
the amount of code code needed to support recycling to about half of
what it currently is.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e10677a8

include/linux/io_uring_types.h

+1 −1
Original line number Diff line number Diff line
@@ -220,7 +220,7 @@ struct io_ev_fd { 
};



struct io_alloc_cache {

	struct io_wq_work_node	list;

	void			**entries;

	unsigned int		nr_cached;

	unsigned int		max_cached;

	size_t			elem_size;

io_uring/alloc_cache.h

+26 −31
Original line number Diff line number Diff line
@@ -6,61 +6,56 @@ 
 */

#define IO_ALLOC_CACHE_MAX	128



struct io_cache_entry {

	struct io_wq_work_node node;

};



static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,

				      struct io_cache_entry *entry)

				      void *entry)

{

	if (cache->nr_cached < cache->max_cached) {

		cache->nr_cached++;

		wq_stack_add_head(&entry->node, &cache->list);

		kasan_mempool_poison_object(entry);

		if (!kasan_mempool_poison_object(entry))

			return false;

		cache->entries[cache->nr_cached++] = entry;

		return true;

	}

	return false;

}



static inline bool io_alloc_cache_empty(struct io_alloc_cache *cache)

{

	return !cache->list.next;

}



static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache)

static inline void *io_alloc_cache_get(struct io_alloc_cache *cache)

{

	if (cache->list.next) {

		struct io_cache_entry *entry;

	if (cache->nr_cached) {

		void *entry = cache->entries[--cache->nr_cached];



		entry = container_of(cache->list.next, struct io_cache_entry, node);

		kasan_mempool_unpoison_object(entry, cache->elem_size);

		cache->list.next = cache->list.next->next;

		cache->nr_cached--;

		return entry;

	}



	return NULL;

}



static inline void io_alloc_cache_init(struct io_alloc_cache *cache,

/* returns false if the cache was initialized properly */

static inline bool io_alloc_cache_init(struct io_alloc_cache *cache,

				       unsigned max_nr, size_t size)

{

	cache->list.next = NULL;

	cache->entries = kvmalloc_array(max_nr, sizeof(void *), GFP_KERNEL);

	if (cache->entries) {

		cache->nr_cached = 0;

		cache->max_cached = max_nr;

		cache->elem_size = size;

		return false;

	}

	return true;

}



static inline void io_alloc_cache_free(struct io_alloc_cache *cache,

					void (*free)(struct io_cache_entry *))

				       void (*free)(const void *))

{

	while (1) {

		struct io_cache_entry *entry = io_alloc_cache_get(cache);

	void *entry;



	if (!cache->entries)

		return;



		if (!entry)

			break;

	while ((entry = io_alloc_cache_get(cache)) != NULL)

		free(entry);

	}

	cache->nr_cached = 0;



	kvfree(cache->entries);

	cache->entries = NULL;

}

#endif

io_uring/futex.c

+12 −18
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ 


#include "../kernel/futex/futex.h"

#include "io_uring.h"

#include "rsrc.h"

#include "alloc_cache.h"

#include "futex.h"



struct io_futex {
@@ -27,27 +27,21 @@ struct io_futex { 
};



struct io_futex_data {

	union {

	struct futex_q	q;

		struct io_cache_entry	cache;

	};

	struct io_kiocb	*req;

};



void io_futex_cache_init(struct io_ring_ctx *ctx)

{

	io_alloc_cache_init(&ctx->futex_cache, IO_NODE_ALLOC_CACHE_MAX,

				sizeof(struct io_futex_data));

}

#define IO_FUTEX_ALLOC_CACHE_MAX	32



static void io_futex_cache_entry_free(struct io_cache_entry *entry)

bool io_futex_cache_init(struct io_ring_ctx *ctx)

{

	kfree(container_of(entry, struct io_futex_data, cache));

	return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX,

				sizeof(struct io_futex_data));

}



void io_futex_cache_free(struct io_ring_ctx *ctx)

{

	io_alloc_cache_free(&ctx->futex_cache, io_futex_cache_entry_free);

	io_alloc_cache_free(&ctx->futex_cache, kfree);

}



static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
@@ -63,7 +57,7 @@ static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) 
	struct io_ring_ctx *ctx = req->ctx;



	io_tw_lock(ctx, ts);

	if (!io_alloc_cache_put(&ctx->futex_cache, &ifd->cache))

	if (!io_alloc_cache_put(&ctx->futex_cache, ifd))

		kfree(ifd);

	__io_futex_complete(req, ts);

}
@@ -259,11 +253,11 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) 


static struct io_futex_data *io_alloc_ifd(struct io_ring_ctx *ctx)

{

	struct io_cache_entry *entry;

	struct io_futex_data *ifd;



	entry = io_alloc_cache_get(&ctx->futex_cache);

	if (entry)

		return container_of(entry, struct io_futex_data, cache);

	ifd = io_alloc_cache_get(&ctx->futex_cache);

	if (ifd)

		return ifd;



	return kmalloc(sizeof(struct io_futex_data), GFP_NOWAIT);

}

io_uring/futex.h

+3 −2
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, 
		    unsigned int issue_flags);

bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,

			 bool cancel_all);

void io_futex_cache_init(struct io_ring_ctx *ctx);

bool io_futex_cache_init(struct io_ring_ctx *ctx);

void io_futex_cache_free(struct io_ring_ctx *ctx);

#else

static inline int io_futex_cancel(struct io_ring_ctx *ctx,
@@ -27,8 +27,9 @@ static inline bool io_futex_remove_all(struct io_ring_ctx *ctx, 
{

	return false;

}

static inline void io_futex_cache_init(struct io_ring_ctx *ctx)

static inline bool io_futex_cache_init(struct io_ring_ctx *ctx)

{

	return false;

}

static inline void io_futex_cache_free(struct io_ring_ctx *ctx)

{

io_uring/io_uring.c

+19 −15
Original line number Diff line number Diff line
@@ -276,6 +276,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
{

	struct io_ring_ctx *ctx;

	int hash_bits;

	bool ret;



	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);

	if (!ctx)
@@ -305,17 +306,19 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	INIT_LIST_HEAD(&ctx->cq_overflow_list);

	INIT_LIST_HEAD(&ctx->io_buffers_cache);

	INIT_HLIST_HEAD(&ctx->io_buf_list);

	io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,

	ret = io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,

			    sizeof(struct io_rsrc_node));

	io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,

	ret |= io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,

			    sizeof(struct async_poll));

	io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,

	ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,

			    sizeof(struct io_async_msghdr));

	io_alloc_cache_init(&ctx->rw_cache, IO_ALLOC_CACHE_MAX,

	ret |= io_alloc_cache_init(&ctx->rw_cache, IO_ALLOC_CACHE_MAX,

			    sizeof(struct io_async_rw));

	io_alloc_cache_init(&ctx->uring_cache, IO_ALLOC_CACHE_MAX,

	ret |= io_alloc_cache_init(&ctx->uring_cache, IO_ALLOC_CACHE_MAX,

			    sizeof(struct uring_cache));

	io_futex_cache_init(ctx);

	ret |= io_futex_cache_init(ctx);

	if (ret)

		goto err;

	init_completion(&ctx->ref_comp);

	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);

	mutex_init(&ctx->uring_lock);
@@ -345,6 +348,12 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 


	return ctx;

err:

	io_alloc_cache_free(&ctx->rsrc_node_cache, kfree);

	io_alloc_cache_free(&ctx->apoll_cache, kfree);

	io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);

	io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);

	io_alloc_cache_free(&ctx->uring_cache, kfree);

	io_futex_cache_free(ctx);

	kfree(ctx->cancel_table.hbs);

	kfree(ctx->cancel_table_locked.hbs);

	xa_destroy(&ctx->io_bl_xa);
@@ -1482,7 +1491,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, 


				if (apoll->double_poll)

					kfree(apoll->double_poll);

				if (!io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache))

				if (!io_alloc_cache_put(&ctx->apoll_cache, apoll))

					kfree(apoll);

				req->flags &= ~REQ_F_POLLED;

			}
@@ -2778,11 +2787,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) 
	mutex_unlock(&ctx->uring_lock);

}



static void io_rsrc_node_cache_free(struct io_cache_entry *entry)

{

	kfree(container_of(entry, struct io_rsrc_node, cache));

}



static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)

{

	io_sq_thread_finish(ctx);
@@ -2797,10 +2801,10 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 
		__io_sqe_files_unregister(ctx);

	io_cqring_overflow_kill(ctx);

	io_eventfd_unregister(ctx);

	io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);

	io_alloc_cache_free(&ctx->apoll_cache, kfree);

	io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);

	io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);

	io_alloc_cache_free(&ctx->uring_cache, io_uring_cache_free);

	io_alloc_cache_free(&ctx->uring_cache, kfree);

	io_futex_cache_free(ctx);

	io_destroy_buffers(ctx);

	mutex_unlock(&ctx->uring_lock);
@@ -2816,7 +2820,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 
	WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));

	WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));



	io_alloc_cache_free(&ctx->rsrc_node_cache, io_rsrc_node_cache_free);

	io_alloc_cache_free(&ctx->rsrc_node_cache, kfree);

	if (ctx->mm_account) {

		mmdrop(ctx->mm_account);

		ctx->mm_account = NULL;