Commit 56112578 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: delay sqarray static branch disablement 



io_key_has_sqarray static branch can be easily switched on/off by the
user every time patching the kernel. That can be very disruptive as it
might require heavy synchronisation across all CPUs. Use deferred static
keys, which can rate-limit it by deferring, batching and potentially
effectively eliminating dec+inc pairs.

Fixes: 9b296c62 ("io_uring: static_key for !IORING_SETUP_NO_SQARRAY")
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6b34f8ed

io_uring/io_uring.c

+4 −4
Original line number Diff line number Diff line
@@ -119,7 +119,7 @@ 
static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);

static void __io_req_caches_free(struct io_ring_ctx *ctx);



static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray);

static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ);



struct kmem_cache *req_cachep;

static struct workqueue_struct *iou_wq __ro_after_init;
@@ -1978,7 +1978,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) 
	unsigned mask = ctx->sq_entries - 1;

	unsigned head = ctx->cached_sq_head++ & mask;



	if (static_branch_unlikely(&io_key_has_sqarray) &&

	if (static_branch_unlikely(&io_key_has_sqarray.key) &&

	    (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {

		head = READ_ONCE(ctx->sq_array[head]);

		if (unlikely(head >= ctx->sq_entries)) {
@@ -2173,7 +2173,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 
	io_rings_free(ctx);



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		static_branch_dec(&io_key_has_sqarray);

		static_branch_slow_dec_deferred(&io_key_has_sqarray);



	percpu_ref_exit(&ctx->refs);

	free_uid(ctx->user);
@@ -2946,7 +2946,7 @@ static __cold int io_uring_create(struct io_ctx_config *config) 
	ctx->clock_offset = 0;



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		static_branch_inc(&io_key_has_sqarray);

		static_branch_deferred_inc(&io_key_has_sqarray);



	if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&

	    !(ctx->flags & IORING_SETUP_IOPOLL))