Commit 9b296c62 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: static_key for !IORING_SETUP_NO_SQARRAY 



IORING_SETUP_NO_SQARRAY should be preferred and used by default by
liburing, optimise flag checking in io_get_sqe() with a static key.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c164a48542fbb080115e2377ecf160c758562742.1729264988.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 1e6e7602

io_uring/io_uring.c

+11 −1
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ 
#include <linux/io_uring/cmd.h>

#include <linux/audit.h>

#include <linux/security.h>

#include <linux/jump_label.h>

#include <asm/shmparam.h>



#define CREATE_TRACE_POINTS
@@ -149,6 +150,8 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, 


static void io_queue_sqe(struct io_kiocb *req);



static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray);



struct kmem_cache *req_cachep;

static struct workqueue_struct *iou_wq __ro_after_init;
@@ -2254,7 +2257,8 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) 
	unsigned mask = ctx->sq_entries - 1;

	unsigned head = ctx->cached_sq_head++ & mask;



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) {

	if (static_branch_unlikely(&io_key_has_sqarray) &&

	    (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {

		head = READ_ONCE(ctx->sq_array[head]);

		if (unlikely(head >= ctx->sq_entries)) {

			/* drop invalid entries */
@@ -2758,6 +2762,9 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 
	}

	io_rings_free(ctx);



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		static_branch_dec(&io_key_has_sqarray);



	percpu_ref_exit(&ctx->refs);

	free_uid(ctx->user);

	io_req_caches_free(ctx);
@@ -3549,6 +3556,9 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, 
	ctx->clockid = CLOCK_MONOTONIC;

	ctx->clock_offset = 0;



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		static_branch_inc(&io_key_has_sqarray);



	if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&

	    !(ctx->flags & IORING_SETUP_IOPOLL) &&

	    !(ctx->flags & IORING_SETUP_SQPOLL))