Commit eb76ff6a authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: pre-calculate scq layout 



Move ring layouts calculations into io_prepare_config(), so that more
misconfiguration checking can be done earlier before creating a ctx.
It also deduplicates some code with ring resizing. And as a bonus, now
it initialises params->sq_off.array, which is closer to all other user
offset init, and also applies it to ring resizing, which was previously
missing it.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 001b76b7

io_uring/io_uring.c

+14 −13
Original line number Diff line number Diff line
@@ -2757,7 +2757,7 @@ static void io_rings_free(struct io_ring_ctx *ctx) 
	ctx->sq_sqes = NULL;

}



int rings_size(unsigned int flags, unsigned int sq_entries,

static int rings_size(unsigned int flags, unsigned int sq_entries,

		      unsigned int cq_entries, struct io_rings_layout *rl)

{

	struct io_rings *rings;
@@ -3353,10 +3353,11 @@ bool io_is_uring_fops(struct file *file) 
}



static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,

					 struct io_uring_params *p)

					 struct io_ctx_config *config)

{

	struct io_uring_params *p = &config->p;

	struct io_rings_layout *rl = &config->layout;

	struct io_uring_region_desc rd;

	struct io_rings_layout __rl, *rl = &__rl;

	struct io_rings *rings;

	int ret;
@@ -3364,10 +3365,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, 
	ctx->sq_entries = p->sq_entries;

	ctx->cq_entries = p->cq_entries;



	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);

	if (ret)

		return ret;



	memset(&rd, 0, sizeof(rd));

	rd.size = PAGE_ALIGN(rl->rings_size);

	if (ctx->flags & IORING_SETUP_NO_MMAP) {
@@ -3378,7 +3375,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, 
	if (ret)

		return ret;

	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
@@ -3560,6 +3556,14 @@ int io_prepare_config(struct io_ctx_config *config) 
	if (ret)

		return ret;



	ret = rings_size(p->flags, p->sq_entries, p->cq_entries,

			 &config->layout);

	if (ret)

		return ret;



	if (!(p->flags & IORING_SETUP_NO_SQARRAY))

		p->sq_off.array = config->layout.sq_array_offset;



	return 0;

}
@@ -3632,13 +3636,10 @@ static __cold int io_uring_create(struct io_ctx_config *config) 
	mmgrab(current->mm);

	ctx->mm_account = current->mm;



	ret = io_allocate_scq_urings(ctx, p);

	ret = io_allocate_scq_urings(ctx, config);

	if (ret)

		goto err;



	if (!(p->flags & IORING_SETUP_NO_SQARRAY))

		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;



	ret = io_sq_offload_create(ctx, p);

	if (ret)

		goto err;

io_uring/io_uring.h

+1 −2
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ struct io_rings_layout { 


struct io_ctx_config {

	struct io_uring_params p;

	struct io_rings_layout layout;

	struct io_uring_params __user *uptr;

};
@@ -147,8 +148,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) 
#define IORING_MAX_ENTRIES	32768

#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)



int rings_size(unsigned int flags, unsigned int sq_entries,

		unsigned int cq_entries, struct io_rings_layout *rl);

int io_prepare_config(struct io_ctx_config *config);



bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

io_uring/register.c

+0 −4
Original line number Diff line number Diff line
@@ -423,10 +423,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) 
	if (unlikely(ret))

		return ret;



	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);

	if (ret)

		return ret;



	memset(&rd, 0, sizeof(rd));

	rd.size = PAGE_ALIGN(rl->rings_size);

	if (p->flags & IORING_SETUP_NO_MMAP) {