Commit 001b76b7 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: keep ring laoyut in a structure 



Add a structure keeping SQ/CQ sizes and offsets. For now it only records
data previously returned from rings_size and the SQ size.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 0f4b5373

io_uring/io_uring.c

+28 −26
Original line number Diff line number Diff line
@@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx) 
	ctx->sq_sqes = NULL;

}



unsigned long rings_size(unsigned int flags, unsigned int sq_entries,

			 unsigned int cq_entries, size_t *sq_offset)

int rings_size(unsigned int flags, unsigned int sq_entries,

		unsigned int cq_entries, struct io_rings_layout *rl)

{

	struct io_rings *rings;

	size_t sqe_size;

	size_t off;



	*sq_offset = SIZE_MAX;



	if (flags & IORING_SETUP_CQE_MIXED) {

		if (cq_entries < 2)

			return SIZE_MAX;

			return -EOVERFLOW;

	}

	if (flags & IORING_SETUP_SQE_MIXED) {

		if (sq_entries < 2)

			return SIZE_MAX;

			return -EOVERFLOW;

	}



	rl->sq_array_offset = SIZE_MAX;



	sqe_size = sizeof(struct io_uring_sqe);

	if (flags & IORING_SETUP_SQE128)

		sqe_size *= 2;



	rl->sq_size = array_size(sqe_size, sq_entries);

	if (rl->sq_size == SIZE_MAX)

		return -EOVERFLOW;



	off = struct_size(rings, cqes, cq_entries);

	if (flags & IORING_SETUP_CQE32)

		off = size_mul(off, 2);

	if (off == SIZE_MAX)

		return SIZE_MAX;

		return -EOVERFLOW;



#ifdef CONFIG_SMP

	off = ALIGN(off, SMP_CACHE_BYTES);

	if (off == 0)

		return SIZE_MAX;

		return -EOVERFLOW;

#endif



	if (!(flags & IORING_SETUP_NO_SQARRAY)) {

		size_t sq_array_size;



		*sq_offset = off;

		rl->sq_array_offset = off;



		sq_array_size = array_size(sizeof(u32), sq_entries);

		off = size_add(off, sq_array_size);

		if (off == SIZE_MAX)

			return SIZE_MAX;

			return -EOVERFLOW;

	}



	return off;

	rl->rings_size = off;

	return 0;

}



static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, 
					 struct io_uring_params *p)

{

	struct io_uring_region_desc rd;

	struct io_rings_layout __rl, *rl = &__rl;

	struct io_rings *rings;

	size_t sq_array_offset;

	size_t sq_size, cq_size, sqe_size;

	int ret;



	/* make sure these are sane, as we already accounted them */

	ctx->sq_entries = p->sq_entries;

	ctx->cq_entries = p->cq_entries;



	sqe_size = sizeof(struct io_uring_sqe);

	if (p->flags & IORING_SETUP_SQE128)

		sqe_size *= 2;

	sq_size = array_size(sqe_size, p->sq_entries);

	if (sq_size == SIZE_MAX)

		return -EOVERFLOW;

	cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,

			  &sq_array_offset);

	if (cq_size == SIZE_MAX)

		return -EOVERFLOW;

	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);

	if (ret)

		return ret;



	memset(&rd, 0, sizeof(rd));

	rd.size = PAGE_ALIGN(cq_size);

	rd.size = PAGE_ALIGN(rl->rings_size);

	if (ctx->flags & IORING_SETUP_NO_MMAP) {

		rd.user_addr = p->cq_off.user_addr;

		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, 
	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);



	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);

		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);



	memset(&rd, 0, sizeof(rd));

	rd.size = PAGE_ALIGN(sq_size);

	rd.size = PAGE_ALIGN(rl->sq_size);

	if (ctx->flags & IORING_SETUP_NO_MMAP) {

		rd.user_addr = p->sq_off.user_addr;

		rd.flags |= IORING_MEM_REGION_TYPE_USER;

io_uring/io_uring.h

+10 −2
Original line number Diff line number Diff line
@@ -17,6 +17,14 @@ 
#include <trace/events/io_uring.h>

#endif



struct io_rings_layout {

	/* size of CQ + headers + SQ offset array */

	size_t rings_size;

	size_t sq_size;



	size_t sq_array_offset;

};



struct io_ctx_config {

	struct io_uring_params p;

	struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) 
#define IORING_MAX_ENTRIES	32768

#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)



unsigned long rings_size(unsigned int flags, unsigned int sq_entries,

			 unsigned int cq_entries, size_t *sq_offset);

int rings_size(unsigned int flags, unsigned int sq_entries,

		unsigned int cq_entries, struct io_rings_layout *rl);

int io_prepare_config(struct io_ctx_config *config);



bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

io_uring/register.c

+7 −17
Original line number Diff line number Diff line
@@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) 
	struct io_ctx_config config;

	struct io_uring_region_desc rd;

	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;

	size_t size, sq_array_offset;

	unsigned i, tail, old_head;

	struct io_uring_params *p = &config.p;

	struct io_rings_layout __rl, *rl = &__rl;

	int ret;



	memset(&config, 0, sizeof(config));
@@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) 
	if (unlikely(ret))

		return ret;



	size = rings_size(p->flags, p->sq_entries, p->cq_entries,

				&sq_array_offset);

	if (size == SIZE_MAX)

		return -EOVERFLOW;

	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);

	if (ret)

		return ret;



	memset(&rd, 0, sizeof(rd));

	rd.size = PAGE_ALIGN(size);

	rd.size = PAGE_ALIGN(rl->rings_size);

	if (p->flags & IORING_SETUP_NO_MMAP) {

		rd.user_addr = p->cq_off.user_addr;

		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) 
		return -EFAULT;

	}



	if (p->flags & IORING_SETUP_SQE128)

		size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);

	else

		size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);

	if (size == SIZE_MAX) {

		io_register_free_rings(ctx, &n);

		return -EOVERFLOW;

	}



	memset(&rd, 0, sizeof(rd));

	rd.size = PAGE_ALIGN(size);

	rd.size = PAGE_ALIGN(rl->sq_size);

	if (p->flags & IORING_SETUP_NO_MMAP) {

		rd.user_addr = p->sq_off.user_addr;

		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -551,7 +541,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) 


	/* all done, store old pointers and assign new ones */

	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))

		ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);

		ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);



	ctx->sq_entries = p->sq_entries;

	ctx->cq_entries = p->cq_entries;