Commit 5c686456 authored by David Wei's avatar David Wei Committed by Jens Axboe
Browse files

io_uring/zcrx: add user_struct and mm_struct to io_zcrx_ifq 



In preparation for removing ifq->ctx and making ifq lifetime independent
of ring ctx, add user_struct and mm_struct to io_zcrx_ifq.

In the ifq cleanup path, these are the only fields used from the main
ring ctx to do accounting. Taking a copy in the ifq allows ifq->ctx to
be removed later, including the ctx->refs held by the ifq.

Signed-off-by: default avatarDavid Wei <dw@davidwei.uk>
Reviewed-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent edd706ed

io_uring/zcrx.c

+18 −6
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq, 
	}



	mem->account_pages = io_count_account_pages(pages, nr_pages);

	ret = io_account_mem(ifq->ctx->user, ifq->ctx->mm_account, mem->account_pages);

	ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages);

	if (ret < 0)

		mem->account_pages = 0;
@@ -344,7 +344,8 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov) 
	atomic_inc(io_get_user_counter(niov));

}



static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,

static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,

				 struct io_zcrx_ifq *ifq,

				 struct io_uring_zcrx_ifq_reg *reg,

				 struct io_uring_region_desc *rd,

				 u32 id)
@@ -362,7 +363,7 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, 
	mmap_offset = IORING_MAP_OFF_ZCRX_REGION;

	mmap_offset += id << IORING_OFF_PBUF_SHIFT;



	ret = io_create_region(ifq->ctx, &ifq->region, rd, mmap_offset);

	ret = io_create_region(ctx, &ifq->region, rd, mmap_offset);

	if (ret < 0)

		return ret;
@@ -378,7 +379,7 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, 


static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)

{

	io_free_region(ifq->ctx->user, &ifq->region);

	io_free_region(ifq->user, &ifq->region);

	ifq->rq_ring = NULL;

	ifq->rqes = NULL;

}
@@ -390,7 +391,7 @@ static void io_zcrx_free_area(struct io_zcrx_ifq *ifq, 
	io_release_area_mem(&area->mem);



	if (area->mem.account_pages)

		io_unaccount_mem(area->ifq->ctx->user, area->ifq->ctx->mm_account,

		io_unaccount_mem(ifq->user, ifq->mm_account,

				 area->mem.account_pages);



	kvfree(area->freelist);
@@ -525,6 +526,9 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) 


	if (ifq->area)

		io_zcrx_free_area(ifq, ifq->area);

	free_uid(ifq->user);

	if (ifq->mm_account)

		mmdrop(ifq->mm_account);

	if (ifq->dev)

		put_device(ifq->dev);
@@ -588,6 +592,14 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 
	ifq = io_zcrx_ifq_alloc(ctx);

	if (!ifq)

		return -ENOMEM;

	if (ctx->user) {

		get_uid(ctx->user);

		ifq->user = ctx->user;

	}

	if (ctx->mm_account) {

		mmgrab(ctx->mm_account);

		ifq->mm_account = ctx->mm_account;

	}

	ifq->rq_entries = reg.rq_entries;



	scoped_guard(mutex, &ctx->mmap_lock) {
@@ -597,7 +609,7 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 
			goto ifq_free;

	}



	ret = io_allocate_rbuf_ring(ifq, &reg, &rd, id);

	ret = io_allocate_rbuf_ring(ctx, ifq, &reg, &rd, id);

	if (ret)

		goto err;

io_uring/zcrx.h

+2 −0
Original line number Diff line number Diff line
@@ -42,6 +42,8 @@ struct io_zcrx_ifq { 
	struct io_ring_ctx		*ctx;

	struct io_zcrx_area		*area;

	unsigned			niov_shift;

	struct user_struct		*user;

	struct mm_struct		*mm_account;



	spinlock_t			rq_lock ____cacheline_aligned_in_smp;

	struct io_uring			*rq_ring;