Commit 99ebe4ef authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: pre-initialise some of req fields 



Most of requests are allocated from an internal cache, so it's waste of
time fully initialising them every time. Instead, let's pre-init some of
the fields we can during initial allocation (e.g. kmalloc(), see
io_alloc_req()) and keep them valid on request recycling. There are four
of them in this patch:

->ctx is always stays the same
->link is NULL on free, it's an invariant
->result is not even needed to init, just a precaution
->async_data we now clean in io_dismantle_req() as it's likely to
   never be allocated.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/892ba0e71309bba9fe9e0142472330bbf9d8f05d.1624739600.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5182ed2e

fs/io_uring.c

+18 −6
Original line number Diff line number Diff line
@@ -1711,7 +1711,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) 


	if (!state->free_reqs) {

		gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;

		int ret;

		int ret, i;



		if (io_flush_cached_reqs(ctx))

			goto got_req;
@@ -1729,6 +1729,20 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) 
				return NULL;

			ret = 1;

		}



		/*

		 * Don't initialise the fields below on every allocation, but

		 * do that in advance and keep valid on free.

		 */

		for (i = 0; i < ret; i++) {

			struct io_kiocb *req = state->reqs[i];



			req->ctx = ctx;

			req->link = NULL;

			req->async_data = NULL;

			/* not necessary, but safer to zero */

			req->result = 0;

		}

		state->free_reqs = ret;

	}

got_req:
@@ -1752,8 +1766,10 @@ static void io_dismantle_req(struct io_kiocb *req) 
		io_put_file(req->file);

	if (req->fixed_rsrc_refs)

		percpu_ref_put(req->fixed_rsrc_refs);

	if (req->async_data)

	if (req->async_data) {

		kfree(req->async_data);

		req->async_data = NULL;

	}

}



/* must to be called somewhat shortly after putting a request */
@@ -6534,15 +6550,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, 
	/* same numerical values with corresponding REQ_F_*, safe to copy */

	req->flags = sqe_flags = READ_ONCE(sqe->flags);

	req->user_data = READ_ONCE(sqe->user_data);

	req->async_data = NULL;

	req->file = NULL;

	req->ctx = ctx;

	req->link = NULL;

	req->fixed_rsrc_refs = NULL;

	/* one is dropped after submission, the other at completion */

	atomic_set(&req->refs, 2);

	req->task = current;

	req->result = 0;



	/* enforce forwards compatibility on users */

	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))