Commit 3f302388 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring/rsrc: improve code generation for fixed file assignment 



For the normal read/write path, we have already locked the ring
submission side when assigning the file. This causes branch
mispredictions when we then check and try and lock again in
io_req_set_rsrc_node(). As this is a very hot path, this matters.

Add a basic helper that already assumes we already have it locked,
and use that in io_file_get_fixed().

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent fe80eb15

io_uring/io_uring.c

+3 −2
Original line number Diff line number Diff line
@@ -2000,9 +2000,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 
		goto out;

	fd = array_index_nospec(fd, ctx->nr_user_files);

	slot = io_fixed_file_slot(&ctx->file_table, fd);

	file = io_slot_file(slot);

	if (!req->rsrc_node)

		__io_req_set_rsrc_node(req, ctx);

	req->flags |= io_slot_flags(slot);

	io_req_set_rsrc_node(req, ctx, 0);

	file = io_slot_file(slot);

out:

	io_ring_submit_unlock(ctx, issue_flags);

	return file;

io_uring/rsrc.h

+9 −5
Original line number Diff line number Diff line
@@ -102,17 +102,21 @@ static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx, 
	node->refs++;

}



static inline void __io_req_set_rsrc_node(struct io_kiocb *req,

					  struct io_ring_ctx *ctx)

{

	lockdep_assert_held(&ctx->uring_lock);

	req->rsrc_node = ctx->rsrc_node;

	io_charge_rsrc_node(ctx, ctx->rsrc_node);

}



static inline void io_req_set_rsrc_node(struct io_kiocb *req,

					struct io_ring_ctx *ctx,

					unsigned int issue_flags)

{

	if (!req->rsrc_node) {

		io_ring_submit_lock(ctx, issue_flags);



		lockdep_assert_held(&ctx->uring_lock);



		req->rsrc_node = ctx->rsrc_node;

		io_charge_rsrc_node(ctx, ctx->rsrc_node);

		__io_req_set_rsrc_node(req, ctx);

		io_ring_submit_unlock(ctx, issue_flags);

	}

}