Commit 6f94cbc2 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring/rsrc: split io_kiocb node type assignments 



Currently the io_rsrc_node assignment in io_kiocb is an array of two
pointers, as two nodes may be assigned to a request - one file node,
and one buffer node. However, the buffer node can co-exist with the
provided buffers, as currently it's not supported to use both provided
and registered buffers at the same time.

This crucially brings struct io_kiocb down to 4 cache lines again, as
before it spilled into the 5th cacheline.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6af82f76

include/linux/io_uring_types.h

+6 −1
Original line number Diff line number Diff line
@@ -475,6 +475,7 @@ enum { 
	REQ_F_BL_EMPTY_BIT,

	REQ_F_BL_NO_RECYCLE_BIT,

	REQ_F_BUFFERS_COMMIT_BIT,

	REQ_F_BUF_NODE_BIT,



	/* not a real bit, just to check we're not overflowing the space */

	__REQ_F_LAST_BIT,
@@ -553,6 +554,8 @@ enum { 
	REQ_F_BL_NO_RECYCLE	= IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),

	/* buffer ring head needs incrementing on put */

	REQ_F_BUFFERS_COMMIT	= IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),

	/* buf node is valid */

	REQ_F_BUF_NODE		= IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),

};



typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
@@ -633,6 +636,8 @@ struct io_kiocb { 
		 * REQ_F_BUFFER_RING is set.

		 */

		struct io_buffer_list	*buf_list;



		struct io_rsrc_node	*buf_node;

	};



	union {
@@ -642,7 +647,7 @@ struct io_kiocb { 
		__poll_t apoll_events;

	};



	struct io_rsrc_node		*rsrc_nodes[2];

	struct io_rsrc_node		*file_node;



	atomic_t			refs;

	bool				cancel_seq_set;

io_uring/io_uring.c

+3 −3
Original line number Diff line number Diff line
@@ -948,8 +948,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) 
static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)

{

	req->ctx = ctx;

	req->rsrc_nodes[IORING_RSRC_FILE] = NULL;

	req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;

	req->buf_node = NULL;

	req->file_node = NULL;

	req->link = NULL;

	req->async_data = NULL;

	/* not necessary, but safer to zero */
@@ -1882,7 +1882,7 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 
	io_ring_submit_lock(ctx, issue_flags);

	node = io_rsrc_node_lookup(&ctx->file_table.data, fd);

	if (node) {

		io_req_assign_rsrc_node(req, node);

		io_req_assign_rsrc_node(&req->file_node, node);

		req->flags |= io_slot_flags(node);

		file = io_slot_file(node);

	}

io_uring/net.c

+2 −1
Original line number Diff line number Diff line
@@ -1348,7 +1348,8 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) 
		io_ring_submit_lock(ctx, issue_flags);

		node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);

		if (node) {

			io_req_assign_rsrc_node(sr->notif, node);

			io_req_assign_rsrc_node(&sr->notif->buf_node, node);

			sr->notif->flags |= REQ_F_BUF_NODE;

			ret = 0;

		}

		io_ring_submit_unlock(ctx, issue_flags);

io_uring/nop.c

+2 −1
Original line number Diff line number Diff line
@@ -67,7 +67,8 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) 
		io_ring_submit_lock(ctx, issue_flags);

		node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);

		if (node) {

			io_req_assign_rsrc_node(req, node);

			io_req_assign_rsrc_node(&req->buf_node, node);

			req->flags |= REQ_F_BUF_NODE;

			ret = 0;

		}

		io_ring_submit_unlock(ctx, issue_flags);

io_uring/notif.c

+2 −2
Original line number Diff line number Diff line
@@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) 
	notif->file = NULL;

	notif->task = current;

	io_get_task_refs(1);

	notif->rsrc_nodes[IORING_RSRC_FILE] = NULL;

	notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;

	notif->file_node = NULL;

	notif->buf_node = NULL;



	nd = io_notif_to_data(notif);

	nd->zc_report = false;