Commit 4a0fef62 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: optimize io_uring_task layout 



task_work bits of io_uring_task are split into two cache lines causing
extra cache bouncing, place them into a separate cache line. Also move
the most used submission path fields closer together, so there are hot.

Cc: stable@vger.kernel.org # 5.15+
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent bce5d70c

io_uring/tctx.h

+18 −16
Original line number Diff line number Diff line
@@ -8,21 +8,23 @@ 
struct io_uring_task {

	/* submission side */

	int				cached_refs;

	struct xarray		xa;

	struct wait_queue_head	wait;

	const struct io_ring_ctx 	*last;

	struct io_wq			*io_wq;

	struct percpu_counter	inflight;

	atomic_t		inflight_tracked;

	struct file			*registered_rings[IO_RINGFD_REG_MAX];



	struct xarray			xa;

	struct wait_queue_head		wait;

	atomic_t			in_idle;

	atomic_t			inflight_tracked;

	struct percpu_counter		inflight;



	struct { /* task_work */

		spinlock_t		task_lock;

		bool			task_running;

		struct io_wq_work_list	task_list;

		struct io_wq_work_list	prio_task_list;

		struct callback_head	task_work;

	bool			task_running;



	struct file		*registered_rings[IO_RINGFD_REG_MAX];

	} ____cacheline_aligned_in_smp;

};



struct io_tctx_node {