Commit 3c75635f authored by Max Kellermann's avatar Max Kellermann Committed by Jens Axboe
Browse files

io_uring/io-wq: eliminate redundant io_work_get_acct() calls 



Instead of calling io_work_get_acct() again, pass acct to
io_wq_insert_work() and io_wq_remove_pending().

This atomic access in io_work_get_acct() was done under the
`acct->lock`, and optimizing it away reduces lock contention a bit.

Signed-off-by: default avatarMax Kellermann <max.kellermann@ionos.com>
Link: https://lore.kernel.org/r/20250128133927.3989681-2-max.kellermann@ionos.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 0ad2507d

io_uring/io-wq.c

+4 −5
Original line number Diff line number Diff line
@@ -903,9 +903,8 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq) 
	} while (work);

}



static void io_wq_insert_work(struct io_wq *wq, struct io_wq_work *work)

static void io_wq_insert_work(struct io_wq *wq, struct io_wq_acct *acct, struct io_wq_work *work)

{

	struct io_wq_acct *acct = io_work_get_acct(wq, work);

	unsigned int hash;

	struct io_wq_work *tail;
@@ -951,7 +950,7 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) 
	}



	raw_spin_lock(&acct->lock);

	io_wq_insert_work(wq, work);

	io_wq_insert_work(wq, acct, work);

	clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);

	raw_spin_unlock(&acct->lock);
@@ -1021,10 +1020,10 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) 
}



static inline void io_wq_remove_pending(struct io_wq *wq,

					struct io_wq_acct *acct,

					 struct io_wq_work *work,

					 struct io_wq_work_node *prev)

{

	struct io_wq_acct *acct = io_work_get_acct(wq, work);

	unsigned int hash = io_get_work_hash(work);

	struct io_wq_work *prev_work = NULL;
@@ -1051,7 +1050,7 @@ static bool io_acct_cancel_pending_work(struct io_wq *wq, 
		work = container_of(node, struct io_wq_work, list);

		if (!match->fn(work, match->data))

			continue;

		io_wq_remove_pending(wq, work, prev);

		io_wq_remove_pending(wq, acct, work, prev);

		raw_spin_unlock(&acct->lock);

		io_run_cancel(work, wq);

		match->nr_pending++;