Commit d34b1b0b authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: use tw for putting rsrc 



Use task_work for completing rsrc removals, it'll be needed later for
spinlock optimisations.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/cbba5d53a11ee6fc2194dacea262c1d733c8b529.1670384893.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 17add5ce

include/linux/io_uring_types.h

+1 −0
Original line number Diff line number Diff line
@@ -328,6 +328,7 @@ struct io_ring_ctx { 
	struct io_rsrc_data		*buf_data;



	struct delayed_work		rsrc_put_work;

	struct callback_head		rsrc_put_tw;

	struct llist_head		rsrc_put_llist;

	struct list_head		rsrc_ref_list;

	spinlock_t			rsrc_ref_lock;

io_uring/io_uring.c

+1 −0
Original line number Diff line number Diff line
@@ -326,6 +326,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	spin_lock_init(&ctx->rsrc_ref_lock);

	INIT_LIST_HEAD(&ctx->rsrc_ref_list);

	INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);

	init_task_work(&ctx->rsrc_put_tw, io_rsrc_put_tw);

	init_llist_head(&ctx->rsrc_put_llist);

	init_llist_head(&ctx->work_llist);

	INIT_LIST_HEAD(&ctx->tctx_list);

io_uring/rsrc.c

+17 −2
Original line number Diff line number Diff line
@@ -204,6 +204,14 @@ void io_rsrc_put_work(struct work_struct *work) 
	}

}



void io_rsrc_put_tw(struct callback_head *cb)

{

	struct io_ring_ctx *ctx = container_of(cb, struct io_ring_ctx,

					       rsrc_put_tw);



	io_rsrc_put_work(&ctx->rsrc_put_work.work);

}



void io_wait_rsrc_data(struct io_rsrc_data *data)

{

	if (data && !atomic_dec_and_test(&data->refs))
@@ -242,7 +250,14 @@ static __cold void io_rsrc_node_ref_zero(struct percpu_ref *ref) 
	}

	spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);



	if (first_add)

	if (!first_add)

		return;



	if (ctx->submitter_task) {

		if (!task_work_add(ctx->submitter_task, &ctx->rsrc_put_tw,

				   ctx->notify_method))

			return;

	}

	mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);

}

io_uring/rsrc.h

+1 −0
Original line number Diff line number Diff line
@@ -53,6 +53,7 @@ struct io_mapped_ubuf { 
	struct bio_vec	bvec[];

};



void io_rsrc_put_tw(struct callback_head *cb);

void io_rsrc_put_work(struct work_struct *work);

void io_rsrc_refs_refill(struct io_ring_ctx *ctx);

void io_wait_rsrc_data(struct io_rsrc_data *data);