Commit e73c5c7c authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: extract a helper for ctx quiesce 



Refactor __io_uring_register() by extracting a helper responsible for
ctx queisce. Looks better and will make it easier to add more
optimisations.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/0339e0027504176be09237eefa7945bf9a6f153d.1628471125.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 90291099

fs/io_uring.c

+29 −24
Original line number Diff line number Diff line
@@ -10074,6 +10074,33 @@ static bool io_register_op_must_quiesce(int op) 
	}

}



static int io_ctx_quiesce(struct io_ring_ctx *ctx)

{

	long ret;



	percpu_ref_kill(&ctx->refs);



	/*

	 * Drop uring mutex before waiting for references to exit. If another

	 * thread is currently inside io_uring_enter() it might need to grab the

	 * uring_lock to make progress. If we hold it here across the drain

	 * wait, then we can deadlock. It's safe to drop the mutex here, since

	 * no new references will come in after we've killed the percpu ref.

	 */

	mutex_unlock(&ctx->uring_lock);

	do {

		ret = wait_for_completion_interruptible(&ctx->ref_comp);

		if (!ret)

			break;

		ret = io_run_task_work_sig();

	} while (ret >= 0);

	mutex_lock(&ctx->uring_lock);



	if (ret)

		io_refs_resurrect(&ctx->refs, &ctx->ref_comp);

	return ret;

}



static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,

			       void __user *arg, unsigned nr_args)

	__releases(ctx->uring_lock)
@@ -10098,32 +10125,10 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, 
	}



	if (io_register_op_must_quiesce(opcode)) {

		percpu_ref_kill(&ctx->refs);



		/*

		 * Drop uring mutex before waiting for references to exit. If

		 * another thread is currently inside io_uring_enter() it might

		 * need to grab the uring_lock to make progress. If we hold it

		 * here across the drain wait, then we can deadlock. It's safe

		 * to drop the mutex here, since no new references will come in

		 * after we've killed the percpu ref.

		 */

		mutex_unlock(&ctx->uring_lock);

		do {

			ret = wait_for_completion_interruptible(&ctx->ref_comp);

			if (!ret)

				break;

			ret = io_run_task_work_sig();

			if (ret < 0)

				break;

		} while (1);

		mutex_lock(&ctx->uring_lock);



		if (ret) {

			io_refs_resurrect(&ctx->refs, &ctx->ref_comp);

		ret = io_ctx_quiesce(ctx);

		if (ret)

			return ret;

	}

	}



	switch (opcode) {

	case IORING_REGISTER_BUFFERS: