Commit a3396b99 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: add a rq_list type 



Replace the semi-open coded request list helpers with a proper rq_list
type that mirrors the bio_list and has head and tail pointers.  Besides
better type safety this actually allows to insert at the tail of the
list, which will be useful soon.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/20241113152050.157179-5-hch@lst.de


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e8225ab1

block/blk-core.c

+3 −3
Original line number Diff line number Diff line
@@ -1120,8 +1120,8 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) 
		return;



	plug->cur_ktime = 0;

	plug->mq_list = NULL;

	plug->cached_rq = NULL;

	rq_list_init(&plug->mq_list);

	rq_list_init(&plug->cached_rqs);

	plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);

	plug->rq_count = 0;

	plug->multiple_queues = false;
@@ -1217,7 +1217,7 @@ void __blk_flush_plug(struct blk_plug *plug, bool from_schedule) 
	 * queue for cached requests, we don't want a blocked task holding

	 * up a queue freeze/quiesce event.

	 */

	if (unlikely(!rq_list_empty(plug->cached_rq)))

	if (unlikely(!rq_list_empty(&plug->cached_rqs)))

		blk_mq_free_plug_rqs(plug);



	plug->cur_ktime = 0;

block/blk-merge.c

+1 −1
Original line number Diff line number Diff line
@@ -1179,7 +1179,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 
	struct blk_plug *plug = current->plug;

	struct request *rq;



	if (!plug || rq_list_empty(plug->mq_list))

	if (!plug || rq_list_empty(&plug->mq_list))

		return false;



	rq_list_for_each(&plug->mq_list, rq) {

block/blk-mq.c

+19 −21
Original line number Diff line number Diff line
@@ -478,7 +478,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) 
		prefetch(tags->static_rqs[tag]);

		tag_mask &= ~(1UL << i);

		rq = blk_mq_rq_ctx_init(data, tags, tag);

		rq_list_add(data->cached_rq, rq);

		rq_list_add_head(data->cached_rqs, rq);

		nr++;

	}

	if (!(data->rq_flags & RQF_SCHED_TAGS))
@@ -487,7 +487,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) 
	percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);

	data->nr_tags -= nr;



	return rq_list_pop(data->cached_rq);

	return rq_list_pop(data->cached_rqs);

}



static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
@@ -584,7 +584,7 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, 
		.flags		= flags,

		.cmd_flags	= opf,

		.nr_tags	= plug->nr_ios,

		.cached_rq	= &plug->cached_rq,

		.cached_rqs	= &plug->cached_rqs,

	};

	struct request *rq;
@@ -609,14 +609,14 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, 
	if (!plug)

		return NULL;



	if (rq_list_empty(plug->cached_rq)) {

	if (rq_list_empty(&plug->cached_rqs)) {

		if (plug->nr_ios == 1)

			return NULL;

		rq = blk_mq_rq_cache_fill(q, plug, opf, flags);

		if (!rq)

			return NULL;

	} else {

		rq = rq_list_peek(&plug->cached_rq);

		rq = rq_list_peek(&plug->cached_rqs);

		if (!rq || rq->q != q)

			return NULL;
@@ -625,7 +625,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, 
		if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))

			return NULL;



		plug->cached_rq = rq_list_next(rq);

		rq_list_pop(&plug->cached_rqs);

		blk_mq_rq_time_init(rq, blk_time_get_ns());

	}
@@ -802,7 +802,7 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) 
{

	struct request *rq;



	while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)

	while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL)

		blk_mq_free_request(rq);

}
@@ -1392,8 +1392,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 
	 */

	if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))

		plug->has_elevator = true;

	rq->rq_next = NULL;

	rq_list_add(&plug->mq_list, rq);

	rq_list_add_head(&plug->mq_list, rq);

	plug->rq_count++;

}
@@ -2785,7 +2784,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug) 
	blk_status_t ret = BLK_STS_OK;



	while ((rq = rq_list_pop(&plug->mq_list))) {

		bool last = rq_list_empty(plug->mq_list);

		bool last = rq_list_empty(&plug->mq_list);



		if (hctx != rq->mq_hctx) {

			if (hctx) {
@@ -2828,8 +2827,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) 
{

	struct blk_mq_hw_ctx *this_hctx = NULL;

	struct blk_mq_ctx *this_ctx = NULL;

	struct request *requeue_list = NULL;

	struct request **requeue_lastp = &requeue_list;

	struct rq_list requeue_list = {};

	unsigned int depth = 0;

	bool is_passthrough = false;

	LIST_HEAD(list);
@@ -2843,12 +2841,12 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) 
			is_passthrough = blk_rq_is_passthrough(rq);

		} else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||

			   is_passthrough != blk_rq_is_passthrough(rq)) {

			rq_list_add_tail(&requeue_lastp, rq);

			rq_list_add_tail(&requeue_list, rq);

			continue;

		}

		list_add(&rq->queuelist, &list);

		depth++;

	} while (!rq_list_empty(plug->mq_list));

	} while (!rq_list_empty(&plug->mq_list));



	plug->mq_list = requeue_list;

	trace_block_unplug(this_hctx->queue, depth, !from_sched);
@@ -2903,19 +2901,19 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 
		if (q->mq_ops->queue_rqs) {

			blk_mq_run_dispatch_ops(q,

				__blk_mq_flush_plug_list(q, plug));

			if (rq_list_empty(plug->mq_list))

			if (rq_list_empty(&plug->mq_list))

				return;

		}



		blk_mq_run_dispatch_ops(q,

				blk_mq_plug_issue_direct(plug));

		if (rq_list_empty(plug->mq_list))

		if (rq_list_empty(&plug->mq_list))

			return;

	}



	do {

		blk_mq_dispatch_plug_list(plug, from_schedule);

	} while (!rq_list_empty(plug->mq_list));

	} while (!rq_list_empty(&plug->mq_list));

}



static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
@@ -2980,7 +2978,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, 
	if (plug) {

		data.nr_tags = plug->nr_ios;

		plug->nr_ios = 1;

		data.cached_rq = &plug->cached_rq;

		data.cached_rqs = &plug->cached_rqs;

	}



	rq = __blk_mq_alloc_requests(&data);
@@ -3003,7 +3001,7 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, 


	if (!plug)

		return NULL;

	rq = rq_list_peek(&plug->cached_rq);

	rq = rq_list_peek(&plug->cached_rqs);

	if (!rq || rq->q != q)

		return NULL;

	if (type != rq->mq_hctx->type &&
@@ -3017,14 +3015,14 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, 
static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,

		struct bio *bio)

{

	WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq);

	if (rq_list_pop(&plug->cached_rqs) != rq)

		WARN_ON_ONCE(1);



	/*

	 * If any qos ->throttle() end up blocking, we will have flushed the

	 * plug and hence killed the cached_rq list as well. Pop this entry

	 * before we throttle.

	 */

	plug->cached_rq = rq_list_next(rq);

	rq_qos_throttle(rq->q, bio);



	blk_mq_rq_time_init(rq, blk_time_get_ns());

block/blk-mq.h

+1 −1
Original line number Diff line number Diff line
@@ -155,7 +155,7 @@ struct blk_mq_alloc_data { 


	/* allocate multiple requests/tags in one go */

	unsigned int nr_tags;

	struct request **cached_rq;

	struct rq_list *cached_rqs;



	/* input & output parameter */

	struct blk_mq_ctx *ctx;

drivers/block/null_blk/main.c

+4 −5
Original line number Diff line number Diff line
@@ -1638,10 +1638,9 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, 
	return BLK_STS_OK;

}



static void null_queue_rqs(struct request **rqlist)

static void null_queue_rqs(struct rq_list *rqlist)

{

	struct request *requeue_list = NULL;

	struct request **requeue_lastp = &requeue_list;

	struct rq_list requeue_list = {};

	struct blk_mq_queue_data bd = { };

	blk_status_t ret;
@@ -1651,8 +1650,8 @@ static void null_queue_rqs(struct request **rqlist) 
		bd.rq = rq;

		ret = null_queue_rq(rq->mq_hctx, &bd);

		if (ret != BLK_STS_OK)

			rq_list_add_tail(&requeue_lastp, rq);

	} while (!rq_list_empty(*rqlist));

			rq_list_add_tail(&requeue_list, rq);

	} while (!rq_list_empty(rqlist));



	*rqlist = requeue_list;

}