Commit ab57a186 authored by Martin K. Petersen's avatar Martin K. Petersen
Browse files

Merge patch series "Optimize the hot path in the UFS driver" 

Bart Van Assche <bvanassche@acm.org> says:

Hi Martin,

This patch series optimizes the hot path of the UFS driver by making
struct scsi_cmnd and struct ufshcd_lrb adjacent. Making these two data
structures adjacent is realized as follows:

@@ -9040,6 +9046,7 @@ static const struct scsi_host_template ufshcd_driver_template = {
     .name           = UFSHCD,
     .proc_name      = UFSHCD,
     .map_queues     = ufshcd_map_queues,
+    .cmd_size       = sizeof(struct ufshcd_lrb),
     .init_cmd_priv  = ufshcd_init_cmd_priv,
     .queuecommand   = ufshcd_queuecommand,
     .mq_poll        = ufshcd_poll,

The following changes had to be made prior to making these two data
structures adjacent:
* Add support for driver-internal and reserved commands in the SCSI core.
* Instead of making the reserved command slot (hba->reserved_slot)
  invisible to the SCSI core, let the SCSI core allocate a reserved command.
* Remove all UFS data structure members that are no longer needed
  because struct scsi_cmnd and struct ufshcd_lrb are now adjacent
* Call ufshcd_init_lrb() from inside the code for queueing a command instead of
  calling this function before I/O starts. This is necessary because
  ufshcd_memory_alloc() allocates fewer instances than the block layer
  allocates requests. See also the following code in the block layer
  core:

    if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
                hctx->numa_node))

  Although the UFS driver could be modified such that ufshcd_init_lrb()
  is called from ufshcd_init_cmd_priv(), realizing this would require
  moving the memory allocations that happen from inside
  ufshcd_memory_alloc() into ufshcd_init_cmd_priv(). That would make
  this patch series even larger. Although ufshcd_init_lrb() is called for each
  command, the benefits of reduced indirection and better cache efficiency
  outweigh the small overhead of per-command lrb initialization.
* ufshcd_add_scsi_host() happens now before any device management
  commands are submitted. This change is necessary because this patch
  makes device management command allocation happen when the SCSI host
  is allocated.
* Allocate as many command slots as the host controller supports. Decrease
  host->cmds_per_lun if necessary once it is clear whether or not the UFS
  device supports less command slots than the host controller.

Please consider this patch series for the next merge window.

Thanks,

Bart.

Link: https://patch.msgid.link/20251031204029.2883185-1-bvanassche@acm.org


Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parents c53a741a 08b12cda

drivers/scsi/hosts.c

+15 −0
Original line number Diff line number Diff line
@@ -231,6 +231,12 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, 
		goto fail;

	}



	if (shost->nr_reserved_cmds && !sht->queue_reserved_command) {

		shost_printk(KERN_ERR, shost,

			     "nr_reserved_cmds set but no method to queue\n");

		goto fail;

	}



	/* Use min_t(int, ...) in case shost->can_queue exceeds SHRT_MAX */

	shost->cmd_per_lun = min_t(int, shost->cmd_per_lun,

				   shost->can_queue);
@@ -307,6 +313,14 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, 
	if (error)

		goto out_del_dev;



	if (shost->nr_reserved_cmds) {

		shost->pseudo_sdev = scsi_get_pseudo_sdev(shost);

		if (!shost->pseudo_sdev) {

			error = -ENOMEM;

			goto out_del_dev;

		}

	}



	scsi_proc_host_add(shost);

	scsi_autopm_put_host(shost);

	return error;
@@ -436,6 +450,7 @@ struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int priv 
	shost->hostt = sht;

	shost->this_id = sht->this_id;

	shost->can_queue = sht->can_queue;

	shost->nr_reserved_cmds = sht->nr_reserved_cmds;

	shost->sg_tablesize = sht->sg_tablesize;

	shost->sg_prot_tablesize = sht->sg_prot_tablesize;

	shost->cmd_per_lun = sht->cmd_per_lun;

drivers/scsi/scsi.c

+10 −2
Original line number Diff line number Diff line
@@ -216,6 +216,9 @@ int scsi_device_max_queue_depth(struct scsi_device *sdev) 
 */

int scsi_change_queue_depth(struct scsi_device *sdev, int depth)

{

	if (!sdev->budget_map.map)

		return -EINVAL;



	depth = min_t(int, depth, scsi_device_max_queue_depth(sdev));



	if (depth > 0) {
@@ -255,6 +258,8 @@ EXPORT_SYMBOL(scsi_change_queue_depth); 
 */

int scsi_track_queue_full(struct scsi_device *sdev, int depth)

{

	if (!sdev->budget_map.map)

		return 0;



	/*

	 * Don't let QUEUE_FULLs on the same
@@ -826,8 +831,11 @@ struct scsi_device *__scsi_iterate_devices(struct Scsi_Host *shost, 
	spin_lock_irqsave(shost->host_lock, flags);

	while (list->next != &shost->__devices) {

		next = list_entry(list->next, struct scsi_device, siblings);

		/* skip devices that we can't get a reference to */

		if (!scsi_device_get(next))

		/*

		 * Skip pseudo devices and also devices we can't get a

		 * reference to.

		 */

		if (!scsi_device_is_pseudo_dev(next) && !scsi_device_get(next))

			break;

		next = NULL;

		list = list->next;

drivers/scsi/scsi_debug.c

+105 −11
Original line number Diff line number Diff line
@@ -6752,20 +6752,59 @@ static bool scsi_debug_stop_cmnd(struct scsi_cmnd *cmnd) 
	return false;

}



struct sdebug_abort_cmd {

	u32 unique_tag;

};



enum sdebug_internal_cmd_type {

	SCSI_DEBUG_ABORT_CMD,

};



struct sdebug_internal_cmd {

	enum sdebug_internal_cmd_type type;



	union {

		struct sdebug_abort_cmd abort_cmd;

	};

};



union sdebug_priv {

	struct sdebug_scsi_cmd cmd;

	struct sdebug_internal_cmd internal_cmd;

};



/*

 * Called from scsi_debug_abort() only, which is for timed-out cmd.

 * Abort SCSI command @cmnd. Only called from scsi_debug_abort(). Although

 * it would be possible to call scsi_debug_stop_cmnd() directly, an internal

 * command is allocated and submitted to trigger the reserved command

 * infrastructure.

 */

static bool scsi_debug_abort_cmnd(struct scsi_cmnd *cmnd)

{

	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd);

	unsigned long flags;

	bool res;



	spin_lock_irqsave(&sdsc->lock, flags);

	res = scsi_debug_stop_cmnd(cmnd);

	spin_unlock_irqrestore(&sdsc->lock, flags);



	return res;

	struct Scsi_Host *shost = cmnd->device->host;

	struct request *rq = scsi_cmd_to_rq(cmnd);

	u32 unique_tag = blk_mq_unique_tag(rq);

	struct sdebug_internal_cmd *internal_cmd;

	struct scsi_cmnd *abort_cmd;

	struct request *abort_rq;

	blk_status_t res;



	abort_cmd = scsi_get_internal_cmd(shost->pseudo_sdev, DMA_NONE,

					  BLK_MQ_REQ_RESERVED);

	if (!abort_cmd)

		return false;

	internal_cmd = scsi_cmd_priv(abort_cmd);

	*internal_cmd = (struct sdebug_internal_cmd) {

		.type = SCSI_DEBUG_ABORT_CMD,

		.abort_cmd = {

			.unique_tag = unique_tag,

		},

	};

	abort_rq = scsi_cmd_to_rq(abort_cmd);

	abort_rq->timeout = secs_to_jiffies(3);

	res = blk_execute_rq(abort_rq, true);

	scsi_put_internal_cmd(abort_cmd);

	return res == BLK_STS_OK;

}



/*
@@ -9220,6 +9259,56 @@ static int sdebug_fail_cmd(struct scsi_cmnd *cmnd, int *retval, 
	return ret;

}



/* Process @scp, a request to abort a SCSI command by tag. */

static void scsi_debug_abort_cmd(struct Scsi_Host *shost, struct scsi_cmnd *scp)

{

	struct sdebug_internal_cmd *internal_cmd = scsi_cmd_priv(scp);

	struct sdebug_abort_cmd *abort_cmd = &internal_cmd->abort_cmd;

	const u32 unique_tag = abort_cmd->unique_tag;

	struct scsi_cmnd *to_be_aborted_scmd =

		scsi_host_find_tag(shost, unique_tag);

	struct sdebug_scsi_cmd *to_be_aborted_sdsc =

		scsi_cmd_priv(to_be_aborted_scmd);

	bool res = false;



	if (!to_be_aborted_scmd) {

		pr_err("%s: command with tag %#x not found\n", __func__,

		       unique_tag);

		return;

	}



	scoped_guard(spinlock_irqsave, &to_be_aborted_sdsc->lock)

		res = scsi_debug_stop_cmnd(to_be_aborted_scmd);



	if (res)

		pr_info("%s: aborted command with tag %#x\n",

			__func__, unique_tag);

	else

		pr_err("%s: failed to abort command with tag %#x\n",

		       __func__, unique_tag);



	set_host_byte(scp, res ? DID_OK : DID_ERROR);

}



static int scsi_debug_process_reserved_command(struct Scsi_Host *shost,

					       struct scsi_cmnd *scp)

{

	struct sdebug_internal_cmd *internal_cmd = scsi_cmd_priv(scp);



	switch (internal_cmd->type) {

	case SCSI_DEBUG_ABORT_CMD:

		scsi_debug_abort_cmd(shost, scp);

		break;

	default:

		WARN_ON_ONCE(true);

		set_host_byte(scp, DID_ERROR);

		break;

	}



	scsi_done(scp);

	return 0;

}



static int scsi_debug_queuecommand(struct Scsi_Host *shost,

				   struct scsi_cmnd *scp)

{
@@ -9420,6 +9509,9 @@ static int sdebug_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 
	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmd);

	struct sdebug_defer *sd_dp = &sdsc->sd_dp;



	if (blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)))

		return 0;



	spin_lock_init(&sdsc->lock);

	hrtimer_setup(&sd_dp->hrt, sdebug_q_cmd_hrt_complete, CLOCK_MONOTONIC,

		      HRTIMER_MODE_REL_PINNED);
@@ -9439,6 +9531,7 @@ static const struct scsi_host_template sdebug_driver_template = { 
	.sdev_destroy =		scsi_debug_sdev_destroy,

	.ioctl =		scsi_debug_ioctl,

	.queuecommand =		scsi_debug_queuecommand,

	.queue_reserved_command = scsi_debug_process_reserved_command,

	.change_queue_depth =	sdebug_change_qdepth,

	.map_queues =		sdebug_map_queues,

	.mq_poll =		sdebug_blk_mq_poll,
@@ -9448,6 +9541,7 @@ static const struct scsi_host_template sdebug_driver_template = { 
	.eh_bus_reset_handler = scsi_debug_bus_reset,

	.eh_host_reset_handler = scsi_debug_host_reset,

	.can_queue =		SDEBUG_CANQUEUE,

	.nr_reserved_cmds =	1,

	.this_id =		7,

	.sg_tablesize =		SG_MAX_SEGMENTS,

	.cmd_per_lun =		DEF_CMD_PER_LUN,
@@ -9456,7 +9550,7 @@ static const struct scsi_host_template sdebug_driver_template = { 
	.module =		THIS_MODULE,

	.skip_settle_delay =	1,

	.track_queue_depth =	1,

	.cmd_size = sizeof(struct sdebug_scsi_cmd),

	.cmd_size = sizeof(union sdebug_priv),

	.init_cmd_priv = sdebug_init_cmd_priv,

	.target_alloc =		sdebug_target_alloc,

	.target_destroy =	sdebug_target_destroy,

drivers/scsi/scsi_error.c

+3 −0
Original line number Diff line number Diff line
@@ -749,6 +749,9 @@ static void scsi_handle_queue_ramp_up(struct scsi_device *sdev) 
	const struct scsi_host_template *sht = sdev->host->hostt;

	struct scsi_device *tmp_sdev;



	if (!sdev->budget_map.map)

		return;



	if (!sht->track_queue_depth ||

	    sdev->queue_depth >= sdev->max_queue_depth)

		return;

drivers/scsi/scsi_lib.c

+85 −19
Original line number Diff line number Diff line
@@ -396,6 +396,7 @@ void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd) 
	if (starget->can_queue > 0)

		atomic_dec(&starget->target_busy);



	if (sdev->budget_map.map)

		sbitmap_put(&sdev->budget_map, cmd->budget_token);

	cmd->budget_token = -1;

}
@@ -1360,6 +1361,9 @@ static inline int scsi_dev_queue_ready(struct request_queue *q, 
{

	int token;



	if (!sdev->budget_map.map)

		return INT_MAX;



	token = sbitmap_get(&sdev->budget_map);

	if (token < 0)

		return -1;
@@ -1530,6 +1534,14 @@ static void scsi_complete(struct request *rq) 
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);

	enum scsi_disposition disposition;



	if (blk_mq_is_reserved_rq(rq)) {

		/* Only pass-through requests are supported in this code path. */

		WARN_ON_ONCE(!blk_rq_is_passthrough(scsi_cmd_to_rq(cmd)));

		scsi_mq_uninit_cmd(cmd);

		__blk_mq_end_request(rq, scsi_result_to_blk_status(cmd->result));

		return;

	}



	INIT_LIST_HEAD(&cmd->eh_entry);



	atomic_inc(&cmd->device->iodone_cnt);
@@ -1749,6 +1761,7 @@ static void scsi_mq_put_budget(struct request_queue *q, int budget_token) 
{

	struct scsi_device *sdev = q->queuedata;



	if (sdev->budget_map.map)

		sbitmap_put(&sdev->budget_map, budget_token);

}
@@ -1818,8 +1831,13 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 
	WARN_ON_ONCE(cmd->budget_token < 0);



	/*

	 * If the device is not in running state we will reject some or all

	 * commands.

	 * Bypass the SCSI device, SCSI target and SCSI host checks for

	 * reserved commands.

	 */

	if (!blk_mq_is_reserved_rq(req)) {

		/*

		 * If the device is not in running state we will reject some or

		 * all commands.

		 */

		if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {

			ret = scsi_device_state_check(sdev, req);
@@ -1837,6 +1855,7 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 
		}

		if (!scsi_host_queue_ready(q, shost, sdev, cmd))

			goto out_dec_target_busy;

	}



	/*

	 * Only clear the driver-private command data if the LLD does not supply
@@ -1865,6 +1884,14 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 
	cmd->submitter = SUBMITTED_BY_BLOCK_LAYER;



	blk_mq_start_request(req);

	if (blk_mq_is_reserved_rq(req)) {

		reason = shost->hostt->queue_reserved_command(shost, cmd);

		if (reason) {

			ret = BLK_STS_RESOURCE;

			goto out_put_budget;

		}

		return BLK_STS_OK;

	}

	reason = scsi_dispatch_cmd(cmd);

	if (reason) {

		scsi_set_blocked(cmd, reason);
@@ -2083,7 +2110,8 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) 
		tag_set->ops = &scsi_mq_ops_no_commit;

	tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;

	tag_set->nr_maps = shost->nr_maps ? : 1;

	tag_set->queue_depth = shost->can_queue;

	tag_set->queue_depth = shost->can_queue + shost->nr_reserved_cmds;

	tag_set->reserved_tags = shost->nr_reserved_cmds;

	tag_set->cmd_size = cmd_size;

	tag_set->numa_node = dev_to_node(shost->dma_dev);

	if (shost->hostt->tag_alloc_policy_rr)
@@ -2106,6 +2134,44 @@ void scsi_mq_free_tags(struct kref *kref) 
	complete(&shost->tagset_freed);

}



/**

 * scsi_get_internal_cmd() - Allocate an internal SCSI command.

 * @sdev: SCSI device from which to allocate the command

 * @data_direction: Data direction for the allocated command

 * @flags: request allocation flags, e.g. BLK_MQ_REQ_RESERVED or

 *	BLK_MQ_REQ_NOWAIT.

 *

 * Allocates a SCSI command for internal LLDD use.

 */

struct scsi_cmnd *scsi_get_internal_cmd(struct scsi_device *sdev,

					enum dma_data_direction data_direction,

					blk_mq_req_flags_t flags)

{

	enum req_op op = data_direction == DMA_TO_DEVICE ? REQ_OP_DRV_OUT :

							   REQ_OP_DRV_IN;

	struct scsi_cmnd *scmd;

	struct request *rq;



	rq = scsi_alloc_request(sdev->request_queue, op, flags);

	if (IS_ERR(rq))

		return NULL;

	scmd = blk_mq_rq_to_pdu(rq);

	scmd->device = sdev;



	return scmd;

}

EXPORT_SYMBOL_GPL(scsi_get_internal_cmd);



/**

 * scsi_put_internal_cmd() - Free an internal SCSI command.

 * @scmd: SCSI command to be freed

 */

void scsi_put_internal_cmd(struct scsi_cmnd *scmd)

{

	blk_mq_free_request(blk_mq_rq_from_pdu(scmd));

}

EXPORT_SYMBOL_GPL(scsi_put_internal_cmd);



/**

 * scsi_device_from_queue - return sdev associated with a request_queue

 * @q: The request queue to return the sdev from