Commit 22089c21 authored by Bart Van Assche's avatar Bart Van Assche Committed by Martin K. Petersen
Browse files

scsi: ufs: core: Optimize the hot path 



Set .cmd_size in the SCSI host template such that the SCSI core makes
struct scsi_cmnd and struct ufshcd_lrb adjacent. Convert the cmd->lrbp
and lrbp->cmd memory loads into pointer offset calculations. Remove the
data structure members that became superfluous, namely ufshcd_lrb.cmd
and ufs_hba.lrb. Since ufshcd_lrb.cmd is removed, this pointer cannot be
used anymore to test whether or not a command is a SCSI command.
Introduce a new function for this purpose, namely ufshcd_is_scsi_cmd().

Signed-off-by: default avatarBart Van Assche <bvanassche@acm.org>
Link: https://patch.msgid.link/20251031204029.2883185-24-bvanassche@acm.org


Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent e5f9cc2a

drivers/ufs/core/ufs-mcq.c

+5 −4
Original line number Diff line number Diff line
@@ -534,8 +534,8 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 
 */

int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)

{

	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];

	struct scsi_cmnd *cmd = lrbp->cmd;

	struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);

	struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);

	struct ufs_hw_queue *hwq;

	void __iomem *reg, *opr_sqd_base;

	u32 nexus, id, val;
@@ -618,7 +618,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) 
static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,

				  struct ufs_hw_queue *hwq, int task_tag)

{

	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];

	struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);

	struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);

	struct utp_transfer_req_desc *utrd;

	__le64  cmd_desc_base_addr;

	bool ret = false;
@@ -669,7 +670,7 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd) 
	struct Scsi_Host *host = cmd->device->host;

	struct ufs_hba *hba = shost_priv(host);

	int tag = scsi_cmd_to_rq(cmd)->tag;

	struct ufshcd_lrb *lrbp = &hba->lrb[tag];

	struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);

	struct ufs_hw_queue *hwq;

	int err;

drivers/ufs/core/ufshcd-crypto.h

+11 −7
Original line number Diff line number Diff line
@@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, 
}



static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,

					  struct ufshcd_lrb *lrbp)

					  struct scsi_cmnd *cmd)

{

	struct scsi_cmnd *cmd = lrbp->cmd;

	const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;

	struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);



	if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)

		return hba->vops->fill_crypto_prdt(hba, crypt_ctx,
@@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, 
}



static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,

					    struct ufshcd_lrb *lrbp)

					    struct scsi_cmnd *cmd)

{

	struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);



	if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))

		return;



	if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))

	if (!(scsi_cmd_to_rq(cmd)->crypt_ctx))

		return;



	/* Zeroize the PRDT because it can contain cryptographic keys. */

	memzero_explicit(lrbp->ucd_prdt_ptr,

			 ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd));

			 ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd));

}



bool ufshcd_crypto_enable(struct ufs_hba *hba);
@@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, 
				   struct request_desc_header *h) { }



static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,

					  struct ufshcd_lrb *lrbp)

					  struct scsi_cmnd *cmd)

{

	return 0;

}



static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,

					    struct ufshcd_lrb *lrbp) { }

					    struct scsi_cmnd *cmd)

{

}



static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)

{

drivers/ufs/core/ufshcd-priv.h

+39 −2
Original line number Diff line number Diff line
@@ -77,8 +77,7 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd); 
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);

int ufshcd_mcq_abort(struct scsi_cmnd *cmd);

int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);

void ufshcd_release_scsi_cmd(struct ufs_hba *hba,

			     struct ufshcd_lrb *lrbp);

void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);



#define SD_ASCII_STD true

#define SD_RAW false
@@ -363,6 +362,44 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8 
	return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported);

}



/*

 * Convert a block layer tag into a SCSI command pointer. This function is

 * called once per I/O completion path and is also called from error paths.

 */

static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag)

{

	struct blk_mq_tags *tags = hba->host->tag_set.shared_tags;

	struct request *rq;



	/*

	 * Handle reserved tags differently because the UFS driver does not

	 * call blk_mq_alloc_request() for allocating reserved requests.

	 * Allocating reserved tags with blk_mq_alloc_request() would require

	 * the following:

	 * - Allocate an additional request queue from &hba->host->tag_set for

	 *   allocating reserved requests from.

	 * - For that request queue, allocate a SCSI device.

	 * - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT,

	 *   BLK_MQ_REQ_RESERVED) for allocating a reserved request and

	 *   blk_mq_free_request() for freeing reserved requests.

	 * - Set the .device pointer for these reserved requests.

	 * - Submit reserved requests with blk_execute_rq().

	 * - Modify ufshcd_queuecommand() such that it handles reserved requests

	 *   in another way than SCSI requests.

	 * - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for

	 *   device management commands.

	 * - Modify all callback functions called by blk_mq_tagset_busy_iter()

	 *   calls in the UFS driver and skip device management commands.

	 */

	rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] :

					 blk_mq_tag_to_rq(tags, tag);



	if (WARN_ON_ONCE(!rq))

		return NULL;



	return blk_mq_rq_to_pdu(rq);

}



static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)

	__must_hold(&q->sq_lock)

{

drivers/ufs/core/ufshcd.c

+124 −111

File changed.

Preview size limit exceeded, changes collapsed.

include/ufs/ufshcd.h

+0 −5

File changed.

Preview size limit exceeded, changes collapsed.