Commit 31e4cdde authored by Eric Biggers's avatar Eric Biggers
Browse files

scsi: target: iscsi: switch to using the crc32c library 



Now that the crc32c() library function directly takes advantage of
architecture-specific optimizations, it is unnecessary to go through the
crypto API.  Just use crc32c().  This is much simpler, and it improves
performance due to eliminating the crypto API overhead.

Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Reviewed-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
Link: https://lore.kernel.org/r/20241202010844.144356-20-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
parent 3ca4bec4

drivers/target/iscsi/Kconfig

+2 −1
Original line number Diff line number Diff line
@@ -2,8 +2,9 @@ 
config ISCSI_TARGET

	tristate "SCSI Target Mode Stack"

	depends on INET

	select CRC32

	select CRYPTO

	select CRYPTO_CRC32C

	select CRYPTO_HASH

	help

	Say M to enable the SCSI target mode stack. A SCSI target mode stack

	is software that makes local storage available over a storage network

drivers/target/iscsi/iscsi_target.c

+45 −108
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ 
 *

 ******************************************************************************/



#include <crypto/hash.h>

#include <linux/crc32c.h>

#include <linux/string.h>

#include <linux/kthread.h>

#include <linux/completion.h>
@@ -490,8 +490,8 @@ void iscsit_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 
}

EXPORT_SYMBOL(iscsit_aborted_task);



static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,

				      u32, u32, const void *, void *);

static u32 iscsit_crc_buf(const void *buf, u32 payload_length,

			  u32 padding, const void *pad_bytes);

static void iscsit_tx_thread_wait_for_tcp(struct iscsit_conn *);



static int
@@ -510,9 +510,7 @@ iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
	if (conn->conn_ops->HeaderDigest) {

		u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];



		iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,

					  ISCSI_HDR_LEN, 0, NULL,

					  header_digest);

		*header_digest = iscsit_crc_buf(hdr, ISCSI_HDR_LEN, 0, NULL);



		iov[0].iov_len += ISCSI_CRC_LEN;

		tx_size += ISCSI_CRC_LEN;
@@ -537,11 +535,9 @@ iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
		}



		if (conn->conn_ops->DataDigest) {

			iscsit_do_crypto_hash_buf(conn->conn_tx_hash,

						  data_buf, data_buf_len,

						  padding, &cmd->pad_bytes,

						  &cmd->data_crc);



			cmd->data_crc = iscsit_crc_buf(data_buf, data_buf_len,

						       padding,

						       &cmd->pad_bytes);

			iov[niov].iov_base = &cmd->data_crc;

			iov[niov++].iov_len = ISCSI_CRC_LEN;

			tx_size += ISCSI_CRC_LEN;
@@ -566,8 +562,8 @@ iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
static int iscsit_map_iovec(struct iscsit_cmd *cmd, struct kvec *iov, int nvec,

			    u32 data_offset, u32 data_length);

static void iscsit_unmap_iovec(struct iscsit_cmd *);

static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsit_cmd *,

				    u32, u32, u32, u8 *);

static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length,

			     u32 padding, const u8 *pad_bytes);

static int

iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,

		       const struct iscsi_datain *datain)
@@ -584,10 +580,8 @@ iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
	if (conn->conn_ops->HeaderDigest) {

		u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];



		iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,

					  ISCSI_HDR_LEN, 0, NULL,

					  header_digest);



		*header_digest = iscsit_crc_buf(cmd->pdu, ISCSI_HDR_LEN, 0,

						NULL);

		iov[0].iov_len += ISCSI_CRC_LEN;

		tx_size += ISCSI_CRC_LEN;
@@ -614,12 +608,8 @@ iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
	}



	if (conn->conn_ops->DataDigest) {

		cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,

							 cmd, datain->offset,

							 datain->length,

							 cmd->padding,

							 cmd->pad_bytes);



		cmd->data_crc = iscsit_crc_sglist(cmd, datain->length,

						  cmd->padding, cmd->pad_bytes);

		iov[iov_count].iov_base	= &cmd->data_crc;

		iov[iov_count++].iov_len = ISCSI_CRC_LEN;

		tx_size += ISCSI_CRC_LEN;
@@ -1404,77 +1394,45 @@ iscsit_handle_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
	return iscsit_get_immediate_data(cmd, hdr, dump_payload);

}



static u32 iscsit_do_crypto_hash_sg(

	struct ahash_request *hash,

	struct iscsit_cmd *cmd,

	u32 data_offset,

	u32 data_length,

	u32 padding,

	u8 *pad_bytes)

static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length,

			     u32 padding, const u8 *pad_bytes)

{

	u32 data_crc;

	struct scatterlist *sg;

	unsigned int page_off;



	crypto_ahash_init(hash);



	sg = cmd->first_data_sg;

	page_off = cmd->first_data_sg_off;



	if (data_length && page_off) {

		struct scatterlist first_sg;

		u32 len = min_t(u32, data_length, sg->length - page_off);



		sg_init_table(&first_sg, 1);

		sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off);



		ahash_request_set_crypt(hash, &first_sg, NULL, len);

		crypto_ahash_update(hash);



		data_length -= len;

		sg = sg_next(sg);

	}

	struct scatterlist *sg = cmd->first_data_sg;

	unsigned int page_off = cmd->first_data_sg_off;

	u32 crc = ~0;



	while (data_length) {

		u32 cur_len = min_t(u32, data_length, sg->length);

		u32 cur_len = min_t(u32, data_length, sg->length - page_off);

		const void *virt;



		ahash_request_set_crypt(hash, sg, NULL, cur_len);

		crypto_ahash_update(hash);

		virt = kmap_local_page(sg_page(sg)) + sg->offset + page_off;

		crc = crc32c(crc, virt, cur_len);

		kunmap_local(virt);



		data_length -= cur_len;

		/* iscsit_map_iovec has already checked for invalid sg pointers */

		sg = sg_next(sg);

	}



	if (padding) {

		struct scatterlist pad_sg;



		sg_init_one(&pad_sg, pad_bytes, padding);

		ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,

					padding);

		crypto_ahash_finup(hash);

	} else {

		ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);

		crypto_ahash_final(hash);

		page_off = 0;

		data_length -= cur_len;

	}



	return data_crc;

	if (padding)

		crc = crc32c(crc, pad_bytes, padding);



	return ~crc;

}



static void iscsit_do_crypto_hash_buf(struct ahash_request *hash,

	const void *buf, u32 payload_length, u32 padding,

	const void *pad_bytes, void *data_crc)

static u32 iscsit_crc_buf(const void *buf, u32 payload_length,

			  u32 padding, const void *pad_bytes)

{

	struct scatterlist sg[2];

	u32 crc = ~0;



	sg_init_table(sg, ARRAY_SIZE(sg));

	sg_set_buf(sg, buf, payload_length);

	if (padding)

		sg_set_buf(sg + 1, pad_bytes, padding);

	crc = crc32c(crc, buf, payload_length);



	ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);

	if (padding)

		crc = crc32c(crc, pad_bytes, padding);



	crypto_ahash_digest(hash);

	return ~crc;

}



int
@@ -1662,11 +1620,8 @@ iscsit_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
	if (conn->conn_ops->DataDigest) {

		u32 data_crc;



		data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,

						    be32_to_cpu(hdr->offset),

						    payload_length, padding,

		data_crc = iscsit_crc_sglist(cmd, payload_length, padding,

					     cmd->pad_bytes);



		if (checksum != data_crc) {

			pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"

				" DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
@@ -1925,10 +1880,8 @@ static int iscsit_handle_nop_out(struct iscsit_conn *conn, struct iscsit_cmd *cm 
		}



		if (conn->conn_ops->DataDigest) {

			iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data,

						  payload_length, padding,

						  cmd->pad_bytes, &data_crc);



			data_crc = iscsit_crc_buf(ping_data, payload_length,

						  padding, cmd->pad_bytes);

			if (checksum != data_crc) {

				pr_err("Ping data CRC32C DataDigest"

				" 0x%08x does not match computed 0x%08x\n",
@@ -2328,10 +2281,7 @@ iscsit_handle_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, 
			goto reject;



		if (conn->conn_ops->DataDigest) {

			iscsit_do_crypto_hash_buf(conn->conn_rx_hash,

						  text_in, rx_size, 0, NULL,

						  &data_crc);



			data_crc = iscsit_crc_buf(text_in, rx_size, 0, NULL);

			if (checksum != data_crc) {

				pr_err("Text data CRC32C DataDigest"

					" 0x%08x does not match computed"
@@ -2688,10 +2638,8 @@ static int iscsit_handle_immediate_data( 
	if (conn->conn_ops->DataDigest) {

		u32 data_crc;



		data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,

						    cmd->write_data_done, length, padding,

		data_crc = iscsit_crc_sglist(cmd, length, padding,

					     cmd->pad_bytes);



		if (checksum != data_crc) {

			pr_err("ImmediateData CRC32C DataDigest 0x%08x"

				" does not match computed 0x%08x\n", checksum,
@@ -4116,10 +4064,8 @@ static void iscsit_get_rx_pdu(struct iscsit_conn *conn) 
				break;

			}



			iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer,

						  ISCSI_HDR_LEN, 0, NULL,

						  &checksum);



			checksum = iscsit_crc_buf(buffer, ISCSI_HDR_LEN, 0,

						  NULL);

			if (digest != checksum) {

				pr_err("HeaderDigest CRC32C failed,"

					" received 0x%08x, computed 0x%08x\n",
@@ -4406,15 +4352,6 @@ int iscsit_close_connection( 
	 */

	iscsit_check_conn_usage_count(conn);



	ahash_request_free(conn->conn_tx_hash);

	if (conn->conn_rx_hash) {

		struct crypto_ahash *tfm;



		tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);

		ahash_request_free(conn->conn_rx_hash);

		crypto_free_ahash(tfm);

	}



	if (conn->sock)

		sock_release(conn->sock);

drivers/target/iscsi/iscsi_target_login.c

+0 −50
Original line number Diff line number Diff line
@@ -8,7 +8,6 @@ 
 *

 ******************************************************************************/



#include <crypto/hash.h>

#include <linux/module.h>

#include <linux/string.h>

#include <linux/kthread.h>
@@ -71,46 +70,6 @@ static struct iscsi_login *iscsi_login_init_conn(struct iscsit_conn *conn) 
	return NULL;

}



/*

 * Used by iscsi_target_nego.c:iscsi_target_locate_portal() to setup

 * per struct iscsit_conn libcrypto contexts for crc32c and crc32-intel

 */

int iscsi_login_setup_crypto(struct iscsit_conn *conn)

{

	struct crypto_ahash *tfm;



	/*

	 * Setup slicing by CRC32C algorithm for RX and TX libcrypto contexts

	 * which will default to crc32c_intel.ko for cpu_has_xmm4_2, or fallback

	 * to software 1x8 byte slicing from crc32c.ko

	 */

	tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);

	if (IS_ERR(tfm)) {

		pr_err("crypto_alloc_ahash() failed\n");

		return -ENOMEM;

	}



	conn->conn_rx_hash = ahash_request_alloc(tfm, GFP_KERNEL);

	if (!conn->conn_rx_hash) {

		pr_err("ahash_request_alloc() failed for conn_rx_hash\n");

		crypto_free_ahash(tfm);

		return -ENOMEM;

	}

	ahash_request_set_callback(conn->conn_rx_hash, 0, NULL, NULL);



	conn->conn_tx_hash = ahash_request_alloc(tfm, GFP_KERNEL);

	if (!conn->conn_tx_hash) {

		pr_err("ahash_request_alloc() failed for conn_tx_hash\n");

		ahash_request_free(conn->conn_rx_hash);

		conn->conn_rx_hash = NULL;

		crypto_free_ahash(tfm);

		return -ENOMEM;

	}

	ahash_request_set_callback(conn->conn_tx_hash, 0, NULL, NULL);



	return 0;

}



static int iscsi_login_check_initiator_version(

	struct iscsit_conn *conn,

	u8 version_max,
@@ -1165,15 +1124,6 @@ void iscsi_target_login_sess_out(struct iscsit_conn *conn, 
		iscsit_dec_session_usage_count(conn->sess);

	}



	ahash_request_free(conn->conn_tx_hash);

	if (conn->conn_rx_hash) {

		struct crypto_ahash *tfm;



		tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);

		ahash_request_free(conn->conn_rx_hash);

		crypto_free_ahash(tfm);

	}



	if (conn->param_list) {

		iscsi_release_param_list(conn->param_list);

		conn->param_list = NULL;

drivers/target/iscsi/iscsi_target_login.h

+0 −1
Original line number Diff line number Diff line
@@ -9,7 +9,6 @@ struct iscsi_login; 
struct iscsi_np;

struct sockaddr_storage;



extern int iscsi_login_setup_crypto(struct iscsit_conn *);

extern int iscsi_check_for_session_reinstatement(struct iscsit_conn *);

extern int iscsi_login_post_auth_non_zero_tsih(struct iscsit_conn *, u16, u32);

extern int iscsit_setup_np(struct iscsi_np *,

drivers/target/iscsi/iscsi_target_nego.c

+2 −19
Original line number Diff line number Diff line
@@ -1194,14 +1194,7 @@ int iscsi_target_locate_portal( 
			goto get_target;



		sess->sess_ops->SessionType = 1;

		/*

		 * Setup crc32c modules from libcrypto

		 */

		if (iscsi_login_setup_crypto(conn) < 0) {

			pr_err("iscsi_login_setup_crypto() failed\n");

			ret = -1;

			goto out;

		}



		/*

		 * Serialize access across the discovery struct iscsi_portal_group to

		 * process login attempt.
@@ -1258,17 +1251,7 @@ int iscsi_target_locate_portal( 
	}

	conn->tpg_np = tpg_np;

	pr_debug("Located Portal Group Object: %hu\n", conn->tpg->tpgt);

	/*

	 * Setup crc32c modules from libcrypto

	 */

	if (iscsi_login_setup_crypto(conn) < 0) {

		pr_err("iscsi_login_setup_crypto() failed\n");

		kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);

		iscsit_put_tiqn_for_login(tiqn);

		conn->tpg = NULL;

		ret = -1;

		goto out;

	}



	/*

	 * Serialize access across the struct iscsi_portal_group to

	 * process login attempt.