Commit af8e69ef authored by Martin K. Petersen's avatar Martin K. Petersen
Browse files

Merge patch series "Basic inline encryption support for ufs-exynos" 

Eric Biggers <ebiggers@kernel.org> says:

Add support for Flash Memory Protector (FMP), which is the inline
encryption hardware on Exynos and Exynos-based SoCs.

Specifically, add support for the "traditional FMP mode" that works on
many Exynos-based SoCs including gs101.  This is the mode that uses
"software keys" and is compatible with the upstream kernel's existing
inline encryption framework in the block and filesystem layers.  I
plan to add support for the wrapped key support on gs101 at a later
time.

Tested on gs101 (specifically Pixel 6) by running the 'encrypt' group
of xfstests on a filesystem mounted with the 'inlinecrypt' mount
option.

This patchset applies to v6.10-rc6, and it has no prerequisites that
aren't already upstream.

Link: https://lore.kernel.org/r/20240708235330.103590-1-ebiggers@kernel.org


Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parents e30618a4 c96499fc

drivers/ufs/core/ufshcd-crypto.c

+20 −14
Original line number Diff line number Diff line
@@ -95,8 +95,12 @@ static int ufshcd_crypto_keyslot_program(struct blk_crypto_profile *profile, 
	return err;

}



static int ufshcd_clear_keyslot(struct ufs_hba *hba, int slot)

static int ufshcd_crypto_keyslot_evict(struct blk_crypto_profile *profile,

				       const struct blk_crypto_key *key,

				       unsigned int slot)

{

	struct ufs_hba *hba =

		container_of(profile, struct ufs_hba, crypto_profile);

	/*

	 * Clear the crypto cfg on the device. Clearing CFGE

	 * might not be sufficient, so just clear the entire cfg.
@@ -106,16 +110,10 @@ static int ufshcd_clear_keyslot(struct ufs_hba *hba, int slot) 
	return ufshcd_program_key(hba, &cfg, slot);

}



static int ufshcd_crypto_keyslot_evict(struct blk_crypto_profile *profile,

				       const struct blk_crypto_key *key,

				       unsigned int slot)

{

	struct ufs_hba *hba =

		container_of(profile, struct ufs_hba, crypto_profile);



	return ufshcd_clear_keyslot(hba, slot);

}



/*

 * Reprogram the keyslots if needed, and return true if CRYPTO_GENERAL_ENABLE

 * should be used in the host controller initialization sequence.

 */

bool ufshcd_crypto_enable(struct ufs_hba *hba)

{

	if (!(hba->caps & UFSHCD_CAP_CRYPTO))
@@ -123,6 +121,10 @@ bool ufshcd_crypto_enable(struct ufs_hba *hba) 


	/* Reset might clear all keys, so reprogram all the keys. */

	blk_crypto_reprogram_all_keys(&hba->crypto_profile);



	if (hba->quirks & UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE)

		return false;



	return true;

}
@@ -159,6 +161,9 @@ int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba) 
	int err = 0;

	enum blk_crypto_mode_num blk_mode_num;



	if (hba->quirks & UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE)

		return 0;



	/*

	 * Don't use crypto if either the hardware doesn't advertise the

	 * standard crypto capability bit *or* if the vendor specific driver
@@ -228,9 +233,10 @@ void ufshcd_init_crypto(struct ufs_hba *hba) 
	if (!(hba->caps & UFSHCD_CAP_CRYPTO))

		return;



	/* Clear all keyslots - the number of keyslots is (CFGC + 1) */

	for (slot = 0; slot < hba->crypto_capabilities.config_count + 1; slot++)

		ufshcd_clear_keyslot(hba, slot);

	/* Clear all keyslots. */

	for (slot = 0; slot < hba->crypto_profile.num_slots; slot++)

		hba->crypto_profile.ll_ops.keyslot_evict(&hba->crypto_profile,

							 NULL, slot);

}



void ufshcd_crypto_register(struct ufs_hba *hba, struct request_queue *q)

drivers/ufs/core/ufshcd-crypto.h

+36 −0
Original line number Diff line number Diff line
@@ -37,6 +37,33 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, 
	h->dunu = cpu_to_le32(upper_32_bits(lrbp->data_unit_num));

}



static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,

					  struct ufshcd_lrb *lrbp)

{

	struct scsi_cmnd *cmd = lrbp->cmd;

	const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;



	if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)

		return hba->vops->fill_crypto_prdt(hba, crypt_ctx,

						   lrbp->ucd_prdt_ptr,

						   scsi_sg_count(cmd));

	return 0;

}



static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,

					    struct ufshcd_lrb *lrbp)

{

	if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))

		return;



	if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))

		return;



	/* Zeroize the PRDT because it can contain cryptographic keys. */

	memzero_explicit(lrbp->ucd_prdt_ptr,

			 ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd));

}



bool ufshcd_crypto_enable(struct ufs_hba *hba);



int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba);
@@ -54,6 +81,15 @@ static inline void 
ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,

				   struct request_desc_header *h) { }



static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,

					  struct ufshcd_lrb *lrbp)

{

	return 0;

}



static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,

					    struct ufshcd_lrb *lrbp) { }



static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)

{

	return false;

drivers/ufs/core/ufshcd.c

+2 −1
Original line number Diff line number Diff line
@@ -2640,7 +2640,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 


	ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));



	return 0;

	return ufshcd_crypto_fill_prdt(hba, lrbp);

}



/**
@@ -5479,6 +5479,7 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 
	struct scsi_cmnd *cmd = lrbp->cmd;



	scsi_dma_unmap(cmd);

	ufshcd_crypto_clear_prdt(hba, lrbp);

	ufshcd_release(hba);

	ufshcd_clk_scaling_update_busy(hba);

}

drivers/ufs/host/ufs-exynos.c

+234 −6
Original line number Diff line number Diff line
@@ -8,6 +8,9 @@ 
 *

 */



#include <asm/unaligned.h>

#include <crypto/aes.h>

#include <linux/arm-smccc.h>

#include <linux/clk.h>

#include <linux/delay.h>

#include <linux/module.h>
@@ -25,12 +28,13 @@ 


#include "ufs-exynos.h"



#define DATA_UNIT_SIZE		4096



/*

 * Exynos's Vendor specific registers for UFSHCI

 */

#define HCI_TXPRDT_ENTRY_SIZE	0x00

#define PRDT_PREFECT_EN		BIT(31)

#define PRDT_SET_SIZE(x)	((x) & 0x1F)

#define HCI_RXPRDT_ENTRY_SIZE	0x04

#define HCI_1US_TO_CNT_VAL	0x0C

#define CNT_VAL_1US_MASK	0x3FF
@@ -1043,8 +1047,8 @@ static int exynos_ufs_post_link(struct ufs_hba *hba) 
	exynos_ufs_fit_aggr_timeout(ufs);



	hci_writel(ufs, 0xa, HCI_DATA_REORDER);

	hci_writel(ufs, PRDT_SET_SIZE(12), HCI_TXPRDT_ENTRY_SIZE);

	hci_writel(ufs, PRDT_SET_SIZE(12), HCI_RXPRDT_ENTRY_SIZE);

	hci_writel(ufs, ilog2(DATA_UNIT_SIZE), HCI_TXPRDT_ENTRY_SIZE);

	hci_writel(ufs, ilog2(DATA_UNIT_SIZE), HCI_RXPRDT_ENTRY_SIZE);

	hci_writel(ufs, (1 << hba->nutrs) - 1, HCI_UTRL_NEXUS_TYPE);

	hci_writel(ufs, (1 << hba->nutmrs) - 1, HCI_UTMRL_NEXUS_TYPE);

	hci_writel(ufs, 0xf, HCI_AXIDMA_RWDATA_BURST_LEN);
@@ -1151,6 +1155,227 @@ static inline void exynos_ufs_priv_init(struct ufs_hba *hba, 
	hba->quirks = ufs->drv_data->quirks;

}



#ifdef CONFIG_SCSI_UFS_CRYPTO



/*

 * Support for Flash Memory Protector (FMP), which is the inline encryption

 * hardware on Exynos and Exynos-based SoCs.  The interface to this hardware is

 * not compatible with the standard UFS crypto.  It requires that encryption be

 * configured in the PRDT using a nonstandard extension.

 */



enum fmp_crypto_algo_mode {

	FMP_BYPASS_MODE = 0,

	FMP_ALGO_MODE_AES_CBC = 1,

	FMP_ALGO_MODE_AES_XTS = 2,

};

enum fmp_crypto_key_length {

	FMP_KEYLEN_256BIT = 1,

};



/**

 * struct fmp_sg_entry - nonstandard format of PRDT entries when FMP is enabled

 *

 * @base: The standard PRDT entry, but with nonstandard bitfields in the high

 *	bits of the 'size' field, i.e. the last 32-bit word.  When these

 *	nonstandard bitfields are zero, the data segment won't be encrypted or

 *	decrypted.  Otherwise they specify the algorithm and key length with

 *	which the data segment will be encrypted or decrypted.

 * @file_iv: The initialization vector (IV) with all bytes reversed

 * @file_enckey: The first half of the AES-XTS key with all bytes reserved

 * @file_twkey: The second half of the AES-XTS key with all bytes reserved

 * @disk_iv: Unused

 * @reserved: Unused

 */

struct fmp_sg_entry {

	struct ufshcd_sg_entry base;

	__be64 file_iv[2];

	__be64 file_enckey[4];

	__be64 file_twkey[4];

	__be64 disk_iv[2];

	__be64 reserved[2];

};



#define SMC_CMD_FMP_SECURITY	\

	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \

			   ARM_SMCCC_OWNER_SIP, 0x1810)

#define SMC_CMD_SMU		\

	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \

			   ARM_SMCCC_OWNER_SIP, 0x1850)

#define SMC_CMD_FMP_SMU_RESUME	\

	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \

			   ARM_SMCCC_OWNER_SIP, 0x1860)

#define SMU_EMBEDDED			0

#define SMU_INIT			0

#define CFG_DESCTYPE_3			3



static void exynos_ufs_fmp_init(struct ufs_hba *hba, struct exynos_ufs *ufs)

{

	struct blk_crypto_profile *profile = &hba->crypto_profile;

	struct arm_smccc_res res;

	int err;



	/*

	 * Check for the standard crypto support bit, since it's available even

	 * though the rest of the interface to FMP is nonstandard.

	 *

	 * This check should have the effect of preventing the driver from

	 * trying to use FMP on old Exynos SoCs that don't have FMP.

	 */

	if (!(ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES) &

	      MASK_CRYPTO_SUPPORT))

		return;



	/*

	 * The below sequence of SMC calls to enable FMP can be found in the

	 * downstream driver source for gs101 and other Exynos-based SoCs.  It

	 * is the only way to enable FMP that works on SoCs such as gs101 that

	 * don't make the FMP registers accessible to Linux.  It probably works

	 * on other Exynos-based SoCs too, and might even still be the only way

	 * that works.  But this hasn't been properly tested, and this code is

	 * mutually exclusive with exynos_ufs_config_smu().  So for now only

	 * enable FMP support on SoCs with EXYNOS_UFS_OPT_UFSPR_SECURE.

	 */

	if (!(ufs->opts & EXYNOS_UFS_OPT_UFSPR_SECURE))

		return;



	/*

	 * This call (which sets DESCTYPE to 0x3 in the FMPSECURITY0 register)

	 * is needed to make the hardware use the larger PRDT entry size.

	 */

	BUILD_BUG_ON(sizeof(struct fmp_sg_entry) != 128);

	arm_smccc_smc(SMC_CMD_FMP_SECURITY, 0, SMU_EMBEDDED, CFG_DESCTYPE_3,

		      0, 0, 0, 0, &res);

	if (res.a0) {

		dev_warn(hba->dev,

			 "SMC_CMD_FMP_SECURITY failed on init: %ld.  Disabling FMP support.\n",

			 res.a0);

		return;

	}

	ufshcd_set_sg_entry_size(hba, sizeof(struct fmp_sg_entry));



	/*

	 * This is needed to initialize FMP.  Without it, errors occur when

	 * inline encryption is used.

	 */

	arm_smccc_smc(SMC_CMD_SMU, SMU_INIT, SMU_EMBEDDED, 0, 0, 0, 0, 0, &res);

	if (res.a0) {

		dev_err(hba->dev,

			"SMC_CMD_SMU(SMU_INIT) failed: %ld.  Disabling FMP support.\n",

			res.a0);

		return;

	}



	/* Advertise crypto capabilities to the block layer. */

	err = devm_blk_crypto_profile_init(hba->dev, profile, 0);

	if (err) {

		/* Only ENOMEM should be possible here. */

		dev_err(hba->dev, "Failed to initialize crypto profile: %d\n",

			err);

		return;

	}

	profile->max_dun_bytes_supported = AES_BLOCK_SIZE;

	profile->dev = hba->dev;

	profile->modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] =

		DATA_UNIT_SIZE;



	/* Advertise crypto support to ufshcd-core. */

	hba->caps |= UFSHCD_CAP_CRYPTO;



	/* Advertise crypto quirks to ufshcd-core. */

	hba->quirks |= UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE |

		       UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE |

		       UFSHCD_QUIRK_KEYS_IN_PRDT;



}



static void exynos_ufs_fmp_resume(struct ufs_hba *hba)

{

	struct arm_smccc_res res;



	arm_smccc_smc(SMC_CMD_FMP_SECURITY, 0, SMU_EMBEDDED, CFG_DESCTYPE_3,

		      0, 0, 0, 0, &res);

	if (res.a0)

		dev_err(hba->dev,

			"SMC_CMD_FMP_SECURITY failed on resume: %ld\n", res.a0);



	arm_smccc_smc(SMC_CMD_FMP_SMU_RESUME, 0, SMU_EMBEDDED, 0, 0, 0, 0, 0,

		      &res);

	if (res.a0)

		dev_err(hba->dev,

			"SMC_CMD_FMP_SMU_RESUME failed: %ld\n", res.a0);

}



static inline __be64 fmp_key_word(const u8 *key, int j)

{

	return cpu_to_be64(get_unaligned_le64(

			key + AES_KEYSIZE_256 - (j + 1) * sizeof(u64)));

}



/* Fill the PRDT for a request according to the given encryption context. */

static int exynos_ufs_fmp_fill_prdt(struct ufs_hba *hba,

				    const struct bio_crypt_ctx *crypt_ctx,

				    void *prdt, unsigned int num_segments)

{

	struct fmp_sg_entry *fmp_prdt = prdt;

	const u8 *enckey = crypt_ctx->bc_key->raw;

	const u8 *twkey = enckey + AES_KEYSIZE_256;

	u64 dun_lo = crypt_ctx->bc_dun[0];

	u64 dun_hi = crypt_ctx->bc_dun[1];

	unsigned int i;



	/* If FMP wasn't enabled, we shouldn't get any encrypted requests. */

	if (WARN_ON_ONCE(!(hba->caps & UFSHCD_CAP_CRYPTO)))

		return -EIO;



	/* Configure FMP on each segment of the request. */

	for (i = 0; i < num_segments; i++) {

		struct fmp_sg_entry *prd = &fmp_prdt[i];

		int j;



		/* Each segment must be exactly one data unit. */

		if (prd->base.size != cpu_to_le32(DATA_UNIT_SIZE - 1)) {

			dev_err(hba->dev,

				"data segment is misaligned for FMP\n");

			return -EIO;

		}



		/* Set the algorithm and key length. */

		prd->base.size |= cpu_to_le32((FMP_ALGO_MODE_AES_XTS << 28) |

					      (FMP_KEYLEN_256BIT << 26));



		/* Set the IV. */

		prd->file_iv[0] = cpu_to_be64(dun_hi);

		prd->file_iv[1] = cpu_to_be64(dun_lo);



		/* Set the key. */

		for (j = 0; j < AES_KEYSIZE_256 / sizeof(u64); j++) {

			prd->file_enckey[j] = fmp_key_word(enckey, j);

			prd->file_twkey[j] = fmp_key_word(twkey, j);

		}



		/* Increment the data unit number. */

		dun_lo++;

		if (dun_lo == 0)

			dun_hi++;

	}

	return 0;

}



#else /* CONFIG_SCSI_UFS_CRYPTO */



static void exynos_ufs_fmp_init(struct ufs_hba *hba, struct exynos_ufs *ufs)

{

}



static void exynos_ufs_fmp_resume(struct ufs_hba *hba)

{

}



#define exynos_ufs_fmp_fill_prdt NULL



#endif /* !CONFIG_SCSI_UFS_CRYPTO */



static int exynos_ufs_init(struct ufs_hba *hba)

{

	struct device *dev = hba->dev;
@@ -1198,6 +1423,8 @@ static int exynos_ufs_init(struct ufs_hba *hba) 


	exynos_ufs_priv_init(hba, ufs);



	exynos_ufs_fmp_init(hba, ufs);



	if (ufs->drv_data->drv_init) {

		ret = ufs->drv_data->drv_init(dev, ufs);

		if (ret) {
@@ -1213,7 +1440,7 @@ static int exynos_ufs_init(struct ufs_hba *hba) 
	if (!(ufs->opts & EXYNOS_UFS_OPT_UFSPR_SECURE))

		exynos_ufs_config_smu(ufs);



	hba->host->dma_alignment = SZ_4K - 1;

	hba->host->dma_alignment = DATA_UNIT_SIZE - 1;

	return 0;



out:
@@ -1332,7 +1559,7 @@ static int exynos_ufs_hce_enable_notify(struct ufs_hba *hba, 
		 * (ufshcd_async_scan()). Note: this callback may also be called

		 * from other functions than ufshcd_init().

		 */

		hba->host->max_segment_size = SZ_4K;

		hba->host->max_segment_size = DATA_UNIT_SIZE;



		if (ufs->drv_data->pre_hce_enable) {

			ret = ufs->drv_data->pre_hce_enable(ufs);
@@ -1432,7 +1659,7 @@ static int exynos_ufs_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 
		phy_power_on(ufs->phy);



	exynos_ufs_config_smu(ufs);



	exynos_ufs_fmp_resume(hba);

	return 0;

}
@@ -1698,6 +1925,7 @@ static const struct ufs_hba_variant_ops ufs_hba_exynos_ops = { 
	.hibern8_notify			= exynos_ufs_hibern8_notify,

	.suspend			= exynos_ufs_suspend,

	.resume				= exynos_ufs_resume,

	.fill_crypto_prdt		= exynos_ufs_fmp_fill_prdt,

};



static struct ufs_hba_variant_ops ufs_hba_exynosauto_vh_ops = {

include/ufs/ufshcd.h

+28 −0
Original line number Diff line number Diff line
@@ -322,6 +322,7 @@ struct ufs_pwr_mode_info { 
 * @device_reset: called to issue a reset pulse on the UFS device

 * @config_scaling_param: called to configure clock scaling parameters

 * @program_key: program or evict an inline encryption key

 * @fill_crypto_prdt: initialize crypto-related fields in the PRDT

 * @event_notify: called to notify important events

 * @reinit_notify: called to notify reinit of UFSHCD during max gear switch

 * @mcq_config_resource: called to configure MCQ platform resources
@@ -369,6 +370,9 @@ struct ufs_hba_variant_ops { 
				struct devfreq_simple_ondemand_data *data);

	int	(*program_key)(struct ufs_hba *hba,

			       const union ufs_crypto_cfg_entry *cfg, int slot);

	int	(*fill_crypto_prdt)(struct ufs_hba *hba,

				    const struct bio_crypt_ctx *crypt_ctx,

				    void *prdt, unsigned int num_segments);

	void	(*event_notify)(struct ufs_hba *hba,

				enum ufs_event_type evt, void *data);

	void	(*reinit_notify)(struct ufs_hba *);
@@ -648,6 +652,30 @@ enum ufshcd_quirks { 
	 * thus need this quirk to skip related flow.

	 */

	UFSHCD_QUIRK_MCQ_BROKEN_RTC			= 1 << 21,



	/*

	 * This quirk needs to be enabled if the host controller supports inline

	 * encryption but it needs to initialize the crypto capabilities in a

	 * nonstandard way and/or needs to override blk_crypto_ll_ops.  If

	 * enabled, the standard code won't initialize the blk_crypto_profile;

	 * ufs_hba_variant_ops::init() must do it instead.

	 */

	UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE		= 1 << 22,



	/*

	 * This quirk needs to be enabled if the host controller supports inline

	 * encryption but does not support the CRYPTO_GENERAL_ENABLE bit, i.e.

	 * host controller initialization fails if that bit is set.

	 */

	UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE		= 1 << 23,



	/*

	 * This quirk needs to be enabled if the host controller driver copies

	 * cryptographic keys into the PRDT in order to send them to hardware,

	 * and therefore the PRDT should be zeroized after each request (as per

	 * the standard best practice for managing keys).

	 */

	UFSHCD_QUIRK_KEYS_IN_PRDT			= 1 << 24,

};



enum ufshcd_caps {