Commit 6e36be51 authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: arm64/sha256 - implement library instead of shash 



Instead of providing crypto_shash algorithms for the arch-optimized
SHA-256 code, instead implement the SHA-256 library.  This is much
simpler, it makes the SHA-256 library functions be arch-optimized, and
it fixes the longstanding issue where the arch-optimized SHA-256 was
disabled by default.  SHA-256 still remains available through
crypto_shash, but individual architectures no longer need to handle it.

Remove support for SHA-256 finalization from the ARMv8 CE assembly code,
since the library does not yet support architecture-specific overrides
of the finalization.  (Support for that has been omitted for now, for
simplicity and because usually it isn't performance-critical.)

To match sha256_blocks_arch(), change the type of the nblocks parameter
of the assembly functions from int or 'unsigned int' to size_t.  Update
the ARMv8 CE assembly function accordingly.  The scalar and NEON
assembly functions actually already treated it as size_t.

While renaming the assembly files, also fix the naming quirks where
"sha2" meant sha256, and "sha512" meant both sha256 and sha512.

Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 642cfc06

arch/arm64/configs/defconfig

+0 −1
Original line number Diff line number Diff line
@@ -1737,7 +1737,6 @@ CONFIG_CRYPTO_USER_API_RNG=m 
CONFIG_CRYPTO_CHACHA20_NEON=m

CONFIG_CRYPTO_GHASH_ARM64_CE=y

CONFIG_CRYPTO_SHA1_ARM64_CE=y

CONFIG_CRYPTO_SHA2_ARM64_CE=y

CONFIG_CRYPTO_SHA512_ARM64_CE=m

CONFIG_CRYPTO_SHA3_ARM64=m

CONFIG_CRYPTO_SM3_ARM64_CE=m

arch/arm64/crypto/Kconfig

+0 −19
Original line number Diff line number Diff line
@@ -36,25 +36,6 @@ config CRYPTO_SHA1_ARM64_CE 
	  Architecture: arm64 using:

	  - ARMv8 Crypto Extensions



config CRYPTO_SHA256_ARM64

	tristate "Hash functions: SHA-224 and SHA-256"

	select CRYPTO_HASH

	help

	  SHA-224 and SHA-256 secure hash algorithms (FIPS 180)



	  Architecture: arm64



config CRYPTO_SHA2_ARM64_CE

	tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)"

	depends on KERNEL_MODE_NEON

	select CRYPTO_HASH

	select CRYPTO_SHA256_ARM64

	help

	  SHA-224 and SHA-256 secure hash algorithms (FIPS 180)



	  Architecture: arm64 using:

	  - ARMv8 Crypto Extensions



config CRYPTO_SHA512_ARM64

	tristate "Hash functions: SHA-384 and SHA-512"

	select CRYPTO_HASH

arch/arm64/crypto/Makefile

+2 −11
Original line number Diff line number Diff line
@@ -8,9 +8,6 @@ 
obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) += sha1-ce.o

sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o



obj-$(CONFIG_CRYPTO_SHA2_ARM64_CE) += sha2-ce.o

sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o



obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o

sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o
@@ -56,9 +53,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o 
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o

aes-neon-blk-y := aes-glue-neon.o aes-neon.o



obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o

sha256-arm64-y := sha256-glue.o sha256-core.o



obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o

sha512-arm64-y := sha512-glue.o sha512-core.o
@@ -74,10 +68,7 @@ aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o 
quiet_cmd_perlasm = PERLASM $@

      cmd_perlasm = $(PERL) $(<) void $(@)



$(obj)/%-core.S: $(src)/%-armv8.pl

	$(call cmd,perlasm)



$(obj)/sha256-core.S: $(src)/sha512-armv8.pl

$(obj)/sha512-core.S: $(src)/../lib/crypto/sha2-armv8.pl

	$(call cmd,perlasm)



clean-files += sha256-core.S sha512-core.S
 
clean-files += sha512-core.S

arch/arm64/crypto/sha2-ce-glue.c

deleted100644 → 0
+0 −138
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions

 *

 * Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>

 */



#include <asm/neon.h>

#include <crypto/internal/hash.h>

#include <crypto/sha2.h>

#include <crypto/sha256_base.h>

#include <linux/cpufeature.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>



MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("sha224");

MODULE_ALIAS_CRYPTO("sha256");



struct sha256_ce_state {

	struct crypto_sha256_state sst;

	u32			finalize;

};



extern const u32 sha256_ce_offsetof_count;

extern const u32 sha256_ce_offsetof_finalize;



asmlinkage int __sha256_ce_transform(struct sha256_ce_state *sst, u8 const *src,

				     int blocks);



static void sha256_ce_transform(struct crypto_sha256_state *sst, u8 const *src,

				int blocks)

{

	while (blocks) {

		int rem;



		kernel_neon_begin();

		rem = __sha256_ce_transform(container_of(sst,

							 struct sha256_ce_state,

							 sst), src, blocks);

		kernel_neon_end();

		src += (blocks - rem) * SHA256_BLOCK_SIZE;

		blocks = rem;

	}

}



const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state,

					      sst.count);

const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state,

						 finalize);



static int sha256_ce_update(struct shash_desc *desc, const u8 *data,

			    unsigned int len)

{

	struct sha256_ce_state *sctx = shash_desc_ctx(desc);



	sctx->finalize = 0;

	return sha256_base_do_update_blocks(desc, data, len,

					    sha256_ce_transform);

}



static int sha256_ce_finup(struct shash_desc *desc, const u8 *data,

			   unsigned int len, u8 *out)

{

	struct sha256_ce_state *sctx = shash_desc_ctx(desc);

	bool finalize = !(len % SHA256_BLOCK_SIZE) && len;



	/*

	 * Allow the asm code to perform the finalization if there is no

	 * partial data and the input is a round multiple of the block size.

	 */

	sctx->finalize = finalize;



	if (finalize)

		sha256_base_do_update_blocks(desc, data, len,

					     sha256_ce_transform);

	else

		sha256_base_do_finup(desc, data, len, sha256_ce_transform);

	return sha256_base_finish(desc, out);

}



static int sha256_ce_digest(struct shash_desc *desc, const u8 *data,

			    unsigned int len, u8 *out)

{

	sha256_base_init(desc);

	return sha256_ce_finup(desc, data, len, out);

}



static struct shash_alg algs[] = { {

	.init			= sha224_base_init,

	.update			= sha256_ce_update,

	.finup			= sha256_ce_finup,

	.descsize		= sizeof(struct sha256_ce_state),

	.statesize		= sizeof(struct crypto_sha256_state),

	.digestsize		= SHA224_DIGEST_SIZE,

	.base			= {

		.cra_name		= "sha224",

		.cra_driver_name	= "sha224-ce",

		.cra_priority		= 200,

		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

					  CRYPTO_AHASH_ALG_FINUP_MAX,

		.cra_blocksize		= SHA256_BLOCK_SIZE,

		.cra_module		= THIS_MODULE,

	}

}, {

	.init			= sha256_base_init,

	.update			= sha256_ce_update,

	.finup			= sha256_ce_finup,

	.digest			= sha256_ce_digest,

	.descsize		= sizeof(struct sha256_ce_state),

	.statesize		= sizeof(struct crypto_sha256_state),

	.digestsize		= SHA256_DIGEST_SIZE,

	.base			= {

		.cra_name		= "sha256",

		.cra_driver_name	= "sha256-ce",

		.cra_priority		= 200,

		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

					  CRYPTO_AHASH_ALG_FINUP_MAX,

		.cra_blocksize		= SHA256_BLOCK_SIZE,

		.cra_module		= THIS_MODULE,

	}

} };



static int __init sha2_ce_mod_init(void)

{

	return crypto_register_shashes(algs, ARRAY_SIZE(algs));

}



static void __exit sha2_ce_mod_fini(void)

{

	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

}



module_cpu_feature_match(SHA2, sha2_ce_mod_init);

module_exit(sha2_ce_mod_fini);

arch/arm64/crypto/sha256-glue.c

deleted100644 → 0
+0 −156
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

/*

 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64

 *

 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>

 */



#include <asm/neon.h>

#include <crypto/internal/hash.h>

#include <crypto/sha2.h>

#include <crypto/sha256_base.h>

#include <linux/cpufeature.h>

#include <linux/kernel.h>

#include <linux/module.h>



MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");

MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("sha224");

MODULE_ALIAS_CRYPTO("sha256");



asmlinkage void sha256_block_data_order(u32 *digest, const void *data,

					unsigned int num_blks);

EXPORT_SYMBOL(sha256_block_data_order);



static void sha256_arm64_transform(struct crypto_sha256_state *sst,

				   u8 const *src, int blocks)

{

	sha256_block_data_order(sst->state, src, blocks);

}



asmlinkage void sha256_block_neon(u32 *digest, const void *data,

				  unsigned int num_blks);



static void sha256_neon_transform(struct crypto_sha256_state *sst,

				  u8 const *src, int blocks)

{

	kernel_neon_begin();

	sha256_block_neon(sst->state, src, blocks);

	kernel_neon_end();

}



static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data,

				      unsigned int len)

{

	return sha256_base_do_update_blocks(desc, data, len,

					    sha256_arm64_transform);

}



static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data,

				     unsigned int len, u8 *out)

{

	sha256_base_do_finup(desc, data, len, sha256_arm64_transform);

	return sha256_base_finish(desc, out);

}



static struct shash_alg algs[] = { {

	.digestsize		= SHA256_DIGEST_SIZE,

	.init			= sha256_base_init,

	.update			= crypto_sha256_arm64_update,

	.finup			= crypto_sha256_arm64_finup,

	.descsize		= sizeof(struct crypto_sha256_state),

	.base.cra_name		= "sha256",

	.base.cra_driver_name	= "sha256-arm64",

	.base.cra_priority	= 125,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA256_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

}, {

	.digestsize		= SHA224_DIGEST_SIZE,

	.init			= sha224_base_init,

	.update			= crypto_sha256_arm64_update,

	.finup			= crypto_sha256_arm64_finup,

	.descsize		= sizeof(struct crypto_sha256_state),

	.base.cra_name		= "sha224",

	.base.cra_driver_name	= "sha224-arm64",

	.base.cra_priority	= 125,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA224_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

} };



static int sha256_update_neon(struct shash_desc *desc, const u8 *data,

			      unsigned int len)

{

	return sha256_base_do_update_blocks(desc, data, len,

					    sha256_neon_transform);

}



static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,

			     unsigned int len, u8 *out)

{

	if (len >= SHA256_BLOCK_SIZE) {

		int remain = sha256_update_neon(desc, data, len);



		data += len - remain;

		len = remain;

	}

	sha256_base_do_finup(desc, data, len, sha256_neon_transform);

	return sha256_base_finish(desc, out);

}



static struct shash_alg neon_algs[] = { {

	.digestsize		= SHA256_DIGEST_SIZE,

	.init			= sha256_base_init,

	.update			= sha256_update_neon,

	.finup			= sha256_finup_neon,

	.descsize		= sizeof(struct crypto_sha256_state),

	.base.cra_name		= "sha256",

	.base.cra_driver_name	= "sha256-arm64-neon",

	.base.cra_priority	= 150,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA256_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

}, {

	.digestsize		= SHA224_DIGEST_SIZE,

	.init			= sha224_base_init,

	.update			= sha256_update_neon,

	.finup			= sha256_finup_neon,

	.descsize		= sizeof(struct crypto_sha256_state),

	.base.cra_name		= "sha224",

	.base.cra_driver_name	= "sha224-arm64-neon",

	.base.cra_priority	= 150,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA224_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

} };



static int __init sha256_mod_init(void)

{

	int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));

	if (ret)

		return ret;



	if (cpu_have_named_feature(ASIMD)) {

		ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));

		if (ret)

			crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

	}

	return ret;

}



static void __exit sha256_mod_fini(void)

{

	if (cpu_have_named_feature(ASIMD))

		crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));

	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

}



module_init(sha256_mod_init);

module_exit(sha256_mod_fini);