Commit 60e3f1e9 authored by Eric Biggers's avatar Eric Biggers
Browse files

lib/crypto: arm64/sha512: Migrate optimized SHA-512 code to library 



Instead of exposing the arm64-optimized SHA-512 code via arm64-specific
crypto_shash algorithms, instead just implement the sha512_blocks()
library function.  This is much simpler, it makes the SHA-512 (and
SHA-384) library functions be arm64-optimized, and it fixes the
longstanding issue where the arm64-optimized SHA-512 code was disabled
by default.  SHA-512 still remains available through crypto_shash, but
individual architectures no longer need to handle it.

To match sha512_blocks(), change the type of the nblocks parameter of
the assembly functions from int or 'unsigned int' to size_t.  Update the
ARMv8 CE assembly function accordingly.  The scalar assembly function
actually already treated it as size_t.

Acked-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250630160320.2888-9-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent 24c91b62

arch/arm64/configs/defconfig

+0 −1
Original line number Diff line number Diff line
@@ -1744,7 +1744,6 @@ CONFIG_CRYPTO_ANSI_CPRNG=y 
CONFIG_CRYPTO_USER_API_RNG=m

CONFIG_CRYPTO_GHASH_ARM64_CE=y

CONFIG_CRYPTO_SHA1_ARM64_CE=y

CONFIG_CRYPTO_SHA512_ARM64_CE=m

CONFIG_CRYPTO_SHA3_ARM64=m

CONFIG_CRYPTO_SM3_ARM64_CE=m

CONFIG_CRYPTO_AES_ARM64_CE_BLK=y

arch/arm64/crypto/Kconfig

+0 −19
Original line number Diff line number Diff line
@@ -36,25 +36,6 @@ config CRYPTO_SHA1_ARM64_CE 
	  Architecture: arm64 using:

	  - ARMv8 Crypto Extensions



config CRYPTO_SHA512_ARM64

	tristate "Hash functions: SHA-384 and SHA-512"

	select CRYPTO_HASH

	help

	  SHA-384 and SHA-512 secure hash algorithms (FIPS 180)



	  Architecture: arm64



config CRYPTO_SHA512_ARM64_CE

	tristate "Hash functions: SHA-384 and SHA-512 (ARMv8 Crypto Extensions)"

	depends on KERNEL_MODE_NEON

	select CRYPTO_HASH

	select CRYPTO_SHA512_ARM64

	help

	  SHA-384 and SHA-512 secure hash algorithms (FIPS 180)



	  Architecture: arm64 using:

	  - ARMv8 Crypto Extensions



config CRYPTO_SHA3_ARM64

	tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)"

	depends on KERNEL_MODE_NEON

arch/arm64/crypto/Makefile

+0 −14
Original line number Diff line number Diff line
@@ -8,9 +8,6 @@ 
obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) += sha1-ce.o

sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o



obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o

sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o



obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o

sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o
@@ -53,9 +50,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o 
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o

aes-neon-blk-y := aes-glue-neon.o aes-neon.o



obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o

sha512-arm64-y := sha512-glue.o sha512-core.o



obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o

nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
@@ -64,11 +58,3 @@ aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o 


obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o

aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
 


quiet_cmd_perlasm = PERLASM $@

      cmd_perlasm = $(PERL) $(<) void $(@)



$(obj)/sha512-core.S: $(src)/../lib/crypto/sha2-armv8.pl

	$(call cmd,perlasm)



clean-files += sha512-core.S

arch/arm64/crypto/sha512-ce-glue.c

deleted100644 → 0
+0 −96
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions

 *

 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 */



#include <asm/neon.h>

#include <crypto/internal/hash.h>

#include <crypto/sha2.h>

#include <crypto/sha512_base.h>

#include <linux/cpufeature.h>

#include <linux/kernel.h>

#include <linux/module.h>



MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("sha384");

MODULE_ALIAS_CRYPTO("sha512");



asmlinkage int __sha512_ce_transform(struct sha512_state *sst, u8 const *src,

				     int blocks);



static void sha512_ce_transform(struct sha512_state *sst, u8 const *src,

				int blocks)

{

	do {

		int rem;



		kernel_neon_begin();

		rem = __sha512_ce_transform(sst, src, blocks);

		kernel_neon_end();

		src += (blocks - rem) * SHA512_BLOCK_SIZE;

		blocks = rem;

	} while (blocks);

}



static int sha512_ce_update(struct shash_desc *desc, const u8 *data,

			    unsigned int len)

{

	return sha512_base_do_update_blocks(desc, data, len,

					    sha512_ce_transform);

}



static int sha512_ce_finup(struct shash_desc *desc, const u8 *data,

			   unsigned int len, u8 *out)

{

	sha512_base_do_finup(desc, data, len, sha512_ce_transform);

	return sha512_base_finish(desc, out);

}



static struct shash_alg algs[] = { {

	.init			= sha384_base_init,

	.update			= sha512_ce_update,

	.finup			= sha512_ce_finup,

	.descsize		= SHA512_STATE_SIZE,

	.digestsize		= SHA384_DIGEST_SIZE,

	.base.cra_name		= "sha384",

	.base.cra_driver_name	= "sha384-ce",

	.base.cra_priority	= 200,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA512_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

}, {

	.init			= sha512_base_init,

	.update			= sha512_ce_update,

	.finup			= sha512_ce_finup,

	.descsize		= SHA512_STATE_SIZE,

	.digestsize		= SHA512_DIGEST_SIZE,

	.base.cra_name		= "sha512",

	.base.cra_driver_name	= "sha512-ce",

	.base.cra_priority	= 200,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA512_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

} };



static int __init sha512_ce_mod_init(void)

{

	return crypto_register_shashes(algs, ARRAY_SIZE(algs));

}



static void __exit sha512_ce_mod_fini(void)

{

	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

}



module_cpu_feature_match(SHA512, sha512_ce_mod_init);

module_exit(sha512_ce_mod_fini);

arch/arm64/crypto/sha512-glue.c

deleted100644 → 0
+0 −83
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

/*

 * Linux/arm64 port of the OpenSSL SHA512 implementation for AArch64

 *

 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>

 */



#include <crypto/internal/hash.h>

#include <crypto/sha2.h>

#include <crypto/sha512_base.h>

#include <linux/kernel.h>

#include <linux/module.h>



MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64");

MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("sha384");

MODULE_ALIAS_CRYPTO("sha512");



asmlinkage void sha512_blocks_arch(u64 *digest, const void *data,

				   unsigned int num_blks);



static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src,

				   int blocks)

{

	sha512_blocks_arch(sst->state, src, blocks);

}



static int sha512_update_arm64(struct shash_desc *desc, const u8 *data,

			       unsigned int len)

{

	return sha512_base_do_update_blocks(desc, data, len,

					    sha512_arm64_transform);

}



static int sha512_finup(struct shash_desc *desc, const u8 *data,

			unsigned int len, u8 *out)

{

	sha512_base_do_finup(desc, data, len, sha512_arm64_transform);

	return sha512_base_finish(desc, out);

}



static struct shash_alg algs[] = { {

	.digestsize		= SHA512_DIGEST_SIZE,

	.init			= sha512_base_init,

	.update			= sha512_update_arm64,

	.finup			= sha512_finup,

	.descsize		= SHA512_STATE_SIZE,

	.base.cra_name		= "sha512",

	.base.cra_driver_name	= "sha512-arm64",

	.base.cra_priority	= 150,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA512_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

}, {

	.digestsize		= SHA384_DIGEST_SIZE,

	.init			= sha384_base_init,

	.update			= sha512_update_arm64,

	.finup			= sha512_finup,

	.descsize		= SHA512_STATE_SIZE,

	.base.cra_name		= "sha384",

	.base.cra_driver_name	= "sha384-arm64",

	.base.cra_priority	= 150,

	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |

				  CRYPTO_AHASH_ALG_FINUP_MAX,

	.base.cra_blocksize	= SHA384_BLOCK_SIZE,

	.base.cra_module	= THIS_MODULE,

} };



static int __init sha512_mod_init(void)

{

	return crypto_register_shashes(algs, ARRAY_SIZE(algs));

}



static void __exit sha512_mod_fini(void)

{

	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));

}



module_init(sha512_mod_init);

module_exit(sha512_mod_fini);