Commit a229d832 authored by Eric Biggers's avatar Eric Biggers
Browse files

lib/crypto: x86/nh: Migrate optimized code into library 

Migrate the x86_64 implementations of NH into lib/crypto/.  This makes
the nh() function be optimized on x86_64 kernels.

Note: this temporarily makes the adiantum template not utilize the
x86_64 optimized NH code.  This is resolved in a later commit that
converts the adiantum template to use nh() instead of "nhpoly1305".

Link: https://lore.kernel.org/r/20251211011846.8179-6-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent b4a8528d

arch/x86/crypto/Kconfig

+0 −20
Original line number Diff line number Diff line
@@ -333,26 +333,6 @@ config CRYPTO_AEGIS128_AESNI_SSE2 
	  - AES-NI (AES New Instructions)

	  - SSE4.1 (Streaming SIMD Extensions 4.1)



config CRYPTO_NHPOLY1305_SSE2

	tristate "Hash functions: NHPoly1305 (SSE2)"

	depends on 64BIT

	select CRYPTO_NHPOLY1305

	help

	  NHPoly1305 hash function for Adiantum



	  Architecture: x86_64 using:

	  - SSE2 (Streaming SIMD Extensions 2)



config CRYPTO_NHPOLY1305_AVX2

	tristate "Hash functions: NHPoly1305 (AVX2)"

	depends on 64BIT

	select CRYPTO_NHPOLY1305

	help

	  NHPoly1305 hash function for Adiantum



	  Architecture: x86_64 using:

	  - AVX2 (Advanced Vector Extensions 2)



config CRYPTO_SM3_AVX_X86_64

	tristate "Hash functions: SM3 (AVX)"

	depends on 64BIT

arch/x86/crypto/Makefile

+0 −5
Original line number Diff line number Diff line
@@ -53,11 +53,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ 
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o

ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o



obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o

nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o

obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o

nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o



obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o

sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o

arch/x86/crypto/nhpoly1305-avx2-glue.c

deleted100644 → 0
+0 −81
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum

 * (AVX2 accelerated version)

 *

 * Copyright 2018 Google LLC

 */



#include <crypto/internal/hash.h>

#include <crypto/internal/simd.h>

#include <crypto/nhpoly1305.h>

#include <linux/module.h>

#include <linux/sizes.h>

#include <asm/simd.h>



asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,

			__le64 hash[NH_NUM_PASSES]);



static int nhpoly1305_avx2_update(struct shash_desc *desc,

				  const u8 *src, unsigned int srclen)

{

	if (srclen < 64 || !crypto_simd_usable())

		return crypto_nhpoly1305_update(desc, src, srclen);



	do {

		unsigned int n = min_t(unsigned int, srclen, SZ_4K);



		kernel_fpu_begin();

		crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);

		kernel_fpu_end();

		src += n;

		srclen -= n;

	} while (srclen);

	return 0;

}



static int nhpoly1305_avx2_digest(struct shash_desc *desc,

				  const u8 *src, unsigned int srclen, u8 *out)

{

	return crypto_nhpoly1305_init(desc) ?:

	       nhpoly1305_avx2_update(desc, src, srclen) ?:

	       crypto_nhpoly1305_final(desc, out);

}



static struct shash_alg nhpoly1305_alg = {

	.base.cra_name		= "nhpoly1305",

	.base.cra_driver_name	= "nhpoly1305-avx2",

	.base.cra_priority	= 300,

	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),

	.base.cra_module	= THIS_MODULE,

	.digestsize		= POLY1305_DIGEST_SIZE,

	.init			= crypto_nhpoly1305_init,

	.update			= nhpoly1305_avx2_update,

	.final			= crypto_nhpoly1305_final,

	.digest			= nhpoly1305_avx2_digest,

	.setkey			= crypto_nhpoly1305_setkey,

	.descsize		= sizeof(struct nhpoly1305_state),

};



static int __init nhpoly1305_mod_init(void)

{

	if (!boot_cpu_has(X86_FEATURE_AVX2) ||

	    !boot_cpu_has(X86_FEATURE_OSXSAVE))

		return -ENODEV;



	return crypto_register_shash(&nhpoly1305_alg);

}



static void __exit nhpoly1305_mod_exit(void)

{

	crypto_unregister_shash(&nhpoly1305_alg);

}



module_init(nhpoly1305_mod_init);

module_exit(nhpoly1305_mod_exit);



MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");

MODULE_LICENSE("GPL v2");

MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");

MODULE_ALIAS_CRYPTO("nhpoly1305");

MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");

arch/x86/crypto/nhpoly1305-sse2-glue.c

deleted100644 → 0
+0 −80
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum

 * (SSE2 accelerated version)

 *

 * Copyright 2018 Google LLC

 */



#include <crypto/internal/hash.h>

#include <crypto/internal/simd.h>

#include <crypto/nhpoly1305.h>

#include <linux/module.h>

#include <linux/sizes.h>

#include <asm/simd.h>



asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,

			__le64 hash[NH_NUM_PASSES]);



static int nhpoly1305_sse2_update(struct shash_desc *desc,

				  const u8 *src, unsigned int srclen)

{

	if (srclen < 64 || !crypto_simd_usable())

		return crypto_nhpoly1305_update(desc, src, srclen);



	do {

		unsigned int n = min_t(unsigned int, srclen, SZ_4K);



		kernel_fpu_begin();

		crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2);

		kernel_fpu_end();

		src += n;

		srclen -= n;

	} while (srclen);

	return 0;

}



static int nhpoly1305_sse2_digest(struct shash_desc *desc,

				  const u8 *src, unsigned int srclen, u8 *out)

{

	return crypto_nhpoly1305_init(desc) ?:

	       nhpoly1305_sse2_update(desc, src, srclen) ?:

	       crypto_nhpoly1305_final(desc, out);

}



static struct shash_alg nhpoly1305_alg = {

	.base.cra_name		= "nhpoly1305",

	.base.cra_driver_name	= "nhpoly1305-sse2",

	.base.cra_priority	= 200,

	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),

	.base.cra_module	= THIS_MODULE,

	.digestsize		= POLY1305_DIGEST_SIZE,

	.init			= crypto_nhpoly1305_init,

	.update			= nhpoly1305_sse2_update,

	.final			= crypto_nhpoly1305_final,

	.digest			= nhpoly1305_sse2_digest,

	.setkey			= crypto_nhpoly1305_setkey,

	.descsize		= sizeof(struct nhpoly1305_state),

};



static int __init nhpoly1305_mod_init(void)

{

	if (!boot_cpu_has(X86_FEATURE_XMM2))

		return -ENODEV;



	return crypto_register_shash(&nhpoly1305_alg);

}



static void __exit nhpoly1305_mod_exit(void)

{

	crypto_unregister_shash(&nhpoly1305_alg);

}



module_init(nhpoly1305_mod_init);

module_exit(nhpoly1305_mod_exit);



MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)");

MODULE_LICENSE("GPL v2");

MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");

MODULE_ALIAS_CRYPTO("nhpoly1305");

MODULE_ALIAS_CRYPTO("nhpoly1305-sse2");

lib/crypto/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -119,6 +119,7 @@ config CRYPTO_LIB_NH_ARCH 
	depends on CRYPTO_LIB_NH && !UML

	default y if ARM && KERNEL_MODE_NEON

	default y if ARM64 && KERNEL_MODE_NEON

	default y if X86_64



config CRYPTO_LIB_POLY1305

	tristate