Commit 982b72cd authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: x86/sm4 - stop using the SIMD helper 



Stop wrapping skcipher and aead algorithms with the crypto SIMD helper
(crypto/simd.c).  The only purpose of doing so was to work around x86
not always supporting kernel-mode FPU in softirqs.  Specifically, if a
hardirq interrupted a task context kernel-mode FPU section and then a
softirqs were run at the end of that hardirq, those softirqs could not
use kernel-mode FPU.  This has now been fixed.  In combination with the
fact that the skcipher and aead APIs only support task and softirq
contexts, these can now just use kernel-mode FPU unconditionally on x86.

This simplifies the code and improves performance.

Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent cc01d284

arch/x86/crypto/Kconfig

+0 −2
Original line number Diff line number Diff line
@@ -192,7 +192,6 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 
	tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX)"

	depends on X86 && 64BIT

	select CRYPTO_SKCIPHER

	select CRYPTO_SIMD

	select CRYPTO_ALGAPI

	select CRYPTO_SM4

	help
@@ -213,7 +212,6 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64 
	tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX2)"

	depends on X86 && 64BIT

	select CRYPTO_SKCIPHER

	select CRYPTO_SIMD

	select CRYPTO_ALGAPI

	select CRYPTO_SM4

	select CRYPTO_SM4_AESNI_AVX_X86_64

arch/x86/crypto/sm4_aesni_avx2_glue.c

+11 −20
Original line number Diff line number Diff line
@@ -8,11 +8,10 @@ 
 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>

 */



#include <asm/fpu/api.h>

#include <linux/module.h>

#include <linux/crypto.h>

#include <linux/kernel.h>

#include <asm/simd.h>

#include <crypto/internal/simd.h>

#include <crypto/internal/skcipher.h>

#include <crypto/sm4.h>

#include "sm4-avx.h"
@@ -48,10 +47,9 @@ static int ctr_crypt(struct skcipher_request *req) 
static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {

	{

		.base = {

			.cra_name		= "__ecb(sm4)",

			.cra_driver_name	= "__ecb-sm4-aesni-avx2",

			.cra_name		= "ecb(sm4)",

			.cra_driver_name	= "ecb-sm4-aesni-avx2",

			.cra_priority		= 500,

			.cra_flags		= CRYPTO_ALG_INTERNAL,

			.cra_blocksize		= SM4_BLOCK_SIZE,

			.cra_ctxsize		= sizeof(struct sm4_ctx),

			.cra_module		= THIS_MODULE,
@@ -64,10 +62,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { 
		.decrypt	= sm4_avx_ecb_decrypt,

	}, {

		.base = {

			.cra_name		= "__cbc(sm4)",

			.cra_driver_name	= "__cbc-sm4-aesni-avx2",

			.cra_name		= "cbc(sm4)",

			.cra_driver_name	= "cbc-sm4-aesni-avx2",

			.cra_priority		= 500,

			.cra_flags		= CRYPTO_ALG_INTERNAL,

			.cra_blocksize		= SM4_BLOCK_SIZE,

			.cra_ctxsize		= sizeof(struct sm4_ctx),

			.cra_module		= THIS_MODULE,
@@ -81,10 +78,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { 
		.decrypt	= cbc_decrypt,

	}, {

		.base = {

			.cra_name		= "__ctr(sm4)",

			.cra_driver_name	= "__ctr-sm4-aesni-avx2",

			.cra_name		= "ctr(sm4)",

			.cra_driver_name	= "ctr-sm4-aesni-avx2",

			.cra_priority		= 500,

			.cra_flags		= CRYPTO_ALG_INTERNAL,

			.cra_blocksize		= 1,

			.cra_ctxsize		= sizeof(struct sm4_ctx),

			.cra_module		= THIS_MODULE,
@@ -100,9 +96,6 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { 
	}

};



static struct simd_skcipher_alg *

simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];



static int __init sm4_init(void)

{

	const char *feature_name;
@@ -121,16 +114,14 @@ static int __init sm4_init(void) 
		return -ENODEV;

	}



	return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,

					ARRAY_SIZE(sm4_aesni_avx2_skciphers),

					simd_sm4_aesni_avx2_skciphers);

	return crypto_register_skciphers(sm4_aesni_avx2_skciphers,

					 ARRAY_SIZE(sm4_aesni_avx2_skciphers));

}



static void __exit sm4_exit(void)

{

	simd_unregister_skciphers(sm4_aesni_avx2_skciphers,

				ARRAY_SIZE(sm4_aesni_avx2_skciphers),

				simd_sm4_aesni_avx2_skciphers);

	crypto_unregister_skciphers(sm4_aesni_avx2_skciphers,

				    ARRAY_SIZE(sm4_aesni_avx2_skciphers));

}



module_init(sm4_init);

arch/x86/crypto/sm4_aesni_avx_glue.c

+11 −20
Original line number Diff line number Diff line
@@ -8,11 +8,10 @@ 
 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>

 */



#include <asm/fpu/api.h>

#include <linux/module.h>

#include <linux/crypto.h>

#include <linux/kernel.h>

#include <asm/simd.h>

#include <crypto/internal/simd.h>

#include <crypto/internal/skcipher.h>

#include <crypto/sm4.h>

#include "sm4-avx.h"
@@ -263,10 +262,9 @@ static int ctr_crypt(struct skcipher_request *req) 
static struct skcipher_alg sm4_aesni_avx_skciphers[] = {

	{

		.base = {

			.cra_name		= "__ecb(sm4)",

			.cra_driver_name	= "__ecb-sm4-aesni-avx",

			.cra_name		= "ecb(sm4)",

			.cra_driver_name	= "ecb-sm4-aesni-avx",

			.cra_priority		= 400,

			.cra_flags		= CRYPTO_ALG_INTERNAL,

			.cra_blocksize		= SM4_BLOCK_SIZE,

			.cra_ctxsize		= sizeof(struct sm4_ctx),

			.cra_module		= THIS_MODULE,
@@ -279,10 +277,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { 
		.decrypt	= sm4_avx_ecb_decrypt,

	}, {

		.base = {

			.cra_name		= "__cbc(sm4)",

			.cra_driver_name	= "__cbc-sm4-aesni-avx",

			.cra_name		= "cbc(sm4)",

			.cra_driver_name	= "cbc-sm4-aesni-avx",

			.cra_priority		= 400,

			.cra_flags		= CRYPTO_ALG_INTERNAL,

			.cra_blocksize		= SM4_BLOCK_SIZE,

			.cra_ctxsize		= sizeof(struct sm4_ctx),

			.cra_module		= THIS_MODULE,
@@ -296,10 +293,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { 
		.decrypt	= cbc_decrypt,

	}, {

		.base = {

			.cra_name		= "__ctr(sm4)",

			.cra_driver_name	= "__ctr-sm4-aesni-avx",

			.cra_name		= "ctr(sm4)",

			.cra_driver_name	= "ctr-sm4-aesni-avx",

			.cra_priority		= 400,

			.cra_flags		= CRYPTO_ALG_INTERNAL,

			.cra_blocksize		= 1,

			.cra_ctxsize		= sizeof(struct sm4_ctx),

			.cra_module		= THIS_MODULE,
@@ -315,9 +311,6 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { 
	}

};



static struct simd_skcipher_alg *

simd_sm4_aesni_avx_skciphers[ARRAY_SIZE(sm4_aesni_avx_skciphers)];



static int __init sm4_init(void)

{

	const char *feature_name;
@@ -335,16 +328,14 @@ static int __init sm4_init(void) 
		return -ENODEV;

	}



	return simd_register_skciphers_compat(sm4_aesni_avx_skciphers,

					ARRAY_SIZE(sm4_aesni_avx_skciphers),

					simd_sm4_aesni_avx_skciphers);

	return crypto_register_skciphers(sm4_aesni_avx_skciphers,

					 ARRAY_SIZE(sm4_aesni_avx_skciphers));

}



static void __exit sm4_exit(void)

{

	simd_unregister_skciphers(sm4_aesni_avx_skciphers,

					ARRAY_SIZE(sm4_aesni_avx_skciphers),

					simd_sm4_aesni_avx_skciphers);

	crypto_unregister_skciphers(sm4_aesni_avx_skciphers,

				    ARRAY_SIZE(sm4_aesni_avx_skciphers));

}



module_init(sm4_init);