Commit 0892c91b authored by Eric Biggers's avatar Eric Biggers
Browse files

lib/crypto: powerpc/aes: Migrate SPE optimized code into library 



Move the PowerPC SPE AES assembly code into lib/crypto/, wire the key
expansion and single-block en/decryption functions up to the AES library
API, and remove the superseded "aes-ppc-spe" crypto_cipher algorithm.

The result is that both the AES library and crypto_cipher APIs are now
optimized with SPE, whereas previously only crypto_cipher was (and
optimizations weren't enabled by default, which this commit fixes too).

Note that many of the functions in the PowerPC SPE assembly code are
still used by the AES mode implementations in arch/powerpc/crypto/.  For
now, just export these functions.  These exports will go away once the
AES modes are migrated to the library as well.  (Trying to split up the
assembly files seemed like much more trouble than it would be worth.)

Acked-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-13-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent 2b1ef7ae

arch/powerpc/crypto/Kconfig

+1 −1
Original line number Diff line number Diff line
@@ -5,9 +5,9 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" 
config CRYPTO_AES_PPC_SPE

	tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)"

	depends on SPE

	select CRYPTO_LIB_AES

	select CRYPTO_SKCIPHER

	help

	  Block ciphers: AES cipher algorithms (FIPS-197)

	  Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes



	  Architecture: powerpc using:

arch/powerpc/crypto/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o 
obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o

obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o



aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o

aes-ppc-spe-y := aes-spe-glue.o

aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o

vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o

arch/powerpc/crypto/aes-spe-glue.c

+5 −83
Original line number Diff line number Diff line
@@ -51,30 +51,6 @@ struct ppc_xts_ctx { 
	u32 rounds;

};



extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);

extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);

extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,

			    u32 bytes);

extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,

			    u32 bytes);

extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,

			    u32 bytes, u8 *iv);

extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,

			    u32 bytes, u8 *iv);

extern void ppc_crypt_ctr  (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,

			    u32 bytes, u8 *iv);

extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,

			    u32 bytes, u8 *iv, u32 *key_twk);

extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,

			    u32 bytes, u8 *iv, u32 *key_twk);



extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);

extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);

extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);



extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,

				     unsigned int key_len);



static void spe_begin(void)

{

	/* disable preemption and save users SPE registers if required */
@@ -89,10 +65,10 @@ static void spe_end(void) 
	preempt_enable();

}



static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,

		unsigned int key_len)

static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,

				   const u8 *in_key, unsigned int key_len)

{

	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);



	switch (key_len) {

	case AES_KEYSIZE_128:
@@ -116,12 +92,6 @@ static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, 
	return 0;

}



static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,

				   const u8 *in_key, unsigned int key_len)

{

	return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);

}



static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,

		   unsigned int key_len)

{
@@ -159,24 +129,6 @@ static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 
	return 0;

}



static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)

{

	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	spe_begin();

	ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);

	spe_end();

}



static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)

{

	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	spe_begin();

	ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);

	spe_end();

}



static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)

{

	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -410,26 +362,6 @@ static int ppc_xts_decrypt(struct skcipher_request *req) 
 * with kmalloc() in the crypto infrastructure

 */



static struct crypto_alg aes_cipher_alg = {

	.cra_name		=	"aes",

	.cra_driver_name	=	"aes-ppc-spe",

	.cra_priority		=	300,

	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,

	.cra_blocksize		=	AES_BLOCK_SIZE,

	.cra_ctxsize		=	sizeof(struct ppc_aes_ctx),

	.cra_alignmask		=	0,

	.cra_module		=	THIS_MODULE,

	.cra_u			=	{

		.cipher = {

			.cia_min_keysize	=	AES_MIN_KEY_SIZE,

			.cia_max_keysize	=	AES_MAX_KEY_SIZE,

			.cia_setkey		=	ppc_aes_setkey,

			.cia_encrypt		=	ppc_aes_encrypt,

			.cia_decrypt		=	ppc_aes_decrypt

		}

	}

};



static struct skcipher_alg aes_skcipher_algs[] = {

	{

		.base.cra_name		=	"ecb(aes)",
@@ -488,22 +420,12 @@ static struct skcipher_alg aes_skcipher_algs[] = { 


static int __init ppc_aes_mod_init(void)

{

	int err;



	err = crypto_register_alg(&aes_cipher_alg);

	if (err)

		return err;



	err = crypto_register_skciphers(aes_skcipher_algs,

	return crypto_register_skciphers(aes_skcipher_algs,

					 ARRAY_SIZE(aes_skcipher_algs));

	if (err)

		crypto_unregister_alg(&aes_cipher_alg);

	return err;

}



static void __exit ppc_aes_mod_fini(void)

{

	crypto_unregister_alg(&aes_cipher_alg);

	crypto_unregister_skciphers(aes_skcipher_algs,

				    ARRAY_SIZE(aes_skcipher_algs));

}

include/crypto/aes.h

+31 −0
Original line number Diff line number Diff line
@@ -20,10 +20,22 @@ 


union aes_enckey_arch {

	u32 rndkeys[AES_MAX_KEYLENGTH_U32];

#ifdef CONFIG_CRYPTO_LIB_AES_ARCH

#if defined(CONFIG_PPC) && defined(CONFIG_SPE)

	/* Used unconditionally (when SPE AES code is enabled in kconfig) */

	u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8);

#endif

#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */

};



union aes_invkey_arch {

	u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];

#ifdef CONFIG_CRYPTO_LIB_AES_ARCH

#if defined(CONFIG_PPC) && defined(CONFIG_SPE)

	/* Used unconditionally (when SPE AES code is enabled in kconfig) */

	u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8);

#endif

#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */

};



/**
@@ -124,6 +136,25 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 
#ifdef CONFIG_ARM64

int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,

		     unsigned int key_len);

#elif defined(CONFIG_PPC)

void ppc_expand_key_128(u32 *key_enc, const u8 *key);

void ppc_expand_key_192(u32 *key_enc, const u8 *key);

void ppc_expand_key_256(u32 *key_enc, const u8 *key);

void ppc_generate_decrypt_key(u32 *key_dec, u32 *key_enc, unsigned int key_len);

void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,

		     u32 bytes);

void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,

		     u32 bytes);

void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,

		     u8 *iv);

void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,

		     u8 *iv);

void ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,

		   u8 *iv);

void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,

		     u8 *iv, u32 *key_twk);

void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,

		     u8 *iv, u32 *key_twk);

#endif



/**

lib/crypto/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ config CRYPTO_LIB_AES_ARCH 
	depends on CRYPTO_LIB_AES && !UML && !KMSAN

	default y if ARM

	default y if ARM64

	default y if PPC && SPE



config CRYPTO_LIB_AESCFB

	tristate