Commit 7cf2082e authored by Eric Biggers's avatar Eric Biggers
Browse files

lib/crypto: powerpc/aes: Migrate POWER8 optimized code into library 



Move the POWER8 AES assembly code into lib/crypto/, wire the key
expansion and single-block en/decryption functions up to the AES library
API, and remove the superseded "p8_aes" crypto_cipher algorithm.

The result is that both the AES library and crypto_cipher APIs are now
optimized for POWER8, whereas previously only crypto_cipher was (and
optimizations weren't enabled by default, which this commit fixes too).

Note that many of the functions in the POWER8 assembly code are still
used by the AES mode implementations in arch/powerpc/crypto/.  For now,
just export these functions.  These exports will go away once the AES
modes are migrated to the library as well.  (Trying to split up the
assembly file seemed like much more trouble than it would be worth.)

Another challenge with this code is that the POWER8 assembly code uses a
custom format for the expanded AES key.  Since that code is imported
from OpenSSL and is also targeted to POWER8 (rather than POWER9 which
has better data movement and byteswap instructions), that is not easily
changed.  For now I've just kept the custom format.  To maintain full
correctness, this requires executing some slow fallback code in the case
where the usability of VSX changes between key expansion and use.  This
should be tolerable, as this case shouldn't happen in practice.

Acked-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-14-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent 0892c91b

arch/powerpc/crypto/Makefile

+3 −4
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o 


aes-ppc-spe-y := aes-spe-glue.o

aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o

vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o

vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o



ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)

override flavour := linux-ppc64le
@@ -26,15 +26,14 @@ endif 
quiet_cmd_perl = PERL    $@

      cmd_perl = $(PERL) $< $(flavour) > $@



targets += aesp10-ppc.S ghashp10-ppc.S aesp8-ppc.S ghashp8-ppc.S

targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S



$(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE

	$(call if_changed,perl)



$(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE

$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE

	$(call if_changed,perl)



OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y

OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y

OBJECT_FILES_NON_STANDARD_aesp8-ppc.o := y

OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y

arch/powerpc/crypto/aes.c

deleted100644 → 0
+0 −134
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * AES routines supporting VMX instructions on the Power 8

 *

 * Copyright (C) 2015 International Business Machines Inc.

 *

 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>

 */



#include <asm/simd.h>

#include <asm/switch_to.h>

#include <crypto/aes.h>

#include <crypto/internal/cipher.h>

#include <crypto/internal/simd.h>

#include <linux/err.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/uaccess.h>



#include "aesp8-ppc.h"



struct p8_aes_ctx {

	struct crypto_cipher *fallback;

	struct p8_aes_key enc_key;

	struct p8_aes_key dec_key;

};



static int p8_aes_init(struct crypto_tfm *tfm)

{

	const char *alg = crypto_tfm_alg_name(tfm);

	struct crypto_cipher *fallback;

	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);

	if (IS_ERR(fallback)) {

		printk(KERN_ERR

		       "Failed to allocate transformation for '%s': %ld\n",

		       alg, PTR_ERR(fallback));

		return PTR_ERR(fallback);

	}



	crypto_cipher_set_flags(fallback,

				crypto_cipher_get_flags((struct

							 crypto_cipher *)

							tfm));

	ctx->fallback = fallback;



	return 0;

}



static void p8_aes_exit(struct crypto_tfm *tfm)

{

	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	if (ctx->fallback) {

		crypto_free_cipher(ctx->fallback);

		ctx->fallback = NULL;

	}

}



static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key,

			 unsigned int keylen)

{

	int ret;

	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	preempt_disable();

	pagefault_disable();

	enable_kernel_vsx();

	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);

	ret |= aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);

	disable_kernel_vsx();

	pagefault_enable();

	preempt_enable();



	ret |= crypto_cipher_setkey(ctx->fallback, key, keylen);



	return ret ? -EINVAL : 0;

}



static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	if (!crypto_simd_usable()) {

		crypto_cipher_encrypt_one(ctx->fallback, dst, src);

	} else {

		preempt_disable();

		pagefault_disable();

		enable_kernel_vsx();

		aes_p8_encrypt(src, dst, &ctx->enc_key);

		disable_kernel_vsx();

		pagefault_enable();

		preempt_enable();

	}

}



static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)

{

	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);



	if (!crypto_simd_usable()) {

		crypto_cipher_decrypt_one(ctx->fallback, dst, src);

	} else {

		preempt_disable();

		pagefault_disable();

		enable_kernel_vsx();

		aes_p8_decrypt(src, dst, &ctx->dec_key);

		disable_kernel_vsx();

		pagefault_enable();

		preempt_enable();

	}

}



struct crypto_alg p8_aes_alg = {

	.cra_name = "aes",

	.cra_driver_name = "p8_aes",

	.cra_module = THIS_MODULE,

	.cra_priority = 1000,

	.cra_type = NULL,

	.cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK,

	.cra_alignmask = 0,

	.cra_blocksize = AES_BLOCK_SIZE,

	.cra_ctxsize = sizeof(struct p8_aes_ctx),

	.cra_init = p8_aes_init,

	.cra_exit = p8_aes_exit,

	.cra_cipher = {

		       .cia_min_keysize = AES_MIN_KEY_SIZE,

		       .cia_max_keysize = AES_MAX_KEY_SIZE,

		       .cia_setkey = p8_aes_setkey,

		       .cia_encrypt = p8_aes_encrypt,

		       .cia_decrypt = p8_aes_decrypt,

	},

};

arch/powerpc/crypto/aesp8-ppc.h

+0 −23
Original line number Diff line number Diff line
@@ -2,30 +2,7 @@ 
#include <linux/types.h>

#include <crypto/aes.h>



struct p8_aes_key {

	u8 key[AES_MAX_KEYLENGTH];

	int rounds;

};



extern struct shash_alg p8_ghash_alg;

extern struct crypto_alg p8_aes_alg;

extern struct skcipher_alg p8_aes_cbc_alg;

extern struct skcipher_alg p8_aes_ctr_alg;

extern struct skcipher_alg p8_aes_xts_alg;
 


int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,

			   struct p8_aes_key *key);

int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,

			   struct p8_aes_key *key);

void aes_p8_encrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);

void aes_p8_decrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);

void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,

			const struct p8_aes_key *key, u8 *iv, const int enc);

void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len,

				 const struct p8_aes_key *key, const u8 *iv);

void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,

			const struct p8_aes_key *key1,

			const struct p8_aes_key *key2, u8 *iv);

void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,

			const struct p8_aes_key *key1,

			const struct p8_aes_key *key2, u8 *iv);

arch/powerpc/crypto/vmx.c

+1 −9
Original line number Diff line number Diff line
@@ -27,13 +27,9 @@ static int __init p8_init(void) 
	if (ret)

		goto err;



	ret = crypto_register_alg(&p8_aes_alg);

	if (ret)

		goto err_unregister_ghash;



	ret = crypto_register_skcipher(&p8_aes_cbc_alg);

	if (ret)

		goto err_unregister_aes;

		goto err_unregister_ghash;



	ret = crypto_register_skcipher(&p8_aes_ctr_alg);

	if (ret)
@@ -49,8 +45,6 @@ static int __init p8_init(void) 
	crypto_unregister_skcipher(&p8_aes_ctr_alg);

err_unregister_aes_cbc:

	crypto_unregister_skcipher(&p8_aes_cbc_alg);

err_unregister_aes:

	crypto_unregister_alg(&p8_aes_alg);

err_unregister_ghash:

	crypto_unregister_shash(&p8_ghash_alg);

err:
@@ -62,7 +56,6 @@ static void __exit p8_exit(void) 
	crypto_unregister_skcipher(&p8_aes_xts_alg);

	crypto_unregister_skcipher(&p8_aes_ctr_alg);

	crypto_unregister_skcipher(&p8_aes_cbc_alg);

	crypto_unregister_alg(&p8_aes_alg);

	crypto_unregister_shash(&p8_ghash_alg);

}
@@ -74,4 +67,3 @@ MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions " 
		   "support on Power 8");

MODULE_LICENSE("GPL");

MODULE_VERSION("1.0.0");
 
MODULE_IMPORT_NS("CRYPTO_INTERNAL");

include/crypto/aes.h

+41 −0
Original line number Diff line number Diff line
@@ -18,12 +18,34 @@ 
#define AES_MAX_KEYLENGTH	(15 * 16)

#define AES_MAX_KEYLENGTH_U32	(AES_MAX_KEYLENGTH / sizeof(u32))



/*

 * The POWER8 VSX optimized AES assembly code is borrowed from OpenSSL and

 * inherits OpenSSL's AES_KEY format, which stores the number of rounds after

 * the round keys.  That assembly code is difficult to change.  So for

 * compatibility purposes we reserve space for the extra nrounds field on PPC64.

 *

 * Note: when prepared for decryption, the round keys are just the reversed

 * standard round keys, not the round keys for the Equivalent Inverse Cipher.

 */

struct p8_aes_key {

	u32 rndkeys[AES_MAX_KEYLENGTH_U32];

	int nrounds;

};



union aes_enckey_arch {

	u32 rndkeys[AES_MAX_KEYLENGTH_U32];

#ifdef CONFIG_CRYPTO_LIB_AES_ARCH

#if defined(CONFIG_PPC) && defined(CONFIG_SPE)

	/* Used unconditionally (when SPE AES code is enabled in kconfig) */

	u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8);

#elif defined(CONFIG_PPC)

	/*

	 * Kernels that include the POWER8 VSX optimized AES code use this field

	 * when that code is usable at key preparation time.  Otherwise they

	 * fall back to rndkeys.  In the latter case, p8.nrounds (which doesn't

	 * overlap rndkeys) is set to 0 to differentiate the two formats.

	 */

	struct p8_aes_key p8;

#endif

#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */

};
@@ -34,6 +56,9 @@ union aes_invkey_arch { 
#if defined(CONFIG_PPC) && defined(CONFIG_SPE)

	/* Used unconditionally (when SPE AES code is enabled in kconfig) */

	u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8);

#elif defined(CONFIG_PPC)

	/* Used conditionally, analogous to aes_enckey_arch::p8 */

	struct p8_aes_key p8;

#endif

#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */

};
@@ -155,6 +180,22 @@ void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, 
		     u8 *iv, u32 *key_twk);

void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,

		     u8 *iv, u32 *key_twk);

int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,

			   struct p8_aes_key *key);

int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,

			   struct p8_aes_key *key);

void aes_p8_encrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);

void aes_p8_decrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);

void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,

			const struct p8_aes_key *key, u8 *iv, const int enc);

void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len,

				 const struct p8_aes_key *key, const u8 *iv);

void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,

			const struct p8_aes_key *key1,

			const struct p8_aes_key *key2, u8 *iv);

void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,

			const struct p8_aes_key *key1,

			const struct p8_aes_key *key2, u8 *iv);

#endif



/**