Commit 378a337a authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: powerpc/poly1305 - implement library instead of shash 



Currently the Power10 optimized Poly1305 is only wired up to the
crypto_shash API, which makes it unavailable to users of the library
API.  The crypto_shash API for Poly1305 is going to change to be
implemented on top of the library API, so the library API needs to be
supported.  And of course it's needed anyway to serve the library users.

Therefore, change the Power10 optimized Poly1305 code to implement the
library API instead of the crypto_shash API.

Cc: Danny Tsen <dtsen@linux.ibm.com>
Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent b03892c2

arch/powerpc/crypto/Kconfig

+3 −8
Original line number Diff line number Diff line
@@ -100,16 +100,11 @@ config CRYPTO_CHACHA20_P10 
	default CRYPTO_LIB_CHACHA_INTERNAL



config CRYPTO_POLY1305_P10

	tristate "Hash functions: Poly1305 (P10 or later)"

	tristate

	depends on PPC64 && CPU_LITTLE_ENDIAN && VSX

	select CRYPTO_HASH

	select CRYPTO_ARCH_HAVE_LIB_POLY1305

	select CRYPTO_LIB_POLY1305_GENERIC

	help

	  Poly1305 authenticator algorithm (RFC7539)



	  Architecture: PowerPC64

	  - Power10 or later

	  - Little-endian

	default CRYPTO_LIB_POLY1305_INTERNAL



config CRYPTO_DEV_VMX

        bool "Support for VMX cryptographic acceleration instructions"

arch/powerpc/crypto/poly1305-p10-glue.c

+36 −96
Original line number Diff line number Diff line
@@ -4,15 +4,11 @@ 
 *

 * Copyright 2023- IBM Corp. All rights reserved.

 */



#include <crypto/algapi.h>

#include <linux/crypto.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/jump_label.h>

#include <crypto/internal/hash.h>

#include <crypto/internal/poly1305.h>

#include <crypto/internal/simd.h>

#include <crypto/poly1305.h>

#include <linux/cpufeature.h>

#include <linux/unaligned.h>

#include <asm/simd.h>
@@ -22,6 +18,8 @@ asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); 
asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);

asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);



static void vsx_begin(void)

{

	preempt_disable();
@@ -34,51 +32,29 @@ static void vsx_end(void) 
	preempt_enable();

}



static int crypto_poly1305_p10_init(struct shash_desc *desc)

void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])

{

	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);



	poly1305_core_init(&dctx->h);

	if (!static_key_enabled(&have_p10))

		return poly1305_init_generic(dctx, key);



	dctx->h = (struct poly1305_state){};

	dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0);

	dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8);

	dctx->s[0] = get_unaligned_le32(key + 16);

	dctx->s[1] = get_unaligned_le32(key + 20);

	dctx->s[2] = get_unaligned_le32(key + 24);

	dctx->s[3] = get_unaligned_le32(key + 28);

	dctx->buflen = 0;

	dctx->rset = 0;

	dctx->sset = false;



	return 0;

}



static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,

					       const u8 *inp, unsigned int len)

{

	unsigned int acc = 0;



	if (unlikely(!dctx->sset)) {

		if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) {

			struct poly1305_core_key *key = &dctx->core_r;



			key->key.r64[0] = get_unaligned_le64(&inp[0]);

			key->key.r64[1] = get_unaligned_le64(&inp[8]);

			inp += POLY1305_BLOCK_SIZE;

			len -= POLY1305_BLOCK_SIZE;

			acc += POLY1305_BLOCK_SIZE;

			dctx->rset = 1;

		}

		if (len >= POLY1305_BLOCK_SIZE) {

			dctx->s[0] = get_unaligned_le32(&inp[0]);

			dctx->s[1] = get_unaligned_le32(&inp[4]);

			dctx->s[2] = get_unaligned_le32(&inp[8]);

			dctx->s[3] = get_unaligned_le32(&inp[12]);

			acc += POLY1305_BLOCK_SIZE;

			dctx->sset = true;

		}

	}

	return acc;

}

EXPORT_SYMBOL(poly1305_init_arch);



static int crypto_poly1305_p10_update(struct shash_desc *desc,

void poly1305_update_arch(struct poly1305_desc_ctx *dctx,

			  const u8 *src, unsigned int srclen)

{

	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);

	unsigned int bytes, used;

	unsigned int bytes;



	if (!static_key_enabled(&have_p10))

		return poly1305_update_generic(dctx, src, srclen);



	if (unlikely(dctx->buflen)) {

		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
@@ -86,26 +62,16 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, 
		src += bytes;

		srclen -= bytes;

		dctx->buflen += bytes;



		if (dctx->buflen == POLY1305_BLOCK_SIZE) {

			if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf,

							       POLY1305_BLOCK_SIZE))) {

		if (dctx->buflen < POLY1305_BLOCK_SIZE)

			return;

		vsx_begin();

				poly1305_64s(&dctx->h, dctx->buf,

						  POLY1305_BLOCK_SIZE, 1);

		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);

		vsx_end();

			}

		dctx->buflen = 0;

	}

	}



	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {

		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);

		used = crypto_poly1305_setdctxkey(dctx, src, bytes);

		if (likely(used)) {

			srclen -= used;

			src += used;

		}

		if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {

			vsx_begin();

			poly1305_p10le_4blocks(&dctx->h, src, srclen);
@@ -126,61 +92,35 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, 
		dctx->buflen = srclen;

		memcpy(dctx->buf, src, srclen);

	}



	return 0;

}

EXPORT_SYMBOL(poly1305_update_arch);



static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst)

void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)

{

	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);

	if (!static_key_enabled(&have_p10))

		return poly1305_final_generic(dctx, dst);



	if (unlikely(!dctx->sset))

		return -ENOKEY;



	if ((dctx->buflen)) {

	if (dctx->buflen) {

		dctx->buf[dctx->buflen++] = 1;

		memset(dctx->buf + dctx->buflen, 0,

		       POLY1305_BLOCK_SIZE - dctx->buflen);

		vsx_begin();

		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);

		vsx_end();

		dctx->buflen = 0;

	}



	poly1305_emit_64(&dctx->h, &dctx->s, dst);

	return 0;

}



static struct shash_alg poly1305_alg = {

	.digestsize	= POLY1305_DIGEST_SIZE,

	.init		= crypto_poly1305_p10_init,

	.update		= crypto_poly1305_p10_update,

	.final		= crypto_poly1305_p10_final,

	.descsize	= sizeof(struct poly1305_desc_ctx),

	.base		= {

		.cra_name		= "poly1305",

		.cra_driver_name	= "poly1305-p10",

		.cra_priority		= 300,

		.cra_blocksize		= POLY1305_BLOCK_SIZE,

		.cra_module		= THIS_MODULE,

	},

};

EXPORT_SYMBOL(poly1305_final_arch);



static int __init poly1305_p10_init(void)

{

	return crypto_register_shash(&poly1305_alg);

}



static void __exit poly1305_p10_exit(void)

{

	crypto_unregister_shash(&poly1305_alg);

	if (cpu_has_feature(CPU_FTR_ARCH_31))

		static_branch_enable(&have_p10);

	return 0;

}



module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init);

module_exit(poly1305_p10_exit);

arch_initcall(poly1305_p10_init);



MODULE_LICENSE("GPL");

MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");

MODULE_DESCRIPTION("Optimized Poly1305 for P10");
 
MODULE_ALIAS_CRYPTO("poly1305");

MODULE_ALIAS_CRYPTO("poly1305-p10");