Commit 10a6d72e authored by Herbert Xu's avatar Herbert Xu
Browse files

crypto: lib/poly1305 - Use block-only interface 



Now that every architecture provides a block function, use that
to implement the lib/poly1305 and remove the old per-arch code.

Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent ceef731b

arch/arm/lib/crypto/poly1305-glue.c

+0 −57
Original line number Diff line number Diff line
@@ -12,7 +12,6 @@ 
#include <linux/jump_label.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>

#include <linux/unaligned.h>



asmlinkage void poly1305_block_init_arch(
@@ -35,17 +34,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state, 


static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);



void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])

{

	dctx->s[0] = get_unaligned_le32(key + 16);

	dctx->s[1] = get_unaligned_le32(key + 20);

	dctx->s[2] = get_unaligned_le32(key + 24);

	dctx->s[3] = get_unaligned_le32(key + 28);

	dctx->buflen = 0;

	poly1305_block_init_arch(&dctx->state, key);

}

EXPORT_SYMBOL(poly1305_init_arch);



void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,

			  unsigned int len, u32 padbit)

{
@@ -67,51 +55,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 
}

EXPORT_SYMBOL_GPL(poly1305_blocks_arch);



void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,

			  unsigned int nbytes)

{

	if (unlikely(dctx->buflen)) {

		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);



		memcpy(dctx->buf + dctx->buflen, src, bytes);

		src += bytes;

		nbytes -= bytes;

		dctx->buflen += bytes;



		if (dctx->buflen == POLY1305_BLOCK_SIZE) {

			poly1305_blocks_arch(&dctx->state, dctx->buf,

					     POLY1305_BLOCK_SIZE, 1);

			dctx->buflen = 0;

		}

	}



	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {

		poly1305_blocks_arch(&dctx->state, src, nbytes, 1);

		src += round_down(nbytes, POLY1305_BLOCK_SIZE);

		nbytes %= POLY1305_BLOCK_SIZE;

	}



	if (unlikely(nbytes)) {

		dctx->buflen = nbytes;

		memcpy(dctx->buf, src, nbytes);

	}

}

EXPORT_SYMBOL(poly1305_update_arch);



void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)

{

	if (unlikely(dctx->buflen)) {

		dctx->buf[dctx->buflen++] = 1;

		memset(dctx->buf + dctx->buflen, 0,

		       POLY1305_BLOCK_SIZE - dctx->buflen);

		poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0);

	}



	poly1305_emit_arch(&dctx->h, dst, dctx->s);

	*dctx = (struct poly1305_desc_ctx){};

}

EXPORT_SYMBOL(poly1305_final_arch);



bool poly1305_is_arch_optimized(void)

{

	/* We always can use at least the ARM scalar implementation. */

arch/arm64/lib/crypto/poly1305-glue.c

+0 −58
Original line number Diff line number Diff line
@@ -12,7 +12,6 @@ 
#include <linux/jump_label.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>

#include <linux/unaligned.h>



asmlinkage void poly1305_block_init_arch(
@@ -30,17 +29,6 @@ EXPORT_SYMBOL_GPL(poly1305_emit_arch); 


static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);



void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])

{

	dctx->s[0] = get_unaligned_le32(key + 16);

	dctx->s[1] = get_unaligned_le32(key + 20);

	dctx->s[2] = get_unaligned_le32(key + 24);

	dctx->s[3] = get_unaligned_le32(key + 28);

	dctx->buflen = 0;

	poly1305_block_init_arch(&dctx->state, key);

}

EXPORT_SYMBOL(poly1305_init_arch);



void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,

			  unsigned int len, u32 padbit)

{
@@ -61,52 +49,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 
}

EXPORT_SYMBOL_GPL(poly1305_blocks_arch);



void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,

			  unsigned int nbytes)

{

	if (unlikely(dctx->buflen)) {

		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);



		memcpy(dctx->buf + dctx->buflen, src, bytes);

		src += bytes;

		nbytes -= bytes;

		dctx->buflen += bytes;



		if (dctx->buflen == POLY1305_BLOCK_SIZE) {

			poly1305_blocks_arch(&dctx->state, dctx->buf,

					     POLY1305_BLOCK_SIZE, 1);

			dctx->buflen = 0;

		}

	}



	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {

		poly1305_blocks_arch(&dctx->state, src, nbytes, 1);

		src += round_down(nbytes, POLY1305_BLOCK_SIZE);

		nbytes %= POLY1305_BLOCK_SIZE;

	}



	if (unlikely(nbytes)) {

		dctx->buflen = nbytes;

		memcpy(dctx->buf, src, nbytes);

	}

}

EXPORT_SYMBOL(poly1305_update_arch);



void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)

{

	if (unlikely(dctx->buflen)) {

		dctx->buf[dctx->buflen++] = 1;

		memset(dctx->buf + dctx->buflen, 0,

		       POLY1305_BLOCK_SIZE - dctx->buflen);

		poly1305_blocks_arch(&dctx->state, dctx->buf,

				     POLY1305_BLOCK_SIZE, 0);

	}



	poly1305_emit_arch(&dctx->h, dst, dctx->s);

	memzero_explicit(dctx, sizeof(*dctx));

}

EXPORT_SYMBOL(poly1305_final_arch);



bool poly1305_is_arch_optimized(void)

{

	/* We always can use at least the ARM64 scalar implementation. */

arch/mips/lib/crypto/poly1305-glue.c

+0 −60
Original line number Diff line number Diff line
@@ -9,7 +9,6 @@ 
#include <linux/cpufeature.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>

#include <linux/unaligned.h>



asmlinkage void poly1305_block_init_arch(
@@ -24,65 +23,6 @@ asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, 
				   const u32 nonce[4]);

EXPORT_SYMBOL_GPL(poly1305_emit_arch);



void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])

{

	dctx->s[0] = get_unaligned_le32(key + 16);

	dctx->s[1] = get_unaligned_le32(key + 20);

	dctx->s[2] = get_unaligned_le32(key + 24);

	dctx->s[3] = get_unaligned_le32(key + 28);

	dctx->buflen = 0;

	poly1305_block_init_arch(&dctx->state, key);

}

EXPORT_SYMBOL(poly1305_init_arch);



void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,

			  unsigned int nbytes)

{

	if (unlikely(dctx->buflen)) {

		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);



		memcpy(dctx->buf + dctx->buflen, src, bytes);

		src += bytes;

		nbytes -= bytes;

		dctx->buflen += bytes;



		if (dctx->buflen == POLY1305_BLOCK_SIZE) {

			poly1305_blocks_arch(&dctx->state, dctx->buf,

					     POLY1305_BLOCK_SIZE, 1);

			dctx->buflen = 0;

		}

	}



	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {

		unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);



		poly1305_blocks_arch(&dctx->state, src, len, 1);

		src += len;

		nbytes %= POLY1305_BLOCK_SIZE;

	}



	if (unlikely(nbytes)) {

		dctx->buflen = nbytes;

		memcpy(dctx->buf, src, nbytes);

	}

}

EXPORT_SYMBOL(poly1305_update_arch);



void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)

{

	if (unlikely(dctx->buflen)) {

		dctx->buf[dctx->buflen++] = 1;

		memset(dctx->buf + dctx->buflen, 0,

		       POLY1305_BLOCK_SIZE - dctx->buflen);

		poly1305_blocks_arch(&dctx->state, dctx->buf,

				     POLY1305_BLOCK_SIZE, 0);

	}



	poly1305_emit_arch(&dctx->h, dst, dctx->s);

	*dctx = (struct poly1305_desc_ctx){};

}

EXPORT_SYMBOL(poly1305_final_arch);



bool poly1305_is_arch_optimized(void)

{

	return true;

arch/powerpc/lib/crypto/poly1305-p10-glue.c

+0 −63
Original line number Diff line number Diff line
@@ -10,7 +10,6 @@ 
#include <linux/jump_label.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>

#include <linux/unaligned.h>



asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
@@ -45,17 +44,6 @@ void poly1305_block_init_arch(struct poly1305_block_state *dctx, 
}

EXPORT_SYMBOL_GPL(poly1305_block_init_arch);



void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])

{

	dctx->s[0] = get_unaligned_le32(key + 16);

	dctx->s[1] = get_unaligned_le32(key + 20);

	dctx->s[2] = get_unaligned_le32(key + 24);

	dctx->s[3] = get_unaligned_le32(key + 28);

	dctx->buflen = 0;

	poly1305_block_init_arch(&dctx->state, key);

}

EXPORT_SYMBOL(poly1305_init_arch);



void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,

			  unsigned int len, u32 padbit)

{
@@ -76,57 +64,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 
}

EXPORT_SYMBOL_GPL(poly1305_blocks_arch);



void poly1305_update_arch(struct poly1305_desc_ctx *dctx,

			  const u8 *src, unsigned int srclen)

{

	unsigned int bytes;



	if (!static_key_enabled(&have_p10))

		return poly1305_update_generic(dctx, src, srclen);



	if (unlikely(dctx->buflen)) {

		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);

		memcpy(dctx->buf + dctx->buflen, src, bytes);

		src += bytes;

		srclen -= bytes;

		dctx->buflen += bytes;

		if (dctx->buflen < POLY1305_BLOCK_SIZE)

			return;

		poly1305_blocks_arch(&dctx->state, dctx->buf,

				     POLY1305_BLOCK_SIZE, 1);

		dctx->buflen = 0;

	}



	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {

		poly1305_blocks_arch(&dctx->state, src, srclen, 1);

		src += srclen - (srclen % POLY1305_BLOCK_SIZE);

		srclen %= POLY1305_BLOCK_SIZE;

	}



	if (unlikely(srclen)) {

		dctx->buflen = srclen;

		memcpy(dctx->buf, src, srclen);

	}

}

EXPORT_SYMBOL(poly1305_update_arch);



void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)

{

	if (!static_key_enabled(&have_p10))

		return poly1305_final_generic(dctx, dst);



	if (dctx->buflen) {

		dctx->buf[dctx->buflen++] = 1;

		memset(dctx->buf + dctx->buflen, 0,

		       POLY1305_BLOCK_SIZE - dctx->buflen);

		poly1305_blocks_arch(&dctx->state, dctx->buf,

				     POLY1305_BLOCK_SIZE, 0);

	}



	poly1305_emit_arch(&dctx->h, dst, dctx->s);

}

EXPORT_SYMBOL(poly1305_final_arch);



bool poly1305_is_arch_optimized(void)

{

	return static_key_enabled(&have_p10);

arch/x86/lib/crypto/poly1305_glue.c

+0 −60
Original line number Diff line number Diff line
@@ -10,7 +10,6 @@ 
#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/sizes.h>

#include <linux/string.h>

#include <linux/unaligned.h>



struct poly1305_arch_internal {
@@ -96,65 +95,6 @@ void poly1305_emit_arch(const struct poly1305_state *ctx, 
}

EXPORT_SYMBOL_GPL(poly1305_emit_arch);



void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])

{

	dctx->s[0] = get_unaligned_le32(&key[16]);

	dctx->s[1] = get_unaligned_le32(&key[20]);

	dctx->s[2] = get_unaligned_le32(&key[24]);

	dctx->s[3] = get_unaligned_le32(&key[28]);

	dctx->buflen = 0;

	poly1305_block_init_arch(&dctx->state, key);

}

EXPORT_SYMBOL(poly1305_init_arch);



void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,

			  unsigned int srclen)

{

	unsigned int bytes;



	if (unlikely(dctx->buflen)) {

		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);

		memcpy(dctx->buf + dctx->buflen, src, bytes);

		src += bytes;

		srclen -= bytes;

		dctx->buflen += bytes;



		if (dctx->buflen == POLY1305_BLOCK_SIZE) {

			poly1305_blocks_arch(&dctx->state, dctx->buf,

					     POLY1305_BLOCK_SIZE, 1);

			dctx->buflen = 0;

		}

	}



	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {

		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);

		poly1305_blocks_arch(&dctx->state, src, bytes, 1);

		src += bytes;

		srclen -= bytes;

	}



	if (unlikely(srclen)) {

		dctx->buflen = srclen;

		memcpy(dctx->buf, src, srclen);

	}

}

EXPORT_SYMBOL(poly1305_update_arch);



void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)

{

	if (unlikely(dctx->buflen)) {

		dctx->buf[dctx->buflen++] = 1;

		memset(dctx->buf + dctx->buflen, 0,

		       POLY1305_BLOCK_SIZE - dctx->buflen);

		poly1305_blocks_arch(&dctx->state, dctx->buf,

				     POLY1305_BLOCK_SIZE, 0);

	}



	poly1305_emit_arch(&dctx->h, dst, dctx->s);

	memzero_explicit(dctx, sizeof(*dctx));

}

EXPORT_SYMBOL(poly1305_final_arch);



bool poly1305_is_arch_optimized(void)

{

	return static_key_enabled(&poly1305_use_avx);