Commit 4d8da355 authored by Eric Biggers's avatar Eric Biggers
Browse files

lib/crypto: x86/polyval: Migrate optimized code into library 



Migrate the x86_64 implementation of POLYVAL into lib/crypto/, wiring it
up to the POLYVAL library interface.  This makes the POLYVAL library be
properly optimized on x86_64.

This drops the x86_64 optimizations of polyval in the crypto_shash API.
That's fine, since polyval will be removed from crypto_shash entirely
since it is unneeded there.  But even if it comes back, the crypto_shash
API could just be implemented on top of the library API, as usual.

Adjust the names and prototypes of the assembly functions to align more
closely with the rest of the library code.

Also replace a movaps instruction with movups to remove the assumption
that the key struct is 16-byte aligned.  Users can still align the key
if they want (and at least in this case, movups is just as fast as
movaps), but it's inconvenient to require it.

Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20251109234726.638437-6-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent 37919e23

arch/x86/crypto/Kconfig

+0 −10
Original line number Diff line number Diff line
@@ -353,16 +353,6 @@ config CRYPTO_NHPOLY1305_AVX2 
	  Architecture: x86_64 using:

	  - AVX2 (Advanced Vector Extensions 2)



config CRYPTO_POLYVAL_CLMUL_NI

	tristate "Hash functions: POLYVAL (CLMUL-NI)"

	depends on 64BIT

	select CRYPTO_POLYVAL

	help

	  POLYVAL hash function for HCTR2



	  Architecture: x86_64 using:

	  - CLMUL-NI (carry-less multiplication new instructions)



config CRYPTO_SM3_AVX_X86_64

	tristate "Hash functions: SM3 (AVX)"

	depends on 64BIT

arch/x86/crypto/Makefile

+0 −3
Original line number Diff line number Diff line
@@ -52,9 +52,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ 
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o

ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o



obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o

polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o



obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o

nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o

obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o

arch/x86/crypto/polyval-clmulni_glue.c

deleted100644 → 0
+0 −180
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Glue code for POLYVAL using PCMULQDQ-NI

 *

 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>

 * Copyright (c) 2009 Intel Corp.

 *   Author: Huang Ying <ying.huang@intel.com>

 * Copyright 2021 Google LLC

 */



/*

 * Glue code based on ghash-clmulni-intel_glue.c.

 *

 * This implementation of POLYVAL uses montgomery multiplication

 * accelerated by PCLMULQDQ-NI to implement the finite field

 * operations.

 */



#include <asm/cpu_device_id.h>

#include <asm/fpu/api.h>

#include <crypto/internal/hash.h>

#include <crypto/polyval.h>

#include <crypto/utils.h>

#include <linux/errno.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>



#define POLYVAL_ALIGN	16

#define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN)

#define POLYVAL_ALIGN_EXTRA ((POLYVAL_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))

#define POLYVAL_CTX_SIZE (sizeof(struct polyval_tfm_ctx) + POLYVAL_ALIGN_EXTRA)

#define NUM_KEY_POWERS	8



struct polyval_tfm_ctx {

	/*

	 * These powers must be in the order h^8, ..., h^1.

	 */

	u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE] POLYVAL_ALIGN_ATTR;

};



struct polyval_desc_ctx {

	u8 buffer[POLYVAL_BLOCK_SIZE];

};



asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,

	const u8 *in, size_t nblocks, u8 *accumulator);

asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);



static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm)

{

	return PTR_ALIGN(crypto_shash_ctx(tfm), POLYVAL_ALIGN);

}



static void internal_polyval_update(const struct polyval_tfm_ctx *keys,

	const u8 *in, size_t nblocks, u8 *accumulator)

{

	kernel_fpu_begin();

	clmul_polyval_update(keys, in, nblocks, accumulator);

	kernel_fpu_end();

}



static void internal_polyval_mul(u8 *op1, const u8 *op2)

{

	kernel_fpu_begin();

	clmul_polyval_mul(op1, op2);

	kernel_fpu_end();

}



static int polyval_x86_setkey(struct crypto_shash *tfm,

			const u8 *key, unsigned int keylen)

{

	struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm);

	int i;



	if (keylen != POLYVAL_BLOCK_SIZE)

		return -EINVAL;



	memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);



	for (i = NUM_KEY_POWERS-2; i >= 0; i--) {

		memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);

		internal_polyval_mul(tctx->key_powers[i],

				     tctx->key_powers[i+1]);

	}



	return 0;

}



static int polyval_x86_init(struct shash_desc *desc)

{

	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);



	memset(dctx, 0, sizeof(*dctx));



	return 0;

}



static int polyval_x86_update(struct shash_desc *desc,

			 const u8 *src, unsigned int srclen)

{

	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);

	const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);

	unsigned int nblocks;



	do {

		/* Allow rescheduling every 4K bytes. */

		nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;

		internal_polyval_update(tctx, src, nblocks, dctx->buffer);

		srclen -= nblocks * POLYVAL_BLOCK_SIZE;

		src += nblocks * POLYVAL_BLOCK_SIZE;

	} while (srclen >= POLYVAL_BLOCK_SIZE);



	return srclen;

}



static int polyval_x86_finup(struct shash_desc *desc, const u8 *src,

			     unsigned int len, u8 *dst)

{

	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);

	const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);



	if (len) {

		crypto_xor(dctx->buffer, src, len);

		internal_polyval_mul(dctx->buffer,

				     tctx->key_powers[NUM_KEY_POWERS-1]);

	}



	memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);



	return 0;

}



static struct shash_alg polyval_alg = {

	.digestsize	= POLYVAL_DIGEST_SIZE,

	.init		= polyval_x86_init,

	.update		= polyval_x86_update,

	.finup		= polyval_x86_finup,

	.setkey		= polyval_x86_setkey,

	.descsize	= sizeof(struct polyval_desc_ctx),

	.base		= {

		.cra_name		= "polyval",

		.cra_driver_name	= "polyval-clmulni",

		.cra_priority		= 200,

		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,

		.cra_blocksize		= POLYVAL_BLOCK_SIZE,

		.cra_ctxsize		= POLYVAL_CTX_SIZE,

		.cra_module		= THIS_MODULE,

	},

};



__maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = {

	X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),

	{}

};

MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);



static int __init polyval_clmulni_mod_init(void)

{

	if (!x86_match_cpu(pcmul_cpu_id))

		return -ENODEV;



	if (!boot_cpu_has(X86_FEATURE_AVX))

		return -ENODEV;



	return crypto_register_shash(&polyval_alg);

}



static void __exit polyval_clmulni_mod_exit(void)

{

	crypto_unregister_shash(&polyval_alg);

}



module_init(polyval_clmulni_mod_init);

module_exit(polyval_clmulni_mod_exit);



MODULE_LICENSE("GPL");

MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");

MODULE_ALIAS_CRYPTO("polyval");

MODULE_ALIAS_CRYPTO("polyval-clmulni");

include/crypto/polyval.h

+3 −0
Original line number Diff line number Diff line
@@ -48,6 +48,9 @@ struct polyval_key { 
#ifdef CONFIG_ARM64

	/** @h_powers: Powers of the hash key H^8 through H^1 */

	struct polyval_elem h_powers[8];

#elif defined(CONFIG_X86)

	/** @h_powers: Powers of the hash key H^8 through H^1 */

	struct polyval_elem h_powers[8];

#else

#error "Unhandled arch"

#endif

lib/crypto/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -145,6 +145,7 @@ config CRYPTO_LIB_POLYVAL_ARCH 
	bool

	depends on CRYPTO_LIB_POLYVAL && !UML

	default y if ARM64 && KERNEL_MODE_NEON

	default y if X86_64



config CRYPTO_LIB_CHACHA20POLY1305

	tristate