Commit 98066f2f authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: lib/chacha - strongly type the ChaCha state 



The ChaCha state matrix is 16 32-bit words.  Currently it is represented
in the code as a raw u32 array, or even just a pointer to u32.  This
weak typing is error-prone.  Instead, introduce struct chacha_state:

    struct chacha_state {
            u32 x[16];
    };

Convert all ChaCha and HChaCha functions to use struct chacha_state.
No functional changes.

Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Acked-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 97855e7f

arch/arm/lib/crypto/chacha-glue.c

+17 −13
Original line number Diff line number Diff line
@@ -17,15 +17,18 @@ 
#include <asm/neon.h>

#include <asm/simd.h>



asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,

				      int nrounds);

asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,

asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,

				      u8 *dst, const u8 *src, int nrounds);

asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state,

				       u8 *dst, const u8 *src,

				       int nrounds, unsigned int nbytes);

asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);

asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);

asmlinkage void hchacha_block_arm(const struct chacha_state *state,

				  u32 *out, int nrounds);

asmlinkage void hchacha_block_neon(const struct chacha_state *state,

				   u32 *out, int nrounds);



asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,

			     const u32 *state, int nrounds);

			     const struct chacha_state *state, int nrounds);



static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
@@ -34,7 +37,7 @@ static inline bool neon_usable(void) 
	return static_branch_likely(&use_neon) && crypto_simd_usable();

}



static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,

static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src,

			  unsigned int bytes, int nrounds)

{

	u8 buf[CHACHA_BLOCK_SIZE];
@@ -46,7 +49,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, 
		bytes -= l;

		src += l;

		dst += l;

		state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);

		state->x[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);

	}

	if (bytes) {

		const u8 *s = src;
@@ -57,11 +60,12 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, 
		chacha_block_xor_neon(state, d, s, nrounds);

		if (d != dst)

			memcpy(dst, buf, bytes);

		state[12]++;

		state->x[12]++;

	}

}



void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)

void hchacha_block_arch(const struct chacha_state *state, u32 *stream,

			int nrounds)

{

	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {

		hchacha_block_arm(state, stream, nrounds);
@@ -73,13 +77,13 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) 
}

EXPORT_SYMBOL(hchacha_block_arch);



void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,

		       int nrounds)

void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src,

		       unsigned int bytes, int nrounds)

{

	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||

	    bytes <= CHACHA_BLOCK_SIZE) {

		chacha_doarm(dst, src, bytes, state, nrounds);

		state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);

		state->x[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);

		return;

	}

arch/arm/lib/crypto/chacha-scalar-core.S

+3 −2
Original line number Diff line number Diff line
@@ -367,7 +367,7 @@ 


/*

 * void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,

 *		     const u32 *state, int nrounds);

 *		     const struct chacha_state *state, int nrounds);

 */

ENTRY(chacha_doarm)

	cmp		r2, #0			// len == 0?
@@ -407,7 +407,8 @@ ENTRY(chacha_doarm) 
ENDPROC(chacha_doarm)



/*

 * void hchacha_block_arm(const u32 state[16], u32 out[8], int nrounds);

 * void hchacha_block_arm(const struct chacha_state *state,

 *			  u32 out[8], int nrounds);

 */

ENTRY(hchacha_block_arm)

	push		{r1,r4-r11,lr}

arch/arm64/lib/crypto/chacha-neon-glue.c

+13 −10
Original line number Diff line number Diff line
@@ -28,15 +28,17 @@ 
#include <asm/neon.h>

#include <asm/simd.h>



asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src,

				      int nrounds);

asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src,

asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,

				      u8 *dst, const u8 *src, int nrounds);

asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state,

				       u8 *dst, const u8 *src,

				       int nrounds, int bytes);

asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);

asmlinkage void hchacha_block_neon(const struct chacha_state *state,

				   u32 *out, int nrounds);



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);



static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,

static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src,

			  int bytes, int nrounds)

{

	while (bytes > 0) {
@@ -48,18 +50,19 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, 
			memcpy(buf, src, l);

			chacha_block_xor_neon(state, buf, buf, nrounds);

			memcpy(dst, buf, l);

			state[12] += 1;

			state->x[12] += 1;

			break;

		}

		chacha_4block_xor_neon(state, dst, src, nrounds, l);

		bytes -= l;

		src += l;

		dst += l;

		state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);

		state->x[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);

	}

}



void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)

void hchacha_block_arch(const struct chacha_state *state, u32 *stream,

			int nrounds)

{

	if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {

		hchacha_block_generic(state, stream, nrounds);
@@ -71,8 +74,8 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) 
}

EXPORT_SYMBOL(hchacha_block_arch);



void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,

		       int nrounds)

void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src,

		       unsigned int bytes, int nrounds)

{

	if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE ||

	    !crypto_simd_usable())

arch/mips/lib/crypto/chacha-glue.c

+4 −2
Original line number Diff line number Diff line
@@ -9,11 +9,13 @@ 
#include <linux/kernel.h>

#include <linux/module.h>



asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src,

asmlinkage void chacha_crypt_arch(struct chacha_state *state,

				  u8 *dst, const u8 *src,

				  unsigned int bytes, int nrounds);

EXPORT_SYMBOL(chacha_crypt_arch);



asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds);

asmlinkage void hchacha_block_arch(const struct chacha_state *state,

				   u32 *stream, int nrounds);

EXPORT_SYMBOL(hchacha_block_arch);



bool chacha_is_arch_optimized(void)

arch/powerpc/lib/crypto/chacha-p10-glue.c

+8 −7
Original line number Diff line number Diff line
@@ -14,8 +14,8 @@ 
#include <asm/simd.h>

#include <asm/switch_to.h>



asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src,

				unsigned int len, int nrounds);

asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst,

				const u8 *src, unsigned int len, int nrounds);



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
@@ -31,7 +31,7 @@ static void vsx_end(void) 
	preempt_enable();

}



static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src,

static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src,

			     unsigned int bytes, int nrounds)

{

	unsigned int l = bytes & ~0x0FF;
@@ -41,21 +41,22 @@ static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src, 
		bytes -= l;

		src += l;

		dst += l;

		state[12] += l / CHACHA_BLOCK_SIZE;

		state->x[12] += l / CHACHA_BLOCK_SIZE;

	}



	if (bytes > 0)

		chacha_crypt_generic(state, dst, src, bytes, nrounds);

}



void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)

void hchacha_block_arch(const struct chacha_state *state,

			u32 *stream, int nrounds)

{

	hchacha_block_generic(state, stream, nrounds);

}

EXPORT_SYMBOL(hchacha_block_arch);



void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,

		       int nrounds)

void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src,

		       unsigned int bytes, int nrounds)

{

	if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||

	    !crypto_simd_usable())