Commit 8dcac98a authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Eric Biggers
Browse files

lib/crypto: arm64: Move remaining algorithms to scoped ksimd API 



Move the arm64 implementations of SHA-3 and POLYVAL to the newly
introduced scoped ksimd API, which replaces kernel_neon_begin() and
kernel_neon_end(). On arm64, this is needed because the latter API
will change in an incompatible manner.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent c0d597e0

lib/crypto/arm64/polyval.h

+11 −13
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ 
 *

 * Copyright 2025 Google LLC

 */

#include <asm/neon.h>

#include <asm/simd.h>

#include <linux/cpufeature.h>
@@ -24,13 +23,14 @@ static void polyval_preparekey_arch(struct polyval_key *key, 
	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);

	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);

	if (static_branch_likely(&have_pmull) && may_use_simd()) {

		kernel_neon_begin();

		scoped_ksimd() {

			for (int i = NUM_H_POWERS - 2; i >= 0; i--) {

				key->h_powers[i] = key->h_powers[i + 1];

			polyval_mul_pmull(&key->h_powers[i],

				polyval_mul_pmull(

					&key->h_powers[i],

					&key->h_powers[NUM_H_POWERS - 1]);

			}

		kernel_neon_end();

		}

	} else {

		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {

			key->h_powers[i] = key->h_powers[i + 1];
@@ -44,9 +44,8 @@ static void polyval_mul_arch(struct polyval_elem *acc, 
			     const struct polyval_key *key)

{

	if (static_branch_likely(&have_pmull) && may_use_simd()) {

		kernel_neon_begin();

		scoped_ksimd()

			polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);

		kernel_neon_end();

	} else {

		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);

	}
@@ -62,9 +61,8 @@ static void polyval_blocks_arch(struct polyval_elem *acc, 
			size_t n = min_t(size_t, nblocks,

					 4096 / POLYVAL_BLOCK_SIZE);



			kernel_neon_begin();

			scoped_ksimd()

				polyval_blocks_pmull(acc, key, data, n);

			kernel_neon_end();

			data += n * POLYVAL_BLOCK_SIZE;

			nblocks -= n;

		} while (nblocks);

lib/crypto/arm64/sha3.h

+5 −8
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ 
 * published by the Free Software Foundation.

 */



#include <asm/neon.h>

#include <asm/simd.h>

#include <linux/cpufeature.h>
@@ -23,10 +22,9 @@ static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, 
		do {

			size_t rem;



			kernel_neon_begin();

			scoped_ksimd()

				rem = sha3_ce_transform(state, data, nblocks,

							block_size);

			kernel_neon_end();

			data += (nblocks - rem) * block_size;

			nblocks = rem;

		} while (nblocks);
@@ -46,9 +44,8 @@ static void sha3_keccakf(struct sha3_state *state) 
		 */

		static const u8 zeroes[SHA3_512_BLOCK_SIZE];



		kernel_neon_begin();

		scoped_ksimd()

			sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));

		kernel_neon_end();

	} else {

		sha3_keccakf_generic(state);

	}