Commit f53d18a4 authored by Ard Biesheuvel's avatar Ard Biesheuvel
Browse files

lib/crypto: Switch ARM and arm64 to 'ksimd' scoped guard API 



Before modifying the prototypes of kernel_neon_begin() and
kernel_neon_end() to accommodate kernel mode FP/SIMD state buffers
allocated on the stack, move arm64 to the new 'ksimd' scoped guard API,
which encapsulates the calls to those functions.

For symmetry, do the same for 32-bit ARM too.

Reviewed-by: default avatarEric Biggers <ebiggers@kernel.org>
Reviewed-by: default avatarJonathan Cameron <jonathan.cameron@huawei.com>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
parent 814f5415

lib/crypto/arm/chacha.h

+4 −7
Original line number Diff line number Diff line
@@ -12,7 +12,6 @@ 


#include <asm/cputype.h>

#include <asm/hwcap.h>

#include <asm/neon.h>

#include <asm/simd.h>



asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
@@ -68,9 +67,8 @@ static void hchacha_block_arch(const struct chacha_state *state, 
	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {

		hchacha_block_arm(state, out, nrounds);

	} else {

		kernel_neon_begin();

		scoped_ksimd()

			hchacha_block_neon(state, out, nrounds);

		kernel_neon_end();

	}

}
@@ -87,9 +85,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 
	do {

		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);



		kernel_neon_begin();

		scoped_ksimd()

			chacha_doneon(state, dst, src, todo, nrounds);

		kernel_neon_end();



		bytes -= todo;

		src += todo;

lib/crypto/arm/curve25519.h

+2 −3
Original line number Diff line number Diff line
@@ -25,9 +25,8 @@ static void curve25519_arch(u8 out[CURVE25519_KEY_SIZE], 
			    const u8 point[CURVE25519_KEY_SIZE])

{

	if (static_branch_likely(&have_neon) && crypto_simd_usable()) {

		kernel_neon_begin();

		scoped_ksimd()

			curve25519_neon(out, scalar, point);

		kernel_neon_end();

	} else {

		curve25519_generic(out, scalar, point);

	}

lib/crypto/arm/poly1305.h

+2 −4
Original line number Diff line number Diff line
@@ -6,7 +6,6 @@ 
 */



#include <asm/hwcap.h>

#include <asm/neon.h>

#include <asm/simd.h>

#include <linux/cpufeature.h>

#include <linux/jump_label.h>
@@ -32,9 +31,8 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, 
		do {

			unsigned int todo = min_t(unsigned int, len, SZ_4K);



			kernel_neon_begin();

			scoped_ksimd()

				poly1305_blocks_neon(state, src, todo, padbit);

			kernel_neon_end();



			len -= todo;

			src += todo;

lib/crypto/arm/sha1.h

+6 −7
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ 
 *

 * Copyright 2025 Google LLC

 */

#include <asm/neon.h>

#include <asm/simd.h>



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
@@ -22,12 +21,12 @@ static void sha1_blocks(struct sha1_block_state *state, 
{

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&

	    static_branch_likely(&have_neon) && likely(may_use_simd())) {

		kernel_neon_begin();

		scoped_ksimd() {

			if (static_branch_likely(&have_ce))

				sha1_ce_transform(state, data, nblocks);

			else

				sha1_transform_neon(state, data, nblocks);

		kernel_neon_end();

		}

	} else {

		sha1_block_data_order(state, data, nblocks);

	}

lib/crypto/arm/sha256.h

+6 −6
Original line number Diff line number Diff line
@@ -22,12 +22,12 @@ static void sha256_blocks(struct sha256_block_state *state, 
{

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&

	    static_branch_likely(&have_neon) && likely(may_use_simd())) {

		kernel_neon_begin();

		scoped_ksimd() {

			if (static_branch_likely(&have_ce))

				sha256_ce_transform(state, data, nblocks);

			else

				sha256_block_data_order_neon(state, data, nblocks);

		kernel_neon_end();

		}

	} else {

		sha256_block_data_order(state, data, nblocks);

	}