Commit 065f0400 authored by Eric Biggers's avatar Eric Biggers
Browse files

Merge tag 'scoped-ksimd-for-arm-arm64' into libcrypto-fpsimd-on-stack 



Pull scoped ksimd API for ARM and arm64 from Ard Biesheuvel:

  "Introduce a more strict replacement API for
   kernel_neon_begin()/kernel_neon_end() on both ARM and arm64, and
   replace occurrences of the latter pair appearing in lib/crypto"

Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parents 2dbb6f4a f53d18a4

arch/arm/include/asm/simd.h

+7 −0
Original line number Diff line number Diff line
@@ -2,14 +2,21 @@ 
#ifndef _ASM_SIMD_H

#define _ASM_SIMD_H



#include <linux/cleanup.h>

#include <linux/compiler_attributes.h>

#include <linux/preempt.h>

#include <linux/types.h>



#include <asm/neon.h>



static __must_check inline bool may_use_simd(void)

{

	return IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !in_hardirq()

	       && !irqs_disabled();

}



DEFINE_LOCK_GUARD_0(ksimd, kernel_neon_begin(), kernel_neon_end())



#define scoped_ksimd()	scoped_guard(ksimd)



#endif	/* _ASM_SIMD_H */

arch/arm64/include/asm/simd.h

+7 −0
Original line number Diff line number Diff line
@@ -6,12 +6,15 @@ 
#ifndef __ASM_SIMD_H

#define __ASM_SIMD_H



#include <linux/cleanup.h>

#include <linux/compiler.h>

#include <linux/irqflags.h>

#include <linux/percpu.h>

#include <linux/preempt.h>

#include <linux/types.h>



#include <asm/neon.h>



#ifdef CONFIG_KERNEL_MODE_NEON



/*
@@ -40,4 +43,8 @@ static __must_check inline bool may_use_simd(void) { 


#endif /* ! CONFIG_KERNEL_MODE_NEON */



DEFINE_LOCK_GUARD_0(ksimd, kernel_neon_begin(), kernel_neon_end())



#define scoped_ksimd()	scoped_guard(ksimd)



#endif

lib/crypto/arm/chacha.h

+4 −7
Original line number Diff line number Diff line
@@ -12,7 +12,6 @@ 


#include <asm/cputype.h>

#include <asm/hwcap.h>

#include <asm/neon.h>

#include <asm/simd.h>



asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
@@ -68,9 +67,8 @@ static void hchacha_block_arch(const struct chacha_state *state, 
	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {

		hchacha_block_arm(state, out, nrounds);

	} else {

		kernel_neon_begin();

		scoped_ksimd()

			hchacha_block_neon(state, out, nrounds);

		kernel_neon_end();

	}

}
@@ -87,9 +85,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 
	do {

		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);



		kernel_neon_begin();

		scoped_ksimd()

			chacha_doneon(state, dst, src, todo, nrounds);

		kernel_neon_end();



		bytes -= todo;

		src += todo;

lib/crypto/arm/curve25519.h

+2 −3
Original line number Diff line number Diff line
@@ -25,9 +25,8 @@ static void curve25519_arch(u8 out[CURVE25519_KEY_SIZE], 
			    const u8 point[CURVE25519_KEY_SIZE])

{

	if (static_branch_likely(&have_neon) && crypto_simd_usable()) {

		kernel_neon_begin();

		scoped_ksimd()

			curve25519_neon(out, scalar, point);

		kernel_neon_end();

	} else {

		curve25519_generic(out, scalar, point);

	}

lib/crypto/arm/poly1305.h

+2 −4
Original line number Diff line number Diff line
@@ -6,7 +6,6 @@ 
 */



#include <asm/hwcap.h>

#include <asm/neon.h>

#include <asm/simd.h>

#include <linux/cpufeature.h>

#include <linux/jump_label.h>
@@ -32,9 +31,8 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, 
		do {

			unsigned int todo = min_t(unsigned int, len, SZ_4K);



			kernel_neon_begin();

			scoped_ksimd()

				poly1305_blocks_neon(state, src, todo, padbit);

			kernel_neon_end();



			len -= todo;

			src += todo;