Commit 3be3f70e authored by Herbert Xu's avatar Herbert Xu
Browse files

crypto: x86/chacha - Restore SSSE3 fallback path 



The chacha_use_simd static branch is required for x86 machines that
lack SSSE3 support.  Restore it and the generic fallback code.

Reported-by: default avatarEric Biggers <ebiggers@kernel.org>
Fixes: 9b440021 ("crypto: x86/chacha - Remove SIMD fallback path")
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 53225843

arch/x86/crypto/chacha_glue.c

+17 −5
Original line number Diff line number Diff line
@@ -5,11 +5,12 @@ 
 * Copyright (C) 2015 Martin Willi

 */



#include <asm/simd.h>

#include <crypto/chacha.h>

#include <linux/jump_label.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/sizes.h>

#include <asm/simd.h>



asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,

				       unsigned int len, int nrounds);
@@ -31,6 +32,7 @@ asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src, 
asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,

					   unsigned int len, int nrounds);



static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl);
@@ -117,15 +119,23 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, 


void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)

{

	if (!static_branch_likely(&chacha_use_simd)) {

		hchacha_block_generic(state, stream, nrounds);

	} else {

		kernel_fpu_begin();

		hchacha_block_ssse3(state, stream, nrounds);

		kernel_fpu_end();

	}

}

EXPORT_SYMBOL(hchacha_block_arch);



void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,

		       int nrounds)

{

	if (!static_branch_likely(&chacha_use_simd) ||

	    bytes <= CHACHA_BLOCK_SIZE)

		return chacha_crypt_generic(state, dst, src, bytes, nrounds);



	do {

		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
@@ -142,7 +152,7 @@ EXPORT_SYMBOL(chacha_crypt_arch); 


bool chacha_is_arch_optimized(void)

{

	return true;

	return static_key_enabled(&chacha_use_simd);

}

EXPORT_SYMBOL(chacha_is_arch_optimized);
@@ -151,6 +161,8 @@ static int __init chacha_simd_mod_init(void) 
	if (!boot_cpu_has(X86_FEATURE_SSSE3))

		return 0;



	static_branch_enable(&chacha_use_simd);



	if (boot_cpu_has(X86_FEATURE_AVX) &&

	    boot_cpu_has(X86_FEATURE_AVX2) &&

	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {