Commit 4fb62307 authored by Ard Biesheuvel's avatar Ard Biesheuvel
Browse files

lib/crc: Switch ARM and arm64 to 'ksimd' scoped guard API 



Before modifying the prototypes of kernel_neon_begin() and
kernel_neon_end() to accommodate kernel mode FP/SIMD state buffers
allocated on the stack, move arm64 to the new 'ksimd' scoped guard API,
which encapsulates the calls to those functions.

For symmetry, do the same for 32-bit ARM too.

Reviewed-by: default avatarEric Biggers <ebiggers@kernel.org>
Reviewed-by: default avatarJonathan Cameron <jonathan.cameron@huawei.com>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
parent f53d18a4

lib/crc/arm/crc-t10dif.h

+6 −13
Original line number Diff line number Diff line
@@ -5,7 +5,6 @@ 
 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>

 */



#include <asm/neon.h>

#include <asm/simd.h>



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
@@ -19,22 +18,16 @@ asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 


static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)

{

	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {

	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && likely(may_use_simd())) {

		if (static_branch_likely(&have_pmull)) {

			if (likely(may_use_simd())) {

				kernel_neon_begin();

				crc = crc_t10dif_pmull64(crc, data, length);

				kernel_neon_end();

				return crc;

			}

			scoped_ksimd()

				return crc_t10dif_pmull64(crc, data, length);

		} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&

			   static_branch_likely(&have_neon) &&

			   likely(may_use_simd())) {

			   static_branch_likely(&have_neon)) {

			u8 buf[16] __aligned(16);



			kernel_neon_begin();

			scoped_ksimd()

				crc_t10dif_pmull8(crc, data, length, buf);

			kernel_neon_end();



			return crc_t10dif_generic(0, buf, sizeof(buf));

		}

lib/crc/arm/crc32.h

+4 −7
Original line number Diff line number Diff line
@@ -8,7 +8,6 @@ 
#include <linux/cpufeature.h>



#include <asm/hwcap.h>

#include <asm/neon.h>

#include <asm/simd.h>



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
@@ -42,9 +41,8 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 
			len -= n;

		}

		n = round_down(len, 16);

		kernel_neon_begin();

		scoped_ksimd()

			crc = crc32_pmull_le(p, n, crc);

		kernel_neon_end();

		p += n;

		len -= n;

	}
@@ -71,9 +69,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 
			len -= n;

		}

		n = round_down(len, 16);

		kernel_neon_begin();

		scoped_ksimd()

			crc = crc32c_pmull_le(p, n, crc);

		kernel_neon_end();

		p += n;

		len -= n;

	}

lib/crc/arm64/crc-t10dif.h

+6 −13
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ 


#include <linux/cpufeature.h>



#include <asm/neon.h>

#include <asm/simd.h>



static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
@@ -21,22 +20,16 @@ asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 


static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)

{

	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {

	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && likely(may_use_simd())) {

		if (static_branch_likely(&have_pmull)) {

			if (likely(may_use_simd())) {

				kernel_neon_begin();

				crc = crc_t10dif_pmull_p64(crc, data, length);

				kernel_neon_end();

				return crc;

			}

			scoped_ksimd()

				return crc_t10dif_pmull_p64(crc, data, length);

		} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&

			   static_branch_likely(&have_asimd) &&

			   likely(may_use_simd())) {

			   static_branch_likely(&have_asimd)) {

			u8 buf[16];



			kernel_neon_begin();

			scoped_ksimd()

				crc_t10dif_pmull_p8(crc, data, length, buf);

			kernel_neon_end();



			return crc_t10dif_generic(0, buf, sizeof(buf));

		}

lib/crc/arm64/crc32.h

+6 −10
Original line number Diff line number Diff line
@@ -2,7 +2,6 @@ 


#include <asm/alternative.h>

#include <asm/cpufeature.h>

#include <asm/neon.h>

#include <asm/simd.h>



// The minimum input length to consider the 4-way interleaved code path
@@ -23,9 +22,8 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 


	if (len >= min_len && cpu_have_named_feature(PMULL) &&

	    likely(may_use_simd())) {

		kernel_neon_begin();

		scoped_ksimd()

			crc = crc32_le_arm64_4way(crc, p, len);

		kernel_neon_end();



		p += round_down(len, 64);

		len %= 64;
@@ -44,9 +42,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 


	if (len >= min_len && cpu_have_named_feature(PMULL) &&

	    likely(may_use_simd())) {

		kernel_neon_begin();

		scoped_ksimd()

			crc = crc32c_le_arm64_4way(crc, p, len);

		kernel_neon_end();



		p += round_down(len, 64);

		len %= 64;
@@ -65,9 +62,8 @@ static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 


	if (len >= min_len && cpu_have_named_feature(PMULL) &&

	    likely(may_use_simd())) {

		kernel_neon_begin();

		scoped_ksimd()

			crc = crc32_be_arm64_4way(crc, p, len);

		kernel_neon_end();



		p += round_down(len, 64);

		len %= 64;