Commit 187d0801 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'libcrypto-fixes-for-linus' of... 

Merge tag 'libcrypto-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library fixes from Eric Biggers:
 "Fixes for some recent regressions as well as some longstanding issues:

   - Fix incorrect output from the arm64 NEON implementation of GHASH

   - Merge the ksimd scopes in the arm64 XTS code to reduce stack usage

   - Roll up the BLAKE2b round loop on 32-bit kernels to greatly reduce
     code size and stack usage

   - Add missing RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS dependency

   - Fix chacha-riscv64-zvkb.S to not use frame pointer for data"

* tag 'libcrypto-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux:
  crypto: arm64/ghash - Fix incorrect output from ghash-neon
  crypto/arm64: sm4/xts - Merge ksimd scopes to reduce stack bloat
  crypto/arm64: aes/xts - Use single ksimd scope to reduce stack bloat
  lib/crypto: blake2s: Replace manual unrolling with unrolled_full
  lib/crypto: blake2b: Roll up BLAKE2b round loop on 32-bit
  lib/crypto: riscv: Depend on RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
  lib/crypto: riscv/chacha: Avoid s0/fp register
parents 35ebee7e f6a45874

arch/arm64/crypto/aes-glue.c

+36 −39
Original line number Diff line number Diff line
@@ -549,13 +549,13 @@ static int __maybe_unused xts_encrypt(struct skcipher_request *req) 
		tail = 0;

	}



	scoped_ksimd() {

		for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {

			int nbytes = walk.nbytes;



			if (walk.nbytes < walk.total)

				nbytes &= ~(AES_BLOCK_SIZE - 1);



		scoped_ksimd()

			aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,

					ctx->key1.key_enc, rounds, nbytes,

					ctx->key2.key_enc, walk.iv, first);
@@ -576,11 +576,10 @@ static int __maybe_unused xts_encrypt(struct skcipher_request *req) 
		if (err)

			return err;



	scoped_ksimd()

		aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,

				ctx->key1.key_enc, rounds, walk.nbytes,

				ctx->key2.key_enc, walk.iv, first);



	}

	return skcipher_walk_done(&walk, 0);

}
@@ -619,13 +618,13 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) 
		tail = 0;

	}



	scoped_ksimd() {

		for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {

			int nbytes = walk.nbytes;



			if (walk.nbytes < walk.total)

				nbytes &= ~(AES_BLOCK_SIZE - 1);



		scoped_ksimd()

			aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,

					ctx->key1.key_dec, rounds, nbytes,

					ctx->key2.key_enc, walk.iv, first);
@@ -646,12 +645,10 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) 
		if (err)

			return err;





	scoped_ksimd()

		aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,

				ctx->key1.key_dec, rounds, walk.nbytes,

				ctx->key2.key_enc, walk.iv, first);



	}

	return skcipher_walk_done(&walk, 0);

}

arch/arm64/crypto/aes-neonbs-glue.c

+21 −23
Original line number Diff line number Diff line
@@ -312,13 +312,13 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, 
	if (err)

		return err;



	scoped_ksimd() {

		while (walk.nbytes >= AES_BLOCK_SIZE) {

			int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;

			out = walk.dst.virt.addr;

			in = walk.src.virt.addr;

			nbytes = walk.nbytes;



		scoped_ksimd() {

			if (blocks >= 8) {

				if (first == 1)

					neon_aes_ecb_encrypt(walk.iv, walk.iv,
@@ -344,7 +344,6 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, 
							     ctx->twkey, walk.iv, first);

				nbytes = first = 0;

			}

		}

			err = skcipher_walk_done(&walk, nbytes);

		}
@@ -367,7 +366,6 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, 
		in = walk.src.virt.addr;

		nbytes = walk.nbytes;



	scoped_ksimd() {

		if (encrypt)

			neon_aes_xts_encrypt(out, in, ctx->cts.key_enc,

					     ctx->key.rounds, nbytes, ctx->twkey,

arch/arm64/crypto/ghash-ce-glue.c

+1 −1
Original line number Diff line number Diff line
@@ -133,7 +133,7 @@ static int ghash_finup(struct shash_desc *desc, const u8 *src, 
		u8 buf[GHASH_BLOCK_SIZE] = {};



		memcpy(buf, src, len);

		ghash_do_simd_update(1, ctx->digest, src, key, NULL,

		ghash_do_simd_update(1, ctx->digest, buf, key, NULL,

				     pmull_ghash_update_p8);

		memzero_explicit(buf, sizeof(buf));

	}

arch/arm64/crypto/sm4-ce-glue.c

+20 −22
Original line number Diff line number Diff line
@@ -346,11 +346,11 @@ static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt) 
		tail = 0;

	}



	scoped_ksimd() {

		while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) {

			if (nbytes < walk.total)

				nbytes &= ~(SM4_BLOCK_SIZE - 1);



		scoped_ksimd() {

			if (encrypt)

				sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,

						walk.src.virt.addr, walk.iv, nbytes,
@@ -359,7 +359,6 @@ static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt) 
				sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,

						walk.src.virt.addr, walk.iv, nbytes,

						rkey2_enc);

		}



			rkey2_enc = NULL;
@@ -377,14 +376,13 @@ static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt) 
		if (req->dst != req->src)

			dst = scatterwalk_ffwd(sg_dst, req->dst, subreq.cryptlen);



	skcipher_request_set_crypt(&subreq, src, dst, SM4_BLOCK_SIZE + tail,

				   req->iv);

		skcipher_request_set_crypt(&subreq, src, dst,

					   SM4_BLOCK_SIZE + tail, req->iv);



		err = skcipher_walk_virt(&walk, &subreq, false);

		if (err)

			return err;



	scoped_ksimd() {

		if (encrypt)

			sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,

					walk.src.virt.addr, walk.iv, walk.nbytes,

arch/riscv/crypto/Kconfig

+8 −4
Original line number Diff line number Diff line
@@ -4,7 +4,8 @@ menu "Accelerated Cryptographic Algorithms for CPU (riscv)" 


config CRYPTO_AES_RISCV64

	tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS"

	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO

	depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \

		   RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS

	select CRYPTO_ALGAPI

	select CRYPTO_LIB_AES

	select CRYPTO_SKCIPHER
@@ -20,7 +21,8 @@ config CRYPTO_AES_RISCV64 


config CRYPTO_GHASH_RISCV64

	tristate "Hash functions: GHASH"

	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO

	depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \

		   RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS

	select CRYPTO_GCM

	help

	  GCM GHASH function (NIST SP 800-38D)
@@ -30,7 +32,8 @@ config CRYPTO_GHASH_RISCV64 


config CRYPTO_SM3_RISCV64

	tristate "Hash functions: SM3 (ShangMi 3)"

	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO

	depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \

		   RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS

	select CRYPTO_HASH

	select CRYPTO_LIB_SM3

	help
@@ -42,7 +45,8 @@ config CRYPTO_SM3_RISCV64 


config CRYPTO_SM4_RISCV64

	tristate "Ciphers: SM4 (ShangMi 4)"

	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO

	depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \

		   RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS

	select CRYPTO_ALGAPI

	select CRYPTO_SM4

	help