Unverified Commit 67daf842 authored by Palmer Dabbelt's avatar Palmer Dabbelt
Browse files

Merge patch series "RISC-V crypto with reworked asm files"

Eric Biggers <ebiggers@kernel.org> says:

This patchset, which applies to v6.8-rc1, adds cryptographic algorithm
implementations accelerated using the RISC-V vector crypto extensions
(https://github.com/riscv/riscv-crypto/releases/download/v1.0.0/riscv-crypto-spec-vector.pdf)
and RISC-V vector extension
(https://github.com/riscv/riscv-v-spec/releases/download/v1.0/riscv-v-spec-1.0.pdf).
The following algorithms are included: AES in ECB, CBC, CTR, and XTS modes;
ChaCha20; GHASH; SHA-2; SM3; and SM4.

In general, the assembly code requires a 64-bit RISC-V CPU with VLEN >= 128,
little endian byte order, and vector unaligned access support.  The ECB, CTR,
XTS, and ChaCha20 code is designed to naturally scale up to larger VLEN values.
Building the assembly code requires tip-of-tree binutils (future 2.42) or
tip-of-tree clang (future 18.x).  All algorithms pass testing in QEMU, using
CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y.  Much of the assembly code is derived from
OpenSSL code that was added by https://github.com/openssl/openssl/pull/21923.
It's been cleaned up for integration with the kernel, e.g. reducing code
duplication, eliminating use of .inst and perlasm, and fixing a few bugs.

This patchset incorporates the work of multiple people, including Jerry Shih,
Heiko Stuebner, Christoph Müllner, Phoebe Chen, Charalampos Mitrodimas, and
myself.  This patchset went through several versions from Heiko (last version
https://lore.kernel.org/linux-crypto/20230711153743.1970625-1-heiko@sntech.de),
then several versions from Jerry (last version:
https://lore.kernel.org/linux-crypto/20231231152743.6304-1-jerry.shih@sifive.com),
then finally several versions from me.  Thanks to everyone who has contributed
to this patchset or its prerequisites.

* b4-shazam-merge:
  crypto: riscv - add vector crypto accelerated SM4
  crypto: riscv - add vector crypto accelerated SM3
  crypto: riscv - add vector crypto accelerated SHA-{512,384}
  crypto: riscv - add vector crypto accelerated SHA-{256,224}
  crypto: riscv - add vector crypto accelerated GHASH
  crypto: riscv - add vector crypto accelerated ChaCha20
  crypto: riscv - add vector crypto accelerated AES-{ECB,CBC,CTR,XTS}
  RISC-V: hook new crypto subdir into build-system
  RISC-V: add TOOLCHAIN_HAS_VECTOR_CRYPTO
  RISC-V: add helper function to read the vector VLEN

Link: https://lore.kernel.org/r/20240122002024.27477-1-ebiggers@kernel.org


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parents 021d2342 b8d06352
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@

obj-y += kernel/ mm/ net/
obj-$(CONFIG_BUILTIN_DTB) += boot/dts/
obj-$(CONFIG_CRYPTO) += crypto/
obj-y += errata/
obj-$(CONFIG_KVM) += kvm/

+7 −0
Original line number Diff line number Diff line
@@ -581,6 +581,13 @@ config TOOLCHAIN_HAS_ZBB
	depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900
	depends on AS_HAS_OPTION_ARCH

# This symbol indicates that the toolchain supports all v1.0 vector crypto
# extensions, including Zvk*, Zvbb, and Zvbc.  LLVM added all of these at once.
# binutils added all except Zvkb, then added Zvkb.  So we just check for Zvkb.
config TOOLCHAIN_HAS_VECTOR_CRYPTO
	def_bool $(as-instr, .option arch$(comma) +zvkb)
	depends on AS_HAS_OPTION_ARCH

config RISCV_ISA_ZBB
	bool "Zbb extension support for bit manipulation instructions"
	depends on TOOLCHAIN_HAS_ZBB
+93 −0
Original line number Diff line number Diff line
# SPDX-License-Identifier: GPL-2.0

menu "Accelerated Cryptographic Algorithms for CPU (riscv)"

config CRYPTO_AES_RISCV64
	tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_ALGAPI
	select CRYPTO_LIB_AES
	select CRYPTO_SKCIPHER
	help
	  Block cipher: AES cipher algorithms
	  Length-preserving ciphers: AES with ECB, CBC, CTR, XTS

	  Architecture: riscv64 using:
	  - Zvkned vector crypto extension
	  - Zvbb vector extension (XTS)
	  - Zvkb vector crypto extension (CTR)
	  - Zvkg vector crypto extension (XTS)

config CRYPTO_CHACHA_RISCV64
	tristate "Ciphers: ChaCha"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_SKCIPHER
	select CRYPTO_LIB_CHACHA_GENERIC
	help
	  Length-preserving ciphers: ChaCha20 stream cipher algorithm

	  Architecture: riscv64 using:
	  - Zvkb vector crypto extension

config CRYPTO_GHASH_RISCV64
	tristate "Hash functions: GHASH"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_GCM
	help
	  GCM GHASH function (NIST SP 800-38D)

	  Architecture: riscv64 using:
	  - Zvkg vector crypto extension

config CRYPTO_SHA256_RISCV64
	tristate "Hash functions: SHA-224 and SHA-256"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_SHA256
	help
	  SHA-224 and SHA-256 secure hash algorithm (FIPS 180)

	  Architecture: riscv64 using:
	  - Zvknha or Zvknhb vector crypto extensions
	  - Zvkb vector crypto extension

config CRYPTO_SHA512_RISCV64
	tristate "Hash functions: SHA-384 and SHA-512"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_SHA512
	help
	  SHA-384 and SHA-512 secure hash algorithm (FIPS 180)

	  Architecture: riscv64 using:
	  - Zvknhb vector crypto extension
	  - Zvkb vector crypto extension

config CRYPTO_SM3_RISCV64
	tristate "Hash functions: SM3 (ShangMi 3)"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_HASH
	select CRYPTO_SM3
	help
	  SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)

	  Architecture: riscv64 using:
	  - Zvksh vector crypto extension
	  - Zvkb vector crypto extension

config CRYPTO_SM4_RISCV64
	tristate "Ciphers: SM4 (ShangMi 4)"
	depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
	select CRYPTO_ALGAPI
	select CRYPTO_SM4
	help
	  SM4 block cipher algorithm (OSCCA GB/T 32907-2016,
	  ISO/IEC 18033-3:2010/Amd 1:2021)

	  SM4 (GBT.32907-2016) is a cryptographic standard issued by the
	  Organization of State Commercial Administration of China (OSCCA)
	  as an authorized cryptographic algorithm for use within China.

	  Architecture: riscv64 using:
	  - Zvksed vector crypto extension
	  - Zvkb vector crypto extension

endmenu
+23 −0
Original line number Diff line number Diff line
# SPDX-License-Identifier: GPL-2.0-only

obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
		 aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o

obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o
chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o

obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o

obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o
sha256-riscv64-y := sha256-riscv64-glue.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o

obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o
sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o

obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o

obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o
+156 −0
Original line number Diff line number Diff line
/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
//
// This file is dual-licensed, meaning that you can use it under your
// choice of either of the following two licenses:
//
// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
//
// Licensed under the Apache License 2.0 (the "License"). You can obtain
// a copy in the file LICENSE in the source distribution or at
// https://www.openssl.org/source/license.html
//
// or
//
// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com>
// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
// Copyright 2024 Google LLC
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// This file contains macros that are shared by the other aes-*.S files.  The
// generated code of these macros depends on the following RISC-V extensions:
// - RV64I
// - RISC-V Vector ('V') with VLEN >= 128
// - RISC-V Vector AES block cipher extension ('Zvkned')

// Loads the AES round keys from \keyp into vector registers and jumps to code
// specific to the length of the key.  Specifically:
//   - If AES-128, loads round keys into v1-v11 and jumps to \label128.
//   - If AES-192, loads round keys into v1-v13 and jumps to \label192.
//   - If AES-256, loads round keys into v1-v15 and continues onwards.
//
// Also sets vl=4 and vtype=e32,m1,ta,ma.  Clobbers t0 and t1.
.macro	aes_begin	keyp, label128, label192
	lwu		t0, 480(\keyp)	// t0 = key length in bytes
	li		t1, 24		// t1 = key length for AES-192
	vsetivli	zero, 4, e32, m1, ta, ma
	vle32.v		v1, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v2, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v3, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v4, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v5, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v6, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v7, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v8, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v9, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v10, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v11, (\keyp)
	blt		t0, t1, \label128	// If AES-128, goto label128.
	addi		\keyp, \keyp, 16
	vle32.v		v12, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v13, (\keyp)
	beq		t0, t1, \label192	// If AES-192, goto label192.
	// Else, it's AES-256.
	addi		\keyp, \keyp, 16
	vle32.v		v14, (\keyp)
	addi		\keyp, \keyp, 16
	vle32.v		v15, (\keyp)
.endm

// Encrypts \data using zvkned instructions, using the round keys loaded into
// v1-v11 (for AES-128), v1-v13 (for AES-192), or v1-v15 (for AES-256).  \keylen
// is the AES key length in bits.  vl and vtype must already be set
// appropriately.  Note that if vl > 4, multiple blocks are encrypted.
.macro	aes_encrypt	data, keylen
	vaesz.vs	\data, v1
	vaesem.vs	\data, v2
	vaesem.vs	\data, v3
	vaesem.vs	\data, v4
	vaesem.vs	\data, v5
	vaesem.vs	\data, v6
	vaesem.vs	\data, v7
	vaesem.vs	\data, v8
	vaesem.vs	\data, v9
	vaesem.vs	\data, v10
.if \keylen == 128
	vaesef.vs	\data, v11
.elseif \keylen == 192
	vaesem.vs	\data, v11
	vaesem.vs	\data, v12
	vaesef.vs	\data, v13
.else
	vaesem.vs	\data, v11
	vaesem.vs	\data, v12
	vaesem.vs	\data, v13
	vaesem.vs	\data, v14
	vaesef.vs	\data, v15
.endif
.endm

// Same as aes_encrypt, but decrypts instead of encrypts.
.macro	aes_decrypt	data, keylen
.if \keylen == 128
	vaesz.vs	\data, v11
.elseif \keylen == 192
	vaesz.vs	\data, v13
	vaesdm.vs	\data, v12
	vaesdm.vs	\data, v11
.else
	vaesz.vs	\data, v15
	vaesdm.vs	\data, v14
	vaesdm.vs	\data, v13
	vaesdm.vs	\data, v12
	vaesdm.vs	\data, v11
.endif
	vaesdm.vs	\data, v10
	vaesdm.vs	\data, v9
	vaesdm.vs	\data, v8
	vaesdm.vs	\data, v7
	vaesdm.vs	\data, v6
	vaesdm.vs	\data, v5
	vaesdm.vs	\data, v4
	vaesdm.vs	\data, v3
	vaesdm.vs	\data, v2
	vaesdf.vs	\data, v1
.endm

// Expands to aes_encrypt or aes_decrypt according to \enc, which is 1 or 0.
.macro	aes_crypt	data, enc, keylen
.if \enc
	aes_encrypt	\data, \keylen
.else
	aes_decrypt	\data, \keylen
.endif
.endm
Loading