Commit bef9c755 authored by Zhihang Shao's avatar Zhihang Shao Committed by Eric Biggers
Browse files

lib/crypto: riscv/poly1305: Import OpenSSL/CRYPTOGAMS implementation 

This is a straight import of the OpenSSL/CRYPTOGAMS Poly1305
implementation for riscv authored by Andy Polyakov.  The file
'poly1305-riscv.pl' is taken straight from
https://github.com/dot-asm/cryptogams

 commit
5e3fba73576244708a752fa61a8e93e587f271bb.  This patch was tested on
SpacemiT X60, with 2~2.5x improvement over generic implementation.

Signed-off-by: default avatarChunyan Zhang <zhangchunyan@iscas.ac.cn>
Signed-off-by: default avatarZhihang Shao <zhihang.shao.iscas@gmail.com>
[EB: ported to lib/crypto/riscv/]
Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250829152513.92459-4-ebiggers@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent b646b782

lib/crypto/Kconfig

+2 −1
Original line number Diff line number Diff line
@@ -128,6 +128,7 @@ config CRYPTO_LIB_POLY1305_ARCH 
	default y if MIPS

	# The PPC64 code needs to be fixed to work in softirq context.

	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN

	default y if RISCV

	default y if X86_64



# This symbol controls the inclusion of the Poly1305 generic code.  This differs
@@ -143,7 +144,7 @@ config CRYPTO_LIB_POLY1305_GENERIC 


config CRYPTO_LIB_POLY1305_RSIZE

	int

	default 2 if MIPS

	default 2 if MIPS || RISCV

	default 11 if X86_64

	default 9 if ARM || ARM64

	default 1

lib/crypto/Makefile

+14 −0
Original line number Diff line number Diff line
@@ -112,6 +112,19 @@ endif 


libpoly1305-$(CONFIG_PPC) += powerpc/poly1305-p10le_64.o



ifeq ($(CONFIG_RISCV),y)

libpoly1305-y += riscv/poly1305-core.o

poly1305-perlasm-flavour-$(CONFIG_32BIT) := 32

poly1305-perlasm-flavour-$(CONFIG_64BIT) := 64

quiet_cmd_perlasm_poly1305 = PERLASM $@

      cmd_perlasm_poly1305 = $(PERL) $< $(poly1305-perlasm-flavour-y) $@

# Use if_changed instead of cmd, in case the flavour changed.

$(obj)/riscv/poly1305-core.S: $(src)/riscv/poly1305-riscv.pl FORCE

	$(call if_changed,perlasm_poly1305)

targets += riscv/poly1305-core.S

AFLAGS_riscv/poly1305-core.o += -Dpoly1305_init=poly1305_block_init

endif



ifeq ($(CONFIG_X86),y)

libpoly1305-y += x86/poly1305-x86_64-cryptogams.o

$(obj)/x86/poly1305-x86_64-cryptogams.S: $(src)/x86/poly1305-x86_64-cryptogams.pl
@@ -124,6 +137,7 @@ endif # CONFIG_CRYPTO_LIB_POLY1305_ARCH 
clean-files += arm/poly1305-core.S \

	       arm64/poly1305-core.S \

	       mips/poly1305-core.S \

	       riscv/poly1305-core.S \

	       x86/poly1305-x86_64-cryptogams.S



################################################################################

lib/crypto/riscv/poly1305-riscv.pl

0 → 100644
+847 −0
Original line number Diff line number Diff line 
#!/usr/bin/env perl

# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause

#

# ====================================================================

# Written by Andy Polyakov, @dot-asm, initially for use with OpenSSL.

# ====================================================================

#

# Poly1305 hash for RISC-V.

#

# February 2019

#

# In the essence it's pretty straightforward transliteration of MIPS

# module [without big-endian option].

#

# 1.8 cycles per byte on U74, >100% faster than compiler-generated

# code. 1.9 cpb on C910, ~75% improvement. 3.3 on Spacemit X60, ~69%

# improvement.

#

# June 2024.

#

# Add CHERI support.

#

######################################################################

#

($zero,$ra,$sp,$gp,$tp)=map("x$_",(0..4));

($t0,$t1,$t2,$t3,$t4,$t5,$t6)=map("x$_",(5..7,28..31));

($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("x$_",(10..17));

($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("x$_",(8,9,18..27));

#

######################################################################



$flavour = shift || "64";



for (@ARGV) {   $output=$_ if (/\w[\w\-]*\.\w+$/);   }

open STDOUT,">$output";



$code.=<<___;

#ifdef __KERNEL__

# ifdef __riscv_zicfilp

#  undef __riscv_zicfilp // calls are expected to be direct

# endif

#endif



#if defined(__CHERI_PURE_CAPABILITY__) && !defined(__riscv_misaligned_fast)

# define __riscv_misaligned_fast 1

#endif

___



if ($flavour =~ /64/) {{{

######################################################################

# 64-bit code path...

#

my ($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3);

my ($in0,$in1,$tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = ($a4,$a5,$a6,$a7,$t0,$t1,$t2);



$code.=<<___;

#if __riscv_xlen == 64

# if __SIZEOF_POINTER__ == 16

#  define PUSH	csc

#  define POP	clc

# else

#  define PUSH	sd

#  define POP	ld

# endif

#else

# error "unsupported __riscv_xlen"

#endif



.option	pic

.text



.globl	poly1305_init

.type	poly1305_init,\@function

poly1305_init:

#ifdef	__riscv_zicfilp

	lpad	0

#endif

	sd	$zero,0($ctx)

	sd	$zero,8($ctx)

	sd	$zero,16($ctx)



	beqz	$inp,.Lno_key



#ifndef	__riscv_misaligned_fast

	andi	$tmp0,$inp,7		# $inp % 8

	andi	$inp,$inp,-8		# align $inp

	slli	$tmp0,$tmp0,3		# byte to bit offset

#endif

	ld	$in0,0($inp)

	ld	$in1,8($inp)

#ifndef	__riscv_misaligned_fast

	beqz	$tmp0,.Laligned_key



	ld	$tmp2,16($inp)

	neg	$tmp1,$tmp0		# implicit &63 in sll

	srl	$in0,$in0,$tmp0

	sll	$tmp3,$in1,$tmp1

	srl	$in1,$in1,$tmp0

	sll	$tmp2,$tmp2,$tmp1

	or	$in0,$in0,$tmp3

	or	$in1,$in1,$tmp2



.Laligned_key:

#endif

	li	$tmp0,1

	slli	$tmp0,$tmp0,32		# 0x0000000100000000

	addi	$tmp0,$tmp0,-63		# 0x00000000ffffffc1

	slli	$tmp0,$tmp0,28		# 0x0ffffffc10000000

	addi	$tmp0,$tmp0,-1		# 0x0ffffffc0fffffff



	and	$in0,$in0,$tmp0

	addi	$tmp0,$tmp0,-3		# 0x0ffffffc0ffffffc

	and	$in1,$in1,$tmp0



	sd	$in0,24($ctx)

	srli	$tmp0,$in1,2

	sd	$in1,32($ctx)

	add	$tmp0,$tmp0,$in1	# s1 = r1 + (r1 >> 2)

	sd	$tmp0,40($ctx)



.Lno_key:

	li	$a0,0			# return 0

	ret

.size	poly1305_init,.-poly1305_init

___

{

my ($h0,$h1,$h2,$r0,$r1,$rs1,$d0,$d1,$d2) =

   ($s0,$s1,$s2,$s3,$t3,$t4,$in0,$in1,$t2);

my ($shr,$shl) = ($t5,$t6);		# used on R6



$code.=<<___;

.globl	poly1305_blocks

.type	poly1305_blocks,\@function

poly1305_blocks:

#ifdef	__riscv_zicfilp

	lpad	0

#endif

	andi	$len,$len,-16		# complete blocks only

	beqz	$len,.Lno_data



	caddi	$sp,$sp,-4*__SIZEOF_POINTER__

	PUSH	$s0,3*__SIZEOF_POINTER__($sp)

	PUSH	$s1,2*__SIZEOF_POINTER__($sp)

	PUSH	$s2,1*__SIZEOF_POINTER__($sp)

	PUSH	$s3,0*__SIZEOF_POINTER__($sp)



#ifndef	__riscv_misaligned_fast

	andi	$shr,$inp,7

	andi	$inp,$inp,-8		# align $inp

	slli	$shr,$shr,3		# byte to bit offset

	neg	$shl,$shr		# implicit &63 in sll

#endif



	ld	$h0,0($ctx)		# load hash value

	ld	$h1,8($ctx)

	ld	$h2,16($ctx)



	ld	$r0,24($ctx)		# load key

	ld	$r1,32($ctx)

	ld	$rs1,40($ctx)



	add	$len,$len,$inp		# end of buffer



.Loop:

	ld	$in0,0($inp)		# load input

	ld	$in1,8($inp)

#ifndef	__riscv_misaligned_fast

	beqz	$shr,.Laligned_inp



	ld	$tmp2,16($inp)

	srl	$in0,$in0,$shr

	sll	$tmp3,$in1,$shl

	srl	$in1,$in1,$shr

	sll	$tmp2,$tmp2,$shl

	or	$in0,$in0,$tmp3

	or	$in1,$in1,$tmp2



.Laligned_inp:

#endif

	caddi	$inp,$inp,16



	andi	$tmp0,$h2,-4		# modulo-scheduled reduction

	srli	$tmp1,$h2,2

	andi	$h2,$h2,3



	add	$d0,$h0,$in0		# accumulate input

	 add	$tmp1,$tmp1,$tmp0

	sltu	$tmp0,$d0,$h0

	add	$d0,$d0,$tmp1		# ... and residue

	sltu	$tmp1,$d0,$tmp1

	add	$d1,$h1,$in1

	add	$tmp0,$tmp0,$tmp1

	sltu	$tmp1,$d1,$h1

	add	$d1,$d1,$tmp0



	 add	$d2,$h2,$padbit

	 sltu	$tmp0,$d1,$tmp0

	mulhu	$h1,$r0,$d0		# h0*r0

	mul	$h0,$r0,$d0



	 add	$d2,$d2,$tmp1

	 add	$d2,$d2,$tmp0

	mulhu	$tmp1,$rs1,$d1		# h1*5*r1

	mul	$tmp0,$rs1,$d1



	mulhu	$h2,$r1,$d0		# h0*r1

	mul	$tmp2,$r1,$d0

	 add	$h0,$h0,$tmp0

	 add	$h1,$h1,$tmp1

	 sltu	$tmp0,$h0,$tmp0



	 add	$h1,$h1,$tmp0

	 add	$h1,$h1,$tmp2

	mulhu	$tmp1,$r0,$d1		# h1*r0

	mul	$tmp0,$r0,$d1



	 sltu	$tmp2,$h1,$tmp2

	 add	$h2,$h2,$tmp2

	mul	$tmp2,$rs1,$d2		# h2*5*r1



	 add	$h1,$h1,$tmp0

	 add	$h2,$h2,$tmp1

	mul	$tmp3,$r0,$d2		# h2*r0

	 sltu	$tmp0,$h1,$tmp0

	 add	$h2,$h2,$tmp0



	add	$h1,$h1,$tmp2

	sltu	$tmp2,$h1,$tmp2

	add	$h2,$h2,$tmp2

	add	$h2,$h2,$tmp3



	bne	$inp,$len,.Loop



	sd	$h0,0($ctx)		# store hash value

	sd	$h1,8($ctx)

	sd	$h2,16($ctx)



	POP	$s0,3*__SIZEOF_POINTER__($sp)		# epilogue

	POP	$s1,2*__SIZEOF_POINTER__($sp)

	POP	$s2,1*__SIZEOF_POINTER__($sp)

	POP	$s3,0*__SIZEOF_POINTER__($sp)

	caddi	$sp,$sp,4*__SIZEOF_POINTER__



.Lno_data:

	ret

.size	poly1305_blocks,.-poly1305_blocks

___

}

{

my ($ctx,$mac,$nonce) = ($a0,$a1,$a2);



$code.=<<___;

.globl	poly1305_emit

.type	poly1305_emit,\@function

poly1305_emit:

#ifdef	__riscv_zicfilp

	lpad	0

#endif

	ld	$tmp2,16($ctx)

	ld	$tmp0,0($ctx)

	ld	$tmp1,8($ctx)



	andi	$in0,$tmp2,-4		# final reduction

	srl	$in1,$tmp2,2

	andi	$tmp2,$tmp2,3

	add	$in0,$in0,$in1



	add	$tmp0,$tmp0,$in0

	sltu	$in1,$tmp0,$in0

	 addi	$in0,$tmp0,5		# compare to modulus

	add	$tmp1,$tmp1,$in1

	 sltiu	$tmp3,$in0,5

	sltu	$tmp4,$tmp1,$in1

	 add	$in1,$tmp1,$tmp3

	add	$tmp2,$tmp2,$tmp4

	 sltu	$tmp3,$in1,$tmp3

	 add	$tmp2,$tmp2,$tmp3



	srli	$tmp2,$tmp2,2		# see if it carried/borrowed

	neg	$tmp2,$tmp2



	xor	$in0,$in0,$tmp0

	xor	$in1,$in1,$tmp1

	and	$in0,$in0,$tmp2

	and	$in1,$in1,$tmp2

	xor	$in0,$in0,$tmp0

	xor	$in1,$in1,$tmp1



	lwu	$tmp0,0($nonce)		# load nonce

	lwu	$tmp1,4($nonce)

	lwu	$tmp2,8($nonce)

	lwu	$tmp3,12($nonce)

	slli	$tmp1,$tmp1,32

	slli	$tmp3,$tmp3,32

	or	$tmp0,$tmp0,$tmp1

	or	$tmp2,$tmp2,$tmp3



	add	$in0,$in0,$tmp0		# accumulate nonce

	add	$in1,$in1,$tmp2

	sltu	$tmp0,$in0,$tmp0

	add	$in1,$in1,$tmp0



#ifdef	__riscv_misaligned_fast

	sd	$in0,0($mac)		# write mac value

	sd	$in1,8($mac)

#else

	srli	$tmp0,$in0,8		# write mac value

	srli	$tmp1,$in0,16

	srli	$tmp2,$in0,24

	sb	$in0,0($mac)

	srli	$tmp3,$in0,32

	sb	$tmp0,1($mac)

	srli	$tmp0,$in0,40

	sb	$tmp1,2($mac)

	srli	$tmp1,$in0,48

	sb	$tmp2,3($mac)

	srli	$tmp2,$in0,56

	sb	$tmp3,4($mac)

	srli	$tmp3,$in1,8

	sb	$tmp0,5($mac)

	srli	$tmp0,$in1,16

	sb	$tmp1,6($mac)

	srli	$tmp1,$in1,24

	sb	$tmp2,7($mac)



	sb	$in1,8($mac)

	srli	$tmp2,$in1,32

	sb	$tmp3,9($mac)

	srli	$tmp3,$in1,40

	sb	$tmp0,10($mac)

	srli	$tmp0,$in1,48

	sb	$tmp1,11($mac)

	srli	$tmp1,$in1,56

	sb	$tmp2,12($mac)

	sb	$tmp3,13($mac)

	sb	$tmp0,14($mac)

	sb	$tmp1,15($mac)

#endif



	ret

.size	poly1305_emit,.-poly1305_emit

.string	"Poly1305 for RISC-V, CRYPTOGAMS by \@dot-asm"

___

}

}}} else {{{

######################################################################

# 32-bit code path

#



my ($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3);

my ($in0,$in1,$in2,$in3,$tmp0,$tmp1,$tmp2,$tmp3) =

   ($a4,$a5,$a6,$a7,$t0,$t1,$t2,$t3);



$code.=<<___;

#if __riscv_xlen == 32

# if __SIZEOF_POINTER__ == 8

#  define PUSH	csc

#  define POP	clc

# else

#  define PUSH	sw

#  define POP	lw

# endif

# define MULX(hi,lo,a,b)	mulhu hi,a,b; mul lo,a,b

# define srliw	srli

# define srlw	srl

# define sllw	sll

# define addw	add

# define addiw	addi

# define mulw	mul

#elif __riscv_xlen == 64

# if __SIZEOF_POINTER__ == 16

#  define PUSH	csc

#  define POP	clc

# else

#  define PUSH	sd

#  define POP	ld

# endif

# define MULX(hi,lo,a,b)	slli b,b,32; srli b,b,32; mul hi,a,b; addiw lo,hi,0; srai hi,hi,32

#else

# error "unsupported __riscv_xlen"

#endif



.option	pic

.text



.globl	poly1305_init

.type	poly1305_init,\@function

poly1305_init:

#ifdef	__riscv_zicfilp

	lpad	0

#endif

	sw	$zero,0($ctx)

	sw	$zero,4($ctx)

	sw	$zero,8($ctx)

	sw	$zero,12($ctx)

	sw	$zero,16($ctx)



	beqz	$inp,.Lno_key



#ifndef	__riscv_misaligned_fast

	andi	$tmp0,$inp,3		# $inp % 4

	sub	$inp,$inp,$tmp0		# align $inp

	sll	$tmp0,$tmp0,3		# byte to bit offset

#endif

	lw	$in0,0($inp)

	lw	$in1,4($inp)

	lw	$in2,8($inp)

	lw	$in3,12($inp)

#ifndef	__riscv_misaligned_fast

	beqz	$tmp0,.Laligned_key



	lw	$tmp2,16($inp)

	sub	$tmp1,$zero,$tmp0

	srlw	$in0,$in0,$tmp0

	sllw	$tmp3,$in1,$tmp1

	srlw	$in1,$in1,$tmp0

	or	$in0,$in0,$tmp3

	sllw	$tmp3,$in2,$tmp1

	srlw	$in2,$in2,$tmp0

	or	$in1,$in1,$tmp3

	sllw	$tmp3,$in3,$tmp1

	srlw	$in3,$in3,$tmp0

	or	$in2,$in2,$tmp3

	sllw	$tmp2,$tmp2,$tmp1

	or	$in3,$in3,$tmp2

.Laligned_key:

#endif



	lui	$tmp0,0x10000

	addi	$tmp0,$tmp0,-1		# 0x0fffffff

	and	$in0,$in0,$tmp0

	addi	$tmp0,$tmp0,-3		# 0x0ffffffc

	and	$in1,$in1,$tmp0

	and	$in2,$in2,$tmp0

	and	$in3,$in3,$tmp0



	sw	$in0,20($ctx)

	sw	$in1,24($ctx)

	sw	$in2,28($ctx)

	sw	$in3,32($ctx)



	srlw	$tmp1,$in1,2

	srlw	$tmp2,$in2,2

	srlw	$tmp3,$in3,2

	addw	$in1,$in1,$tmp1		# s1 = r1 + (r1 >> 2)

	addw	$in2,$in2,$tmp2

	addw	$in3,$in3,$tmp3

	sw	$in1,36($ctx)

	sw	$in2,40($ctx)

	sw	$in3,44($ctx)

.Lno_key:

	li	$a0,0

	ret

.size	poly1305_init,.-poly1305_init

___

{

my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $rs1,$rs2,$rs3) =

   ($s0,$s1,$s2,$s3,$s4, $s5,$s6,$s7,$s8, $t0,$t1,$t2);

my ($d0,$d1,$d2,$d3) =

   ($a4,$a5,$a6,$a7);

my $shr = $ra;		# used on R6



$code.=<<___;

.globl	poly1305_blocks

.type	poly1305_blocks,\@function

poly1305_blocks:

#ifdef	__riscv_zicfilp

	lpad	0

#endif

	andi	$len,$len,-16		# complete blocks only

	beqz	$len,.Labort



#ifdef	__riscv_zcmp

	cm.push	{ra,s0-s8}, -48

#else

	caddi	$sp,$sp,-__SIZEOF_POINTER__*12

	PUSH	$ra, __SIZEOF_POINTER__*11($sp)

	PUSH	$s0, __SIZEOF_POINTER__*10($sp)

	PUSH	$s1, __SIZEOF_POINTER__*9($sp)

	PUSH	$s2, __SIZEOF_POINTER__*8($sp)

	PUSH	$s3, __SIZEOF_POINTER__*7($sp)

	PUSH	$s4, __SIZEOF_POINTER__*6($sp)

	PUSH	$s5, __SIZEOF_POINTER__*5($sp)

	PUSH	$s6, __SIZEOF_POINTER__*4($sp)

	PUSH	$s7, __SIZEOF_POINTER__*3($sp)

	PUSH	$s8, __SIZEOF_POINTER__*2($sp)

#endif



#ifndef	__riscv_misaligned_fast

	andi	$shr,$inp,3

	andi	$inp,$inp,-4		# align $inp

	slli	$shr,$shr,3		# byte to bit offset

#endif



	lw	$h0,0($ctx)		# load hash value

	lw	$h1,4($ctx)

	lw	$h2,8($ctx)

	lw	$h3,12($ctx)

	lw	$h4,16($ctx)



	lw	$r0,20($ctx)		# load key

	lw	$r1,24($ctx)

	lw	$r2,28($ctx)

	lw	$r3,32($ctx)

	lw	$rs1,36($ctx)

	lw	$rs2,40($ctx)

	lw	$rs3,44($ctx)



	add	$len,$len,$inp		# end of buffer



.Loop:

	lw	$d0,0($inp)		# load input

	lw	$d1,4($inp)

	lw	$d2,8($inp)

	lw	$d3,12($inp)

#ifndef	__riscv_misaligned_fast

	beqz	$shr,.Laligned_inp



	lw	$t4,16($inp)

	sub	$t5,$zero,$shr

	srlw	$d0,$d0,$shr

	sllw	$t3,$d1,$t5

	srlw	$d1,$d1,$shr

	or	$d0,$d0,$t3

	sllw	$t3,$d2,$t5

	srlw	$d2,$d2,$shr

	or	$d1,$d1,$t3

	sllw	$t3,$d3,$t5

	srlw	$d3,$d3,$shr

	or	$d2,$d2,$t3

	sllw	$t4,$t4,$t5

	or	$d3,$d3,$t4



.Laligned_inp:

#endif

	srliw	$t3,$h4,2		# modulo-scheduled reduction

	andi	$t4,$h4,-4

	andi	$h4,$h4,3



	addw	$d0,$d0,$h0		# accumulate input

	 addw	$t4,$t4,$t3

	sltu	$h0,$d0,$h0

	addw	$d0,$d0,$t4		# ... and residue

	sltu	$t4,$d0,$t4



	addw	$d1,$d1,$h1

	 addw	$h0,$h0,$t4		# carry

	sltu	$h1,$d1,$h1

	addw	$d1,$d1,$h0

	sltu	$h0,$d1,$h0



	addw	$d2,$d2,$h2

	 addw	$h1,$h1,$h0		# carry

	sltu	$h2,$d2,$h2

	addw	$d2,$d2,$h1

	sltu	$h1,$d2,$h1



	addw	$d3,$d3,$h3

	 addw	$h2,$h2,$h1		# carry

	sltu	$h3,$d3,$h3

	addw	$d3,$d3,$h2



	MULX	($h1,$h0,$r0,$d0)	# d0*r0



	 sltu	$h2,$d3,$h2

	 addw	$h3,$h3,$h2		# carry



	MULX	($t4,$t3,$rs3,$d1)	# d1*s3



	 addw	$h4,$h4,$padbit

	 caddi	$inp,$inp,16

	 addw	$h4,$h4,$h3



	MULX	($t6,$a3,$rs2,$d2)	# d2*s2

	 addw	$h0,$h0,$t3

	 addw	$h1,$h1,$t4

	 sltu	$t3,$h0,$t3

	 addw	$h1,$h1,$t3



	MULX	($t4,$t3,$rs1,$d3)	# d3*s1

	 addw	$h0,$h0,$a3

	 addw	$h1,$h1,$t6

	 sltu	$a3,$h0,$a3

	 addw	$h1,$h1,$a3





	MULX	($h2,$a3,$r1,$d0)	# d0*r1

	 addw	$h0,$h0,$t3

	 addw	$h1,$h1,$t4

	 sltu	$t3,$h0,$t3

	 addw	$h1,$h1,$t3



	MULX	($t4,$t3,$r0,$d1)	# d1*r0

	 addw	$h1,$h1,$a3

	 sltu	$a3,$h1,$a3

	 addw	$h2,$h2,$a3



	MULX	($t6,$a3,$rs3,$d2)	# d2*s3

	 addw	$h1,$h1,$t3

	 addw	$h2,$h2,$t4

	 sltu	$t3,$h1,$t3

	 addw	$h2,$h2,$t3



	MULX	($t4,$t3,$rs2,$d3)	# d3*s2

	 addw	$h1,$h1,$a3

	 addw	$h2,$h2,$t6

	 sltu	$a3,$h1,$a3

	 addw	$h2,$h2,$a3



	mulw	$a3,$rs1,$h4		# h4*s1

	 addw	$h1,$h1,$t3

	 addw	$h2,$h2,$t4

	 sltu	$t3,$h1,$t3

	 addw	$h2,$h2,$t3





	MULX	($h3,$t3,$r2,$d0)	# d0*r2

	 addw	$h1,$h1,$a3

	 sltu	$a3,$h1,$a3

	 addw	$h2,$h2,$a3



	MULX	($t6,$a3,$r1,$d1)	# d1*r1

	 addw	$h2,$h2,$t3

	 sltu	$t3,$h2,$t3

	 addw	$h3,$h3,$t3



	MULX	($t4,$t3,$r0,$d2)	# d2*r0

	 addw	$h2,$h2,$a3

	 addw	$h3,$h3,$t6

	 sltu	$a3,$h2,$a3

	 addw	$h3,$h3,$a3



	MULX	($t6,$a3,$rs3,$d3)	# d3*s3

	 addw	$h2,$h2,$t3

	 addw	$h3,$h3,$t4

	 sltu	$t3,$h2,$t3

	 addw	$h3,$h3,$t3



	mulw	$t3,$rs2,$h4		# h4*s2

	 addw	$h2,$h2,$a3

	 addw	$h3,$h3,$t6

	 sltu	$a3,$h2,$a3

	 addw	$h3,$h3,$a3





	MULX	($t6,$a3,$r3,$d0)	# d0*r3

	 addw	$h2,$h2,$t3

	 sltu	$t3,$h2,$t3

	 addw	$h3,$h3,$t3



	MULX	($t4,$t3,$r2,$d1)	# d1*r2

	 addw	$h3,$h3,$a3

	 sltu	$a3,$h3,$a3

	 addw	$t6,$t6,$a3



	MULX	($a3,$d3,$r0,$d3)	# d3*r0

	 addw	$h3,$h3,$t3

	 addw	$t6,$t6,$t4

	 sltu	$t3,$h3,$t3

	 addw	$t6,$t6,$t3



	MULX	($t4,$t3,$r1,$d2)	# d2*r1

	 addw	$h3,$h3,$d3

	 addw	$t6,$t6,$a3

	 sltu	$d3,$h3,$d3

	 addw	$t6,$t6,$d3



	mulw	$a3,$rs3,$h4		# h4*s3

	 addw	$h3,$h3,$t3

	 addw	$t6,$t6,$t4

	 sltu	$t3,$h3,$t3

	 addw	$t6,$t6,$t3





	mulw	$h4,$r0,$h4		# h4*r0

	 addw	$h3,$h3,$a3

	 sltu	$a3,$h3,$a3

	 addw	$t6,$t6,$a3

	addw	$h4,$t6,$h4



	li	$padbit,1		# if we loop, padbit is 1



	bne	$inp,$len,.Loop



	sw	$h0,0($ctx)		# store hash value

	sw	$h1,4($ctx)

	sw	$h2,8($ctx)

	sw	$h3,12($ctx)

	sw	$h4,16($ctx)



#ifdef	__riscv_zcmp

	cm.popret	{ra,s0-s8}, 48

#else

	POP	$ra, __SIZEOF_POINTER__*11($sp)

	POP	$s0, __SIZEOF_POINTER__*10($sp)

	POP	$s1, __SIZEOF_POINTER__*9($sp)

	POP	$s2, __SIZEOF_POINTER__*8($sp)

	POP	$s3, __SIZEOF_POINTER__*7($sp)

	POP	$s4, __SIZEOF_POINTER__*6($sp)

	POP	$s5, __SIZEOF_POINTER__*5($sp)

	POP	$s6, __SIZEOF_POINTER__*4($sp)

	POP	$s7, __SIZEOF_POINTER__*3($sp)

	POP	$s8, __SIZEOF_POINTER__*2($sp)

	caddi	$sp,$sp,__SIZEOF_POINTER__*12

#endif

.Labort:

	ret

.size	poly1305_blocks,.-poly1305_blocks

___

}

{

my ($ctx,$mac,$nonce,$tmp4) = ($a0,$a1,$a2,$a3);



$code.=<<___;

.globl	poly1305_emit

.type	poly1305_emit,\@function

poly1305_emit:

#ifdef	__riscv_zicfilp

	lpad	0

#endif

	lw	$tmp4,16($ctx)

	lw	$tmp0,0($ctx)

	lw	$tmp1,4($ctx)

	lw	$tmp2,8($ctx)

	lw	$tmp3,12($ctx)



	srliw	$ctx,$tmp4,2		# final reduction

	andi	$in0,$tmp4,-4

	andi	$tmp4,$tmp4,3

	addw	$ctx,$ctx,$in0



	addw	$tmp0,$tmp0,$ctx

	sltu	$ctx,$tmp0,$ctx

	 addiw	$in0,$tmp0,5		# compare to modulus

	addw	$tmp1,$tmp1,$ctx

	 sltiu	$in1,$in0,5

	sltu	$ctx,$tmp1,$ctx

	 addw	$in1,$in1,$tmp1

	addw	$tmp2,$tmp2,$ctx

	 sltu	$in2,$in1,$tmp1

	sltu	$ctx,$tmp2,$ctx

	 addw	$in2,$in2,$tmp2

	addw	$tmp3,$tmp3,$ctx

	 sltu	$in3,$in2,$tmp2

	sltu	$ctx,$tmp3,$ctx

	 addw	$in3,$in3,$tmp3

	addw	$tmp4,$tmp4,$ctx

	 sltu	$ctx,$in3,$tmp3

	 addw	$ctx,$ctx,$tmp4



	srl	$ctx,$ctx,2		# see if it carried/borrowed

	sub	$ctx,$zero,$ctx



	xor	$in0,$in0,$tmp0

	xor	$in1,$in1,$tmp1

	xor	$in2,$in2,$tmp2

	xor	$in3,$in3,$tmp3

	and	$in0,$in0,$ctx

	and	$in1,$in1,$ctx

	and	$in2,$in2,$ctx

	and	$in3,$in3,$ctx

	xor	$in0,$in0,$tmp0

	xor	$in1,$in1,$tmp1

	xor	$in2,$in2,$tmp2

	xor	$in3,$in3,$tmp3



	lw	$tmp0,0($nonce)		# load nonce

	lw	$tmp1,4($nonce)

	lw	$tmp2,8($nonce)

	lw	$tmp3,12($nonce)



	addw	$in0,$in0,$tmp0		# accumulate nonce

	sltu	$ctx,$in0,$tmp0



	addw	$in1,$in1,$tmp1

	sltu	$tmp1,$in1,$tmp1

	addw	$in1,$in1,$ctx

	sltu	$ctx,$in1,$ctx

	addw	$ctx,$ctx,$tmp1



	addw	$in2,$in2,$tmp2

	sltu	$tmp2,$in2,$tmp2

	addw	$in2,$in2,$ctx

	sltu	$ctx,$in2,$ctx

	addw	$ctx,$ctx,$tmp2



	addw	$in3,$in3,$tmp3

	addw	$in3,$in3,$ctx



#ifdef	__riscv_misaligned_fast

	sw	$in0,0($mac)		# write mac value

	sw	$in1,4($mac)

	sw	$in2,8($mac)

	sw	$in3,12($mac)

#else

	srl	$tmp0,$in0,8		# write mac value

	srl	$tmp1,$in0,16

	srl	$tmp2,$in0,24

	sb	$in0, 0($mac)

	sb	$tmp0,1($mac)

	srl	$tmp0,$in1,8

	sb	$tmp1,2($mac)

	srl	$tmp1,$in1,16

	sb	$tmp2,3($mac)

	srl	$tmp2,$in1,24

	sb	$in1, 4($mac)

	sb	$tmp0,5($mac)

	srl	$tmp0,$in2,8

	sb	$tmp1,6($mac)

	srl	$tmp1,$in2,16

	sb	$tmp2,7($mac)

	srl	$tmp2,$in2,24

	sb	$in2, 8($mac)

	sb	$tmp0,9($mac)

	srl	$tmp0,$in3,8

	sb	$tmp1,10($mac)

	srl	$tmp1,$in3,16

	sb	$tmp2,11($mac)

	srl	$tmp2,$in3,24

	sb	$in3, 12($mac)

	sb	$tmp0,13($mac)

	sb	$tmp1,14($mac)

	sb	$tmp2,15($mac)

#endif



	ret

.size	poly1305_emit,.-poly1305_emit

.string	"Poly1305 for RISC-V, CRYPTOGAMS by \@dot-asm"

___

}

}}}



foreach (split("\n", $code)) {

    if ($flavour =~ /^cheri/) {

	s/\(x([0-9]+)\)/(c$1)/ and s/\b([ls][bhwd]u?)\b/c$1/;

	s/\b(PUSH|POP)(\s+)x([0-9]+)/$1$2c$3/ or

	s/\b(ret|jal)\b/c$1/;

	s/\bcaddi?\b/cincoffset/ and s/\bx([0-9]+,)/c$1/g or

	m/\bcmove\b/ and s/\bx([0-9]+)/c$1/g;

    } else {

	s/\bcaddi?\b/add/ or

	s/\bcmove\b/mv/;

    }

    print $_, "\n";

}



close STDOUT;

lib/crypto/riscv/poly1305.h

0 → 100644
+14 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * OpenSSL/Cryptogams accelerated Poly1305 transform for riscv

 *

 * Copyright (C) 2025 Institute of Software, CAS.

 */



asmlinkage void poly1305_block_init(struct poly1305_block_state *state,

				    const u8 raw_key[POLY1305_BLOCK_SIZE]);

asmlinkage void poly1305_blocks(struct poly1305_block_state *state,

				const u8 *src, u32 len, u32 hibit);

asmlinkage void poly1305_emit(const struct poly1305_state *state,

			      u8 digest[POLY1305_DIGEST_SIZE],

			      const u32 nonce[4]);