Commit adcadab3 authored by Dave Watson's avatar Dave Watson Committed by Herbert Xu
Browse files

crypto: aesni - Add GCM_COMPLETE macro 



Merge encode and decode tag calculations in GCM_COMPLETE macro.
Scatter/gather routines will call this once at the end of encryption
or decryption.

Signed-off-by: default avatarDave Watson <davejwatson@fb.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 7af964c2

arch/x86/crypto/aesni-intel_asm.S

+63 −109
Original line number Diff line number Diff line
@@ -223,6 +223,67 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff 
	mov	%r13, %r12

.endm



# GCM_COMPLETE Finishes update of tag of last partial block

# Output: Authorization Tag (AUTH_TAG)

# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15

.macro GCM_COMPLETE

	mov	arg8, %r12		  # %r13 = aadLen (number of bytes)

	shl	$3, %r12		  # convert into number of bits

	movd	%r12d, %xmm15		  # len(A) in %xmm15

	shl	$3, %arg4		  # len(C) in bits (*128)

	MOVQ_R64_XMM	%arg4, %xmm1

	pslldq	$8, %xmm15		  # %xmm15 = len(A)||0x0000000000000000

	pxor	%xmm1, %xmm15		  # %xmm15 = len(A)||len(C)

	pxor	%xmm15, %xmm8

	GHASH_MUL	%xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6

	# final GHASH computation

	movdqa SHUF_MASK(%rip), %xmm10

	PSHUFB_XMM %xmm10, %xmm8



	mov	%arg5, %rax		  # %rax = *Y0

	movdqu	(%rax), %xmm0		  # %xmm0 = Y0

	ENCRYPT_SINGLE_BLOCK	%xmm0,  %xmm1	  # E(K, Y0)

	pxor	%xmm8, %xmm0

_return_T_\@:

	mov	arg9, %r10                     # %r10 = authTag

	mov	arg10, %r11                    # %r11 = auth_tag_len

	cmp	$16, %r11

	je	_T_16_\@

	cmp	$8, %r11

	jl	_T_4_\@

_T_8_\@:

	MOVQ_R64_XMM	%xmm0, %rax

	mov	%rax, (%r10)

	add	$8, %r10

	sub	$8, %r11

	psrldq	$8, %xmm0

	cmp	$0, %r11

	je	_return_T_done_\@

_T_4_\@:

	movd	%xmm0, %eax

	mov	%eax, (%r10)

	add	$4, %r10

	sub	$4, %r11

	psrldq	$4, %xmm0

	cmp	$0, %r11

	je	_return_T_done_\@

_T_123_\@:

	movd	%xmm0, %eax

	cmp	$2, %r11

	jl	_T_1_\@

	mov	%ax, (%r10)

	cmp	$2, %r11

	je	_return_T_done_\@

	add	$2, %r10

	sar	$16, %eax

_T_1_\@:

	mov	%al, (%r10)

	jmp	_return_T_done_\@

_T_16_\@:

	movdqu	%xmm0, (%r10)

_return_T_done_\@:

.endm



#ifdef __x86_64__

/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)

*
@@ -1272,61 +1333,7 @@ _less_than_8_bytes_left_decrypt: 
	sub	$1, %r13

	jne	_less_than_8_bytes_left_decrypt

_multiple_of_16_bytes_decrypt:

	mov	arg8, %r12		  # %r13 = aadLen (number of bytes)

	shl	$3, %r12		  # convert into number of bits

	movd	%r12d, %xmm15		  # len(A) in %xmm15

	shl	$3, %arg4		  # len(C) in bits (*128)

	MOVQ_R64_XMM	%arg4, %xmm1

	pslldq	$8, %xmm15		  # %xmm15 = len(A)||0x0000000000000000

	pxor	%xmm1, %xmm15		  # %xmm15 = len(A)||len(C)

	pxor	%xmm15, %xmm8

	GHASH_MUL	%xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6

	         # final GHASH computation

        movdqa SHUF_MASK(%rip), %xmm10

	PSHUFB_XMM %xmm10, %xmm8



	mov	%arg5, %rax		  # %rax = *Y0

	movdqu	(%rax), %xmm0		  # %xmm0 = Y0

	ENCRYPT_SINGLE_BLOCK	%xmm0,  %xmm1	  # E(K, Y0)

	pxor	%xmm8, %xmm0

_return_T_decrypt:

	mov	arg9, %r10                # %r10 = authTag

	mov	arg10, %r11               # %r11 = auth_tag_len

	cmp	$16, %r11

	je	_T_16_decrypt

	cmp	$8, %r11

	jl	_T_4_decrypt

_T_8_decrypt:

	MOVQ_R64_XMM	%xmm0, %rax

	mov	%rax, (%r10)

	add	$8, %r10

	sub	$8, %r11

	psrldq	$8, %xmm0

	cmp	$0, %r11

	je	_return_T_done_decrypt

_T_4_decrypt:

	movd	%xmm0, %eax

	mov	%eax, (%r10)

	add	$4, %r10

	sub	$4, %r11

	psrldq	$4, %xmm0

	cmp	$0, %r11

	je	_return_T_done_decrypt

_T_123_decrypt:

	movd	%xmm0, %eax

	cmp	$2, %r11

	jl	_T_1_decrypt

	mov	%ax, (%r10)

	cmp	$2, %r11

	je	_return_T_done_decrypt

	add	$2, %r10

	sar	$16, %eax

_T_1_decrypt:

	mov	%al, (%r10)

	jmp	_return_T_done_decrypt

_T_16_decrypt:

	movdqu	%xmm0, (%r10)

_return_T_done_decrypt:

	GCM_COMPLETE

	FUNC_RESTORE

	ret

ENDPROC(aesni_gcm_dec)
@@ -1502,61 +1509,8 @@ _less_than_8_bytes_left_encrypt: 
	sub $1, %r13

	jne _less_than_8_bytes_left_encrypt

_multiple_of_16_bytes_encrypt:

	mov	arg8, %r12    # %r12 = addLen (number of bytes)

	shl	$3, %r12

	movd	%r12d, %xmm15       # len(A) in %xmm15

	shl	$3, %arg4               # len(C) in bits (*128)

	MOVQ_R64_XMM	%arg4, %xmm1

	pslldq	$8, %xmm15          # %xmm15 = len(A)||0x0000000000000000

	pxor	%xmm1, %xmm15       # %xmm15 = len(A)||len(C)

	pxor	%xmm15, %xmm8

	GHASH_MUL	%xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6

	# final GHASH computation

        movdqa SHUF_MASK(%rip), %xmm10

	PSHUFB_XMM %xmm10, %xmm8         # perform a 16 byte swap



	mov	%arg5, %rax		       # %rax  = *Y0

	movdqu	(%rax), %xmm0		       # %xmm0 = Y0

	ENCRYPT_SINGLE_BLOCK	%xmm0, %xmm15         # Encrypt(K, Y0)

	pxor	%xmm8, %xmm0

_return_T_encrypt:

	mov	arg9, %r10                     # %r10 = authTag

	mov	arg10, %r11                    # %r11 = auth_tag_len

	cmp	$16, %r11

	je	_T_16_encrypt

	cmp	$8, %r11

	jl	_T_4_encrypt

_T_8_encrypt:

	MOVQ_R64_XMM	%xmm0, %rax

	mov	%rax, (%r10)

	add	$8, %r10

	sub	$8, %r11

	psrldq	$8, %xmm0

	cmp	$0, %r11

	je	_return_T_done_encrypt

_T_4_encrypt:

	movd	%xmm0, %eax

	mov	%eax, (%r10)

	add	$4, %r10

	sub	$4, %r11

	psrldq	$4, %xmm0

	cmp	$0, %r11

	je	_return_T_done_encrypt

_T_123_encrypt:

	movd	%xmm0, %eax

	cmp	$2, %r11

	jl	_T_1_encrypt

	mov	%ax, (%r10)

	cmp	$2, %r11

	je	_return_T_done_encrypt

	add	$2, %r10

	sar	$16, %eax

_T_1_encrypt:

	mov	%al, (%r10)

	jmp	_return_T_done_encrypt

_T_16_encrypt:

	movdqu	%xmm0, (%r10)

_return_T_done_encrypt:

	GCM_COMPLETE

	FUNC_RESTORE

	ret

ENDPROC(aesni_gcm_enc)