Commit 8072b39c authored by Christophe Leroy's avatar Christophe Leroy Committed by Jason A. Donenfeld
Browse files

powerpc/vdso: Wire up getrandom() vDSO implementation on VDSO64 



Extend getrandom() vDSO implementation to VDSO64.

Tested on QEMU on both ppc64_defconfig and ppc64le_defconfig.

Results from a Power9 (PowerNV):
~ # ./vdso_test_getrandom bench-single
   vdso: 25000000 times in 0.787943615 seconds
   libc: 25000000 times in 14.101887252 seconds
   syscall: 25000000 times in 14.047475082 seconds

Signed-off-by: default avatarChristophe Leroy <christophe.leroy@csgroup.eu>
Tested-by: default avatarMadhavan Srinivasan <maddy@linux.ibm.com>
Acked-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Signed-off-by: default avatarJason A. Donenfeld <Jason@zx2c4.com>
parent 53cee505

arch/powerpc/Kconfig

+1 −1
Original line number Diff line number Diff line
@@ -311,7 +311,7 @@ config PPC 
	select SYSCTL_EXCEPTION_TRACE

	select THREAD_INFO_IN_TASK

	select TRACE_IRQFLAGS_SUPPORT

	select VDSO_GETRANDOM			if VDSO32

	select VDSO_GETRANDOM

	#

	# Please keep this list sorted alphabetically.

	#

arch/powerpc/kernel/vdso/Makefile

+6 −2
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ obj-vdso32 = sigtramp32-32.o gettimeofday-32.o datapage-32.o cacheflush-32.o not 
obj-vdso64 = sigtramp64-64.o gettimeofday-64.o datapage-64.o cacheflush-64.o note-64.o getcpu-64.o



obj-vdso32 += getrandom-32.o vgetrandom-chacha-32.o

obj-vdso64 += getrandom-64.o vgetrandom-chacha-64.o



ifneq ($(c-gettimeofday-y),)

  CFLAGS_vgettimeofday-32.o += -include $(c-gettimeofday-y)
@@ -21,6 +22,7 @@ endif 


ifneq ($(c-getrandom-y),)

  CFLAGS_vgetrandom-32.o += -include $(c-getrandom-y)

  CFLAGS_vgetrandom-64.o += -include $(c-getrandom-y) $(call cc-option, -ffixed-r30)

endif



# Build rules
@@ -34,7 +36,7 @@ endif 
targets := $(obj-vdso32) vdso32.so.dbg vgettimeofday-32.o vgetrandom-32.o

targets += crtsavres-32.o

obj-vdso32 := $(addprefix $(obj)/, $(obj-vdso32))

targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o

targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o vgetrandom-64.o

obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64))



ccflags-y := -fno-common -fno-builtin -DBUILD_VDSO
@@ -71,7 +73,7 @@ CPPFLAGS_vdso64.lds += -P -C 
# link rule for the .so file, .lds has to be first

$(obj)/vdso32.so.dbg: $(obj)/vdso32.lds $(obj-vdso32) $(obj)/vgettimeofday-32.o $(obj)/vgetrandom-32.o $(obj)/crtsavres-32.o FORCE

	$(call if_changed,vdso32ld_and_check)

$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o FORCE

$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o $(obj)/vgetrandom-64.o FORCE

	$(call if_changed,vdso64ld_and_check)



# assembly rules for the .S files
@@ -87,6 +89,8 @@ $(obj-vdso64): %-64.o: %.S FORCE 
	$(call if_changed_dep,vdso64as)

$(obj)/vgettimeofday-64.o: %-64.o: %.c FORCE

	$(call if_changed_dep,cc_o_c)

$(obj)/vgetrandom-64.o: %-64.o: %.c FORCE

	$(call if_changed_dep,cc_o_c)



# Generate VDSO offsets using helper script

gen-vdso32sym := $(src)/gen_vdso32_offsets.sh

arch/powerpc/kernel/vdso/getrandom.S

+8 −0
Original line number Diff line number Diff line
@@ -27,10 +27,18 @@ 
  .cfi_adjust_cfa_offset PPC_MIN_STKFRM

	PPC_STL		r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)

  .cfi_rel_offset lr, PPC_MIN_STKFRM + PPC_LR_STKOFF

#ifdef __powerpc64__

	PPC_STL		r2, PPC_MIN_STKFRM + STK_GOT(r1)

  .cfi_rel_offset r2, PPC_MIN_STKFRM + STK_GOT

#endif

	get_realdatapage	r8, r11

	addi		r8, r8, VDSO_RNG_DATA_OFFSET

	bl		CFUNC(DOTSYM(\funct))

	PPC_LL		r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)

#ifdef __powerpc64__

	PPC_LL		r2, PPC_MIN_STKFRM + STK_GOT(r1)

  .cfi_restore r2

#endif

	cmpwi		r3, 0

	mtlr		r0

	addi		r1, r1, 2 * PPC_MIN_STKFRM

arch/powerpc/kernel/vdso/vdso64.lds.S

+1 −0
Original line number Diff line number Diff line
@@ -123,6 +123,7 @@ VERSION 
		__kernel_sigtramp_rt64;

		__kernel_getcpu;

		__kernel_time;

		__kernel_getrandom;



	local: *;

	};

arch/powerpc/kernel/vdso/vgetrandom-chacha.S

+53 −0
Original line number Diff line number Diff line
@@ -124,6 +124,26 @@ 
 */

SYM_FUNC_START(__arch_chacha20_blocks_nostack)

#ifdef __powerpc64__

	std	counter, -216(r1)



	std	r14, -144(r1)

	std	r15, -136(r1)

	std	r16, -128(r1)

	std	r17, -120(r1)

	std	r18, -112(r1)

	std	r19, -104(r1)

	std	r20, -96(r1)

	std	r21, -88(r1)

	std	r22, -80(r1)

	std	r23, -72(r1)

	std	r24, -64(r1)

	std	r25, -56(r1)

	std	r26, -48(r1)

	std	r27, -40(r1)

	std	r28, -32(r1)

	std	r29, -24(r1)

	std	r30, -16(r1)

	std	r31, -8(r1)

#else

	stwu	r1, -96(r1)

	stw	counter, 20(r1)
@@ -149,9 +169,13 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) 
	stw	r30, 88(r1)

	stw	r31, 92(r1)

#endif

#endif	/* __powerpc64__ */



	lwz	counter0, 0(counter)

	lwz	counter1, 4(counter)

#ifdef __powerpc64__

	rldimi	counter0, counter1, 32, 0

#endif

	mr	idx_r0, nblocks

	subi	dst_bytes, dst_bytes, 4
@@ -267,12 +291,21 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) 


	subic.	idx_r0, idx_r0, 1	/* subi. can't use r0 as source */



#ifdef __powerpc64__

	addi	counter0, counter0, 1

	srdi	counter1, counter0, 32

#else

	addic	counter0, counter0, 1

	addze	counter1, counter1

#endif



	bne	.Lblock



#ifdef __powerpc64__

	ld	counter, -216(r1)

#else

	lwz	counter, 20(r1)

#endif

	stw	counter0, 0(counter)

	stw	counter1, 4(counter)
@@ -284,6 +317,26 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) 
	li	r11, 0

	li	r12, 0



#ifdef __powerpc64__

	ld	r14, -144(r1)

	ld	r15, -136(r1)

	ld	r16, -128(r1)

	ld	r17, -120(r1)

	ld	r18, -112(r1)

	ld	r19, -104(r1)

	ld	r20, -96(r1)

	ld	r21, -88(r1)

	ld	r22, -80(r1)

	ld	r23, -72(r1)

	ld	r24, -64(r1)

	ld	r25, -56(r1)

	ld	r26, -48(r1)

	ld	r27, -40(r1)

	ld	r28, -32(r1)

	ld	r29, -24(r1)

	ld	r30, -16(r1)

	ld	r31, -8(r1)

#else

#ifdef __BIG_ENDIAN__

	lmw	r14, 24(r1)

#else