Commit 712676ea authored by Adhemerval Zanella's avatar Adhemerval Zanella Committed by Jason A. Donenfeld
Browse files

arm64: vDSO: Wire up getrandom() vDSO implementation 



Hook up the generic vDSO implementation to the aarch64 vDSO data page.
The _vdso_rng_data required data is placed within the _vdso_data vvar
page, by using a offset larger than the vdso_data.

The vDSO function requires a ChaCha20 implementation that does not write
to the stack, and that can do an entire ChaCha20 permutation.  The one
provided uses NEON on the permute operation, with a fallback to the
syscall for chips that do not support AdvSIMD.

This also passes the vdso_test_chacha test along with
vdso_test_getrandom. The vdso_test_getrandom bench-single result on
Neoverse-N1 shows:

   vdso: 25000000 times in 0.783884250 seconds
   libc: 25000000 times in 8.780275399 seconds
syscall: 25000000 times in 8.786581518 seconds

A small fixup to arch/arm64/include/asm/mman.h was required to avoid
pulling kernel code into the vDSO, similar to what's already done in
arch/arm64/include/asm/rwonce.h.

Signed-off-by: default avatarAdhemerval Zanella <adhemerval.zanella@linaro.org>
Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Acked-by: default avatarWill Deacon <will@kernel.org>
Signed-off-by: default avatarJason A. Donenfeld <Jason@zx2c4.com>
parent 2c2ca341

arch/arm64/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -262,6 +262,7 @@ config ARM64 
	select TRACE_IRQFLAGS_NMI_SUPPORT

	select HAVE_SOFTIRQ_ON_OWN_STACK

	select USER_STACKTRACE_SUPPORT

	select VDSO_GETRANDOM

	help

	  ARM 64-bit (AArch64) Linux support.

arch/arm64/include/asm/mman.h

+5 −1
Original line number Diff line number Diff line
@@ -2,9 +2,11 @@ 
#ifndef __ASM_MMAN_H__

#define __ASM_MMAN_H__



#include <uapi/asm/mman.h>



#ifndef BUILD_VDSO

#include <linux/compiler.h>

#include <linux/types.h>

#include <uapi/asm/mman.h>



static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,

	unsigned long pkey __always_unused)
@@ -60,4 +62,6 @@ static inline bool arch_validate_flags(unsigned long vm_flags) 
}

#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags)



#endif /* !BUILD_VDSO */



#endif /* ! __ASM_MMAN_H__ */

arch/arm64/include/asm/vdso/getrandom.h

0 → 100644
+50 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#ifndef __ASM_VDSO_GETRANDOM_H

#define __ASM_VDSO_GETRANDOM_H



#ifndef __ASSEMBLY__



#include <asm/unistd.h>

#include <asm/vdso/vsyscall.h>

#include <vdso/datapage.h>



/**

 * getrandom_syscall - Invoke the getrandom() syscall.

 * @buffer:	Destination buffer to fill with random bytes.

 * @len:	Size of @buffer in bytes.

 * @flags:	Zero or more GRND_* flags.

 * Returns:	The number of random bytes written to @buffer, or a negative value indicating an error.

 */

static __always_inline ssize_t getrandom_syscall(void *_buffer, size_t _len, unsigned int _flags)

{

	register void *buffer asm ("x0") = _buffer;

	register size_t len asm ("x1") = _len;

	register unsigned int flags asm ("x2") = _flags;

	register long ret asm ("x0");

	register long nr asm ("x8") = __NR_getrandom;



	asm volatile(

	"       svc #0\n"

	: "=r" (ret)

	: "r" (buffer), "r" (len), "r" (flags), "r" (nr)

	: "memory");



	return ret;

}



static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)

{

	/*

	 * The RNG data is in the real VVAR data page, but if a task belongs to a time namespace

	 * then VVAR_DATA_PAGE_OFFSET points to the namespace-specific VVAR page and VVAR_TIMENS_

	 * PAGE_OFFSET points to the real VVAR page.

	 */

	if (IS_ENABLED(CONFIG_TIME_NS) && _vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)

		return (void *)&_vdso_rng_data + VVAR_TIMENS_PAGE_OFFSET * (1UL << CONFIG_PAGE_SHIFT);

	return &_vdso_rng_data;

}



#endif /* !__ASSEMBLY__ */



#endif /* __ASM_VDSO_GETRANDOM_H */

arch/arm64/include/asm/vdso/vsyscall.h

+15 −0
Original line number Diff line number Diff line
@@ -2,11 +2,19 @@ 
#ifndef __ASM_VDSO_VSYSCALL_H

#define __ASM_VDSO_VSYSCALL_H



#define __VDSO_RND_DATA_OFFSET  480



#ifndef __ASSEMBLY__



#include <linux/timekeeper_internal.h>

#include <vdso/datapage.h>



enum vvar_pages {

	VVAR_DATA_PAGE_OFFSET,

	VVAR_TIMENS_PAGE_OFFSET,

	VVAR_NR_PAGES,

};



#define VDSO_PRECISION_MASK	~(0xFF00ULL<<48)



extern struct vdso_data *vdso_data;
@@ -21,6 +29,13 @@ struct vdso_data *__arm64_get_k_vdso_data(void) 
}

#define __arch_get_k_vdso_data __arm64_get_k_vdso_data



static __always_inline

struct vdso_rng_data *__arm64_get_k_vdso_rnd_data(void)

{

	return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;

}

#define __arch_get_k_vdso_rng_data __arm64_get_k_vdso_rnd_data



static __always_inline

void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk)

{

arch/arm64/kernel/vdso.c

+0 −6
Original line number Diff line number Diff line
@@ -34,12 +34,6 @@ enum vdso_abi { 
	VDSO_ABI_AA32,

};



enum vvar_pages {

	VVAR_DATA_PAGE_OFFSET,

	VVAR_TIMENS_PAGE_OFFSET,

	VVAR_NR_PAGES,

};



struct vdso_abi_info {

	const char *name;

	const char *vdso_code_start;