Commit 5b629e64 authored by Heiko Carstens's avatar Heiko Carstens Committed by Alexander Gordeev
Browse files

s390/uaccess: Implement __get_kernel_nofault()/__put_kernel_nofault() with mvc 



Use the mvc instruction in order to implement __get_kernel_nofault() and
__put_kernel_nofault(). Both functions have a source and destination
address where the code is supposed to read from and write to. Use the mvc
instruction to copy from source to destination instead of lg/stg like
instructions. This generates slightly better code.

Acked-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
Signed-off-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
parent 722926ec

arch/s390/include/asm/uaccess.h

+44 −78
Original line number Diff line number Diff line
@@ -341,109 +341,75 @@ static inline void *s390_kernel_write(void *dst, const void *src, size_t size) 
	return __s390_kernel_write(dst, src, size);

}



int __noreturn __put_kernel_bad(void);

void __noreturn __mvc_kernel_nofault_bad(void);



#define __put_kernel_asm(val, to, insn)					\

({									\

	int __rc;							\

									\

	asm volatile(							\

		"0:   " insn "  %[_val],%[_to]\n"			\

		"1:	xr	%[rc],%[rc]\n"				\

		"2:\n"							\

		EX_TABLE_UA_FAULT(0b, 2b, %[rc])			\

		EX_TABLE_UA_FAULT(1b, 2b, %[rc])			\

		: [rc] "=d" (__rc), [_to] "+Q" (*(to))			\

		: [_val] "d" (val)					\

		: "cc");						\

	__rc;								\

})

#ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS



#define __put_kernel_nofault(dst, src, type, err_label)			\

#define __mvc_kernel_nofault(dst, src, type, err_label)			\

do {									\

	unsigned long __x = (unsigned long)(*((type *)(src)));		\

	int __pk_err;							\

	int __rc;							\

									\

	switch (sizeof(type)) {						\

	case 1:								\

		__pk_err = __put_kernel_asm(__x, (type *)(dst), "stc"); \

		break;							\

	case 2:								\

		__pk_err = __put_kernel_asm(__x, (type *)(dst), "sth"); \

		break;							\

	case 4:								\

		__pk_err = __put_kernel_asm(__x, (type *)(dst), "st");	\

		break;							\

	case 8:								\

		__pk_err = __put_kernel_asm(__x, (type *)(dst), "stg"); \

		asm_inline volatile(					\

			"0:	mvc	%O[_dst](%[_len],%R[_dst]),%[_src]\n" \

			"1:	lhi	%[_rc],0\n"			\

			"2:\n"						\

			EX_TABLE_UA_FAULT(0b, 2b, %[_rc])		\

			EX_TABLE_UA_FAULT(1b, 2b, %[_rc])		\

			: [_rc] "=d" (__rc),				\

			  [_dst] "=Q" (*(type *)dst)			\

			: [_src] "Q" (*(type *)(src)),			\

			  [_len] "I" (sizeof(type)));			\

		if (__rc)						\

			goto err_label;					\

		break;							\

	default:							\

		__pk_err = __put_kernel_bad();				\

		__mvc_kernel_nofault_bad();				\

		break;							\

	}								\

	if (unlikely(__pk_err))						\

		goto err_label;						\

} while (0)



int __noreturn __get_kernel_bad(void);

#else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */



#define __get_kernel_asm(val, from, insn)				\

({									\

	int __rc;							\

									\

	asm volatile(							\

		"0:   " insn "  %[_val],%[_from]\n"			\

		"1:	xr	%[rc],%[rc]\n"				\

		"2:\n"							\

		EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val])		\

		EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val])		\

		: [rc] "=d" (__rc), [_val] "=d" (val)			\

		: [_from] "Q" (*(from))					\

		: "cc");						\

	__rc;								\

})



#define __get_kernel_nofault(dst, src, type, err_label)			\

#define __mvc_kernel_nofault(dst, src, type, err_label)			\

do {									\

	int __gk_err;							\

	type *(__dst) = (type *)(dst);					\

	int __rc;							\

									\

	switch (sizeof(type)) {						\

	case 1: {							\

		unsigned char __x;					\

									\

		__gk_err = __get_kernel_asm(__x, (type *)(src), "ic");	\

		*((type *)(dst)) = (type)__x;				\

		break;							\

	};								\

	case 2: {							\

		unsigned short __x;					\

									\

		__gk_err = __get_kernel_asm(__x, (type *)(src), "lh");	\

		*((type *)(dst)) = (type)__x;				\

		break;							\

	};								\

	case 4: {							\

		unsigned int __x;					\

									\

		__gk_err = __get_kernel_asm(__x, (type *)(src), "l");	\

		*((type *)(dst)) = (type)__x;				\

		break;							\

	};								\

	case 8: {							\

		unsigned long __x;					\

									\

		__gk_err = __get_kernel_asm(__x, (type *)(src), "lg");	\

		*((type *)(dst)) = (type)__x;				\

	case 1:								\

	case 2:								\

	case 4:								\

	case 8:								\

		asm_inline volatile(					\

			"0:	mvc	0(%[_len],%[_dst]),%[_src]\n"	\

			"1:	lhi	%[_rc],0\n"			\

			"2:\n"						\

			EX_TABLE_UA_FAULT(0b, 2b, %[_rc])		\

			EX_TABLE_UA_FAULT(1b, 2b, %[_rc])		\

			: [_rc] "=d" (__rc),				\

			  "=m" (*__dst)					\

			: [_src] "Q" (*(type *)(src)),			\

			[_dst] "a" (__dst),				\

			[_len] "I" (sizeof(type)));			\

		if (__rc)						\

			goto err_label;					\

		break;							\

	};								\

	default:							\

		__gk_err = __get_kernel_bad();				\

		__mvc_kernel_nofault_bad();				\

		break;							\

	}								\

	if (unlikely(__gk_err))						\

		goto err_label;						\

} while (0)



#endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */



#define __get_kernel_nofault __mvc_kernel_nofault

#define __put_kernel_nofault __mvc_kernel_nofault



void __cmpxchg_user_key_called_with_bad_pointer(void);



#define CMPXCHG_USER_KEY_MAX_LOOPS 128