Commit 884f0582 authored by Heiko Carstens's avatar Heiko Carstens Committed by Alexander Gordeev
Browse files

s390/uaccess: Remove INLINE_COPY_FROM_USER and INLINE_COPY_TO_USER 



The s390 implementations of raw_copy_from_user() and raw_copy_to_user() are
never inlined. However INLINE_COPY_FROM_USER and INLINE_COPY_TO_USER are
still set. This leads to the odd situation that only the error handling
(memset to zero of the not copied bytes) of copy_from_user() is inlined,
while the actual fast path code is out-of-line.

This would make sense if raw_copy_from_user() and raw_copy_to_user() were
implemented in assembler files, where inlining is not possible. But the
current s390 setup does not make any sense.

Address this by moving the raw uaccess copy inline assemblies to the
uaccess header file, and remove INLINE_COPY_FROM_USER and
INLINE_COPY_TO_USER definitions. This way the uaccess code, but now
including error handling, is still out-of-line with the common code
_copy_from_user() and _copy_to_user() variants, which inline the raw
uaccess functions via _inline_copy_from_user() and _inline_copy_to_user().

This reduces the size of the kernel image by ~17kb.
(defconfig, gcc 14.2.0)

Acked-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
Signed-off-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
parent ea5ae3a7

arch/s390/include/asm/uaccess.h

+109 −30
Original line number Diff line number Diff line
@@ -22,16 +22,117 @@ 


void debug_user_asce(int exit);



unsigned long __must_check

raw_copy_from_user(void *to, const void __user *from, unsigned long n);

union oac {

	unsigned int val;

	struct {

		struct {

			unsigned short key : 4;

			unsigned short	   : 4;

			unsigned short as  : 2;

			unsigned short	   : 4;

			unsigned short k   : 1;

			unsigned short a   : 1;

		} oac1;

		struct {

			unsigned short key : 4;

			unsigned short	   : 4;

			unsigned short as  : 2;

			unsigned short	   : 4;

			unsigned short k   : 1;

			unsigned short a   : 1;

		} oac2;

	};

};



unsigned long __must_check

raw_copy_to_user(void __user *to, const void *from, unsigned long n);

static __always_inline __must_check unsigned long

raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)

{

	unsigned long rem;

	union oac spec = {

		.oac2.key = key,

		.oac2.as = PSW_BITS_AS_SECONDARY,

		.oac2.k = 1,

		.oac2.a = 1,

	};



#ifndef CONFIG_KASAN

#define INLINE_COPY_FROM_USER

#define INLINE_COPY_TO_USER

#endif

	asm_inline volatile(

		"	lr	%%r0,%[spec]\n"

		"0:	mvcos	0(%[to]),0(%[from]),%[size]\n"

		"1:	jz	5f\n"

		"	algr	%[size],%[val]\n"

		"	slgr	%[from],%[val]\n"

		"	slgr	%[to],%[val]\n"

		"	j	0b\n"

		"2:	la	%[rem],4095(%[from])\n" /* rem = from + 4095 */

		"	nr	%[rem],%[val]\n"	/* rem = (from + 4095) & -4096 */

		"	slgr	%[rem],%[from]\n"

		"	clgr	%[size],%[rem]\n"	/* copy crosses next page boundary? */

		"	jnh	6f\n"

		"3:	mvcos	0(%[to]),0(%[from]),%[rem]\n"

		"4:	slgr	%[size],%[rem]\n"

		"	j	6f\n"

		"5:	lghi	%[size],0\n"

		"6:\n"

		EX_TABLE(0b, 2b)

		EX_TABLE(1b, 2b)

		EX_TABLE(3b, 6b)

		EX_TABLE(4b, 6b)

		: [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem)

		: [val] "a" (-4096UL), [spec] "d" (spec.val)

		: "cc", "memory", "0");

	return size;

}



static __always_inline __must_check unsigned long

raw_copy_from_user(void *to, const void __user *from, unsigned long n)

{

	return raw_copy_from_user_key(to, from, n, 0);

}



static __always_inline __must_check unsigned long

raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)

{

	unsigned long rem;

	union oac spec = {

		.oac1.key = key,

		.oac1.as = PSW_BITS_AS_SECONDARY,

		.oac1.k = 1,

		.oac1.a = 1,

	};



	asm_inline volatile(

		"	lr	%%r0,%[spec]\n"

		"0:	mvcos	0(%[to]),0(%[from]),%[size]\n"

		"1:	jz	5f\n"

		"	algr	%[size],%[val]\n"

		"	slgr	%[to],%[val]\n"

		"	slgr	%[from],%[val]\n"

		"	j	0b\n"

		"2:	la	%[rem],4095(%[to])\n"	/* rem = to + 4095 */

		"	nr	%[rem],%[val]\n"	/* rem = (to + 4095) & -4096 */

		"	slgr	%[rem],%[to]\n"

		"	clgr	%[size],%[rem]\n"	/* copy crosses next page boundary? */

		"	jnh	6f\n"

		"3:	mvcos	0(%[to]),0(%[from]),%[rem]\n"

		"4:	slgr	%[size],%[rem]\n"

		"	j	6f\n"

		"5:	lghi	%[size],0\n"

		"6:\n"

		EX_TABLE(0b, 2b)

		EX_TABLE(1b, 2b)

		EX_TABLE(3b, 6b)

		EX_TABLE(4b, 6b)

		: [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem)

		: [val] "a" (-4096UL), [spec] "d" (spec.val)

		: "cc", "memory", "0");

	return size;

}



static __always_inline __must_check unsigned long

raw_copy_to_user(void __user *to, const void *from, unsigned long n)

{

	return raw_copy_to_user_key(to, from, n, 0);

}



unsigned long __must_check

_copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key);
@@ -55,28 +156,6 @@ copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned lo 
	return n;

}



union oac {

	unsigned int val;

	struct {

		struct {

			unsigned short key : 4;

			unsigned short	   : 4;

			unsigned short as  : 2;

			unsigned short	   : 4;

			unsigned short k   : 1;

			unsigned short a   : 1;

		} oac1;

		struct {

			unsigned short key : 4;

			unsigned short	   : 4;

			unsigned short as  : 2;

			unsigned short	   : 4;

			unsigned short k   : 1;

			unsigned short a   : 1;

		} oac2;

	};

};



int __noreturn __put_user_bad(void);



#ifdef CONFIG_KMSAN

arch/s390/lib/uaccess.c

+0 −90
Original line number Diff line number Diff line
@@ -31,51 +31,6 @@ void debug_user_asce(int exit) 
}

#endif /*CONFIG_DEBUG_ENTRY */



static unsigned long raw_copy_from_user_key(void *to, const void __user *from,

					    unsigned long size, unsigned long key)

{

	unsigned long rem;

	union oac spec = {

		.oac2.key = key,

		.oac2.as = PSW_BITS_AS_SECONDARY,

		.oac2.k = 1,

		.oac2.a = 1,

	};



	asm volatile(

		"	lr	0,%[spec]\n"

		"0:	mvcos	0(%[to]),0(%[from]),%[size]\n"

		"1:	jz	5f\n"

		"	algr	%[size],%[val]\n"

		"	slgr	%[from],%[val]\n"

		"	slgr	%[to],%[val]\n"

		"	j	0b\n"

		"2:	la	%[rem],4095(%[from])\n"	/* rem = from + 4095 */

		"	nr	%[rem],%[val]\n"	/* rem = (from + 4095) & -4096 */

		"	slgr	%[rem],%[from]\n"

		"	clgr	%[size],%[rem]\n"	/* copy crosses next page boundary? */

		"	jnh	6f\n"

		"3:	mvcos	0(%[to]),0(%[from]),%[rem]\n"

		"4:	slgr	%[size],%[rem]\n"

		"	j	6f\n"

		"5:	slgr	%[size],%[size]\n"

		"6:\n"

		EX_TABLE(0b, 2b)

		EX_TABLE(1b, 2b)

		EX_TABLE(3b, 6b)

		EX_TABLE(4b, 6b)

		: [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem)

		: [val] "a" (-4096UL), [spec] "d" (spec.val)

		: "cc", "memory", "0");

	return size;

}



unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)

{

	return raw_copy_from_user_key(to, from, n, 0);

}

EXPORT_SYMBOL(raw_copy_from_user);



unsigned long _copy_from_user_key(void *to, const void __user *from,

				  unsigned long n, unsigned long key)

{
@@ -93,51 +48,6 @@ unsigned long _copy_from_user_key(void *to, const void __user *from, 
}

EXPORT_SYMBOL(_copy_from_user_key);



static unsigned long raw_copy_to_user_key(void __user *to, const void *from,

					  unsigned long size, unsigned long key)

{

	unsigned long rem;

	union oac spec = {

		.oac1.key = key,

		.oac1.as = PSW_BITS_AS_SECONDARY,

		.oac1.k = 1,

		.oac1.a = 1,

	};



	asm volatile(

		"	lr	0,%[spec]\n"

		"0:	mvcos	0(%[to]),0(%[from]),%[size]\n"

		"1:	jz	5f\n"

		"	algr	%[size],%[val]\n"

		"	slgr	%[to],%[val]\n"

		"	slgr	%[from],%[val]\n"

		"	j	0b\n"

		"2:	la	%[rem],4095(%[to])\n"	/* rem = to + 4095 */

		"	nr	%[rem],%[val]\n"	/* rem = (to + 4095) & -4096 */

		"	slgr	%[rem],%[to]\n"

		"	clgr	%[size],%[rem]\n"	/* copy crosses next page boundary? */

		"	jnh	6f\n"

		"3:	mvcos	0(%[to]),0(%[from]),%[rem]\n"

		"4:	slgr	%[size],%[rem]\n"

		"	j	6f\n"

		"5:	slgr	%[size],%[size]\n"

		"6:\n"

		EX_TABLE(0b, 2b)

		EX_TABLE(1b, 2b)

		EX_TABLE(3b, 6b)

		EX_TABLE(4b, 6b)

		: [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem)

		: [val] "a" (-4096UL), [spec] "d" (spec.val)

		: "cc", "memory", "0");

	return size;

}



unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)

{

	return raw_copy_to_user_key(to, from, n, 0);

}

EXPORT_SYMBOL(raw_copy_to_user);



unsigned long _copy_to_user_key(void __user *to, const void *from,

				unsigned long n, unsigned long key)

{