Commit 59bec00a authored by Uros Bizjak's avatar Uros Bizjak Committed by Ingo Molnar
Browse files

x86/percpu: Introduce %rip-relative addressing to PER_CPU_VAR() 



Introduce x86_64 %rip-relative addressing to the PER_CPU_VAR() macro.
Instructions using %rip-relative address operand are one byte shorter
than their absolute address counterparts and are also compatible with
position independent executable (-fpie) builds. The patch reduces
code size of a test kernel build by 150 bytes.

The PER_CPU_VAR() macro is intended to be applied to a symbol and should
not be used with register operands. Introduce the new __percpu macro and
use it in cmpxchg{8,16}b_emu.S instead.

Also add a missing function comment to this_cpu_cmpxchg8b_emu().

No functional changes intended.

Signed-off-by: default avatarUros Bizjak <ubizjak@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: linux-kernel@vger.kernel.org
Cc: Brian Gerst <brgerst@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Sean Christopherson <seanjc@google.com>
parent aa47f90c

arch/x86/include/asm/percpu.h

+8 −4
Original line number Diff line number Diff line
@@ -4,17 +4,21 @@ 


#ifdef CONFIG_X86_64

#define __percpu_seg		gs

#define __percpu_rel		(%rip)

#else

#define __percpu_seg		fs

#define __percpu_rel

#endif



#ifdef __ASSEMBLY__



#ifdef CONFIG_SMP

#define PER_CPU_VAR(var)	%__percpu_seg:var

#else /* ! SMP */

#define PER_CPU_VAR(var)	var

#endif	/* SMP */

#define __percpu		%__percpu_seg:

#else

#define __percpu

#endif



#define PER_CPU_VAR(var)	__percpu(var)__percpu_rel



#ifdef CONFIG_X86_64_SMP

#define INIT_PER_CPU_VAR(var)  init_per_cpu__##var

arch/x86/lib/cmpxchg16b_emu.S

+6 −6
Original line number Diff line number Diff line
@@ -23,14 +23,14 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu) 
	cli



	/* if (*ptr == old) */

	cmpq	PER_CPU_VAR(0(%rsi)), %rax

	cmpq	__percpu (%rsi), %rax

	jne	.Lnot_same

	cmpq	PER_CPU_VAR(8(%rsi)), %rdx

	cmpq	__percpu 8(%rsi), %rdx

	jne	.Lnot_same



	/* *ptr = new */

	movq	%rbx, PER_CPU_VAR(0(%rsi))

	movq	%rcx, PER_CPU_VAR(8(%rsi))

	movq	%rbx, __percpu (%rsi)

	movq	%rcx, __percpu 8(%rsi)



	/* set ZF in EFLAGS to indicate success */

	orl	$X86_EFLAGS_ZF, (%rsp)
@@ -42,8 +42,8 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu) 
	/* *ptr != old */



	/* old = *ptr */

	movq	PER_CPU_VAR(0(%rsi)), %rax

	movq	PER_CPU_VAR(8(%rsi)), %rdx

	movq	__percpu (%rsi), %rax

	movq	__percpu 8(%rsi), %rdx



	/* clear ZF in EFLAGS to indicate failure */

	andl	$(~X86_EFLAGS_ZF), (%rsp)

arch/x86/lib/cmpxchg8b_emu.S

+21 −9
Original line number Diff line number Diff line
@@ -24,12 +24,12 @@ SYM_FUNC_START(cmpxchg8b_emu) 
	pushfl

	cli



	cmpl	0(%esi), %eax

	cmpl	(%esi), %eax

	jne	.Lnot_same

	cmpl	4(%esi), %edx

	jne	.Lnot_same



	movl	%ebx, 0(%esi)

	movl	%ebx, (%esi)

	movl	%ecx, 4(%esi)



	orl	$X86_EFLAGS_ZF, (%esp)
@@ -38,7 +38,7 @@ SYM_FUNC_START(cmpxchg8b_emu) 
	RET



.Lnot_same:

	movl	0(%esi), %eax

	movl	(%esi), %eax

	movl	4(%esi), %edx



	andl	$(~X86_EFLAGS_ZF), (%esp)
@@ -53,18 +53,30 @@ EXPORT_SYMBOL(cmpxchg8b_emu) 


#ifndef CONFIG_UML



/*

 * Emulate 'cmpxchg8b %fs:(%rsi)'

 *

 * Inputs:

 * %esi : memory location to compare

 * %eax : low 32 bits of old value

 * %edx : high 32 bits of old value

 * %ebx : low 32 bits of new value

 * %ecx : high 32 bits of new value

 *

 * Notably this is not LOCK prefixed and is not safe against NMIs

 */

SYM_FUNC_START(this_cpu_cmpxchg8b_emu)



	pushfl

	cli



	cmpl	PER_CPU_VAR(0(%esi)), %eax

	cmpl	__percpu (%esi), %eax

	jne	.Lnot_same2

	cmpl	PER_CPU_VAR(4(%esi)), %edx

	cmpl	__percpu 4(%esi), %edx

	jne	.Lnot_same2



	movl	%ebx, PER_CPU_VAR(0(%esi))

	movl	%ecx, PER_CPU_VAR(4(%esi))

	movl	%ebx, __percpu (%esi)

	movl	%ecx, __percpu 4(%esi)



	orl	$X86_EFLAGS_ZF, (%esp)
@@ -72,8 +84,8 @@ SYM_FUNC_START(this_cpu_cmpxchg8b_emu) 
	RET



.Lnot_same2:

	movl	PER_CPU_VAR(0(%esi)), %eax

	movl	PER_CPU_VAR(4(%esi)), %edx

	movl	__percpu (%esi), %eax

	movl	__percpu 4(%esi), %edx



	andl	$(~X86_EFLAGS_ZF), (%esp)