Commit 385f72c8 authored by Brian Gerst's avatar Brian Gerst Committed by Ingo Molnar
Browse files

x86/percpu: Move top_of_stack to percpu hot section 



No functional change.

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-9-brgerst@gmail.com
parent c6a09180

arch/x86/entry/entry_32.S

+2 −2
Original line number Diff line number Diff line
@@ -1153,7 +1153,7 @@ SYM_CODE_START(asm_exc_nmi) 
	 * is using the thread stack right now, so it's safe for us to use it.

	 */

	movl	%esp, %ebx

	movl	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp

	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esp

	call	exc_nmi

	movl	%ebx, %esp
@@ -1217,7 +1217,7 @@ SYM_CODE_START(rewind_stack_and_make_dead) 
	/* Prevent any naive code from trying to unwind to our caller. */

	xorl	%ebp, %ebp



	movl	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi

	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esi

	leal	-TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp



	call	make_task_dead

arch/x86/entry/entry_64.S

+3 −3
Original line number Diff line number Diff line
@@ -92,7 +92,7 @@ SYM_CODE_START(entry_SYSCALL_64) 
	/* tss.sp2 is scratch space. */

	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp



SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR
@@ -1168,7 +1168,7 @@ SYM_CODE_START(asm_exc_nmi) 
	FENCE_SWAPGS_USER_ENTRY

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx

	movq	%rsp, %rdx

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	UNWIND_HINT_IRET_REGS base=%rdx offset=8

	pushq	5*8(%rdx)	/* pt_regs->ss */

	pushq	4*8(%rdx)	/* pt_regs->rsp */
@@ -1486,7 +1486,7 @@ SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead) 
	/* Prevent any naive code from trying to unwind to our caller. */

	xorl	%ebp, %ebp



	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax

	leaq	-PTREGS_SIZE(%rax), %rsp

	UNWIND_HINT_REGS

arch/x86/entry/entry_64_compat.S

+2 −2
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ SYM_CODE_START(entry_SYSENTER_compat) 
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax

	popq	%rax



	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp



	/* Construct struct pt_regs on stack */

	pushq	$__USER_DS		/* pt_regs->ss */
@@ -193,7 +193,7 @@ SYM_CODE_START(entry_SYSCALL_compat) 
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp



	/* Switch to the kernel stack */

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp



SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR

arch/x86/include/asm/current.h

+0 −1
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ struct task_struct; 


struct pcpu_hot {

	struct task_struct	*current_task;

	unsigned long		top_of_stack;

};



DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);

arch/x86/include/asm/percpu.h

+1 −1
Original line number Diff line number Diff line
@@ -551,7 +551,7 @@ do { \ 
 * it is accessed while this_cpu_read_stable() allows the value to be cached.

 * this_cpu_read_stable() is more efficient and can be used if its value

 * is guaranteed to be valid across CPUs.  The current users include

 * pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are

 * pcpu_hot.current_task and cpu_current_top_of_stack, both of which are

 * actually per-thread variables implemented as per-CPU variables and

 * thus stable for the duration of the respective task.

 */