Commit c6a09180 authored by Brian Gerst's avatar Brian Gerst Committed by Ingo Molnar
Browse files

x86/irq: Move irq stacks to percpu hot section 



No functional change.

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-8-brgerst@gmail.com
parent c8f1ac2b

arch/x86/include/asm/current.h

+0 −6
Original line number Diff line number Diff line
@@ -15,12 +15,6 @@ struct task_struct; 
struct pcpu_hot {

	struct task_struct	*current_task;

	unsigned long		top_of_stack;

	void			*hardirq_stack_ptr;

#ifdef CONFIG_X86_64

	bool			hardirq_stack_inuse;

#else

	void			*softirq_stack_ptr;

#endif

};



DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);

arch/x86/include/asm/irq_stack.h

+6 −6
Original line number Diff line number Diff line
@@ -116,7 +116,7 @@ 
	ASM_CALL_ARG2



#define call_on_irqstack(func, asm_call, argconstr...)			\

	call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr),	\

	call_on_stack(__this_cpu_read(hardirq_stack_ptr),		\

		      func, asm_call, argconstr)



/* Macros to assert type correctness for run_*_on_irqstack macros */
@@ -135,7 +135,7 @@ 
	 * User mode entry and interrupt on the irq stack do not	\

	 * switch stacks. If from user mode the task stack is empty.	\

	 */								\

	if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \

	if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {	\

		irq_enter_rcu();					\

		func(c_args);						\

		irq_exit_rcu();						\
@@ -146,9 +146,9 @@ 
		 * places. Invoke the stack switch macro with the call	\

		 * sequence which matches the above direct invocation.	\

		 */							\

		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);	\

		__this_cpu_write(hardirq_stack_inuse, true);		\

		call_on_irqstack(func, asm_call, constr);		\

		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);	\

		__this_cpu_write(hardirq_stack_inuse, false);		\

	}								\

}
@@ -212,9 +212,9 @@ 
 */

#define do_softirq_own_stack()						\

{									\

	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);		\

	__this_cpu_write(hardirq_stack_inuse, true);			\

	call_on_irqstack(__do_softirq, ASM_CALL_ARG0);			\

	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);		\

	__this_cpu_write(hardirq_stack_inuse, false);			\

}



#endif

arch/x86/include/asm/processor.h

+7 −0
Original line number Diff line number Diff line
@@ -415,6 +415,13 @@ struct irq_stack { 
	char		stack[IRQ_STACK_SIZE];

} __aligned(IRQ_STACK_SIZE);



DECLARE_PER_CPU_CACHE_HOT(struct irq_stack *, hardirq_stack_ptr);

#ifdef CONFIG_X86_64

DECLARE_PER_CPU_CACHE_HOT(bool, hardirq_stack_inuse);

#else

DECLARE_PER_CPU_CACHE_HOT(struct irq_stack *, softirq_stack_ptr);

#endif



#ifdef CONFIG_X86_64

static inline unsigned long cpu_kernelmode_gs_base(int cpu)

{

arch/x86/kernel/dumpstack_32.c

+2 −2
Original line number Diff line number Diff line
@@ -37,7 +37,7 @@ const char *stack_type_name(enum stack_type type) 


static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);

	unsigned long *begin = (unsigned long *)this_cpu_read(hardirq_stack_ptr);

	unsigned long *end   = begin + (THREAD_SIZE / sizeof(long));



	/*
@@ -62,7 +62,7 @@ static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info) 


static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.softirq_stack_ptr);

	unsigned long *begin = (unsigned long *)this_cpu_read(softirq_stack_ptr);

	unsigned long *end   = begin + (THREAD_SIZE / sizeof(long));



	/*

arch/x86/kernel/dumpstack_64.c

+1 −1
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac 


static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long *end = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);

	unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr);

	unsigned long *begin;



	/*