Unverified Commit 82982fdd authored by Sami Tolvanen's avatar Sami Tolvanen Committed by Palmer Dabbelt
Browse files

riscv: Deduplicate IRQ stack switching 



With CONFIG_IRQ_STACKS, we switch to a separate per-CPU IRQ stack
before calling handle_riscv_irq or __do_softirq. We currently
have duplicate inline assembly snippets for stack switching in
both code paths. Now that we can access per-CPU variables in
assembly, implement call_on_irq_stack in assembly, and use that
instead of redundant inline assembly.

Signed-off-by: default avatarSami Tolvanen <samitolvanen@google.com>
Tested-by: default avatarNathan Chancellor <nathan@kernel.org>
Reviewed-by: default avatarGuo Ren <guoren@kernel.org>
Link: https://lore.kernel.org/r/20230927224757.1154247-10-samitolvanen@google.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent be97d0db

arch/riscv/include/asm/asm.h

+5 −0
Original line number Diff line number Diff line
@@ -104,6 +104,11 @@ 
.endm

#endif /* CONFIG_SMP */



.macro load_per_cpu dst ptr tmp

	asm_per_cpu \dst \ptr \tmp

	REG_L \dst, 0(\dst)

.endm



	/* save all GPs except x1 ~ x5 */

	.macro save_from_x6_to_x31

	REG_S x6,  PT_T1(sp)

arch/riscv/include/asm/irq_stack.h

+3 −0
Original line number Diff line number Diff line
@@ -12,6 +12,9 @@ 


DECLARE_PER_CPU(ulong *, irq_stack_ptr);



asmlinkage void call_on_irq_stack(struct pt_regs *regs,

				  void (*func)(struct pt_regs *));



#ifdef CONFIG_VMAP_STACK

/*

 * To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd

arch/riscv/kernel/asm-offsets.c

+5 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
#include <asm/thread_info.h>

#include <asm/ptrace.h>

#include <asm/cpu_ops_sbi.h>

#include <asm/stacktrace.h>

#include <asm/suspend.h>



void asm_offsets(void);
@@ -480,4 +481,8 @@ void asm_offsets(void) 
	OFFSET(KERNEL_MAP_VIRT_ADDR, kernel_mapping, virt_addr);

	OFFSET(SBI_HART_BOOT_TASK_PTR_OFFSET, sbi_hart_boot_data, task_ptr);

	OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr);



	DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN));

	OFFSET(STACKFRAME_FP, stackframe, fp);

	OFFSET(STACKFRAME_RA, stackframe, ra);

}

arch/riscv/kernel/entry.S

+30 −0
Original line number Diff line number Diff line
@@ -218,6 +218,36 @@ SYM_CODE_START(ret_from_fork) 
	tail syscall_exit_to_user_mode

SYM_CODE_END(ret_from_fork)



#ifdef CONFIG_IRQ_STACKS

/*

 * void call_on_irq_stack(struct pt_regs *regs,

 * 		          void (*func)(struct pt_regs *));

 *

 * Calls func(regs) using the per-CPU IRQ stack.

 */

SYM_FUNC_START(call_on_irq_stack)

	/* Create a frame record to save ra and s0 (fp) */

	addi	sp, sp, -STACKFRAME_SIZE_ON_STACK

	REG_S	ra, STACKFRAME_RA(sp)

	REG_S	s0, STACKFRAME_FP(sp)

	addi	s0, sp, STACKFRAME_SIZE_ON_STACK



	/* Switch to the per-CPU IRQ stack and call the handler */

	load_per_cpu t0, irq_stack_ptr, t1

	li	t1, IRQ_STACK_SIZE

	add	sp, t0, t1

	jalr	a1



	/* Switch back to the thread stack and restore ra and s0 */

	addi	sp, s0, -STACKFRAME_SIZE_ON_STACK

	REG_L	ra, STACKFRAME_RA(sp)

	REG_L	s0, STACKFRAME_FP(sp)

	addi	sp, sp, STACKFRAME_SIZE_ON_STACK



	ret

SYM_FUNC_END(call_on_irq_stack)

#endif /* CONFIG_IRQ_STACKS */



/*

 * Integer register context switch

 * The callee-saved registers must be saved and restored.

arch/riscv/kernel/irq.c

+8 −27
Original line number Diff line number Diff line
@@ -61,35 +61,16 @@ static void init_irq_stacks(void) 
#endif /* CONFIG_VMAP_STACK */



#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK

static void ___do_softirq(struct pt_regs *regs)

{

	__do_softirq();

}



void do_softirq_own_stack(void)

{

#ifdef CONFIG_IRQ_STACKS

	if (on_thread_stack()) {

		ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())

					+ IRQ_STACK_SIZE/sizeof(ulong);

		__asm__ __volatile(

		"addi	sp, sp, -"RISCV_SZPTR  "\n"

		REG_S"  ra, (sp)		\n"

		"addi	sp, sp, -"RISCV_SZPTR  "\n"

		REG_S"  s0, (sp)		\n"

		"addi	s0, sp, 2*"RISCV_SZPTR "\n"

		"move	sp, %[sp]		\n"

		"call	__do_softirq		\n"

		"addi	sp, s0, -2*"RISCV_SZPTR"\n"

		REG_L"  s0, (sp)		\n"

		"addi	sp, sp, "RISCV_SZPTR   "\n"

		REG_L"  ra, (sp)		\n"

		"addi	sp, sp, "RISCV_SZPTR   "\n"

		:

		: [sp] "r" (sp)

		: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",

		  "t0", "t1", "t2", "t3", "t4", "t5", "t6",

#ifndef CONFIG_FRAME_POINTER

		  "s0",

#endif

		  "memory");

	} else

#endif

	if (on_thread_stack())

		call_on_irq_stack(NULL, ___do_softirq);

	else

		__do_softirq();

}

#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */