Unverified Commit c40fef85 authored by Sami Tolvanen's avatar Sami Tolvanen Committed by Palmer Dabbelt
Browse files

riscv: Use separate IRQ shadow call stacks 



When both CONFIG_IRQ_STACKS and SCS are enabled, also use a separate
per-CPU shadow call stack.

Signed-off-by: default avatarSami Tolvanen <samitolvanen@google.com>
Tested-by: default avatarNathan Chancellor <nathan@kernel.org>
Link: https://lore.kernel.org/r/20230927224757.1154247-13-samitolvanen@google.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent d1584d79

arch/riscv/include/asm/scs.h

+7 −0
Original line number Diff line number Diff line
@@ -13,6 +13,11 @@ 
	XIP_FIXUP_OFFSET gp

.endm



/* Load the per-CPU IRQ shadow call stack to gp. */

.macro scs_load_irq_stack tmp

	load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp

.endm



/* Load task_scs_sp(current) to gp. */

.macro scs_load_current

	REG_L	gp, TASK_TI_SCS_SP(tp)
@@ -34,6 +39,8 @@ 


.macro scs_load_init_stack

.endm

.macro scs_load_irq_stack tmp

.endm

.macro scs_load_current

.endm

.macro scs_load_current_if_task_changed prev

arch/riscv/kernel/entry.S

+7 −0
Original line number Diff line number Diff line
@@ -237,12 +237,19 @@ SYM_FUNC_START(call_on_irq_stack) 
	REG_S	s0, STACKFRAME_FP(sp)

	addi	s0, sp, STACKFRAME_SIZE_ON_STACK



	/* Switch to the per-CPU shadow call stack */

	scs_save_current

	scs_load_irq_stack t0



	/* Switch to the per-CPU IRQ stack and call the handler */

	load_per_cpu t0, irq_stack_ptr, t1

	li	t1, IRQ_STACK_SIZE

	add	sp, t0, t1

	jalr	a1



	/* Switch back to the thread shadow call stack */

	scs_load_current



	/* Switch back to the thread stack and restore ra and s0 */

	addi	sp, s0, -STACKFRAME_SIZE_ON_STACK

	REG_L	ra, STACKFRAME_RA(sp)

arch/riscv/kernel/irq.c

+21 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
#include <linux/irqchip.h>

#include <linux/irqdomain.h>

#include <linux/module.h>

#include <linux/scs.h>

#include <linux/seq_file.h>

#include <asm/sbi.h>

#include <asm/smp.h>
@@ -34,6 +35,24 @@ EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode); 
#ifdef CONFIG_IRQ_STACKS

#include <asm/irq_stack.h>



DECLARE_PER_CPU(ulong *, irq_shadow_call_stack_ptr);



#ifdef CONFIG_SHADOW_CALL_STACK

DEFINE_PER_CPU(ulong *, irq_shadow_call_stack_ptr);

#endif



static void init_irq_scs(void)

{

	int cpu;



	if (!scs_is_enabled())

		return;



	for_each_possible_cpu(cpu)

		per_cpu(irq_shadow_call_stack_ptr, cpu) =

			scs_alloc(cpu_to_node(cpu));

}



DEFINE_PER_CPU(ulong *, irq_stack_ptr);



#ifdef CONFIG_VMAP_STACK
@@ -76,6 +95,7 @@ void do_softirq_own_stack(void) 
#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */



#else

static void init_irq_scs(void) {}

static void init_irq_stacks(void) {}

#endif /* CONFIG_IRQ_STACKS */
@@ -87,6 +107,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) 


void __init init_IRQ(void)

{

	init_irq_scs();

	init_irq_stacks();

	irqchip_init();

	if (!handle_arch_irq)