Unverified Commit 2080ff94 authored by Andy Chiu's avatar Andy Chiu Committed by Palmer Dabbelt
Browse files

riscv: vector: allow kernel-mode Vector with preemption 



Add kernel_vstate to keep track of kernel-mode Vector registers when
trap introduced context switch happens. Also, provide riscv_v_flags to
let context save/restore routine track context status. Context tracking
happens whenever the core starts its in-kernel Vector executions. An
active (dirty) kernel task's V contexts will be saved to memory whenever
a trap-introduced context switch happens. Or, when a softirq, which
happens to nest on top of it, uses Vector. Context retoring happens when
the execution transfer back to the original Kernel context where it
first enable preempt_v.

Also, provide a config CONFIG_RISCV_ISA_V_PREEMPTIVE to give users an
option to disable preemptible kernel-mode Vector at build time. Users
with constraint memory may want to disable this config as preemptible
kernel-mode Vector needs extra space for tracking of per thread's
kernel-mode V context. Or, users might as well want to disable it if all
kernel-mode Vector code is time sensitive and cannot tolerate context
switch overhead.

Signed-off-by: default avatarAndy Chiu <andy.chiu@sifive.com>
Tested-by: default avatarBjörn Töpel <bjorn@rivosinc.com>
Tested-by: default avatarLad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
Link: https://lore.kernel.org/r/20240115055929.4736-11-andy.chiu@sifive.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent bd446f5d

arch/riscv/Kconfig

+14 −0
Original line number Diff line number Diff line
@@ -533,6 +533,20 @@ config RISCV_ISA_V_UCOPY_THRESHOLD 
	  Prefer using vectorized copy_to_user()/copy_from_user() when the

	  workload size exceeds this value.



config RISCV_ISA_V_PREEMPTIVE

	bool "Run kernel-mode Vector with kernel preemption"

	depends on PREEMPTION

	depends on RISCV_ISA_V

	default y

	help

	  Usually, in-kernel SIMD routines are run with preemption disabled.

	  Functions which envoke long running SIMD thus must yield core's

	  vector unit to prevent blocking other tasks for too long.



	  This config allows kernel to run SIMD without explicitly disable

	  preemption. Enabling this config will result in higher memory

	  consumption due to the allocation of per-task's kernel Vector context.



config TOOLCHAIN_HAS_ZBB

	bool

	default y

arch/riscv/include/asm/asm-prototypes.h

+5 −0
Original line number Diff line number Diff line
@@ -30,6 +30,11 @@ void xor_regs_5_(unsigned long bytes, unsigned long *__restrict p1, 
		 const unsigned long *__restrict p4,

		 const unsigned long *__restrict p5);



#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE

asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs);

asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs);

#endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */



#endif /* CONFIG_RISCV_ISA_V */



#define DECLARE_DO_ERROR_INFO(name)	asmlinkage void name(struct pt_regs *regs)

arch/riscv/include/asm/processor.h

+29 −1
Original line number Diff line number Diff line
@@ -80,8 +80,35 @@ struct pt_regs; 
 *  - bit 0: indicates whether the in-kernel Vector context is active. The

 *    activation of this state disables the preemption. On a non-RT kernel, it

 *    also disable bh.

 *  - bits 8: is used for tracking preemptible kernel-mode Vector, when

 *    RISCV_ISA_V_PREEMPTIVE is enabled. Calling kernel_vector_begin() does not

 *    disable the preemption if the thread's kernel_vstate.datap is allocated.

 *    Instead, the kernel set this bit field. Then the trap entry/exit code

 *    knows if we are entering/exiting the context that owns preempt_v.

 *     - 0: the task is not using preempt_v

 *     - 1: the task is actively using preempt_v. But whether does the task own

 *          the preempt_v context is decided by bits in RISCV_V_CTX_DEPTH_MASK.

 *  - bit 16-23 are RISCV_V_CTX_DEPTH_MASK, used by context tracking routine

 *     when preempt_v starts:

 *     - 0: the task is actively using, and own preempt_v context.

 *     - non-zero: the task was using preempt_v, but then took a trap within.

 *       Thus, the task does not own preempt_v. Any use of Vector will have to

 *       save preempt_v, if dirty, and fallback to non-preemptible kernel-mode

 *       Vector.

 *  - bit 30: The in-kernel preempt_v context is saved, and requries to be

 *    restored when returning to the context that owns the preempt_v.

 *  - bit 31: The in-kernel preempt_v context is dirty, as signaled by the

 *    trap entry code. Any context switches out-of current task need to save

 *    it to the task's in-kernel V context. Also, any traps nesting on-top-of

 *    preempt_v requesting to use V needs a save.

 */

#define RISCV_KERNEL_MODE_V	0x1

#define RISCV_V_CTX_DEPTH_MASK		0x00ff0000



#define RISCV_V_CTX_UNIT_DEPTH		0x00010000

#define RISCV_KERNEL_MODE_V		0x00000001

#define RISCV_PREEMPT_V			0x00000100

#define RISCV_PREEMPT_V_DIRTY		0x80000000

#define RISCV_PREEMPT_V_NEED_RESTORE	0x40000000



/* CPU-specific state of a task */

struct thread_struct {
@@ -95,6 +122,7 @@ struct thread_struct { 
	u32 vstate_ctrl;

	struct __riscv_v_ext_state vstate;

	unsigned long align_ctl;

	struct __riscv_v_ext_state kernel_vstate;

};



/* Whitelist the fstate from the task_struct for hardened usercopy */

arch/riscv/include/asm/simd.h

+21 −5
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
#include <linux/percpu.h>

#include <linux/preempt.h>

#include <linux/types.h>

#include <linux/thread_info.h>



#include <asm/vector.h>
@@ -28,12 +29,27 @@ static __must_check inline bool may_use_simd(void) 
	/*

	 * RISCV_KERNEL_MODE_V is only set while preemption is disabled,

	 * and is clear whenever preemption is enabled.

	 *

	 * Kernel-mode Vector temporarily disables bh. So we must not return

	 * true on irq_disabled(). Otherwise we would fail the lockdep check

	 * calling local_bh_enable()

	 */

	return !in_hardirq() && !in_nmi() && !irqs_disabled() && !(riscv_v_flags() & RISCV_KERNEL_MODE_V);

	if (in_hardirq() || in_nmi())

		return false;



	/*

	 * Nesting is acheived in preempt_v by spreading the control for

	 * preemptible and non-preemptible kernel-mode Vector into two fields.

	 * Always try to match with prempt_v if kernel V-context exists. Then,

	 * fallback to check non preempt_v if nesting happens, or if the config

	 * is not set.

	 */

	if (IS_ENABLED(CONFIG_RISCV_ISA_V_PREEMPTIVE) && current->thread.kernel_vstate.datap) {

		if (!riscv_preempt_v_started(current))

			return true;

	}

	/*

	 * Non-preemptible kernel-mode Vector temporarily disables bh. So we

	 * must not return true on irq_disabled(). Otherwise we would fail the

	 * lockdep check calling local_bh_enable()

	 */

	return !irqs_disabled() && !(riscv_v_flags() & RISCV_KERNEL_MODE_V);

}



#else /* ! CONFIG_RISCV_ISA_V */

arch/riscv/include/asm/vector.h

+54 −4
Original line number Diff line number Diff line
@@ -28,10 +28,11 @@ void get_cpu_vector_context(void); 
void put_cpu_vector_context(void);

void riscv_v_thread_free(struct task_struct *tsk);

void __init riscv_v_setup_ctx_cache(void);

void riscv_v_thread_alloc(struct task_struct *tsk);



static inline u32 riscv_v_flags(void)

{

	return current->thread.riscv_v_flags;

	return READ_ONCE(current->thread.riscv_v_flags);

}



static __always_inline bool has_vector(void)
@@ -200,13 +201,61 @@ static inline void riscv_v_vstate_set_restore(struct task_struct *task, 
	}

}



#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE

static inline bool riscv_preempt_v_dirty(struct task_struct *task)

{

	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_DIRTY);

}



static inline bool riscv_preempt_v_restore(struct task_struct *task)

{

	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_NEED_RESTORE);

}



static inline void riscv_preempt_v_clear_dirty(struct task_struct *task)

{

	barrier();

	task->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_DIRTY;

}



static inline void riscv_preempt_v_set_restore(struct task_struct *task)

{

	barrier();

	task->thread.riscv_v_flags |= RISCV_PREEMPT_V_NEED_RESTORE;

}



static inline bool riscv_preempt_v_started(struct task_struct *task)

{

	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V);

}



#else /* !CONFIG_RISCV_ISA_V_PREEMPTIVE */

static inline bool riscv_preempt_v_dirty(struct task_struct *task) { return false; }

static inline bool riscv_preempt_v_restore(struct task_struct *task) { return false; }

static inline bool riscv_preempt_v_started(struct task_struct *task) { return false; }

#define riscv_preempt_v_clear_dirty(tsk)	do {} while (0)

#define riscv_preempt_v_set_restore(tsk)	do {} while (0)

#endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */



static inline void __switch_to_vector(struct task_struct *prev,

				      struct task_struct *next)

{

	struct pt_regs *regs;



	if (riscv_preempt_v_started(prev)) {

		if (riscv_preempt_v_dirty(prev)) {

			__riscv_v_vstate_save(&prev->thread.kernel_vstate,

					      prev->thread.kernel_vstate.datap);

			riscv_preempt_v_clear_dirty(prev);

		}

	} else {

		regs = task_pt_regs(prev);

		riscv_v_vstate_save(&prev->thread.vstate, regs);

	}



	if (riscv_preempt_v_started(next))

		riscv_preempt_v_set_restore(next);

	else

		riscv_v_vstate_set_restore(next, task_pt_regs(next));

}
@@ -231,6 +280,7 @@ static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; } 
#define riscv_v_vstate_on(regs)			do {} while (0)

#define riscv_v_thread_free(tsk)		do {} while (0)

#define  riscv_v_setup_ctx_cache()		do {} while (0)

#define riscv_v_thread_alloc(tsk)		do {} while (0)



#endif /* CONFIG_RISCV_ISA_V */