Unverified Commit a894e8ed authored by Palmer Dabbelt's avatar Palmer Dabbelt
Browse files

Merge patch series "riscv: support kernel-mode Vector" 

Andy Chiu <andy.chiu@sifive.com> says:

This series provides support running Vector in kernel mode.
Additionally, kernel-mode Vector can be configured to run without
turnning off preemption on a CONFIG_PREEMPT kernel. Along with the
suport, we add Vector optimized copy_{to,from}_user. And provide a
simple threshold to decide when to run the vectorized functions.

We decided to drop vectorized memcpy/memset/memmove for the moment due
to the concern of memory side-effect in kernel_vector_begin(). The
detailed description can be found at v9[0]

This series is composed by 4 parts:
 patch 1-4: adds basic support for kernel-mode Vector
 patch 5: includes vectorized copy_{to,from}_user into the kernel
 patch 6: refactor context switch code in fpu [1]
 patch 7-10: provides some code refactors and support for preemptible
             kernel-mode Vector.

This series can be merged if we feel any part of {1~4, 5, 6, 7~10} is
mature enough.

This patch is tested on a QEMU with V and verified that booting, normal
userspace operations all work as usual with thresholds set to 0. Also,
we test by launching multiple kernel threads which continuously executes
and verifies Vector operations in the background. The module that tests
these operation is expected to be upstream later.

* b4-shazam-merge:
  riscv: vector: allow kernel-mode Vector with preemption
  riscv: vector: use kmem_cache to manage vector context
  riscv: vector: use a mask to write vstate_ctrl
  riscv: vector: do not pass task_struct into riscv_v_vstate_{save,restore}()
  riscv: fpu: drop SR_SD bit checking
  riscv: lib: vectorize copy_to_user/copy_from_user
  riscv: sched: defer restoring Vector context for user
  riscv: Add vector extension XOR implementation
  riscv: vector: make Vector always available for softirq context
  riscv: Add support for kernel mode vector

Link: https://lore.kernel.org/r/20240115055929.4736-1-andy.chiu@sifive.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parents d4abde52 2080ff94

arch/riscv/Kconfig

+22 −0
Original line number Diff line number Diff line
@@ -527,6 +527,28 @@ config RISCV_ISA_V_DEFAULT_ENABLE 


	  If you don't know what to do here, say Y.



config RISCV_ISA_V_UCOPY_THRESHOLD

	int "Threshold size for vectorized user copies"

	depends on RISCV_ISA_V

	default 768

	help

	  Prefer using vectorized copy_to_user()/copy_from_user() when the

	  workload size exceeds this value.



config RISCV_ISA_V_PREEMPTIVE

	bool "Run kernel-mode Vector with kernel preemption"

	depends on PREEMPTION

	depends on RISCV_ISA_V

	default y

	help

	  Usually, in-kernel SIMD routines are run with preemption disabled.

	  Functions which envoke long running SIMD thus must yield core's

	  vector unit to prevent blocking other tasks for too long.



	  This config allows kernel to run SIMD without explicitly disable

	  preemption. Enabling this config will result in higher memory

	  consumption due to the allocation of per-task's kernel Vector context.



config TOOLCHAIN_HAS_ZBB

	bool

	default y

arch/riscv/include/asm/asm-prototypes.h

+27 −0
Original line number Diff line number Diff line
@@ -9,6 +9,33 @@ long long __lshrti3(long long a, int b); 
long long __ashrti3(long long a, int b);

long long __ashlti3(long long a, int b);



#ifdef CONFIG_RISCV_ISA_V



#ifdef CONFIG_MMU

asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n);

#endif /* CONFIG_MMU  */



void xor_regs_2_(unsigned long bytes, unsigned long *__restrict p1,

		 const unsigned long *__restrict p2);

void xor_regs_3_(unsigned long bytes, unsigned long *__restrict p1,

		 const unsigned long *__restrict p2,

		 const unsigned long *__restrict p3);

void xor_regs_4_(unsigned long bytes, unsigned long *__restrict p1,

		 const unsigned long *__restrict p2,

		 const unsigned long *__restrict p3,

		 const unsigned long *__restrict p4);

void xor_regs_5_(unsigned long bytes, unsigned long *__restrict p1,

		 const unsigned long *__restrict p2,

		 const unsigned long *__restrict p3,

		 const unsigned long *__restrict p4,

		 const unsigned long *__restrict p5);



#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE

asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs);

asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs);

#endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */



#endif /* CONFIG_RISCV_ISA_V */



#define DECLARE_DO_ERROR_INFO(name)	asmlinkage void name(struct pt_regs *regs)

arch/riscv/include/asm/entry-common.h

+17 −0
Original line number Diff line number Diff line
@@ -4,6 +4,23 @@ 
#define _ASM_RISCV_ENTRY_COMMON_H



#include <asm/stacktrace.h>

#include <asm/thread_info.h>

#include <asm/vector.h>



static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,

						  unsigned long ti_work)

{

	if (ti_work & _TIF_RISCV_V_DEFER_RESTORE) {

		clear_thread_flag(TIF_RISCV_V_DEFER_RESTORE);

		/*

		 * We are already called with irq disabled, so go without

		 * keeping track of riscv_v_flags.

		 */

		riscv_v_vstate_restore(&current->thread.vstate, regs);

	}

}



#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare



void handle_page_fault(struct pt_regs *regs);

void handle_break(struct pt_regs *regs);

arch/riscv/include/asm/processor.h

+40 −1
Original line number Diff line number Diff line
@@ -73,6 +73,43 @@ 
struct task_struct;

struct pt_regs;



/*

 * We use a flag to track in-kernel Vector context. Currently the flag has the

 * following meaning:

 *

 *  - bit 0: indicates whether the in-kernel Vector context is active. The

 *    activation of this state disables the preemption. On a non-RT kernel, it

 *    also disable bh.

 *  - bits 8: is used for tracking preemptible kernel-mode Vector, when

 *    RISCV_ISA_V_PREEMPTIVE is enabled. Calling kernel_vector_begin() does not

 *    disable the preemption if the thread's kernel_vstate.datap is allocated.

 *    Instead, the kernel set this bit field. Then the trap entry/exit code

 *    knows if we are entering/exiting the context that owns preempt_v.

 *     - 0: the task is not using preempt_v

 *     - 1: the task is actively using preempt_v. But whether does the task own

 *          the preempt_v context is decided by bits in RISCV_V_CTX_DEPTH_MASK.

 *  - bit 16-23 are RISCV_V_CTX_DEPTH_MASK, used by context tracking routine

 *     when preempt_v starts:

 *     - 0: the task is actively using, and own preempt_v context.

 *     - non-zero: the task was using preempt_v, but then took a trap within.

 *       Thus, the task does not own preempt_v. Any use of Vector will have to

 *       save preempt_v, if dirty, and fallback to non-preemptible kernel-mode

 *       Vector.

 *  - bit 30: The in-kernel preempt_v context is saved, and requries to be

 *    restored when returning to the context that owns the preempt_v.

 *  - bit 31: The in-kernel preempt_v context is dirty, as signaled by the

 *    trap entry code. Any context switches out-of current task need to save

 *    it to the task's in-kernel V context. Also, any traps nesting on-top-of

 *    preempt_v requesting to use V needs a save.

 */

#define RISCV_V_CTX_DEPTH_MASK		0x00ff0000



#define RISCV_V_CTX_UNIT_DEPTH		0x00010000

#define RISCV_KERNEL_MODE_V		0x00000001

#define RISCV_PREEMPT_V			0x00000100

#define RISCV_PREEMPT_V_DIRTY		0x80000000

#define RISCV_PREEMPT_V_NEED_RESTORE	0x40000000



/* CPU-specific state of a task */

struct thread_struct {

	/* Callee-saved registers */
@@ -81,9 +118,11 @@ struct thread_struct { 
	unsigned long s[12];	/* s[0]: frame pointer */

	struct __riscv_d_ext_state fstate;

	unsigned long bad_cause;

	unsigned long vstate_ctrl;

	u32 riscv_v_flags;

	u32 vstate_ctrl;

	struct __riscv_v_ext_state vstate;

	unsigned long align_ctl;

	struct __riscv_v_ext_state kernel_vstate;

};



/* Whitelist the fstate from the task_struct for hardened usercopy */

arch/riscv/include/asm/simd.h

0 → 100644
+64 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>

 * Copyright (C) 2023 SiFive

 */



#ifndef __ASM_SIMD_H

#define __ASM_SIMD_H



#include <linux/compiler.h>

#include <linux/irqflags.h>

#include <linux/percpu.h>

#include <linux/preempt.h>

#include <linux/types.h>

#include <linux/thread_info.h>



#include <asm/vector.h>



#ifdef CONFIG_RISCV_ISA_V

/*

 * may_use_simd - whether it is allowable at this time to issue vector

 *                instructions or access the vector register file

 *

 * Callers must not assume that the result remains true beyond the next

 * preempt_enable() or return from softirq context.

 */

static __must_check inline bool may_use_simd(void)

{

	/*

	 * RISCV_KERNEL_MODE_V is only set while preemption is disabled,

	 * and is clear whenever preemption is enabled.

	 */

	if (in_hardirq() || in_nmi())

		return false;



	/*

	 * Nesting is acheived in preempt_v by spreading the control for

	 * preemptible and non-preemptible kernel-mode Vector into two fields.

	 * Always try to match with prempt_v if kernel V-context exists. Then,

	 * fallback to check non preempt_v if nesting happens, or if the config

	 * is not set.

	 */

	if (IS_ENABLED(CONFIG_RISCV_ISA_V_PREEMPTIVE) && current->thread.kernel_vstate.datap) {

		if (!riscv_preempt_v_started(current))

			return true;

	}

	/*

	 * Non-preemptible kernel-mode Vector temporarily disables bh. So we

	 * must not return true on irq_disabled(). Otherwise we would fail the

	 * lockdep check calling local_bh_enable()

	 */

	return !irqs_disabled() && !(riscv_v_flags() & RISCV_KERNEL_MODE_V);

}



#else /* ! CONFIG_RISCV_ISA_V */



static __must_check inline bool may_use_simd(void)

{

	return false;

}



#endif /* ! CONFIG_RISCV_ISA_V */



#endif