Commit 5b3d6103 authored by Charlie Jenkins's avatar Charlie Jenkins Committed by Thomas Gleixner
Browse files

riscv: entry: Split ret_from_fork() into user and kernel 



This function was unified into a single function in commit ab9164da
("riscv: entry: Consolidate ret_from_kernel_thread into ret_from_fork").
However that imposed a performance degradation.

Partially reverting this commit to have ret_from_fork() split again,
results in a 1% increase on the number of times fork is able to be called
per second.

Signed-off-by: default avatarCharlie Jenkins <charlie@rivosinc.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Link: https://lore.kernel.org/all/20250320-riscv_optimize_entry-v6-2-63e187e26041@rivosinc.com
parent f955aa87

arch/riscv/include/asm/asm-prototypes.h

+2 −1
Original line number Diff line number Diff line
@@ -52,7 +52,8 @@ DECLARE_DO_ERROR_INFO(do_trap_ecall_s); 
DECLARE_DO_ERROR_INFO(do_trap_ecall_m);

DECLARE_DO_ERROR_INFO(do_trap_break);



asmlinkage void ret_from_fork(void *fn_arg, int (*fn)(void *), struct pt_regs *regs);

asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs);

asmlinkage void ret_from_fork_user(struct pt_regs *regs);

asmlinkage void handle_bad_stack(struct pt_regs *regs);

asmlinkage void do_page_fault(struct pt_regs *regs);

asmlinkage void do_irq(struct pt_regs *regs);

arch/riscv/kernel/entry.S

+10 −3
Original line number Diff line number Diff line
@@ -319,14 +319,21 @@ SYM_CODE_END(handle_kernel_stack_overflow) 
ASM_NOKPROBE(handle_kernel_stack_overflow)

#endif



SYM_CODE_START(ret_from_fork_asm)

SYM_CODE_START(ret_from_fork_kernel_asm)

	call schedule_tail

	move a0, s1 /* fn_arg */

	move a1, s0 /* fn */

	move a2, sp /* pt_regs */

	call ret_from_fork

	call ret_from_fork_kernel

	j ret_from_exception

SYM_CODE_END(ret_from_fork_asm)

SYM_CODE_END(ret_from_fork_kernel_asm)



SYM_CODE_START(ret_from_fork_user_asm)

	call schedule_tail

	move a0, sp /* pt_regs */

	call ret_from_fork_user

	j ret_from_exception

SYM_CODE_END(ret_from_fork_user_asm)



#ifdef CONFIG_IRQ_STACKS

/*

arch/riscv/kernel/process.c

+11 −6
Original line number Diff line number Diff line
@@ -38,7 +38,8 @@ unsigned long __stack_chk_guard __read_mostly; 
EXPORT_SYMBOL(__stack_chk_guard);

#endif



extern asmlinkage void ret_from_fork_asm(void);

extern asmlinkage void ret_from_fork_kernel_asm(void);

extern asmlinkage void ret_from_fork_user_asm(void);



void noinstr arch_cpu_idle(void)

{
@@ -208,14 +209,18 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) 
	return 0;

}



asmlinkage void ret_from_fork(void *fn_arg, int (*fn)(void *), struct pt_regs *regs)

asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs)

{

	if (unlikely(fn))

	fn(fn_arg);



	syscall_exit_to_user_mode(regs);

}



asmlinkage void ret_from_fork_user(struct pt_regs *regs)

{

	syscall_exit_to_user_mode(regs);

}



int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)

{

	unsigned long clone_flags = args->flags;
@@ -238,6 +243,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) 


		p->thread.s[0] = (unsigned long)args->fn;

		p->thread.s[1] = (unsigned long)args->fn_arg;

		p->thread.ra = (unsigned long)ret_from_fork_kernel_asm;

	} else {

		*childregs = *(current_pt_regs());

		/* Turn off status.VS */
@@ -247,12 +253,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) 
		if (clone_flags & CLONE_SETTLS)

			childregs->tp = tls;

		childregs->a0 = 0; /* Return value of fork() */

		p->thread.s[0] = 0;

		p->thread.ra = (unsigned long)ret_from_fork_user_asm;

	}

	p->thread.riscv_v_flags = 0;

	if (has_vector() || has_xtheadvector())

		riscv_v_thread_alloc(p);

	p->thread.ra = (unsigned long)ret_from_fork_asm;

	p->thread.sp = (unsigned long)childregs; /* kernel sp */

	return 0;

}