Commit a1e4cc01 authored by Brian Gerst's avatar Brian Gerst Committed by Ingo Molnar
Browse files

x86/percpu: Move current_task to percpu hot section 



No functional change.

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-10-brgerst@gmail.com
parent 385f72c8

arch/x86/include/asm/current.h

+6 −11
Original line number Diff line number Diff line
@@ -12,22 +12,17 @@ 


struct task_struct;



struct pcpu_hot {

	struct task_struct	*current_task;

};



DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);



/* const-qualified alias to pcpu_hot, aliased by linker. */

DECLARE_PER_CPU_CACHE_HOT(const struct pcpu_hot __percpu_seg_override,

			const_pcpu_hot);

DECLARE_PER_CPU_CACHE_HOT(struct task_struct *, current_task);

/* const-qualified alias provided by the linker. */

DECLARE_PER_CPU_CACHE_HOT(struct task_struct * const __percpu_seg_override,

			  const_current_task);



static __always_inline struct task_struct *get_current(void)

{

	if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))

		return this_cpu_read_const(const_pcpu_hot.current_task);

		return this_cpu_read_const(const_current_task);



	return this_cpu_read_stable(pcpu_hot.current_task);

	return this_cpu_read_stable(current_task);

}



#define current get_current()

arch/x86/include/asm/percpu.h

+1 −1
Original line number Diff line number Diff line
@@ -551,7 +551,7 @@ do { \ 
 * it is accessed while this_cpu_read_stable() allows the value to be cached.

 * this_cpu_read_stable() is more efficient and can be used if its value

 * is guaranteed to be valid across CPUs.  The current users include

 * pcpu_hot.current_task and cpu_current_top_of_stack, both of which are

 * current_task and cpu_current_top_of_stack, both of which are

 * actually per-thread variables implemented as per-CPU variables and

 * thus stable for the duration of the respective task.

 */

arch/x86/kernel/asm-offsets.c

+0 −1
Original line number Diff line number Diff line
@@ -107,7 +107,6 @@ static void __used common(void) 
	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);

	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);

	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);

	OFFSET(X86_current_task, pcpu_hot, current_task);

#if IS_ENABLED(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64)

	/* Offset for fields in aria_ctx */

	BLANK();

arch/x86/kernel/cpu/common.c

+3 −5
Original line number Diff line number Diff line
@@ -2064,11 +2064,9 @@ static __init int setup_setcpuid(char *arg) 
}

__setup("setcpuid=", setup_setcpuid);



DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = {

	.current_task	= &init_task,

};

EXPORT_PER_CPU_SYMBOL(pcpu_hot);

EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);

DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;

EXPORT_PER_CPU_SYMBOL(current_task);

EXPORT_PER_CPU_SYMBOL(const_current_task);



DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;

EXPORT_PER_CPU_SYMBOL(__preempt_count);

arch/x86/kernel/head_64.S

+2 −2
Original line number Diff line number Diff line
@@ -322,7 +322,7 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL) 
	 *

	 * RDX contains the per-cpu offset

	 */

	movq	pcpu_hot + X86_current_task(%rdx), %rax

	movq	current_task(%rdx), %rax

	movq	TASK_threadsp(%rax), %rsp



	/*
@@ -433,7 +433,7 @@ SYM_CODE_START(soft_restart_cpu) 
	UNWIND_HINT_END_OF_STACK



	/* Find the idle task stack */

	movq	PER_CPU_VAR(pcpu_hot + X86_current_task), %rcx

	movq	PER_CPU_VAR(current_task), %rcx

	movq	TASK_threadsp(%rcx), %rsp



	jmp	.Ljump_to_C_code