Commit 972f9cdf authored by Brian Gerst's avatar Brian Gerst Committed by Ingo Molnar
Browse files

x86/percpu: Move pcpu_hot to percpu hot section 



Also change the alignment of the percpu hot section:

 -       PERCPU_SECTION(INTERNODE_CACHE_BYTES)
 +       PERCPU_SECTION(L1_CACHE_BYTES)

As vSMP will muck with INTERNODE_CACHE_BYTES that invalidates the
too-large-section assert we do:

  ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large")

[ mingo: Added INTERNODE_CACHE_BYTES fix & explanation. ]

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-3-brgerst@gmail.com
parent ab2bb9c0

arch/x86/include/asm/current.h

+11 −17
Original line number Diff line number Diff line
@@ -13,8 +13,6 @@ 
struct task_struct;



struct pcpu_hot {

	union {

		struct {

	struct task_struct	*current_task;

	int			preempt_count;

	int			cpu_number;
@@ -30,15 +28,11 @@ struct pcpu_hot { 
	void			*softirq_stack_ptr;

#endif

};

		u8	pad[64];

	};

};

static_assert(sizeof(struct pcpu_hot) == 64);



DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);

DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);



/* const-qualified alias to pcpu_hot, aliased by linker. */

DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override,

DECLARE_PER_CPU_CACHE_HOT(const struct pcpu_hot __percpu_seg_override,

			const_pcpu_hot);



static __always_inline struct task_struct *get_current(void)

arch/x86/kernel/cpu/common.c

+1 −1
Original line number Diff line number Diff line
@@ -2064,7 +2064,7 @@ static __init int setup_setcpuid(char *arg) 
}

__setup("setcpuid=", setup_setcpuid);



DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {

DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = {

	.current_task	= &init_task,

	.preempt_count	= INIT_PREEMPT_COUNT,

	.top_of_stack	= TOP_OF_INIT_STACK,

arch/x86/kernel/vmlinux.lds.S

+4 −1
Original line number Diff line number Diff line
@@ -187,6 +187,8 @@ SECTIONS 


		PAGE_ALIGNED_DATA(PAGE_SIZE)



		CACHE_HOT_DATA(L1_CACHE_BYTES)



		CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES)



		DATA_DATA
@@ -327,7 +329,8 @@ SECTIONS 
		EXIT_DATA

	}



	PERCPU_SECTION(INTERNODE_CACHE_BYTES)

	PERCPU_SECTION(L1_CACHE_BYTES)

	ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large")



	RUNTIME_CONST_VARIABLES

	RUNTIME_CONST(ptr, USER_PTR_MAX)