Commit ab2bb9c0 authored by Brian Gerst's avatar Brian Gerst Committed by Ingo Molnar
Browse files

percpu: Introduce percpu hot section 



Add a subsection to the percpu data for frequently accessed variables
that should remain cached on each processor.  These varables should not
be accessed from other processors to avoid cacheline bouncing.

This will replace the pcpu_hot struct on x86, and open up similar
functionality to other architectures and the kernel core.

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-2-brgerst@gmail.com
parent f3a3c29b

include/asm-generic/vmlinux.lds.h

+11 −0
Original line number Diff line number Diff line
@@ -385,6 +385,11 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) 
	. = ALIGN(PAGE_SIZE);						\

	__nosave_end = .;



#define CACHE_HOT_DATA(align)						\

	. = ALIGN(align);						\

	*(SORT_BY_ALIGNMENT(.data..hot.*))				\

	. = ALIGN(align);



#define PAGE_ALIGNED_DATA(page_align)					\

	. = ALIGN(page_align);						\

	*(.data..page_aligned)						\
@@ -1065,6 +1070,11 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) 
	. = ALIGN(PAGE_SIZE);						\

	*(.data..percpu..page_aligned)					\

	. = ALIGN(cacheline);						\

	__per_cpu_hot_start = .;					\

	*(SORT_BY_ALIGNMENT(.data..percpu..hot.*))			\

	__per_cpu_hot_pad = .;						\

	. = ALIGN(cacheline);						\

	__per_cpu_hot_end = .;						\

	*(.data..percpu..read_mostly)					\

	. = ALIGN(cacheline);						\

	*(.data..percpu)						\
@@ -1112,6 +1122,7 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) 
		INIT_TASK_DATA(inittask)				\

		NOSAVE_DATA						\

		PAGE_ALIGNED_DATA(pagealigned)				\

		CACHE_HOT_DATA(cacheline)				\

		CACHELINE_ALIGNED_DATA(cacheline)			\

		READ_MOSTLY_DATA(cacheline)				\

		DATA_DATA						\

include/linux/percpu-defs.h

+13 −0
Original line number Diff line number Diff line
@@ -112,6 +112,19 @@ 
#define DEFINE_PER_CPU(type, name)					\

	DEFINE_PER_CPU_SECTION(type, name, "")



/*

 * Declaration/definition used for per-CPU variables that are frequently

 * accessed and should be in a single cacheline.

 *

 * For use only by architecture and core code.  Only use scalar or pointer

 * types to maximize density.

 */

#define DECLARE_PER_CPU_CACHE_HOT(type, name)				\

	DECLARE_PER_CPU_SECTION(type, name, "..hot.." #name)



#define DEFINE_PER_CPU_CACHE_HOT(type, name)				\

	DEFINE_PER_CPU_SECTION(type, name, "..hot.." #name)



/*

 * Declaration/definition used for per-CPU variables that must be cacheline

 * aligned under SMP conditions so that, whilst a particular instance of the