Commit 46e8fff6 authored by Brian Gerst's avatar Brian Gerst Committed by Ingo Molnar
Browse files

x86/preempt: Move preempt count to percpu hot section 



No functional change.

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-4-brgerst@gmail.com
parent 972f9cdf

arch/x86/include/asm/current.h

+0 −1
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ struct task_struct; 


struct pcpu_hot {

	struct task_struct	*current_task;

	int			preempt_count;

	int			cpu_number;

#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING

	u64			call_depth;

arch/x86/include/asm/preempt.h

+13 −12
Original line number Diff line number Diff line
@@ -4,10 +4,11 @@ 


#include <asm/rmwcc.h>

#include <asm/percpu.h>

#include <asm/current.h>



#include <linux/static_call_types.h>



DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);



/* We use the MSB mostly because its available */

#define PREEMPT_NEED_RESCHED	0x80000000
@@ -23,18 +24,18 @@ 
 */

static __always_inline int preempt_count(void)

{

	return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED;

	return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;

}



static __always_inline void preempt_count_set(int pc)

{

	int old, new;



	old = raw_cpu_read_4(pcpu_hot.preempt_count);

	old = raw_cpu_read_4(__preempt_count);

	do {

		new = (old & PREEMPT_NEED_RESCHED) |

			(pc & ~PREEMPT_NEED_RESCHED);

	} while (!raw_cpu_try_cmpxchg_4(pcpu_hot.preempt_count, &old, new));

	} while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new));

}



/*
@@ -43,7 +44,7 @@ static __always_inline void preempt_count_set(int pc) 
#define init_task_preempt_count(p) do { } while (0)



#define init_idle_preempt_count(p, cpu) do { \

	per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \

	per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \

} while (0)



/*
@@ -57,17 +58,17 @@ static __always_inline void preempt_count_set(int pc) 


static __always_inline void set_preempt_need_resched(void)

{

	raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED);

	raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);

}



static __always_inline void clear_preempt_need_resched(void)

{

	raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED);

	raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);

}



static __always_inline bool test_preempt_need_resched(void)

{

	return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED);

	return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);

}



/*
@@ -76,12 +77,12 @@ static __always_inline bool test_preempt_need_resched(void) 


static __always_inline void __preempt_count_add(int val)

{

	raw_cpu_add_4(pcpu_hot.preempt_count, val);

	raw_cpu_add_4(__preempt_count, val);

}



static __always_inline void __preempt_count_sub(int val)

{

	raw_cpu_add_4(pcpu_hot.preempt_count, -val);

	raw_cpu_add_4(__preempt_count, -val);

}



/*
@@ -91,7 +92,7 @@ static __always_inline void __preempt_count_sub(int val) 
 */

static __always_inline bool __preempt_count_dec_and_test(void)

{

	return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.preempt_count), e,

	return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e,

			       __percpu_arg([var]));

}
@@ -100,7 +101,7 @@ static __always_inline bool __preempt_count_dec_and_test(void) 
 */

static __always_inline bool should_resched(int preempt_offset)

{

	return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset);

	return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);

}



#ifdef CONFIG_PREEMPTION

arch/x86/kernel/cpu/common.c

+3 −1
Original line number Diff line number Diff line
@@ -2066,12 +2066,14 @@ __setup("setcpuid=", setup_setcpuid); 


DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = {

	.current_task	= &init_task,

	.preempt_count	= INIT_PREEMPT_COUNT,

	.top_of_stack	= TOP_OF_INIT_STACK,

};

EXPORT_PER_CPU_SYMBOL(pcpu_hot);

EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);



DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;

EXPORT_PER_CPU_SYMBOL(__preempt_count);



#ifdef CONFIG_X86_64

static void wrmsrl_cstar(unsigned long val)

{

include/linux/preempt.h

+1 −0
Original line number Diff line number Diff line
@@ -319,6 +319,7 @@ do { \ 
#ifdef CONFIG_PREEMPT_NOTIFIERS



struct preempt_notifier;

struct task_struct;



/**

 * preempt_ops - notifiers called when a task is preempted and rescheduled