Commit 288a4ff0 authored by Xin Li (Intel)'s avatar Xin Li (Intel) Committed by Ingo Molnar
Browse files

x86/msr: Move rdtsc{,_ordered}() to <asm/tsc.h> 



Relocate rdtsc{,_ordered}() from <asm/msr.h> to <asm/tsc.h>.

[ mingo: Do not remove the <asm/tsc.h> inclusion from <asm/msr.h>
         just yet, to reduce -next breakages. We can do this later
	 on, separately, shortly before the next -rc1. ]

Signed-off-by: default avatarXin Li (Intel) <xin@zytor.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Uros Bizjak <ubizjak@gmail.com>
Link: https://lore.kernel.org/r/20250427092027.1598740-3-xin@zytor.com
parent efef7f18

arch/x86/include/asm/msr.h

+0 −54
Original line number Diff line number Diff line
@@ -153,60 +153,6 @@ native_write_msr_safe(u32 msr, u32 low, u32 high) 
extern int rdmsr_safe_regs(u32 regs[8]);

extern int wrmsr_safe_regs(u32 regs[8]);



/**

 * rdtsc() - returns the current TSC without ordering constraints

 *

 * rdtsc() returns the result of RDTSC as a 64-bit integer.  The

 * only ordering constraint it supplies is the ordering implied by

 * "asm volatile": it will put the RDTSC in the place you expect.  The

 * CPU can and will speculatively execute that RDTSC, though, so the

 * results can be non-monotonic if compared on different CPUs.

 */

static __always_inline u64 rdtsc(void)

{

	EAX_EDX_DECLARE_ARGS(val, low, high);



	asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));



	return EAX_EDX_VAL(val, low, high);

}



/**

 * rdtsc_ordered() - read the current TSC in program order

 *

 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.

 * It is ordered like a load to a global in-memory counter.  It should

 * be impossible to observe non-monotonic rdtsc_unordered() behavior

 * across multiple CPUs as long as the TSC is synced.

 */

static __always_inline u64 rdtsc_ordered(void)

{

	EAX_EDX_DECLARE_ARGS(val, low, high);



	/*

	 * The RDTSC instruction is not ordered relative to memory

	 * access.  The Intel SDM and the AMD APM are both vague on this

	 * point, but empirically an RDTSC instruction can be

	 * speculatively executed before prior loads.  An RDTSC

	 * immediately after an appropriate barrier appears to be

	 * ordered as a normal load, that is, it provides the same

	 * ordering guarantees as reading from a global memory location

	 * that some other imaginary CPU is updating continuously with a

	 * time stamp.

	 *

	 * Thus, use the preferred barrier on the respective CPU, aiming for

	 * RDTSCP as the default.

	 */

	asm volatile(ALTERNATIVE_2("rdtsc",

				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,

				   "rdtscp", X86_FEATURE_RDTSCP)

			: EAX_EDX_RET(val, low, high)

			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */

			:: "ecx");



	return EAX_EDX_VAL(val, low, high);

}



static inline u64 native_read_pmc(int counter)

{

	EAX_EDX_DECLARE_ARGS(val, low, high);

arch/x86/include/asm/tsc.h

+55 −0
Original line number Diff line number Diff line
@@ -5,10 +5,65 @@ 
#ifndef _ASM_X86_TSC_H

#define _ASM_X86_TSC_H



#include <asm/asm.h>

#include <asm/cpufeature.h>

#include <asm/processor.h>

#include <asm/msr.h>



/**

 * rdtsc() - returns the current TSC without ordering constraints

 *

 * rdtsc() returns the result of RDTSC as a 64-bit integer.  The

 * only ordering constraint it supplies is the ordering implied by

 * "asm volatile": it will put the RDTSC in the place you expect.  The

 * CPU can and will speculatively execute that RDTSC, though, so the

 * results can be non-monotonic if compared on different CPUs.

 */

static __always_inline u64 rdtsc(void)

{

	EAX_EDX_DECLARE_ARGS(val, low, high);



	asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));



	return EAX_EDX_VAL(val, low, high);

}



/**

 * rdtsc_ordered() - read the current TSC in program order

 *

 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.

 * It is ordered like a load to a global in-memory counter.  It should

 * be impossible to observe non-monotonic rdtsc_unordered() behavior

 * across multiple CPUs as long as the TSC is synced.

 */

static __always_inline u64 rdtsc_ordered(void)

{

	EAX_EDX_DECLARE_ARGS(val, low, high);



	/*

	 * The RDTSC instruction is not ordered relative to memory

	 * access.  The Intel SDM and the AMD APM are both vague on this

	 * point, but empirically an RDTSC instruction can be

	 * speculatively executed before prior loads.  An RDTSC

	 * immediately after an appropriate barrier appears to be

	 * ordered as a normal load, that is, it provides the same

	 * ordering guarantees as reading from a global memory location

	 * that some other imaginary CPU is updating continuously with a

	 * time stamp.

	 *

	 * Thus, use the preferred barrier on the respective CPU, aiming for

	 * RDTSCP as the default.

	 */

	asm volatile(ALTERNATIVE_2("rdtsc",

				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,

				   "rdtscp", X86_FEATURE_RDTSCP)

			: EAX_EDX_RET(val, low, high)

			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */

			:: "ecx");



	return EAX_EDX_VAL(val, low, high);

}



/*

 * Standard way to access the cycle counter.

 */