Commit 8e786a85 authored by Borislav Petkov (AMD)'s avatar Borislav Petkov (AMD)
Browse files

x86/process: Move the buffer clearing before MONITOR 



Move the VERW clearing before the MONITOR so that VERW doesn't disarm it
and the machine never enters C1.

Original idea by Kim Phillips <kim.phillips@amd.com>.

Suggested-by: default avatarAndrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
parent 2329f250

arch/x86/include/asm/mwait.h

+15 −10
Original line number Diff line number Diff line
@@ -43,8 +43,6 @@ static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx) 


static __always_inline void __mwait(u32 eax, u32 ecx)

{

	x86_idle_clear_cpu_buffers();



	/*

	 * Use the instruction mnemonic with implicit operands, as the LLVM

	 * assembler fails to assemble the mnemonic with explicit operands:
@@ -98,7 +96,6 @@ static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx) 
 */

static __always_inline void __sti_mwait(u32 eax, u32 ecx)

{

	x86_idle_clear_cpu_buffers();



	asm volatile("sti; mwait" :: "a" (eax), "c" (ecx));

}
@@ -115,13 +112,20 @@ static __always_inline void __sti_mwait(u32 eax, u32 ecx) 
 */

static __always_inline void mwait_idle_with_hints(u32 eax, u32 ecx)

{

	if (need_resched())

		return;



	x86_idle_clear_cpu_buffers();



	if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) {

		const void *addr = &current_thread_info()->flags;



		alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));

		__monitor(addr, 0, 0);



		if (!need_resched()) {

		if (need_resched())

			goto out;



		if (ecx & 1) {

			__mwait(eax, ecx);

		} else {
@@ -129,7 +133,8 @@ static __always_inline void mwait_idle_with_hints(u32 eax, u32 ecx) 
			raw_local_irq_disable();

		}

	}

	}



out:

	current_clr_polling();

}

arch/x86/kernel/process.c

+12 −4
Original line number Diff line number Diff line
@@ -907,16 +907,24 @@ static __init bool prefer_mwait_c1_over_halt(void) 
 */

static __cpuidle void mwait_idle(void)

{

	if (need_resched())

		return;



	x86_idle_clear_cpu_buffers();



	if (!current_set_polling_and_test()) {

		const void *addr = &current_thread_info()->flags;



		alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));

		__monitor(addr, 0, 0);

		if (!need_resched()) {

		if (need_resched())

			goto out;



		__sti_mwait(0, 0);

		raw_local_irq_disable();

	}

	}



out:

	__current_clr_polling();

}