Commit 1eee4ef3 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86_urgent_for_v6.8_rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 fixes from Borislav Petkov:

 - Make sure clearing CPU buffers using VERW happens at the latest
   possible point in the return-to-userspace path, otherwise memory
   accesses after the VERW execution could cause data to land in CPU
   buffers again

* tag 'x86_urgent_for_v6.8_rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  KVM/VMX: Move VERW closer to VMentry for MDS mitigation
  KVM/VMX: Use BT+JNC, i.e. EFLAGS.CF to select VMRESUME vs. VMLAUNCH
  x86/bugs: Use ALTERNATIVE() instead of mds_user_clear static key
  x86/entry_32: Add VERW just before userspace transition
  x86/entry_64: Add VERW just before userspace transition
  x86/bugs: Add asm helpers for executing VERW
parents 8c46ed37 43fb862d

Documentation/arch/x86/mds.rst

+27 −11
Original line number Diff line number Diff line
@@ -95,6 +95,9 @@ The kernel provides a function to invoke the buffer clearing: 


    mds_clear_cpu_buffers()



Also macro CLEAR_CPU_BUFFERS can be used in ASM late in exit-to-user path.

Other than CFLAGS.ZF, this macro doesn't clobber any registers.



The mitigation is invoked on kernel/userspace, hypervisor/guest and C-state

(idle) transitions.
@@ -138,17 +141,30 @@ Mitigation points 


   When transitioning from kernel to user space the CPU buffers are flushed

   on affected CPUs when the mitigation is not disabled on the kernel

   command line. The migitation is enabled through the static key

   mds_user_clear.



   The mitigation is invoked in prepare_exit_to_usermode() which covers

   all but one of the kernel to user space transitions.  The exception

   is when we return from a Non Maskable Interrupt (NMI), which is

   handled directly in do_nmi().



   (The reason that NMI is special is that prepare_exit_to_usermode() can

    enable IRQs.  In NMI context, NMIs are blocked, and we don't want to

    enable IRQs with NMIs blocked.)

   command line. The mitigation is enabled through the feature flag

   X86_FEATURE_CLEAR_CPU_BUF.



   The mitigation is invoked just before transitioning to userspace after

   user registers are restored. This is done to minimize the window in

   which kernel data could be accessed after VERW e.g. via an NMI after

   VERW.



   **Corner case not handled**

   Interrupts returning to kernel don't clear CPUs buffers since the

   exit-to-user path is expected to do that anyways. But, there could be

   a case when an NMI is generated in kernel after the exit-to-user path

   has cleared the buffers. This case is not handled and NMI returning to

   kernel don't clear CPU buffers because:



   1. It is rare to get an NMI after VERW, but before returning to userspace.

   2. For an unprivileged user, there is no known way to make that NMI

      less rare or target it.

   3. It would take a large number of these precisely-timed NMIs to mount

      an actual attack.  There's presumably not enough bandwidth.

   4. The NMI in question occurs after a VERW, i.e. when user state is

      restored and most interesting data is already scrubbed. Whats left

      is only the data that NMI touches, and that may or may not be of

      any interest.





2. C-State transition

arch/x86/entry/entry.S

+23 −0
Original line number Diff line number Diff line
@@ -6,6 +6,9 @@ 
#include <linux/export.h>

#include <linux/linkage.h>

#include <asm/msr-index.h>

#include <asm/unwind_hints.h>

#include <asm/segment.h>

#include <asm/cache.h>



.pushsection .noinstr.text, "ax"
@@ -20,3 +23,23 @@ SYM_FUNC_END(entry_ibpb) 
EXPORT_SYMBOL_GPL(entry_ibpb);



.popsection



/*

 * Define the VERW operand that is disguised as entry code so that

 * it can be referenced with KPTI enabled. This ensure VERW can be

 * used late in exit-to-user path after page tables are switched.

 */

.pushsection .entry.text, "ax"



.align L1_CACHE_BYTES, 0xcc

SYM_CODE_START_NOALIGN(mds_verw_sel)

	UNWIND_HINT_UNDEFINED

	ANNOTATE_NOENDBR

	.word __KERNEL_DS

.align L1_CACHE_BYTES, 0xcc

SYM_CODE_END(mds_verw_sel);

/* For KVM */

EXPORT_SYMBOL_GPL(mds_verw_sel);



.popsection

arch/x86/entry/entry_32.S

+3 −0
Original line number Diff line number Diff line
@@ -885,6 +885,7 @@ SYM_FUNC_START(entry_SYSENTER_32) 
	BUG_IF_WRONG_CR3 no_user_check=1

	popfl

	popl	%eax

	CLEAR_CPU_BUFFERS



	/*

	 * Return back to the vDSO, which will pop ecx and edx.
@@ -954,6 +955,7 @@ restore_all_switch_stack: 


	/* Restore user state */

	RESTORE_REGS pop=4			# skip orig_eax/error_code

	CLEAR_CPU_BUFFERS

.Lirq_return:

	/*

	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
@@ -1146,6 +1148,7 @@ SYM_CODE_START(asm_exc_nmi) 


	/* Not on SYSENTER stack. */

	call	exc_nmi

	CLEAR_CPU_BUFFERS

	jmp	.Lnmi_return



.Lnmi_from_sysenter_stack:

arch/x86/entry/entry_64.S

+11 −0
Original line number Diff line number Diff line
@@ -161,6 +161,7 @@ syscall_return_via_sysret: 
SYM_INNER_LABEL(entry_SYSRETQ_unsafe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR

	swapgs

	CLEAR_CPU_BUFFERS

	sysretq

SYM_INNER_LABEL(entry_SYSRETQ_end, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR
@@ -573,6 +574,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 


.Lswapgs_and_iret:

	swapgs

	CLEAR_CPU_BUFFERS

	/* Assert that the IRET frame indicates user mode. */

	testb	$3, 8(%rsp)

	jnz	.Lnative_iret
@@ -723,6 +725,8 @@ native_irq_return_ldt: 
	 */

	popq	%rax				/* Restore user RAX */



	CLEAR_CPU_BUFFERS



	/*

	 * RSP now points to an ordinary IRET frame, except that the page

	 * is read-only and RSP[31:16] are preloaded with the userspace
@@ -1449,6 +1453,12 @@ nmi_restore: 
	std

	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */



	/*

	 * Skip CLEAR_CPU_BUFFERS here, since it only helps in rare cases like

	 * NMI in kernel after user state is restored. For an unprivileged user

	 * these conditions are hard to meet.

	 */



	/*

	 * iretq reads the "iret" frame and exits the NMI stack in a

	 * single instruction.  We are returning to kernel mode, so this
@@ -1466,6 +1476,7 @@ SYM_CODE_START(entry_SYSCALL32_ignore) 
	UNWIND_HINT_END_OF_STACK

	ENDBR

	mov	$-ENOSYS, %eax

	CLEAR_CPU_BUFFERS

	sysretl

SYM_CODE_END(entry_SYSCALL32_ignore)

arch/x86/entry/entry_64_compat.S

+1 −0
Original line number Diff line number Diff line
@@ -270,6 +270,7 @@ SYM_INNER_LABEL(entry_SYSRETL_compat_unsafe_stack, SYM_L_GLOBAL) 
	xorl	%r9d, %r9d

	xorl	%r10d, %r10d

	swapgs

	CLEAR_CPU_BUFFERS

	sysretl

SYM_INNER_LABEL(entry_SYSRETL_compat_end, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR