Commit 0701c9e1 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Thomas Gleixner
Browse files

x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core 



Move the VMX interrupt dispatch magic into the x86 core code. This
isolates KVM from the FRED/IDT decisions and reduces the amount of
EXPORT_SYMBOL_FOR_KVM().

Suggested-by: default avatarSean Christopherson <seanjc@google.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarThomas Gleixner <tglx@kernel.org>
Tested-by: default avatar"Verma, Vishal L" <vishal.l.verma@intel.com>
Tested-by: default avatarZhao Liu <zhao1.liu@intel.com>
Tested-by: default avatarZhao Liu <zhao1.liu@intel.com>
Tested-by: default avatarSean Christopherson <seanjc@google.com>
Reviewed-by: default avatarBinbin Wu <binbin.wu@linxu.intel.com>
Acked-by: default avatarSean Christopherson <seanjc@google.com>
Link: https://patch.msgid.link/20260508091829.GO3126523@noisy.programming.kicks-ass.net
parent b088fe35

arch/x86/entry/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_FTRACE) 
CFLAGS_syscall_32.o		+= -fno-stack-protector

CFLAGS_syscall_64.o		+= -fno-stack-protector



obj-y				:= entry.o entry_$(BITS).o syscall_$(BITS).o

obj-y				:= entry.o entry_$(BITS).o syscall_$(BITS).o common.o



obj-y				+= vdso/

obj-y				+= vsyscall/

arch/x86/entry/common.c

0 → 100644
+48 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#include <linux/entry-common.h>

#include <linux/kvm_types.h>

#include <asm/fred.h>

#include <asm/desc.h>



#if IS_ENABLED(CONFIG_KVM_INTEL)

/*

 * On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as

 * part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.

 * x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs

 * to the kernel for servicing.  On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is

 * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered

 * the VM-Exit is held pending until it's unblocked in the host.

 */

noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)

{

	if (event_type == EVENT_TYPE_EXTINT) {

#ifdef CONFIG_X86_64

		/*

		 * Use FRED dispatch, even when running IDT. The dispatch

		 * tables are kept in sync between FRED and IDT, and the FRED

		 * dispatch works well with CFI.

		 */

		fred_entry_from_kvm(event_type, vector);

#else

		idt_entry_from_kvm(vector);

#endif

		return;

	}



	WARN_ON_ONCE(event_type != EVENT_TYPE_NMI);



#ifdef CONFIG_X86_64

	if (cpu_feature_enabled(X86_FEATURE_FRED))

		return fred_entry_from_kvm(event_type, vector);

#endif



	/*

	 * Notably, we must use IDT dispatch for NMI when running in IDT mode.

	 * The FRED NMI context is significantly different and will not work

	 * right (specifically FRED fixed the NMI recursion issue).

	 */

	idt_entry_from_kvm(vector);

}

EXPORT_SYMBOL_FOR_KVM(x86_entry_from_kvm);

#endif

arch/x86/entry/entry.S

+46 −0
Original line number Diff line number Diff line
@@ -75,3 +75,49 @@ THUNK warn_thunk_thunk, __warn_thunk 
#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)

EXPORT_SYMBOL(__ref_stack_chk_guard);

#endif



#if IS_ENABLED(CONFIG_KVM_INTEL)

.macro IDT_DO_EVENT_IRQOFF call_insn call_target

	/*

	 * Unconditionally create a stack frame, getting the correct RSP on the

	 * stack (for x86-64) would take two instructions anyways, and RBP can

	 * be used to restore RSP to make objtool happy (see below).

	 */

	push %_ASM_BP

	mov %_ASM_SP, %_ASM_BP



#ifdef CONFIG_X86_64

	/*

	 * Align RSP to a 16-byte boundary (to emulate CPU behavior) before

	 * creating the synthetic interrupt stack frame for the IRQ/NMI.

	 */

	and  $-16, %rsp

	push $__KERNEL_DS

	push %rbp

#endif

	pushf

	push $__KERNEL_CS

	\call_insn \call_target



	/*

	 * "Restore" RSP from RBP, even though IRET has already unwound RSP to

	 * the correct value.  objtool doesn't know the callee will IRET and,

	 * without the explicit restore, thinks the stack is getting walloped.

	 * Using an unwind hint is problematic due to x86-64's dynamic alignment.

	 */

	leave

	RET

.endm



.pushsection .text, "ax"

SYM_FUNC_START(idt_do_interrupt_irqoff)

	IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1

SYM_FUNC_END(idt_do_interrupt_irqoff)

.popsection



.pushsection .noinstr.text, "ax"

SYM_FUNC_START(idt_do_nmi_irqoff)

	IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx

SYM_FUNC_END(idt_do_nmi_irqoff)

.popsection

#endif

arch/x86/entry/entry_64_fred.S

+0 −1
Original line number Diff line number Diff line
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) 
	RET



SYM_FUNC_END(asm_fred_entry_from_kvm)

EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);

#endif

arch/x86/include/asm/desc.h

+4 −0
Original line number Diff line number Diff line
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void); 
extern void idt_setup_apic_and_irq_gates(void);

extern bool idt_is_f00f_address(unsigned long address);



extern void idt_do_interrupt_irqoff(unsigned long address);

extern void idt_do_nmi_irqoff(void);

extern void idt_entry_from_kvm(unsigned int vector);



#ifdef CONFIG_X86_64

extern void idt_setup_early_pf(void);

#else