Commit 2e670358 authored by Xin Li's avatar Xin Li Committed by Borislav Petkov (AMD)
Browse files

x86/entry: Add fred_entry_from_kvm() for VMX to handle IRQ/NMI 



In IRQ/NMI induced VM exits, KVM VMX needs to execute the respective
handlers, which requires the software to create a FRED stack frame,
and use it to invoke the handlers. Add fred_irq_entry_from_kvm() for
this job.

Export fred_entry_from_kvm() because VMX can be compiled as a module.

Suggested-by: default avatarSean Christopherson <seanjc@google.com>
Suggested-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarXin Li <xin3.li@intel.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Tested-by: default avatarShan Kang <shan.kang@intel.com>
Link: https://lore.kernel.org/r/20231205105030.8698-32-xin3.li@intel.com
parent 2333f3c4

arch/x86/entry/entry_64_fred.S

+77 −0
Original line number Diff line number Diff line
@@ -3,8 +3,11 @@ 
 * The actual FRED entry points.

 */



#include <linux/export.h>



#include <asm/asm.h>

#include <asm/fred.h>

#include <asm/segment.h>



#include "calling.h"
@@ -52,3 +55,77 @@ SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) 
	FRED_EXIT

	ERETS

SYM_CODE_END(asm_fred_entrypoint_kernel)



#if IS_ENABLED(CONFIG_KVM_INTEL)

SYM_FUNC_START(asm_fred_entry_from_kvm)

	push %rbp

	mov %rsp, %rbp



	UNWIND_HINT_SAVE



	/*

	 * Both IRQ and NMI from VMX can be handled on current task stack

	 * because there is no need to protect from reentrancy and the call

	 * stack leading to this helper is effectively constant and shallow

	 * (relatively speaking). Do the same when FRED is active, i.e., no

	 * need to check current stack level for a stack switch.

	 *

	 * Emulate the FRED-defined redzone and stack alignment.

	 */

	sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp

	and $FRED_STACK_FRAME_RSP_MASK, %rsp



	/*

	 * Start to push a FRED stack frame, which is always 64 bytes:

	 *

	 * +--------+-----------------+

	 * | Bytes  | Usage           |

	 * +--------+-----------------+

	 * | 63:56  | Reserved        |

	 * | 55:48  | Event Data      |

	 * | 47:40  | SS + Event Info |

	 * | 39:32  | RSP             |

	 * | 31:24  | RFLAGS          |

	 * | 23:16  | CS + Aux Info   |

	 * |  15:8  | RIP             |

	 * |   7:0  | Error Code      |

	 * +--------+-----------------+

	 */

	push $0				/* Reserved, must be 0 */

	push $0				/* Event data, 0 for IRQ/NMI */

	push %rdi			/* fred_ss handed in by the caller */

	push %rbp

	pushf

	mov $__KERNEL_CS, %rax

	push %rax



	/*

	 * Unlike the IDT event delivery, FRED _always_ pushes an error code

	 * after pushing the return RIP, thus the CALL instruction CANNOT be

	 * used here to push the return RIP, otherwise there is no chance to

	 * push an error code before invoking the IRQ/NMI handler.

	 *

	 * Use LEA to get the return RIP and push it, then push an error code.

	 */

	lea 1f(%rip), %rax

	push %rax				/* Return RIP */

	push $0					/* Error code, 0 for IRQ/NMI */



	PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0

	movq %rsp, %rdi				/* %rdi -> pt_regs */

	call __fred_entry_from_kvm		/* Call the C entry point */

	POP_REGS

	ERETS

1:

	/*

	 * Objtool doesn't understand what ERETS does, this hint tells it that

	 * yes, we'll reach here and with what stack state. A save/restore pair

	 * isn't strictly needed, but it's the simplest form.

	 */

	UNWIND_HINT_RESTORE

	pop %rbp

	RET



SYM_FUNC_END(asm_fred_entry_from_kvm)

EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm);

#endif

arch/x86/entry/entry_fred.c

+14 −0
Original line number Diff line number Diff line
@@ -257,3 +257,17 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 


	return fred_bad_type(regs, error_code);

}



#if IS_ENABLED(CONFIG_KVM_INTEL)

__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)

{

	switch (regs->fred_ss.type) {

	case EVENT_TYPE_EXTINT:

		return fred_extint(regs);

	case EVENT_TYPE_NMI:

		return fred_exc_nmi(regs);

	default:

		WARN_ON_ONCE(1);

	}

}

#endif

arch/x86/include/asm/fred.h

+18 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
#include <linux/const.h>



#include <asm/asm.h>

#include <asm/trapnr.h>



/*

 * FRED event return instruction opcodes for ERET{S,U}; supported in
@@ -62,12 +63,29 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs) 


void asm_fred_entrypoint_user(void);

void asm_fred_entrypoint_kernel(void);

void asm_fred_entry_from_kvm(struct fred_ss);



__visible void fred_entry_from_user(struct pt_regs *regs);

__visible void fred_entry_from_kernel(struct pt_regs *regs);

__visible void __fred_entry_from_kvm(struct pt_regs *regs);



/* Can be called from noinstr code, thus __always_inline */

static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector)

{

	struct fred_ss ss = {

		.ss     =__KERNEL_DS,

		.type   = type,

		.vector = vector,

		.nmi    = type == EVENT_TYPE_NMI,

		.lm     = 1,

	};



	asm_fred_entry_from_kvm(ss);

}



#else /* CONFIG_X86_FRED */

static __always_inline unsigned long fred_event_data(struct pt_regs *regs) { return 0; }

static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }

#endif /* CONFIG_X86_FRED */

#endif /* !__ASSEMBLY__ */