Commit 8999ad99 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86_tdx_for_6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 TDX updates from Dave Hansen:
 "The majority of this is a rework of the assembly and C wrappers that
  are used to talk to the TDX module and VMM. This is a nice cleanup in
  general but is also clearing the way for using this code when Linux is
  the TDX VMM.

  There are also some tidbits to make TDX guests play nicer with Hyper-V
  and to take advantage the hardware TSC.

  Summary:

   - Refactor and clean up TDX hypercall/module call infrastructure

   - Handle retrying/resuming page conversion hypercalls

   - Make sure to use the (shockingly) reliable TSC in TDX guests"

[ TLA reminder: TDX is "Trust Domain Extensions", Intel's guest VM
  confidentiality technology ]

* tag 'x86_tdx_for_6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/tdx: Mark TSC reliable
  x86/tdx: Fix __noreturn build warning around __tdx_hypercall_failed()
  x86/virt/tdx: Make TDX_MODULE_CALL handle SEAMCALL #UD and #GP
  x86/virt/tdx: Wire up basic SEAMCALL functions
  x86/tdx: Remove 'struct tdx_hypercall_args'
  x86/tdx: Reimplement __tdx_hypercall() using TDX_MODULE_CALL asm
  x86/tdx: Make TDX_HYPERCALL asm similar to TDX_MODULE_CALL
  x86/tdx: Extend TDX_MODULE_CALL to support more TDCALL/SEAMCALL leafs
  x86/tdx: Pass TDCALL/SEAMCALL input/output registers via a structure
  x86/tdx: Rename __tdx_module_call() to __tdcall()
  x86/tdx: Make macros of TDCALLs consistent with the spec
  x86/tdx: Skip saving output regs when SEAMCALL fails with VMFailInvalid
  x86/tdx: Zero out the missing RSI in TDX_HYPERCALL macro
  x86/tdx: Retry partially-completed page conversion hypercalls
parents f00593e0 9ee4318c

arch/x86/Kconfig

+12 −0
Original line number Diff line number Diff line
@@ -1940,6 +1940,18 @@ config X86_USER_SHADOW_STACK 


	  If unsure, say N.



config INTEL_TDX_HOST

	bool "Intel Trust Domain Extensions (TDX) host support"

	depends on CPU_SUP_INTEL

	depends on X86_64

	depends on KVM_INTEL

	help

	  Intel Trust Domain Extensions (TDX) protects guest VMs from malicious

	  host and certain physical attacks.  This option enables necessary TDX

	  support in the host kernel to run confidential VMs.



	  If unsure, say N.



config EFI

	bool "EFI runtime service support"

	depends on ACPI

arch/x86/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -253,6 +253,8 @@ archheaders: 


libs-y  += arch/x86/lib/



core-y += arch/x86/virt/



# drivers-y are linked after core-y

drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/

drivers-$(CONFIG_PCI)            += arch/x86/pci/

arch/x86/boot/compressed/tdx.c

+3 −3
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ void __tdx_hypercall_failed(void) 


static inline unsigned int tdx_io_in(int size, u16 port)

{

	struct tdx_hypercall_args args = {

	struct tdx_module_args args = {

		.r10 = TDX_HYPERCALL_STANDARD,

		.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),

		.r12 = size,
@@ -26,7 +26,7 @@ static inline unsigned int tdx_io_in(int size, u16 port) 
		.r14 = port,

	};



	if (__tdx_hypercall_ret(&args))

	if (__tdx_hypercall(&args))

		return UINT_MAX;



	return args.r11;
@@ -34,7 +34,7 @@ static inline unsigned int tdx_io_in(int size, u16 port) 


static inline void tdx_io_out(int size, u16 port, u32 value)

{

	struct tdx_hypercall_args args = {

	struct tdx_module_args args = {

		.r10 = TDX_HYPERCALL_STANDARD,

		.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),

		.r12 = size,

arch/x86/coco/tdx/tdcall.S

+29 −205
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#include <asm/asm-offsets.h>

#include <asm/asm.h>

#include <asm/frame.h>

#include <asm/unwind_hints.h>



#include <linux/linkage.h>

#include <linux/bits.h>

#include <linux/errno.h>



#include "../../virt/vmx/tdx/tdxcall.S"



/*

 * Bitmasks of exposed registers (with VMM).

 */

#define TDX_RDX		BIT(2)

#define TDX_RBX		BIT(3)

#define TDX_RSI		BIT(6)

#define TDX_RDI		BIT(7)

#define TDX_R8		BIT(8)

#define TDX_R9		BIT(9)

#define TDX_R10		BIT(10)

#define TDX_R11		BIT(11)

#define TDX_R12		BIT(12)

#define TDX_R13		BIT(13)

#define TDX_R14		BIT(14)

#define TDX_R15		BIT(15)



/*

 * These registers are clobbered to hold arguments for each

 * TDVMCALL. They are safe to expose to the VMM.

 * Each bit in this mask represents a register ID. Bit field

 * details can be found in TDX GHCI specification, section

 * titled "TDCALL [TDG.VP.VMCALL] leaf".

 */

#define TDVMCALL_EXPOSE_REGS_MASK	\

	( TDX_RDX | TDX_RBX | TDX_RSI | TDX_RDI | TDX_R8  | TDX_R9  | \

	  TDX_R10 | TDX_R11 | TDX_R12 | TDX_R13 | TDX_R14 | TDX_R15 )



.section .noinstr.text, "ax"



/*

 * __tdx_module_call()  - Used by TDX guests to request services from

 * the TDX module (does not include VMM services) using TDCALL instruction.

 *

 * Transforms function call register arguments into the TDCALL register ABI.

 * After TDCALL operation, TDX module output is saved in @out (if it is

 * provided by the user).

 *

 *-------------------------------------------------------------------------

 * TDCALL ABI:

 *-------------------------------------------------------------------------

 * Input Registers:

 *

 * RAX                 - TDCALL Leaf number.

 * RCX,RDX,R8-R9       - TDCALL Leaf specific input registers.

 *

 * Output Registers:

 *

 * RAX                 - TDCALL instruction error code.

 * RCX,RDX,R8-R11      - TDCALL Leaf specific output registers.

 * __tdcall()  - Used by TDX guests to request services from the TDX

 * module (does not include VMM services) using TDCALL instruction.

 *

 *-------------------------------------------------------------------------

 *

 * __tdx_module_call() function ABI:

 * __tdcall() function ABI:

 *

 * @fn   (RDI)	- TDCALL Leaf ID, moved to RAX

 * @rcx (RSI)          - Input parameter 1, moved to RCX

 * @rdx (RDX)          - Input parameter 2, moved to RDX

 * @r8  (RCX)          - Input parameter 3, moved to R8

 * @r9  (R8)           - Input parameter 4, moved to R9

 * @args (RSI)	- struct tdx_module_args for input

 *

 * @out (R9)           - struct tdx_module_output pointer

 *                       stored temporarily in R12 (not

 *                       shared with the TDX module). It

 *                       can be NULL.

 * Only RCX/RDX/R8-R11 are used as input registers.

 *

 * Return status of TDCALL via RAX.

 */

SYM_FUNC_START(__tdx_module_call)

	FRAME_BEGIN

SYM_FUNC_START(__tdcall)

	TDX_MODULE_CALL host=0

	FRAME_END

	RET

SYM_FUNC_END(__tdx_module_call)

SYM_FUNC_END(__tdcall)



/*

 * TDX_HYPERCALL - Make hypercalls to a TDX VMM using TDVMCALL leaf of TDCALL

 * instruction

 *

 * Transforms values in  function call argument struct tdx_hypercall_args @args

 * into the TDCALL register ABI. After TDCALL operation, VMM output is saved

 * back in @args, if \ret is 1.

 *

 *-------------------------------------------------------------------------

 * TD VMCALL ABI:

 *-------------------------------------------------------------------------

 * __tdcall_ret() - Used by TDX guests to request services from the TDX

 * module (does not include VMM services) using TDCALL instruction, with

 * saving output registers to the 'struct tdx_module_args' used as input.

 *

 * Input Registers:

 * __tdcall_ret() function ABI:

 *

 * RAX                 - TDCALL instruction leaf number (0 - TDG.VP.VMCALL)

 * RCX                 - BITMAP which controls which part of TD Guest GPR

 *                       is passed as-is to the VMM and back.

 * R10                 - Set 0 to indicate TDCALL follows standard TDX ABI

 *                       specification. Non zero value indicates vendor

 *                       specific ABI.

 * R11                 - VMCALL sub function number

 * RBX, RDX, RDI, RSI  - Used to pass VMCALL sub function specific arguments.

 * R8-R9, R12-R15      - Same as above.

 *

 * Output Registers:

 * @fn   (RDI)	- TDCALL Leaf ID, moved to RAX

 * @args (RSI)	- struct tdx_module_args for input and output

 *

 * RAX                 - TDCALL instruction status (Not related to hypercall

 *                        output).

 * RBX, RDX, RDI, RSI  - Hypercall sub function specific output values.

 * R8-R15              - Same as above.

 * Only RCX/RDX/R8-R11 are used as input/output registers.

 *

 * Return status of TDCALL via RAX.

 */

.macro TDX_HYPERCALL ret:req

	FRAME_BEGIN



	/* Save callee-saved GPRs as mandated by the x86_64 ABI */

	push %r15

	push %r14

	push %r13

	push %r12

	push %rbx



	/* Free RDI to be used as TDVMCALL arguments */

	movq %rdi, %rax



	/* Copy hypercall registers from arg struct: */

	movq TDX_HYPERCALL_r8(%rax),  %r8

	movq TDX_HYPERCALL_r9(%rax),  %r9

	movq TDX_HYPERCALL_r10(%rax), %r10

	movq TDX_HYPERCALL_r11(%rax), %r11

	movq TDX_HYPERCALL_r12(%rax), %r12

	movq TDX_HYPERCALL_r13(%rax), %r13

	movq TDX_HYPERCALL_r14(%rax), %r14

	movq TDX_HYPERCALL_r15(%rax), %r15

	movq TDX_HYPERCALL_rdi(%rax), %rdi

	movq TDX_HYPERCALL_rsi(%rax), %rsi

	movq TDX_HYPERCALL_rbx(%rax), %rbx

	movq TDX_HYPERCALL_rdx(%rax), %rdx



	push %rax



	/* Mangle function call ABI into TDCALL ABI: */

	/* Set TDCALL leaf ID (TDVMCALL (0)) in RAX */

	xor %eax, %eax



	movl $TDVMCALL_EXPOSE_REGS_MASK, %ecx



	tdcall



	/*

	 * RAX!=0 indicates a failure of the TDVMCALL mechanism itself and that

	 * something has gone horribly wrong with the TDX module.

	 *

	 * The return status of the hypercall operation is in a separate

	 * register (in R10). Hypercall errors are a part of normal operation

	 * and are handled by callers.

	 */

	testq %rax, %rax

	jne .Lpanic\@



	pop %rax



	.if \ret

	movq %r8,  TDX_HYPERCALL_r8(%rax)

	movq %r9,  TDX_HYPERCALL_r9(%rax)

	movq %r10, TDX_HYPERCALL_r10(%rax)

	movq %r11, TDX_HYPERCALL_r11(%rax)

	movq %r12, TDX_HYPERCALL_r12(%rax)

	movq %r13, TDX_HYPERCALL_r13(%rax)

	movq %r14, TDX_HYPERCALL_r14(%rax)

	movq %r15, TDX_HYPERCALL_r15(%rax)

	movq %rdi, TDX_HYPERCALL_rdi(%rax)

	movq %rsi, TDX_HYPERCALL_rsi(%rax)

	movq %rbx, TDX_HYPERCALL_rbx(%rax)

	movq %rdx, TDX_HYPERCALL_rdx(%rax)

	.endif



	/* TDVMCALL leaf return code is in R10 */

	movq %r10, %rax



	/*

	 * Zero out registers exposed to the VMM to avoid speculative execution

	 * with VMM-controlled values. This needs to include all registers

	 * present in TDVMCALL_EXPOSE_REGS_MASK, except RBX, and R12-R15 which

	 * will be restored.

	 */

	xor %r8d,  %r8d

	xor %r9d,  %r9d

	xor %r10d, %r10d

	xor %r11d, %r11d

	xor %rdi,  %rdi

	xor %rdx,  %rdx



	/* Restore callee-saved GPRs as mandated by the x86_64 ABI */

	pop %rbx

	pop %r12

	pop %r13

	pop %r14

	pop %r15



	FRAME_END



	RET

.Lpanic\@:

	call __tdx_hypercall_failed

	/* __tdx_hypercall_failed never returns */

	REACHABLE

	jmp .Lpanic\@

.endm

SYM_FUNC_START(__tdcall_ret)

	TDX_MODULE_CALL host=0 ret=1

SYM_FUNC_END(__tdcall_ret)



/*

 * __tdcall_saved_ret() - Used by TDX guests to request services from the

 * TDX module (including VMM services) using TDCALL instruction, with

 * saving output registers to the 'struct tdx_module_args' used as input.

 *

 * __tdx_hypercall() function ABI:

 *

 * @args  (RDI)        - struct tdx_hypercall_args for input

 *

 * On successful completion, return the hypercall error code.

 */

SYM_FUNC_START(__tdx_hypercall)

	TDX_HYPERCALL ret=0

SYM_FUNC_END(__tdx_hypercall)



/*

 * __tdcall_saved_ret() function ABI:

 *

 * __tdx_hypercall_ret() function ABI:

 * @fn   (RDI)	- TDCALL leaf ID, moved to RAX

 * @args (RSI)	- struct tdx_module_args for input/output

 *

 * @args  (RDI)        - struct tdx_hypercall_args for input and output

 * All registers in @args are used as input/output registers.

 *

 * On successful completion, return the hypercall error code.

 */

SYM_FUNC_START(__tdx_hypercall_ret)

	TDX_HYPERCALL ret=1

SYM_FUNC_END(__tdx_hypercall_ret)
 
SYM_FUNC_START(__tdcall_saved_ret)

	TDX_MODULE_CALL host=0 ret=1 saved=1

SYM_FUNC_END(__tdcall_saved_ret)

arch/x86/coco/tdx/tdx-shared.c

+24 −4
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ static unsigned long try_accept_one(phys_addr_t start, unsigned long len, 
				    enum pg_level pg_level)

{

	unsigned long accept_size = page_level_size(pg_level);

	u64 tdcall_rcx;

	struct tdx_module_args args = {};

	u8 page_size;



	if (!IS_ALIGNED(start, accept_size))
@@ -34,8 +34,8 @@ static unsigned long try_accept_one(phys_addr_t start, unsigned long len, 
		return 0;

	}



	tdcall_rcx = start | page_size;

	if (__tdx_module_call(TDX_ACCEPT_PAGE, tdcall_rcx, 0, 0, 0, NULL))

	args.rcx = start | page_size;

	if (__tdcall(TDG_MEM_PAGE_ACCEPT, &args))

		return 0;



	return accept_size;
@@ -45,7 +45,7 @@ bool tdx_accept_memory(phys_addr_t start, phys_addr_t end) 
{

	/*

	 * For shared->private conversion, accept the page using

	 * TDX_ACCEPT_PAGE TDX module call.

	 * TDG_MEM_PAGE_ACCEPT TDX module call.

	 */

	while (start < end) {

		unsigned long len = end - start;
@@ -69,3 +69,23 @@ bool tdx_accept_memory(phys_addr_t start, phys_addr_t end) 


	return true;

}



noinstr u64 __tdx_hypercall(struct tdx_module_args *args)

{

	/*

	 * For TDVMCALL explicitly set RCX to the bitmap of shared registers.

	 * The caller isn't expected to set @args->rcx anyway.

	 */

	args->rcx = TDVMCALL_EXPOSE_REGS_MASK;



	/*

	 * Failure of __tdcall_saved_ret() indicates a failure of the TDVMCALL

	 * mechanism itself and that something has gone horribly wrong with

	 * the TDX module.  __tdx_hypercall_failed() never returns.

	 */

	if (__tdcall_saved_ret(TDG_VP_VMCALL, args))

		__tdx_hypercall_failed();



	/* TDVMCALL leaf return code is in R10 */

	return args->r10;

}