Commit 0e20f1f4 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

x86/hyperv: Clean up hv_do_hypercall() 



What used to be a simple few instructions has turned into a giant mess
(for x86_64). Not only does it use static_branch wrong, it mixes it
with dynamic branches for no apparent reason.

Notably it uses static_branch through an out-of-line function call,
which completely defeats the purpose, since instead of a simple
JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is
absolutely idiotic.

Add to that a dynamic test of hyperv_paravisor_present, something
which is set once and never changed.

Replace all this idiocy with a single direct function call to the
right hypercall variant.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarMichael Kelley <mhklinux@outlook.com>
Tested-by: default avatarMichael Kelley <mhklinux@outlook.com>
Acked-by: default avatarSean Christopherson <seanjc@google.com>
Link: https://lkml.kernel.org/r/20250714103440.897136093@infradead.org
parent a1d34a44

arch/x86/hyperv/hv_init.c

+20 −0
Original line number Diff line number Diff line
@@ -37,7 +37,27 @@ 
#include <linux/export.h>



void *hv_hypercall_pg;



#ifdef CONFIG_X86_64

u64 hv_std_hypercall(u64 control, u64 param1, u64 param2)

{

	u64 hv_status;



	if (!hv_hypercall_pg)

		return U64_MAX;



	register u64 __r8 asm("r8") = param2;

	asm volatile (CALL_NOSPEC

		      : "=a" (hv_status), ASM_CALL_CONSTRAINT,

		        "+c" (control), "+d" (param1), "+r" (__r8)

		      : THUNK_TARGET(hv_hypercall_pg)

		      : "cc", "memory", "r9", "r10", "r11");



	return hv_status;

}

#else

EXPORT_SYMBOL_GPL(hv_hypercall_pg);

#endif



union hv_ghcb * __percpu *hv_ghcb_pg;

arch/x86/hyperv/ivm.c

+15 −0
Original line number Diff line number Diff line
@@ -385,9 +385,23 @@ int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu) 
	return ret;

}



u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2)

{

	u64 hv_status;



	register u64 __r8 asm("r8") = param2;

	asm volatile("vmmcall"

		     : "=a" (hv_status), ASM_CALL_CONSTRAINT,

		       "+c" (control), "+d" (param1), "+r" (__r8)

		     : : "cc", "memory", "r9", "r10", "r11");



	return hv_status;

}



#else

static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}

static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}

u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }

#endif /* CONFIG_AMD_MEM_ENCRYPT */



#ifdef CONFIG_INTEL_TDX_GUEST
@@ -437,6 +451,7 @@ u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) 
#else

static inline void hv_tdx_msr_write(u64 msr, u64 value) {}

static inline void hv_tdx_msr_read(u64 msr, u64 *value) {}

u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }

#endif /* CONFIG_INTEL_TDX_GUEST */



#if defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST)

arch/x86/include/asm/mshyperv.h

+41 −96
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#include <linux/nmi.h>

#include <linux/msi.h>

#include <linux/io.h>

#include <linux/static_call.h>

#include <asm/nospec-branch.h>

#include <asm/paravirt.h>

#include <asm/msr.h>
@@ -39,16 +40,21 @@ static inline unsigned char hv_get_nmi_reason(void) 
	return 0;

}



#if IS_ENABLED(CONFIG_HYPERV)

extern bool hyperv_paravisor_present;

extern u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);

extern u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2);

extern u64 hv_std_hypercall(u64 control, u64 param1, u64 param2);



#if IS_ENABLED(CONFIG_HYPERV)

extern void *hv_hypercall_pg;



extern union hv_ghcb * __percpu *hv_ghcb_pg;



bool hv_isolation_type_snp(void);

bool hv_isolation_type_tdx(void);

u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);



#ifdef CONFIG_X86_64

DECLARE_STATIC_CALL(hv_hypercall, hv_std_hypercall);

#endif



/*

 * DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA
@@ -65,37 +71,15 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output) 
{

	u64 input_address = input ? virt_to_phys(input) : 0;

	u64 output_address = output ? virt_to_phys(output) : 0;

	u64 hv_status;



#ifdef CONFIG_X86_64

	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)

		return hv_tdx_hypercall(control, input_address, output_address);



	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {

		__asm__ __volatile__("mov %[output_address], %%r8\n"

				     "vmmcall"

				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,

				       "+c" (control), "+d" (input_address)

				     : [output_address] "r" (output_address)

				     : "cc", "memory", "r8", "r9", "r10", "r11");

		return hv_status;

	}



	if (!hv_hypercall_pg)

		return U64_MAX;



	__asm__ __volatile__("mov %[output_address], %%r8\n"

			     CALL_NOSPEC

			     : "=a" (hv_status), ASM_CALL_CONSTRAINT,

			       "+c" (control), "+d" (input_address)

			     : [output_address] "r" (output_address),

			       THUNK_TARGET(hv_hypercall_pg)

			     : "cc", "memory", "r8", "r9", "r10", "r11");

	return static_call_mod(hv_hypercall)(control, input_address, output_address);

#else

	u32 input_address_hi = upper_32_bits(input_address);

	u32 input_address_lo = lower_32_bits(input_address);

	u32 output_address_hi = upper_32_bits(output_address);

	u32 output_address_lo = lower_32_bits(output_address);

	u64 hv_status;



	if (!hv_hypercall_pg)

		return U64_MAX;
@@ -108,36 +92,19 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output) 
			       "D"(output_address_hi), "S"(output_address_lo),

			       THUNK_TARGET(hv_hypercall_pg)

			     : "cc", "memory");

#endif /* !x86_64 */

	return hv_status;

#endif /* !x86_64 */

}



/* Fast hypercall with 8 bytes of input and no output */

static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)

{

	u64 hv_status;



#ifdef CONFIG_X86_64

	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)

		return hv_tdx_hypercall(control, input1, 0);



	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {

		__asm__ __volatile__(

				"vmmcall"

				: "=a" (hv_status), ASM_CALL_CONSTRAINT,

				"+c" (control), "+d" (input1)

				:: "cc", "r8", "r9", "r10", "r11");

	} else {

		__asm__ __volatile__(CALL_NOSPEC

				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,

				       "+c" (control), "+d" (input1)

				     : THUNK_TARGET(hv_hypercall_pg)

				     : "cc", "r8", "r9", "r10", "r11");

	}

	return static_call_mod(hv_hypercall)(control, input1, 0);

#else

	{

	u32 input1_hi = upper_32_bits(input1);

	u32 input1_lo = lower_32_bits(input1);

	u64 hv_status;



	__asm__ __volatile__ (CALL_NOSPEC

			      : "=A"(hv_status),
@@ -147,9 +114,8 @@ static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1) 
			      "b" (input1_hi),

			      THUNK_TARGET(hv_hypercall_pg)

			      : "cc", "edi", "esi");

	}

#endif

	return hv_status;

#endif

}



static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
@@ -162,34 +128,14 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) 
/* Fast hypercall with 16 bytes of input */

static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2)

{

	u64 hv_status;



#ifdef CONFIG_X86_64

	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)

		return hv_tdx_hypercall(control, input1, input2);



	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {

		__asm__ __volatile__("mov %[input2], %%r8\n"

				     "vmmcall"

				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,

				       "+c" (control), "+d" (input1)

				     : [input2] "r" (input2)

				     : "cc", "r8", "r9", "r10", "r11");

	} else {

		__asm__ __volatile__("mov %[input2], %%r8\n"

				     CALL_NOSPEC

				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,

				       "+c" (control), "+d" (input1)

				     : [input2] "r" (input2),

				       THUNK_TARGET(hv_hypercall_pg)

				     : "cc", "r8", "r9", "r10", "r11");

	}

	return static_call_mod(hv_hypercall)(control, input1, input2);

#else

	{

	u32 input1_hi = upper_32_bits(input1);

	u32 input1_lo = lower_32_bits(input1);

	u32 input2_hi = upper_32_bits(input2);

	u32 input2_lo = lower_32_bits(input2);

	u64 hv_status;



	__asm__ __volatile__ (CALL_NOSPEC

			      : "=A"(hv_status),
@@ -198,9 +144,8 @@ static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2) 
			      "D"(input2_hi), "S"(input2_lo),

			      THUNK_TARGET(hv_hypercall_pg)

			      : "cc");

	}

#endif

	return hv_status;

#endif

}



static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)

arch/x86/kernel/cpu/mshyperv.c

+13 −6
Original line number Diff line number Diff line
@@ -38,10 +38,6 @@ 
bool hv_nested;

struct ms_hyperv_info ms_hyperv;



/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */

bool hyperv_paravisor_present __ro_after_init;

EXPORT_SYMBOL_GPL(hyperv_paravisor_present);



#if IS_ENABLED(CONFIG_HYPERV)

static inline unsigned int hv_get_nested_msr(unsigned int reg)

{
@@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc_pg_clock(void) 
	old_restore_sched_clock_state = x86_platform.restore_sched_clock_state;

	x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state;

}



#ifdef CONFIG_X86_64

DEFINE_STATIC_CALL(hv_hypercall, hv_std_hypercall);

EXPORT_STATIC_CALL_TRAMP_GPL(hv_hypercall);

#define hypercall_update(hc) static_call_update(hv_hypercall, hc)

#endif

#endif /* CONFIG_HYPERV */



#ifndef hypercall_update

#define hypercall_update(hc) (void)hc

#endif



static uint32_t  __init ms_hyperv_platform(void)

{

	u32 eax;
@@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platform(void) 
			ms_hyperv.shared_gpa_boundary =

				BIT_ULL(ms_hyperv.shared_gpa_boundary_bits);



		hyperv_paravisor_present = !!ms_hyperv.paravisor_present;



		pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n",

			ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);





		if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) {

			static_branch_enable(&isolation_type_snp);

			if (!ms_hyperv.paravisor_present)

				hypercall_update(hv_snp_hypercall);

		} else if (hv_get_isolation_type() == HV_ISOLATION_TYPE_TDX) {

			static_branch_enable(&isolation_type_tdx);
@@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platform(void) 
			ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED;



			if (!ms_hyperv.paravisor_present) {

				hypercall_update(hv_tdx_hypercall);

				/*

				 * Mark the Hyper-V TSC page feature as disabled

				 * in a TDX VM without paravisor so that the