Commit 37dbd78f authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Borislav Petkov (AMD)
Browse files

x86/sev: Move GHCB page based HV communication out of startup code 



Both the decompressor and the core kernel implement an early #VC handler,
which only deals with CPUID instructions, and full featured one, which can
handle any #VC exception.

The former communicates with the hypervisor using the MSR based protocol,
whereas the latter uses a shared GHCB page, which is configured a bit later
during the boot, when the kernel runs from its ordinary virtual mapping,
rather than the 1:1 mapping that the startup code uses.

Accessing this shared GHCB page from the core kernel's startup code is
problematic, because it involves converting the GHCB address provided by the
caller to a physical address. In the startup code, virtual to physical address
translations are problematic, given that the virtual address might be a 1:1
mapped address, and such translations should therefore be avoided.

This means that exposing startup code dealing with the GHCB to callers that
execute from the ordinary kernel virtual mapping should be avoided too. So
move all GHCB page based communication out of the startup code, now that all
communication occurring before the kernel virtual mapping is up relies on the
MSR protocol only.

As an exception, add a flag representing the need to apply the coherency
fix in order to avoid exporting CPUID* helpers because of the code
running too early for the *cpu_has* infrastructure.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/20250828102202.1849035-29-ardb+git@google.com
parent e349241b

arch/x86/boot/compressed/sev-handle-vc.c

+3 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0



#include "misc.h"

#include "error.h"

#include "sev.h"



#include <linux/kernel.h>
@@ -14,6 +15,8 @@ 
#include <asm/fpu/xcr.h>



#define __BOOT_COMPRESSED

#undef __init

#define __init



/* Basic instruction decoding support needed */

#include "../../lib/inat.c"

arch/x86/boot/compressed/sev.c

+2 −0
Original line number Diff line number Diff line
@@ -371,6 +371,8 @@ static int sev_check_cpu_support(void) 
	if (!(eax & BIT(1)))

		return -ENODEV;



	sev_snp_needs_sfw = !(ebx & BIT(31));



	return ebx & 0x3f;

}

arch/x86/boot/cpuflags.c

+0 −13
Original line number Diff line number Diff line
@@ -106,18 +106,5 @@ void get_cpuflags(void) 
			cpuid(0x80000001, &ignored, &ignored, &cpu.flags[6],

			      &cpu.flags[1]);

		}



		if (max_amd_level >= 0x8000001f) {

			u32 ebx;



			/*

			 * The X86_FEATURE_COHERENCY_SFW_NO feature bit is in

			 * the virtualization flags entry (word 8) and set by

			 * scattered.c, so the bit needs to be explicitly set.

			 */

			cpuid(0x8000001f, &ignored, &ebx, &ignored, &ignored);

			if (ebx & BIT(31))

				set_bit(X86_FEATURE_COHERENCY_SFW_NO, cpu.flags);

		}

	}

}

arch/x86/boot/startup/sev-shared.c

+6 −139
Original line number Diff line number Diff line
@@ -13,12 +13,9 @@ 


#ifndef __BOOT_COMPRESSED

#define error(v)			pr_err(v)

#define has_cpuflag(f)			boot_cpu_has(f)

#else

#undef WARN

#define WARN(condition, format...) (!!(condition))

#undef vc_forward_exception

#define vc_forward_exception(c)		panic("SNP: Hypervisor requested exception\n")

#endif



/*
@@ -39,7 +36,7 @@ u64 boot_svsm_caa_pa __ro_after_init; 
 *

 * GHCB protocol version negotiated with the hypervisor.

 */

static u16 ghcb_version __ro_after_init;

u16 ghcb_version __ro_after_init;



/* Copy of the SNP firmware's CPUID page. */

static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
@@ -54,15 +51,7 @@ static u32 cpuid_std_range_max __ro_after_init; 
static u32 cpuid_hyp_range_max __ro_after_init;

static u32 cpuid_ext_range_max __ro_after_init;



bool __init sev_es_check_cpu_features(void)

{

	if (!has_cpuflag(X86_FEATURE_RDRAND)) {

		error("RDRAND instruction not supported - no trusted source of randomness available\n");

		return false;

	}



	return true;

}

bool sev_snp_needs_sfw;



void __head __noreturn

sev_es_terminate(unsigned int set, unsigned int reason)
@@ -100,72 +89,7 @@ u64 get_hv_features(void) 
	return GHCB_MSR_HV_FT_RESP_VAL(val);

}



void snp_register_ghcb_early(unsigned long paddr)

{

	unsigned long pfn = paddr >> PAGE_SHIFT;

	u64 val;



	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));

	VMGEXIT();



	val = sev_es_rd_ghcb_msr();



	/* If the response GPA is not ours then abort the guest */

	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||

	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))

		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);

}



bool sev_es_negotiate_protocol(void)

{

	u64 val;



	/* Do the GHCB protocol version negotiation */

	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);

	VMGEXIT();

	val = sev_es_rd_ghcb_msr();



	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)

		return false;



	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||

	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)

		return false;



	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);



	return true;

}



static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)

{

	u32 ret;



	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);

	if (!ret)

		return ES_OK;



	if (ret == 1) {

		u64 info = ghcb->save.sw_exit_info_2;

		unsigned long v = info & SVM_EVTINJ_VEC_MASK;



		/* Check if exception information from hypervisor is sane. */

		if ((info & SVM_EVTINJ_VALID) &&

		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&

		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {

			ctxt->fi.vector = v;



			if (info & SVM_EVTINJ_VALID_ERR)

				ctxt->fi.error_code = info >> 32;



			return ES_EXCEPTION;

		}

	}



	return ES_VMM_ERROR;

}



static inline int svsm_process_result_codes(struct svsm_call *call)

int svsm_process_result_codes(struct svsm_call *call)

{

	switch (call->rax_out) {

	case SVSM_SUCCESS:
@@ -193,7 +117,7 @@ static inline int svsm_process_result_codes(struct svsm_call *call) 
 *     - RAX specifies the SVSM protocol/callid as input and the return code

 *       as output.

 */

static __always_inline void svsm_issue_call(struct svsm_call *call, u8 *pending)

void svsm_issue_call(struct svsm_call *call, u8 *pending)

{

	register unsigned long rax asm("rax") = call->rax;

	register unsigned long rcx asm("rcx") = call->rcx;
@@ -216,7 +140,7 @@ static __always_inline void svsm_issue_call(struct svsm_call *call, u8 *pending) 
	call->r9_out  = r9;

}



static int svsm_perform_msr_protocol(struct svsm_call *call)

int svsm_perform_msr_protocol(struct svsm_call *call)

{

	u8 pending = 0;

	u64 val, resp;
@@ -247,63 +171,6 @@ static int svsm_perform_msr_protocol(struct svsm_call *call) 
	return svsm_process_result_codes(call);

}



static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)

{

	struct es_em_ctxt ctxt;

	u8 pending = 0;



	vc_ghcb_invalidate(ghcb);



	/*

	 * Fill in protocol and format specifiers. This can be called very early

	 * in the boot, so use rip-relative references as needed.

	 */

	ghcb->protocol_version = ghcb_version;

	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;



	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL);

	ghcb_set_sw_exit_info_1(ghcb, 0);

	ghcb_set_sw_exit_info_2(ghcb, 0);



	sev_es_wr_ghcb_msr(__pa(ghcb));



	svsm_issue_call(call, &pending);



	if (pending)

		return -EINVAL;



	switch (verify_exception_info(ghcb, &ctxt)) {

	case ES_OK:

		break;

	case ES_EXCEPTION:

		vc_forward_exception(&ctxt);

		fallthrough;

	default:

		return -EINVAL;

	}



	return svsm_process_result_codes(call);

}



enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,

				   struct es_em_ctxt *ctxt,

				   u64 exit_code, u64 exit_info_1,

				   u64 exit_info_2)

{

	/* Fill in protocol and format specifiers */

	ghcb->protocol_version = ghcb_version;

	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;



	ghcb_set_sw_exit_code(ghcb, exit_code);

	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);

	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);



	sev_es_wr_ghcb_msr(__pa(ghcb));

	VMGEXIT();



	return verify_exception_info(ghcb, ctxt);

}



static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)

{

	u64 val;
@@ -793,7 +660,7 @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, 
	 * If validating memory (making it private) and affected by the

	 * cache-coherency vulnerability, perform the cache eviction mitigation.

	 */

	if (validate && !has_cpuflag(X86_FEATURE_COHERENCY_SFW_NO))

	if (validate && sev_snp_needs_sfw)

		sev_evict_cache((void *)vaddr, 1);

}

arch/x86/boot/startup/sev-startup.c

+0 −42
Original line number Diff line number Diff line
@@ -41,15 +41,6 @@ 
#include <asm/cpuid/api.h>

#include <asm/cmdline.h>



/* For early boot hypervisor communication in SEV-ES enabled guests */

struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);



/*

 * Needs to be in the .data section because we need it NULL before bss is

 * cleared

 */

struct ghcb *boot_ghcb __section(".data");



/* Bitmap of SEV features supported by the hypervisor */

u64 sev_hv_features __ro_after_init;
@@ -139,39 +130,6 @@ noinstr void __sev_put_ghcb(struct ghcb_state *state) 
	}

}



int svsm_perform_call_protocol(struct svsm_call *call)

{

	struct ghcb_state state;

	unsigned long flags;

	struct ghcb *ghcb;

	int ret;



	/*

	 * This can be called very early in the boot, use native functions in

	 * order to avoid paravirt issues.

	 */

	flags = native_local_irq_save();



	if (sev_cfg.ghcbs_initialized)

		ghcb = __sev_get_ghcb(&state);

	else if (boot_ghcb)

		ghcb = boot_ghcb;

	else

		ghcb = NULL;



	do {

		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)

			   : svsm_perform_msr_protocol(call);

	} while (ret == -EAGAIN);



	if (sev_cfg.ghcbs_initialized)

		__sev_put_ghcb(&state);



	native_local_irq_restore(flags);



	return ret;

}



void __head

early_set_pages_state(unsigned long vaddr, unsigned long paddr,

		      unsigned long npages, enum psc_op op)