Commit 221df25f authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Ingo Molnar
Browse files

x86/sev: Prepare for splitting off early SEV code 



Prepare for splitting off parts of the SEV core.c source file into a
file that carries code that must tolerate being called from the early
1:1 mapping. This will allow special build-time handling of thise code,
to ensure that it gets generated in a way that is compatible with the
early execution context.

So create a de-facto internal SEV API and put the definitions into
sev-internal.h. No attempt is made to allow this header file to be
included in arbitrary other sources - this is explicitly not the intent.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kevin Loughlin <kevinloughlin@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: linux-efi@vger.kernel.org
Link: https://lore.kernel.org/r/20250410134117.3713574-20-ardb+git@google.com
parent bee174b2

arch/x86/boot/compressed/sev.c

+10 −5
Original line number Diff line number Diff line
@@ -126,20 +126,25 @@ static bool fault_in_kernel_space(unsigned long address) 
#include "../../lib/inat.c"

#include "../../lib/insn.c"



/* Include code for early handlers */

#include "../../coco/sev/shared.c"

extern struct svsm_ca *boot_svsm_caa;

extern u64 boot_svsm_caa_pa;



static struct svsm_ca *svsm_get_caa(void)

struct svsm_ca *svsm_get_caa(void)

{

	return boot_svsm_caa;

}



static u64 svsm_get_caa_pa(void)

u64 svsm_get_caa_pa(void)

{

	return boot_svsm_caa_pa;

}



static int svsm_perform_call_protocol(struct svsm_call *call)

int svsm_perform_call_protocol(struct svsm_call *call);



/* Include code for early handlers */

#include "../../coco/sev/shared.c"



int svsm_perform_call_protocol(struct svsm_call *call)

{

	struct ghcb *ghcb;

	int ret;

arch/x86/coco/sev/core.c

+13 −95
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ 
#include <asm/cpu_entry_area.h>

#include <asm/stacktrace.h>

#include <asm/sev.h>

#include <asm/sev-internal.h>

#include <asm/insn-eval.h>

#include <asm/fpu/xcr.h>

#include <asm/processor.h>
@@ -44,8 +45,6 @@ 
#include <asm/cpuid.h>

#include <asm/cmdline.h>



#define DR7_RESET_VALUE        0x400



/* AP INIT values as documented in the APM2  section "Processor Initialization State" */

#define AP_INIT_CS_LIMIT		0xffff

#define AP_INIT_DS_LIMIT		0xffff
@@ -82,16 +81,16 @@ static const char * const sev_status_feat_names[] = { 
};



/* For early boot hypervisor communication in SEV-ES enabled guests */

static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);

struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);



/*

 * Needs to be in the .data section because we need it NULL before bss is

 * cleared

 */

static struct ghcb *boot_ghcb __section(".data");

struct ghcb *boot_ghcb __section(".data");



/* Bitmap of SEV features supported by the hypervisor */

static u64 sev_hv_features __ro_after_init;

u64 sev_hv_features __ro_after_init;



/* Secrets page physical address from the CC blob */

static u64 secrets_pa __ro_after_init;
@@ -105,54 +104,14 @@ static u64 snp_tsc_scale __ro_after_init; 
static u64 snp_tsc_offset __ro_after_init;

static u64 snp_tsc_freq_khz __ro_after_init;



/* #VC handler runtime per-CPU data */

struct sev_es_runtime_data {

	struct ghcb ghcb_page;



	/*

	 * Reserve one page per CPU as backup storage for the unencrypted GHCB.

	 * It is needed when an NMI happens while the #VC handler uses the real

	 * GHCB, and the NMI handler itself is causing another #VC exception. In

	 * that case the GHCB content of the first handler needs to be backed up

	 * and restored.

	 */

	struct ghcb backup_ghcb;



	/*

	 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.

	 * There is no need for it to be atomic, because nothing is written to

	 * the GHCB between the read and the write of ghcb_active. So it is safe

	 * to use it when a nested #VC exception happens before the write.

	 *

	 * This is necessary for example in the #VC->NMI->#VC case when the NMI

	 * happens while the first #VC handler uses the GHCB. When the NMI code

	 * raises a second #VC handler it might overwrite the contents of the

	 * GHCB written by the first handler. To avoid this the content of the

	 * GHCB is saved and restored when the GHCB is detected to be in use

	 * already.

	 */

	bool ghcb_active;

	bool backup_ghcb_active;



	/*

	 * Cached DR7 value - write it on DR7 writes and return it on reads.

	 * That value will never make it to the real hardware DR7 as debugging

	 * is currently unsupported in SEV-ES guests.

	 */

	unsigned long dr7;

};



struct ghcb_state {

	struct ghcb *ghcb;

};



/* For early boot SVSM communication */

static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);

struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);



static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);

static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);

static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);

static DEFINE_PER_CPU(u64, svsm_caa_pa);

DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);

DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);

DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);

DEFINE_PER_CPU(u64, svsm_caa_pa);



static __always_inline bool on_vc_stack(struct pt_regs *regs)

{
@@ -231,7 +190,7 @@ void noinstr __sev_es_ist_exit(void) 
 *

 * Callers must disable local interrupts around it.

 */

static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)

noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)

{

	struct sev_es_runtime_data *data;

	struct ghcb *ghcb;
@@ -274,21 +233,6 @@ static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) 
	return ghcb;

}



static inline u64 sev_es_rd_ghcb_msr(void)

{

	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);

}



static __always_inline void sev_es_wr_ghcb_msr(u64 val)

{

	u32 low, high;



	low  = (u32)(val);

	high = (u32)(val >> 32);



	native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);

}



static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,

				unsigned char *buffer)

{
@@ -601,33 +545,7 @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) 
/* Include code shared with pre-decompression boot stage */

#include "shared.c"



static inline struct svsm_ca *svsm_get_caa(void)

{

	/*

	 * Use rIP-relative references when called early in the boot. If

	 * ->use_cas is set, then it is late in the boot and no need

	 * to worry about rIP-relative references.

	 */

	if (RIP_REL_REF(sev_cfg).use_cas)

		return this_cpu_read(svsm_caa);

	else

		return RIP_REL_REF(boot_svsm_caa);

}



static u64 svsm_get_caa_pa(void)

{

	/*

	 * Use rIP-relative references when called early in the boot. If

	 * ->use_cas is set, then it is late in the boot and no need

	 * to worry about rIP-relative references.

	 */

	if (RIP_REL_REF(sev_cfg).use_cas)

		return this_cpu_read(svsm_caa_pa);

	else

		return RIP_REL_REF(boot_svsm_caa_pa);

}



static noinstr void __sev_put_ghcb(struct ghcb_state *state)

noinstr void __sev_put_ghcb(struct ghcb_state *state)

{

	struct sev_es_runtime_data *data;

	struct ghcb *ghcb;
@@ -652,7 +570,7 @@ static noinstr void __sev_put_ghcb(struct ghcb_state *state) 
	}

}



static int svsm_perform_call_protocol(struct svsm_call *call)

int svsm_perform_call_protocol(struct svsm_call *call)

{

	struct ghcb_state state;

	unsigned long flags;
@@ -761,7 +679,7 @@ static u64 __init get_jump_table_addr(void) 
	return ret;

}



static void __head

void __head

early_set_pages_state(unsigned long vaddr, unsigned long paddr,

		      unsigned long npages, enum psc_op op)

{

arch/x86/coco/sev/shared.c

+12 −52
Original line number Diff line number Diff line
@@ -38,12 +38,8 @@ 
 */

u8 snp_vmpl __ro_after_init;

EXPORT_SYMBOL_GPL(snp_vmpl);

static struct svsm_ca *boot_svsm_caa __ro_after_init;

static u64 boot_svsm_caa_pa __ro_after_init;



static struct svsm_ca *svsm_get_caa(void);

static u64 svsm_get_caa_pa(void);

static int svsm_perform_call_protocol(struct svsm_call *call);

struct svsm_ca *boot_svsm_caa __ro_after_init;

u64 boot_svsm_caa_pa __ro_after_init;



/* I/O parameters for CPUID-related helpers */

struct cpuid_leaf {
@@ -55,36 +51,6 @@ struct cpuid_leaf { 
	u32 edx;

};



/*

 * Individual entries of the SNP CPUID table, as defined by the SNP

 * Firmware ABI, Revision 0.9, Section 7.1, Table 14.

 */

struct snp_cpuid_fn {

	u32 eax_in;

	u32 ecx_in;

	u64 xcr0_in;

	u64 xss_in;

	u32 eax;

	u32 ebx;

	u32 ecx;

	u32 edx;

	u64 __reserved;

} __packed;



/*

 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,

 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit

 * of 64 entries per CPUID table.

 */

#define SNP_CPUID_COUNT_MAX 64



struct snp_cpuid_table {

	u32 count;

	u32 __reserved1;

	u64 __reserved2;

	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];

} __packed;



/*

 * Since feature negotiation related variables are set early in the boot

 * process they must reside in the .data section so as not to be zeroed
@@ -107,7 +73,7 @@ static u32 cpuid_std_range_max __ro_after_init; 
static u32 cpuid_hyp_range_max __ro_after_init;

static u32 cpuid_ext_range_max __ro_after_init;



static bool __init sev_es_check_cpu_features(void)

bool __init sev_es_check_cpu_features(void)

{

	if (!has_cpuflag(X86_FEATURE_RDRAND)) {

		error("RDRAND instruction not supported - no trusted source of randomness available\n");
@@ -117,7 +83,7 @@ static bool __init sev_es_check_cpu_features(void) 
	return true;

}



static void __head __noreturn

void __head __noreturn

sev_es_terminate(unsigned int set, unsigned int reason)

{

	u64 val = GHCB_MSR_TERM_REQ;
@@ -136,7 +102,7 @@ sev_es_terminate(unsigned int set, unsigned int reason) 
/*

 * The hypervisor features are available from GHCB version 2 onward.

 */

static u64 get_hv_features(void)

u64 get_hv_features(void)

{

	u64 val;
@@ -153,7 +119,7 @@ static u64 get_hv_features(void) 
	return GHCB_MSR_HV_FT_RESP_VAL(val);

}



static void snp_register_ghcb_early(unsigned long paddr)

void snp_register_ghcb_early(unsigned long paddr)

{

	unsigned long pfn = paddr >> PAGE_SHIFT;

	u64 val;
@@ -169,7 +135,7 @@ static void snp_register_ghcb_early(unsigned long paddr) 
		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);

}



static bool sev_es_negotiate_protocol(void)

bool sev_es_negotiate_protocol(void)

{

	u64 val;
@@ -190,12 +156,6 @@ static bool sev_es_negotiate_protocol(void) 
	return true;

}



static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)

{

	ghcb->save.sw_exit_code = 0;

	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));

}



static bool vc_decoding_needed(unsigned long exit_code)

{

	/* Exceptions don't require to decode the instruction */
@@ -371,7 +331,7 @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call) 
	return svsm_process_result_codes(call);

}



static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,

enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,

				   struct es_em_ctxt *ctxt,

				   u64 exit_code, u64 exit_info_1,

				   u64 exit_info_2)
@@ -473,7 +433,7 @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid 
 * while running with the initial identity mapping as well as the

 * switch-over to kernel virtual addresses later.

 */

static const struct snp_cpuid_table *snp_cpuid_get_table(void)

const struct snp_cpuid_table *snp_cpuid_get_table(void)

{

	return rip_rel_ptr(&cpuid_table_copy);

}

arch/x86/include/asm/sev-internal.h

0 → 100644
+122 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#define DR7_RESET_VALUE        0x400



extern struct ghcb boot_ghcb_page;

extern struct ghcb *boot_ghcb;

extern u64 sev_hv_features;



/* #VC handler runtime per-CPU data */

struct sev_es_runtime_data {

	struct ghcb ghcb_page;



	/*

	 * Reserve one page per CPU as backup storage for the unencrypted GHCB.

	 * It is needed when an NMI happens while the #VC handler uses the real

	 * GHCB, and the NMI handler itself is causing another #VC exception. In

	 * that case the GHCB content of the first handler needs to be backed up

	 * and restored.

	 */

	struct ghcb backup_ghcb;



	/*

	 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.

	 * There is no need for it to be atomic, because nothing is written to

	 * the GHCB between the read and the write of ghcb_active. So it is safe

	 * to use it when a nested #VC exception happens before the write.

	 *

	 * This is necessary for example in the #VC->NMI->#VC case when the NMI

	 * happens while the first #VC handler uses the GHCB. When the NMI code

	 * raises a second #VC handler it might overwrite the contents of the

	 * GHCB written by the first handler. To avoid this the content of the

	 * GHCB is saved and restored when the GHCB is detected to be in use

	 * already.

	 */

	bool ghcb_active;

	bool backup_ghcb_active;



	/*

	 * Cached DR7 value - write it on DR7 writes and return it on reads.

	 * That value will never make it to the real hardware DR7 as debugging

	 * is currently unsupported in SEV-ES guests.

	 */

	unsigned long dr7;

};



struct ghcb_state {

	struct ghcb *ghcb;

};



extern struct svsm_ca boot_svsm_ca_page;



struct ghcb *__sev_get_ghcb(struct ghcb_state *state);

void __sev_put_ghcb(struct ghcb_state *state);



DECLARE_PER_CPU(struct sev_es_runtime_data*, runtime_data);

DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa);



void early_set_pages_state(unsigned long vaddr, unsigned long paddr,

			   unsigned long npages, enum psc_op op);



void __noreturn sev_es_terminate(unsigned int set, unsigned int reason);



DECLARE_PER_CPU(struct svsm_ca *, svsm_caa);

DECLARE_PER_CPU(u64, svsm_caa_pa);



extern struct svsm_ca *boot_svsm_caa;

extern u64 boot_svsm_caa_pa;



static __always_inline struct svsm_ca *svsm_get_caa(void)

{

	/*

	 * Use rIP-relative references when called early in the boot. If

	 * ->use_cas is set, then it is late in the boot and no need

	 * to worry about rIP-relative references.

	 */

	if (RIP_REL_REF(sev_cfg).use_cas)

		return this_cpu_read(svsm_caa);

	else

		return RIP_REL_REF(boot_svsm_caa);

}



static __always_inline u64 svsm_get_caa_pa(void)

{

	/*

	 * Use rIP-relative references when called early in the boot. If

	 * ->use_cas is set, then it is late in the boot and no need

	 * to worry about rIP-relative references.

	 */

	if (RIP_REL_REF(sev_cfg).use_cas)

		return this_cpu_read(svsm_caa_pa);

	else

		return RIP_REL_REF(boot_svsm_caa_pa);

}



int svsm_perform_call_protocol(struct svsm_call *call);



static inline u64 sev_es_rd_ghcb_msr(void)

{

	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);

}



static __always_inline void sev_es_wr_ghcb_msr(u64 val)

{

	u32 low, high;



	low  = (u32)(val);

	high = (u32)(val >> 32);



	native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);

}



enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,

				   struct es_em_ctxt *ctxt,

				   u64 exit_code, u64 exit_info_1,

				   u64 exit_info_2);



void snp_register_ghcb_early(unsigned long paddr);

bool sev_es_negotiate_protocol(void);

bool sev_es_check_cpu_features(void);

u64 get_hv_features(void);



const struct snp_cpuid_table *snp_cpuid_get_table(void);

arch/x86/include/asm/sev.h

+37 −0
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
#include <asm/sev-common.h>

#include <asm/coco.h>

#include <asm/set_memory.h>

#include <asm/svm.h>



#define GHCB_PROTOCOL_MIN	1ULL

#define GHCB_PROTOCOL_MAX	2ULL
@@ -83,6 +84,36 @@ extern void vc_no_ghcb(void); 
extern void vc_boot_ghcb(void);

extern bool handle_vc_boot_ghcb(struct pt_regs *regs);



/*

 * Individual entries of the SNP CPUID table, as defined by the SNP

 * Firmware ABI, Revision 0.9, Section 7.1, Table 14.

 */

struct snp_cpuid_fn {

	u32 eax_in;

	u32 ecx_in;

	u64 xcr0_in;

	u64 xss_in;

	u32 eax;

	u32 ebx;

	u32 ecx;

	u32 edx;

	u64 __reserved;

} __packed;



/*

 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,

 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit

 * of 64 entries per CPUID table.

 */

#define SNP_CPUID_COUNT_MAX 64



struct snp_cpuid_table {

	u32 count;

	u32 __reserved1;

	u64 __reserved2;

	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];

} __packed;



/* PVALIDATE return codes */

#define PVALIDATE_FAIL_SIZEMISMATCH	6
@@ -484,6 +515,12 @@ int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req 
void __init snp_secure_tsc_prepare(void);

void __init snp_secure_tsc_init(void);



static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)

{

	ghcb->save.sw_exit_code = 0;

	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));

}



#else	/* !CONFIG_AMD_MEM_ENCRYPT */



#define snp_vmpl 0