Commit d4077e6a authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Borislav Petkov (AMD)
Browse files

x86/sev: Move __sev_[get|put]_ghcb() into separate noinstr object 



Rename sev-nmi.c to noinstr.c, and move the get/put GHCB routines into it too,
which are also annotated as 'noinstr' and suffer from the same problem as the
NMI code, i.e., that GCC may ignore the __no_sanitize_address__ function
attribute implied by 'noinstr' and insert KASAN instrumentation anyway.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/20250828102202.1849035-37-ardb+git@google.com
parent 9723dd0c

arch/x86/boot/startup/sev-startup.c

+0 −74
Original line number Diff line number Diff line
@@ -41,83 +41,9 @@ 
#include <asm/cpuid/api.h>

#include <asm/cmdline.h>



/*

 * Nothing shall interrupt this code path while holding the per-CPU

 * GHCB. The backup GHCB is only for NMIs interrupting this path.

 *

 * Callers must disable local interrupts around it.

 */

noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)

{

	struct sev_es_runtime_data *data;

	struct ghcb *ghcb;



	WARN_ON(!irqs_disabled());



	data = this_cpu_read(runtime_data);

	ghcb = &data->ghcb_page;



	if (unlikely(data->ghcb_active)) {

		/* GHCB is already in use - save its contents */



		if (unlikely(data->backup_ghcb_active)) {

			/*

			 * Backup-GHCB is also already in use. There is no way

			 * to continue here so just kill the machine. To make

			 * panic() work, mark GHCBs inactive so that messages

			 * can be printed out.

			 */

			data->ghcb_active        = false;

			data->backup_ghcb_active = false;



			instrumentation_begin();

			panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");

			instrumentation_end();

		}



		/* Mark backup_ghcb active before writing to it */

		data->backup_ghcb_active = true;



		state->ghcb = &data->backup_ghcb;



		/* Backup GHCB content */

		*state->ghcb = *ghcb;

	} else {

		state->ghcb = NULL;

		data->ghcb_active = true;

	}



	return ghcb;

}



/* Include code shared with pre-decompression boot stage */

#include "sev-shared.c"



noinstr void __sev_put_ghcb(struct ghcb_state *state)

{

	struct sev_es_runtime_data *data;

	struct ghcb *ghcb;



	WARN_ON(!irqs_disabled());



	data = this_cpu_read(runtime_data);

	ghcb = &data->ghcb_page;



	if (state->ghcb) {

		/* Restore GHCB from Backup */

		*ghcb = *state->ghcb;

		data->backup_ghcb_active = false;

		state->ghcb = NULL;

	} else {

		/*

		 * Invalidate the GHCB so a VMGEXIT instruction issued

		 * from userspace won't appear to be valid.

		 */

		vc_ghcb_invalidate(ghcb);

		data->ghcb_active = false;

	}

}



void __head

early_set_pages_state(unsigned long vaddr, unsigned long paddr,

		      unsigned long npages, const struct psc_desc *desc)

arch/x86/coco/sev/Makefile

+4 −4
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0



obj-y += core.o sev-nmi.o vc-handle.o

obj-y += core.o noinstr.o vc-handle.o



# Clang 14 and older may fail to respect __no_sanitize_undefined when inlining

UBSAN_SANITIZE_sev-nmi.o	:= n

UBSAN_SANITIZE_noinstr.o	:= n



# GCC may fail to respect __no_sanitize_address or __no_kcsan when inlining

KASAN_SANITIZE_sev-nmi.o	:= n

KCSAN_SANITIZE_sev-nmi.o	:= n
 
KASAN_SANITIZE_noinstr.o	:= n

KCSAN_SANITIZE_noinstr.o	:= n

arch/x86/coco/sev/sev-nmi.c→arch/x86/coco/sev/noinstr.c

+74 −0
Original line number Diff line number Diff line
@@ -106,3 +106,77 @@ void noinstr __sev_es_nmi_complete(void) 


	__sev_put_ghcb(&state);

}



/*

 * Nothing shall interrupt this code path while holding the per-CPU

 * GHCB. The backup GHCB is only for NMIs interrupting this path.

 *

 * Callers must disable local interrupts around it.

 */

noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)

{

	struct sev_es_runtime_data *data;

	struct ghcb *ghcb;



	WARN_ON(!irqs_disabled());



	data = this_cpu_read(runtime_data);

	ghcb = &data->ghcb_page;



	if (unlikely(data->ghcb_active)) {

		/* GHCB is already in use - save its contents */



		if (unlikely(data->backup_ghcb_active)) {

			/*

			 * Backup-GHCB is also already in use. There is no way

			 * to continue here so just kill the machine. To make

			 * panic() work, mark GHCBs inactive so that messages

			 * can be printed out.

			 */

			data->ghcb_active        = false;

			data->backup_ghcb_active = false;



			instrumentation_begin();

			panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");

			instrumentation_end();

		}



		/* Mark backup_ghcb active before writing to it */

		data->backup_ghcb_active = true;



		state->ghcb = &data->backup_ghcb;



		/* Backup GHCB content */

		*state->ghcb = *ghcb;

	} else {

		state->ghcb = NULL;

		data->ghcb_active = true;

	}



	return ghcb;

}



noinstr void __sev_put_ghcb(struct ghcb_state *state)

{

	struct sev_es_runtime_data *data;

	struct ghcb *ghcb;



	WARN_ON(!irqs_disabled());



	data = this_cpu_read(runtime_data);

	ghcb = &data->ghcb_page;



	if (state->ghcb) {

		/* Restore GHCB from Backup */

		*ghcb = *state->ghcb;

		data->backup_ghcb_active = false;

		state->ghcb = NULL;

	} else {

		/*

		 * Invalidate the GHCB so a VMGEXIT instruction issued

		 * from userspace won't appear to be valid.

		 */

		vc_ghcb_invalidate(ghcb);

		data->ghcb_active = false;

	}

}