Commit 2a783066 authored by Ashish Kalra's avatar Ashish Kalra Committed by Borislav Petkov (AMD)
Browse files

x86/mm: Refactor __set_clr_pte_enc() 



Refactor __set_clr_pte_enc() and add two new helper functions to
set/clear PTE C-bit from early SEV/SNP initialization code and later
during shutdown/kexec especially when all CPUs are stopped and
interrupts are disabled and set_memory_xx() interfaces can't be used.

Co-developed-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: default avatarAshish Kalra <ashish.kalra@amd.com>
Reviewed-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/5df4aa450447f28294d1c5a890e27b63ed4ded36.1722520012.git.ashish.kalra@amd.com
parent f30470c1

arch/x86/include/asm/sev.h

+20 −0
Original line number Diff line number Diff line
@@ -322,6 +322,22 @@ struct svsm_attest_call { 
	u8 rsvd[4];

};



/* PTE descriptor used for the prepare_pte_enc() operations. */

struct pte_enc_desc {

	pte_t *kpte;

	int pte_level;

	bool encrypt;

	/* pfn of the kpte above */

	unsigned long pfn;

	/* physical address of @pfn */

	unsigned long pa;

	/* virtual address of @pfn */

	void *va;

	/* memory covered by the pte */

	unsigned long size;

	pgprot_t new_pgprot;

};



/*

 * SVSM protocol structure

 */
@@ -437,6 +453,8 @@ u64 snp_get_unsupported_features(u64 status); 
u64 sev_get_status(void);

void sev_show_status(void);

void snp_update_svsm_ca(void);

int prepare_pte_enc(struct pte_enc_desc *d);

void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot);



#else	/* !CONFIG_AMD_MEM_ENCRYPT */
@@ -474,6 +492,8 @@ static inline u64 snp_get_unsupported_features(u64 status) { return 0; } 
static inline u64 sev_get_status(void) { return 0; }

static inline void sev_show_status(void) { }

static inline void snp_update_svsm_ca(void) { }

static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; }

static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { }



#endif	/* CONFIG_AMD_MEM_ENCRYPT */

arch/x86/mm/mem_encrypt_amd.c

+49 −26
Original line number Diff line number Diff line
@@ -311,59 +311,82 @@ static int amd_enc_status_change_finish(unsigned long vaddr, int npages, bool en 
	return 0;

}



static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)

int prepare_pte_enc(struct pte_enc_desc *d)

{

	pgprot_t old_prot, new_prot;

	unsigned long pfn, pa, size;

	pte_t new_pte;

	pgprot_t old_prot;



	pfn = pg_level_to_pfn(level, kpte, &old_prot);

	if (!pfn)

		return;

	d->pfn = pg_level_to_pfn(d->pte_level, d->kpte, &old_prot);

	if (!d->pfn)

		return 1;



	new_prot = old_prot;

	if (enc)

		pgprot_val(new_prot) |= _PAGE_ENC;

	d->new_pgprot = old_prot;

	if (d->encrypt)

		pgprot_val(d->new_pgprot) |= _PAGE_ENC;

	else

		pgprot_val(new_prot) &= ~_PAGE_ENC;

		pgprot_val(d->new_pgprot) &= ~_PAGE_ENC;



	/* If prot is same then do nothing. */

	if (pgprot_val(old_prot) == pgprot_val(new_prot))

		return;

	if (pgprot_val(old_prot) == pgprot_val(d->new_pgprot))

		return 1;



	pa = pfn << PAGE_SHIFT;

	size = page_level_size(level);

	d->pa = d->pfn << PAGE_SHIFT;

	d->size = page_level_size(d->pte_level);



	/*

	 * We are going to perform in-place en-/decryption and change the

	 * physical page attribute from C=1 to C=0 or vice versa. Flush the

	 * caches to ensure that data gets accessed with the correct C-bit.

	 * In-place en-/decryption and physical page attribute change

	 * from C=1 to C=0 or vice versa will be performed. Flush the

	 * caches to ensure that data gets accessed with the correct

	 * C-bit.

	 */

	clflush_cache_range(__va(pa), size);

	if (d->va)

		clflush_cache_range(d->va, d->size);

	else

		clflush_cache_range(__va(d->pa), d->size);



	return 0;

}



void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot)

{

	pte_t new_pte;



	/* Change the page encryption mask. */

	new_pte = pfn_pte(pfn, new_prot);

	set_pte_atomic(kpte, new_pte);

}



static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)

{

	struct pte_enc_desc d = {

		.kpte	     = kpte,

		.pte_level   = level,

		.encrypt     = enc

	};



	if (prepare_pte_enc(&d))

		return;



	/* Encrypt/decrypt the contents in-place */

	if (enc) {

		sme_early_encrypt(pa, size);

		sme_early_encrypt(d.pa, d.size);

	} else {

		sme_early_decrypt(pa, size);

		sme_early_decrypt(d.pa, d.size);



		/*

		 * ON SNP, the page state in the RMP table must happen

		 * before the page table updates.

		 */

		early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1);

		early_snp_set_memory_shared((unsigned long)__va(d.pa), d.pa, 1);

	}



	/* Change the page encryption mask. */

	new_pte = pfn_pte(pfn, new_prot);

	set_pte_atomic(kpte, new_pte);

	set_pte_enc_mask(kpte, d.pfn, d.new_pgprot);



	/*

	 * If page is set encrypted in the page table, then update the RMP table to

	 * add this page as private.

	 */

	if (enc)

		early_snp_set_memory_private((unsigned long)__va(pa), pa, 1);

		early_snp_set_memory_private((unsigned long)__va(d.pa), d.pa, 1);

}



static int __init early_set_memory_enc_dec(unsigned long vaddr,