Commit c63cf135 authored by Brijesh Singh's avatar Brijesh Singh Committed by Paolo Bonzini
Browse files

KVM: SEV: Add support to handle RMP nested page faults 



When SEV-SNP is enabled in the guest, the hardware places restrictions
on all memory accesses based on the contents of the RMP table. When
hardware encounters RMP check failure caused by the guest memory access
it raises the #NPF. The error code contains additional information on
the access type. See the APM volume 2 for additional information.

When using gmem, RMP faults resulting from mismatches between the state
in the RMP table vs. what the guest expects via its page table result
in KVM_EXIT_MEMORY_FAULTs being forwarded to userspace to handle. This
means the only expected case that needs to be handled in the kernel is
when the page size of the entry in the RMP table is larger than the
mapping in the nested page table, in which case a PSMASH instruction
needs to be issued to split the large RMP entry into individual 4K
entries so that subsequent accesses can succeed.

Signed-off-by: default avatarBrijesh Singh <brijesh.singh@amd.com>
Co-developed-by: default avatarMichael Roth <michael.roth@amd.com>
Signed-off-by: default avatarMichael Roth <michael.roth@amd.com>
Signed-off-by: default avatarAshish Kalra <ashish.kalra@amd.com>
Message-ID: <20240501085210.2213060-12-michael.roth@amd.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 9b54e248

arch/x86/include/asm/kvm_host.h

+1 −0
Original line number Diff line number Diff line
@@ -1942,6 +1942,7 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 
				   const struct kvm_memory_slot *memslot);

void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);

void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);

void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);



int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);

arch/x86/include/asm/sev.h

+3 −0
Original line number Diff line number Diff line
@@ -91,6 +91,9 @@ extern bool handle_vc_boot_ghcb(struct pt_regs *regs); 
/* RMUPDATE detected 4K page and 2MB page overlap. */

#define RMPUPDATE_FAIL_OVERLAP		4



/* PSMASH failed due to concurrent access by another CPU */

#define PSMASH_FAIL_INUSE		3



/* RMP page size */

#define RMP_PG_SIZE_4K			0

#define RMP_PG_SIZE_2M			1

arch/x86/kvm/mmu.h

+0 −2
Original line number Diff line number Diff line
@@ -253,8 +253,6 @@ static inline bool kvm_mmu_honors_guest_mtrrs(struct kvm *kvm) 
	return __kvm_mmu_honors_guest_mtrrs(kvm_arch_has_noncoherent_dma(kvm));

}



void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);



int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu);



int kvm_mmu_post_init_vm(struct kvm *kvm);

arch/x86/kvm/mmu/mmu.c

+1 −0
Original line number Diff line number Diff line
@@ -6802,6 +6802,7 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, 


	return need_tlb_flush;

}

EXPORT_SYMBOL_GPL(kvm_zap_gfn_range);



static void kvm_rmap_zap_collapsible_sptes(struct kvm *kvm,

					   const struct kvm_memory_slot *slot)

arch/x86/kvm/svm/sev.c

+109 −0
Original line number Diff line number Diff line
@@ -3465,6 +3465,23 @@ static void set_ghcb_msr(struct vcpu_svm *svm, u64 value) 
	svm->vmcb->control.ghcb_gpa = value;

}



static int snp_rmptable_psmash(kvm_pfn_t pfn)

{

	int ret;



	pfn = pfn & ~(KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) - 1);



	/*

	 * PSMASH_FAIL_INUSE indicates another processor is modifying the

	 * entry, so retry until that's no longer the case.

	 */

	do {

		ret = psmash(pfn);

	} while (ret == PSMASH_FAIL_INUSE);



	return ret;

}



static int snp_complete_psc_msr(struct kvm_vcpu *vcpu)

{

	struct vcpu_svm *svm = to_svm(vcpu);
@@ -4229,3 +4246,95 @@ struct page *snp_safe_alloc_page(struct kvm_vcpu *vcpu) 


	return p;

}



void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)

{

	struct kvm_memory_slot *slot;

	struct kvm *kvm = vcpu->kvm;

	int order, rmp_level, ret;

	bool assigned;

	kvm_pfn_t pfn;

	gfn_t gfn;



	gfn = gpa >> PAGE_SHIFT;



	/*

	 * The only time RMP faults occur for shared pages is when the guest is

	 * triggering an RMP fault for an implicit page-state change from

	 * shared->private. Implicit page-state changes are forwarded to

	 * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults

	 * for shared pages should not end up here.

	 */

	if (!kvm_mem_is_private(kvm, gfn)) {

		pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n",

				    gpa);

		return;

	}



	slot = gfn_to_memslot(kvm, gfn);

	if (!kvm_slot_can_be_private(slot)) {

		pr_warn_ratelimited("SEV: Unexpected RMP fault, non-private slot for GPA 0x%llx\n",

				    gpa);

		return;

	}



	ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order);

	if (ret) {

		pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n",

				    gpa);

		return;

	}



	ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level);

	if (ret || !assigned) {

		pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n",

				    gpa, pfn, ret);

		goto out_no_trace;

	}



	/*

	 * There are 2 cases where a PSMASH may be needed to resolve an #NPF

	 * with PFERR_GUEST_RMP_BIT set:

	 *

	 * 1) RMPADJUST/PVALIDATE can trigger an #NPF with PFERR_GUEST_SIZEM

	 *    bit set if the guest issues them with a smaller granularity than

	 *    what is indicated by the page-size bit in the 2MB RMP entry for

	 *    the PFN that backs the GPA.

	 *

	 * 2) Guest access via NPT can trigger an #NPF if the NPT mapping is

	 *    smaller than what is indicated by the 2MB RMP entry for the PFN

	 *    that backs the GPA.

	 *

	 * In both these cases, the corresponding 2M RMP entry needs to

	 * be PSMASH'd to 512 4K RMP entries.  If the RMP entry is already

	 * split into 4K RMP entries, then this is likely a spurious case which

	 * can occur when there are concurrent accesses by the guest to a 2MB

	 * GPA range that is backed by a 2MB-aligned PFN who's RMP entry is in

	 * the process of being PMASH'd into 4K entries. These cases should

	 * resolve automatically on subsequent accesses, so just ignore them

	 * here.

	 */

	if (rmp_level == PG_LEVEL_4K)

		goto out;



	ret = snp_rmptable_psmash(pfn);

	if (ret) {

		/*

		 * Look it up again. If it's 4K now then the PSMASH may have

		 * raced with another process and the issue has already resolved

		 * itself.

		 */

		if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) &&

		    assigned && rmp_level == PG_LEVEL_4K)

			goto out;



		pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n",

				    gpa, pfn, ret);

	}



	kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD);

out:

	trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret);

out_no_trace:

	put_page(pfn_to_page(pfn));

}