Commit 35be969d authored by Rick Edgecombe's avatar Rick Edgecombe Committed by Paolo Bonzini
Browse files

KVM: x86/mmu: Zap invalid roots with mmu_lock holding for write at uninit 



Prepare for a future TDX patch which asserts that atomic zapping
(i.e. zapping with mmu_lock taken for read) don't operate on mirror roots.
When tearing down a VM, all roots have to be zapped (including mirror
roots once they're in place) so do that with the mmu_lock taken for write.

kvm_mmu_uninit_tdp_mmu() is invoked either before or after executing any
atomic operations on SPTEs by vCPU threads. Therefore, it will not impact
vCPU threads performance if kvm_tdp_mmu_zap_invalidated_roots() acquires
mmu_lock for write to zap invalid roots.

Co-developed-by: default avatarYan Zhao <yan.y.zhao@intel.com>
Signed-off-by: default avatarYan Zhao <yan.y.zhao@intel.com>
Signed-off-by: default avatarRick Edgecombe <rick.p.edgecombe@intel.com>
Message-ID: <20240718211230.1492011-2-rick.p.edgecombe@intel.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 67b43038

arch/x86/kvm/mmu/mmu.c

+1 −1
Original line number Diff line number Diff line
@@ -6467,7 +6467,7 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) 
	 * lead to use-after-free.

	 */

	if (tdp_mmu_enabled)

		kvm_tdp_mmu_zap_invalidated_roots(kvm);

		kvm_tdp_mmu_zap_invalidated_roots(kvm, true);

}



void kvm_mmu_init_vm(struct kvm *kvm)

arch/x86/kvm/mmu/tdp_mmu.c

+11 −5
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) 
	 * ultimately frees all roots.

	 */

	kvm_tdp_mmu_invalidate_all_roots(kvm);

	kvm_tdp_mmu_zap_invalidated_roots(kvm);

	kvm_tdp_mmu_zap_invalidated_roots(kvm, false);



	WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));

	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
@@ -883,11 +883,14 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) 
 * Zap all invalidated roots to ensure all SPTEs are dropped before the "fast

 * zap" completes.

 */

void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm)

void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm, bool shared)

{

	struct kvm_mmu_page *root;



	if (shared)

		read_lock(&kvm->mmu_lock);

	else

		write_lock(&kvm->mmu_lock);



	for_each_tdp_mmu_root_yield_safe(kvm, root) {

		if (!root->tdp_mmu_scheduled_root_to_zap)
@@ -905,7 +908,7 @@ void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) 
		 * that may be zapped, as such entries are associated with the

		 * ASID on both VMX and SVM.

		 */

		tdp_mmu_zap_root(kvm, root, true);

		tdp_mmu_zap_root(kvm, root, shared);



		/*

		 * The referenced needs to be put *after* zapping the root, as
@@ -915,7 +918,10 @@ void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) 
		kvm_tdp_mmu_put_root(kvm, root);

	}



	if (shared)

		read_unlock(&kvm->mmu_lock);

	else

		write_unlock(&kvm->mmu_lock);

}



/*

arch/x86/kvm/mmu/tdp_mmu.h

+1 −1
Original line number Diff line number Diff line
@@ -23,7 +23,7 @@ bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush); 
bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp);

void kvm_tdp_mmu_zap_all(struct kvm *kvm);

void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm);

void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm);

void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm, bool shared);



int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);