Commit e37d5a2d authored by Lu Baolu's avatar Lu Baolu Committed by Andrew Morton
Browse files

iommu/sva: invalidate stale IOTLB entries for kernel address space 

Introduce a new IOMMU interface to flush IOTLB paging cache entries for
the CPU kernel address space.  This interface is invoked from the x86
architecture code that manages combined user and kernel page tables,
specifically before any kernel page table page is freed and reused.

This addresses the main issue with vfree() which is a common occurrence
and can be triggered by unprivileged users.  While this resolves the
primary problem, it doesn't address some extremely rare case related to
memory unplug of memory that was present as reserved memory at boot, which
cannot be triggered by unprivileged users.  The discussion can be found at
the link below.

Enable SVA on x86 architecture since the IOMMU can now receive
notification to flush the paging cache before freeing the CPU kernel page
table pages.

Link: https://lkml.kernel.org/r/20251022082635.2462433-9-baolu.lu@linux.intel.com
Link: https://lore.kernel.org/linux-iommu/04983c62-3b1d-40d4-93ae-34ca04b827e5@intel.com/


Co-developed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Signed-off-by: default avatarLu Baolu <baolu.lu@linux.intel.com>
Suggested-by: default avatarJann Horn <jannh@google.com>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Reviewed-by: default avatarVasant Hegde <vasant.hegde@amd.com>
Reviewed-by: default avatarKevin Tian <kevin.tian@intel.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Betkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jean-Philippe Brucker <jean-philippe@linaro.org>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Robin Murohy <robin.murphy@arm.com>
Cc: Thomas Gleinxer <tglx@linutronix.de>
Cc: "Uladzislau Rezki (Sony)" <urezki@gmail.com>
Cc: Vinicius Costa Gomes <vinicius.gomes@intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Cc: Yi Lai <yi1.lai@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 5ba2f0a1

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -279,6 +279,7 @@ config X86 
	select HAVE_PCI

	select HAVE_PERF_REGS

	select HAVE_PERF_USER_STACK_DUMP

	select ASYNC_KERNEL_PGTABLE_FREE	if IOMMU_SVA

	select MMU_GATHER_RCU_TABLE_FREE

	select MMU_GATHER_MERGE_VMAS

	select HAVE_POSIX_CPU_TIMERS_TASK_WORK

drivers/iommu/iommu-sva.c

+28 −4
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ 
#include "iommu-priv.h"



static DEFINE_MUTEX(iommu_sva_lock);

static bool iommu_sva_present;

static LIST_HEAD(iommu_sva_mms);

static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,

						   struct mm_struct *mm);
@@ -42,6 +44,7 @@ static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct de 
		return ERR_PTR(-ENOSPC);

	}

	iommu_mm->pasid = pasid;

	iommu_mm->mm = mm;

	INIT_LIST_HEAD(&iommu_mm->sva_domains);

	/*

	 * Make sure the write to mm->iommu_mm is not reordered in front of
@@ -77,9 +80,6 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm 
	if (!group)

		return ERR_PTR(-ENODEV);



	if (IS_ENABLED(CONFIG_X86))

		return ERR_PTR(-EOPNOTSUPP);



	mutex_lock(&iommu_sva_lock);



	/* Allocate mm->pasid if necessary. */
@@ -135,8 +135,13 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm 
	if (ret)

		goto out_free_domain;

	domain->users = 1;

	list_add(&domain->next, &mm->iommu_mm->sva_domains);



	if (list_empty(&iommu_mm->sva_domains)) {

		if (list_empty(&iommu_sva_mms))

			iommu_sva_present = true;

		list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms);

	}

	list_add(&domain->next, &iommu_mm->sva_domains);

out:

	refcount_set(&handle->users, 1);

	mutex_unlock(&iommu_sva_lock);
@@ -178,6 +183,13 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) 
		list_del(&domain->next);

		iommu_domain_free(domain);

	}



	if (list_empty(&iommu_mm->sva_domains)) {

		list_del(&iommu_mm->mm_list_elm);

		if (list_empty(&iommu_sva_mms))

			iommu_sva_present = false;

	}



	mutex_unlock(&iommu_sva_lock);

	kfree(handle);

}
@@ -315,3 +327,15 @@ static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 


	return domain;

}



void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end)

{

	struct iommu_mm_data *iommu_mm;



	guard(mutex)(&iommu_sva_lock);

	if (!iommu_sva_present)

		return;



	list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm)

		mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end);

}

include/linux/iommu.h

+4 −0
Original line number Diff line number Diff line
@@ -1134,7 +1134,9 @@ struct iommu_sva { 


struct iommu_mm_data {

	u32			pasid;

	struct mm_struct	*mm;

	struct list_head	sva_domains;

	struct list_head	mm_list_elm;

};



int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode);
@@ -1615,6 +1617,7 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, 
					struct mm_struct *mm);

void iommu_sva_unbind_device(struct iommu_sva *handle);

u32 iommu_sva_get_pasid(struct iommu_sva *handle);

void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end);

#else

static inline struct iommu_sva *

iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
@@ -1639,6 +1642,7 @@ static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm) 
}



static inline void mm_pasid_drop(struct mm_struct *mm) {}

static inline void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) {}

#endif /* CONFIG_IOMMU_SVA */



#ifdef CONFIG_IOMMU_IOPF

mm/pgtable-generic.c

+2 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include <linux/swap.h>

#include <linux/swapops.h>

#include <linux/mm_inline.h>

#include <linux/iommu.h>

#include <asm/pgalloc.h>

#include <asm/tlb.h>
@@ -430,6 +431,7 @@ static void kernel_pgtable_work_func(struct work_struct *work) 
	list_splice_tail_init(&kernel_pgtable_work.list, &page_list);

	spin_unlock(&kernel_pgtable_work.lock);



	iommu_sva_invalidate_kva_range(PAGE_OFFSET, TLB_FLUSH_ALL);

	list_for_each_entry_safe(pt, next, &page_list, pt_list)

		__pagetable_free(pt);

}