Commit 18954252 authored by Suravee Suthikulpanit's avatar Suravee Suthikulpanit Committed by Joerg Roedel
Browse files

iommu/amd: Move IO page table related functions 



Preparing to migrate to use IO page table framework.
There is no functional change.

Signed-off-by: default avatarSuravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Link: https://lore.kernel.org/r/20201215073705.123786-7-suravee.suthikulpanit@amd.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent f9b4df79

drivers/iommu/amd/amd_iommu.h

+18 −0
Original line number Diff line number Diff line
@@ -131,4 +131,22 @@ void amd_iommu_apply_ivrs_quirks(void); 
static inline void amd_iommu_apply_ivrs_quirks(void) { }

#endif



/* TODO: These are temporary and will be removed once fully transition */

extern void free_pagetable(struct domain_pgtable *pgtable);

extern int iommu_map_page(struct protection_domain *dom,

			  unsigned long bus_addr,

			  unsigned long phys_addr,

			  unsigned long page_size,

			  int prot,

			  gfp_t gfp);

extern unsigned long iommu_unmap_page(struct protection_domain *dom,

				      unsigned long bus_addr,

				      unsigned long page_size);

extern u64 *fetch_pte(struct protection_domain *domain,

		      unsigned long address,

		      unsigned long *page_size);

extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain,

					 struct domain_pgtable *pgtable);

extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,

					 u64 *root, int mode);

#endif

drivers/iommu/amd/io_pgtable.c

+473 −0
Original line number Diff line number Diff line
@@ -44,6 +44,479 @@ static const struct iommu_flush_ops v1_flush_ops = { 
	.tlb_add_page	= v1_tlb_add_page,

};



/*

 * Helper function to get the first pte of a large mapping

 */

static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,

			 unsigned long *count)

{

	unsigned long pte_mask, pg_size, cnt;

	u64 *fpte;



	pg_size  = PTE_PAGE_SIZE(*pte);

	cnt      = PAGE_SIZE_PTE_COUNT(pg_size);

	pte_mask = ~((cnt << 3) - 1);

	fpte     = (u64 *)(((unsigned long)pte) & pte_mask);



	if (page_size)

		*page_size = pg_size;



	if (count)

		*count = cnt;



	return fpte;

}



/****************************************************************************

 *

 * The functions below are used the create the page table mappings for

 * unity mapped regions.

 *

 ****************************************************************************/



static void free_page_list(struct page *freelist)

{

	while (freelist != NULL) {

		unsigned long p = (unsigned long)page_address(freelist);



		freelist = freelist->freelist;

		free_page(p);

	}

}



static struct page *free_pt_page(unsigned long pt, struct page *freelist)

{

	struct page *p = virt_to_page((void *)pt);



	p->freelist = freelist;



	return p;

}



#define DEFINE_FREE_PT_FN(LVL, FN)						\

static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist)	\

{										\

	unsigned long p;							\

	u64 *pt;								\

	int i;									\

										\

	pt = (u64 *)__pt;							\

										\

	for (i = 0; i < 512; ++i) {						\

		/* PTE present? */						\

		if (!IOMMU_PTE_PRESENT(pt[i]))					\

			continue;						\

										\

		/* Large PTE? */						\

		if (PM_PTE_LEVEL(pt[i]) == 0 ||					\

		    PM_PTE_LEVEL(pt[i]) == 7)					\

			continue;						\

										\

		p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);			\

		freelist = FN(p, freelist);					\

	}									\

										\

	return free_pt_page((unsigned long)pt, freelist);			\

}



DEFINE_FREE_PT_FN(l2, free_pt_page)

DEFINE_FREE_PT_FN(l3, free_pt_l2)

DEFINE_FREE_PT_FN(l4, free_pt_l3)

DEFINE_FREE_PT_FN(l5, free_pt_l4)

DEFINE_FREE_PT_FN(l6, free_pt_l5)



static struct page *free_sub_pt(unsigned long root, int mode,

				struct page *freelist)

{

	switch (mode) {

	case PAGE_MODE_NONE:

	case PAGE_MODE_7_LEVEL:

		break;

	case PAGE_MODE_1_LEVEL:

		freelist = free_pt_page(root, freelist);

		break;

	case PAGE_MODE_2_LEVEL:

		freelist = free_pt_l2(root, freelist);

		break;

	case PAGE_MODE_3_LEVEL:

		freelist = free_pt_l3(root, freelist);

		break;

	case PAGE_MODE_4_LEVEL:

		freelist = free_pt_l4(root, freelist);

		break;

	case PAGE_MODE_5_LEVEL:

		freelist = free_pt_l5(root, freelist);

		break;

	case PAGE_MODE_6_LEVEL:

		freelist = free_pt_l6(root, freelist);

		break;

	default:

		BUG();

	}



	return freelist;

}



void free_pagetable(struct domain_pgtable *pgtable)

{

	struct page *freelist = NULL;

	unsigned long root;



	if (pgtable->mode == PAGE_MODE_NONE)

		return;



	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||

	       pgtable->mode > PAGE_MODE_6_LEVEL);



	root = (unsigned long)pgtable->root;

	freelist = free_sub_pt(root, pgtable->mode, freelist);



	free_page_list(freelist);

}



void amd_iommu_domain_set_pgtable(struct protection_domain *domain,

				  u64 *root, int mode)

{

	u64 pt_root;



	/* lowest 3 bits encode pgtable mode */

	pt_root = mode & 7;

	pt_root |= (u64)root;



	amd_iommu_domain_set_pt_root(domain, pt_root);

}



/*

 * This function is used to add another level to an IO page table. Adding

 * another level increases the size of the address space by 9 bits to a size up

 * to 64 bits.

 */

static bool increase_address_space(struct protection_domain *domain,

				   unsigned long address,

				   gfp_t gfp)

{

	struct domain_pgtable pgtable;

	unsigned long flags;

	bool ret = true;

	u64 *pte;



	spin_lock_irqsave(&domain->lock, flags);



	amd_iommu_domain_get_pgtable(domain, &pgtable);



	if (address <= PM_LEVEL_SIZE(pgtable.mode))

		goto out;



	ret = false;

	if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))

		goto out;



	pte = (void *)get_zeroed_page(gfp);

	if (!pte)

		goto out;



	*pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));



	pgtable.root  = pte;

	pgtable.mode += 1;

	amd_iommu_update_and_flush_device_table(domain);

	amd_iommu_domain_flush_complete(domain);



	/*

	 * Device Table needs to be updated and flushed before the new root can

	 * be published.

	 */

	amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);



	ret = true;



out:

	spin_unlock_irqrestore(&domain->lock, flags);



	return ret;

}



static u64 *alloc_pte(struct protection_domain *domain,

		      unsigned long address,

		      unsigned long page_size,

		      u64 **pte_page,

		      gfp_t gfp,

		      bool *updated)

{

	struct domain_pgtable pgtable;

	int level, end_lvl;

	u64 *pte, *page;



	BUG_ON(!is_power_of_2(page_size));



	amd_iommu_domain_get_pgtable(domain, &pgtable);



	while (address > PM_LEVEL_SIZE(pgtable.mode)) {

		/*

		 * Return an error if there is no memory to update the

		 * page-table.

		 */

		if (!increase_address_space(domain, address, gfp))

			return NULL;



		/* Read new values to check if update was successful */

		amd_iommu_domain_get_pgtable(domain, &pgtable);

	}





	level   = pgtable.mode - 1;

	pte     = &pgtable.root[PM_LEVEL_INDEX(level, address)];

	address = PAGE_SIZE_ALIGN(address, page_size);

	end_lvl = PAGE_SIZE_LEVEL(page_size);



	while (level > end_lvl) {

		u64 __pte, __npte;

		int pte_level;



		__pte     = *pte;

		pte_level = PM_PTE_LEVEL(__pte);



		/*

		 * If we replace a series of large PTEs, we need

		 * to tear down all of them.

		 */

		if (IOMMU_PTE_PRESENT(__pte) &&

		    pte_level == PAGE_MODE_7_LEVEL) {

			unsigned long count, i;

			u64 *lpte;



			lpte = first_pte_l7(pte, NULL, &count);



			/*

			 * Unmap the replicated PTEs that still match the

			 * original large mapping

			 */

			for (i = 0; i < count; ++i)

				cmpxchg64(&lpte[i], __pte, 0ULL);



			*updated = true;

			continue;

		}



		if (!IOMMU_PTE_PRESENT(__pte) ||

		    pte_level == PAGE_MODE_NONE) {

			page = (u64 *)get_zeroed_page(gfp);



			if (!page)

				return NULL;



			__npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));



			/* pte could have been changed somewhere. */

			if (cmpxchg64(pte, __pte, __npte) != __pte)

				free_page((unsigned long)page);

			else if (IOMMU_PTE_PRESENT(__pte))

				*updated = true;



			continue;

		}



		/* No level skipping support yet */

		if (pte_level != level)

			return NULL;



		level -= 1;



		pte = IOMMU_PTE_PAGE(__pte);



		if (pte_page && level == end_lvl)

			*pte_page = pte;



		pte = &pte[PM_LEVEL_INDEX(level, address)];

	}



	return pte;

}



/*

 * This function checks if there is a PTE for a given dma address. If

 * there is one, it returns the pointer to it.

 */

u64 *fetch_pte(struct protection_domain *domain,

	       unsigned long address,

	       unsigned long *page_size)

{

	struct domain_pgtable pgtable;

	int level;

	u64 *pte;



	*page_size = 0;



	amd_iommu_domain_get_pgtable(domain, &pgtable);



	if (address > PM_LEVEL_SIZE(pgtable.mode))

		return NULL;



	level	   =  pgtable.mode - 1;

	pte	   = &pgtable.root[PM_LEVEL_INDEX(level, address)];

	*page_size =  PTE_LEVEL_PAGE_SIZE(level);



	while (level > 0) {



		/* Not Present */

		if (!IOMMU_PTE_PRESENT(*pte))

			return NULL;



		/* Large PTE */

		if (PM_PTE_LEVEL(*pte) == 7 ||

		    PM_PTE_LEVEL(*pte) == 0)

			break;



		/* No level skipping support yet */

		if (PM_PTE_LEVEL(*pte) != level)

			return NULL;



		level -= 1;



		/* Walk to the next level */

		pte	   = IOMMU_PTE_PAGE(*pte);

		pte	   = &pte[PM_LEVEL_INDEX(level, address)];

		*page_size = PTE_LEVEL_PAGE_SIZE(level);

	}



	/*

	 * If we have a series of large PTEs, make

	 * sure to return a pointer to the first one.

	 */

	if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)

		pte = first_pte_l7(pte, page_size, NULL);



	return pte;

}



static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)

{

	unsigned long pt;

	int mode;



	while (cmpxchg64(pte, pteval, 0) != pteval) {

		pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");

		pteval = *pte;

	}



	if (!IOMMU_PTE_PRESENT(pteval))

		return freelist;



	pt   = (unsigned long)IOMMU_PTE_PAGE(pteval);

	mode = IOMMU_PTE_MODE(pteval);



	return free_sub_pt(pt, mode, freelist);

}



/*

 * Generic mapping functions. It maps a physical address into a DMA

 * address space. It allocates the page table pages if necessary.

 * In the future it can be extended to a generic mapping function

 * supporting all features of AMD IOMMU page tables like level skipping

 * and full 64 bit address spaces.

 */

int iommu_map_page(struct protection_domain *dom,

		   unsigned long bus_addr,

		   unsigned long phys_addr,

		   unsigned long page_size,

		   int prot,

		   gfp_t gfp)

{

	struct page *freelist = NULL;

	bool updated = false;

	u64 __pte, *pte;

	int ret, i, count;



	BUG_ON(!IS_ALIGNED(bus_addr, page_size));

	BUG_ON(!IS_ALIGNED(phys_addr, page_size));



	ret = -EINVAL;

	if (!(prot & IOMMU_PROT_MASK))

		goto out;



	count = PAGE_SIZE_PTE_COUNT(page_size);

	pte   = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);



	ret = -ENOMEM;

	if (!pte)

		goto out;



	for (i = 0; i < count; ++i)

		freelist = free_clear_pte(&pte[i], pte[i], freelist);



	if (freelist != NULL)

		updated = true;



	if (count > 1) {

		__pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);

		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;

	} else

		__pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;



	if (prot & IOMMU_PROT_IR)

		__pte |= IOMMU_PTE_IR;

	if (prot & IOMMU_PROT_IW)

		__pte |= IOMMU_PTE_IW;



	for (i = 0; i < count; ++i)

		pte[i] = __pte;



	ret = 0;



out:

	if (updated) {

		unsigned long flags;



		spin_lock_irqsave(&dom->lock, flags);

		/*

		 * Flush domain TLB(s) and wait for completion. Any Device-Table

		 * Updates and flushing already happened in

		 * increase_address_space().

		 */

		amd_iommu_domain_flush_tlb_pde(dom);

		amd_iommu_domain_flush_complete(dom);

		spin_unlock_irqrestore(&dom->lock, flags);

	}



	/* Everything flushed out, free pages now */

	free_page_list(freelist);



	return ret;

}



unsigned long iommu_unmap_page(struct protection_domain *dom,

			       unsigned long bus_addr,

			       unsigned long page_size)

{

	unsigned long long unmapped;

	unsigned long unmap_size;

	u64 *pte;



	BUG_ON(!is_power_of_2(page_size));



	unmapped = 0;



	while (unmapped < page_size) {



		pte = fetch_pte(dom, bus_addr, &unmap_size);



		if (pte) {

			int i, count;



			count = PAGE_SIZE_PTE_COUNT(unmap_size);

			for (i = 0; i < count; i++)

				pte[i] = 0ULL;

		}



		bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;

		unmapped += unmap_size;

	}



	BUG_ON(unmapped && !is_power_of_2(unmapped));



	return unmapped;

}



/*

 * ----------------------------------------------------

 */

drivers/iommu/amd/iommu.c

+2 −474
Original line number Diff line number Diff line
@@ -138,7 +138,7 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom) 
	return container_of(dom, struct protection_domain, domain);

}



static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,

void amd_iommu_domain_get_pgtable(struct protection_domain *domain,

				  struct domain_pgtable *pgtable)

{

	u64 pt_root = atomic64_read(&domain->iop.pt_root);
@@ -147,18 +147,6 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain, 
	pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */

}



static void amd_iommu_domain_set_pgtable(struct protection_domain *domain,

					 u64 *root, int mode)

{

	u64 pt_root;



	/* lowest 3 bits encode pgtable mode */

	pt_root = mode & 7;

	pt_root |= (u64)root;



	amd_iommu_domain_set_pt_root(domain, pt_root);

}



static struct iommu_dev_data *alloc_dev_data(u16 devid)

{

	struct iommu_dev_data *dev_data;
@@ -414,29 +402,6 @@ static void amd_iommu_uninit_device(struct device *dev) 
	 */

}



/*

 * Helper function to get the first pte of a large mapping

 */

static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,

			 unsigned long *count)

{

	unsigned long pte_mask, pg_size, cnt;

	u64 *fpte;



	pg_size  = PTE_PAGE_SIZE(*pte);

	cnt      = PAGE_SIZE_PTE_COUNT(pg_size);

	pte_mask = ~((cnt << 3) - 1);

	fpte     = (u64 *)(((unsigned long)pte) & pte_mask);



	if (page_size)

		*page_size = pg_size;



	if (count)

		*count = cnt;



	return fpte;

}



/****************************************************************************

 *

 * Interrupt handling functions
@@ -1359,443 +1324,6 @@ static void domain_flush_devices(struct protection_domain *domain) 
		device_flush_dte(dev_data);

}



/****************************************************************************

 *

 * The functions below are used the create the page table mappings for

 * unity mapped regions.

 *

 ****************************************************************************/



static void free_page_list(struct page *freelist)

{

	while (freelist != NULL) {

		unsigned long p = (unsigned long)page_address(freelist);

		freelist = freelist->freelist;

		free_page(p);

	}

}



static struct page *free_pt_page(unsigned long pt, struct page *freelist)

{

	struct page *p = virt_to_page((void *)pt);



	p->freelist = freelist;



	return p;

}



#define DEFINE_FREE_PT_FN(LVL, FN)						\

static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist)	\

{										\

	unsigned long p;							\

	u64 *pt;								\

	int i;									\

										\

	pt = (u64 *)__pt;							\

										\

	for (i = 0; i < 512; ++i) {						\

		/* PTE present? */						\

		if (!IOMMU_PTE_PRESENT(pt[i]))					\

			continue;						\

										\

		/* Large PTE? */						\

		if (PM_PTE_LEVEL(pt[i]) == 0 ||					\

		    PM_PTE_LEVEL(pt[i]) == 7)					\

			continue;						\

										\

		p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);			\

		freelist = FN(p, freelist);					\

	}									\

										\

	return free_pt_page((unsigned long)pt, freelist);			\

}



DEFINE_FREE_PT_FN(l2, free_pt_page)

DEFINE_FREE_PT_FN(l3, free_pt_l2)

DEFINE_FREE_PT_FN(l4, free_pt_l3)

DEFINE_FREE_PT_FN(l5, free_pt_l4)

DEFINE_FREE_PT_FN(l6, free_pt_l5)



static struct page *free_sub_pt(unsigned long root, int mode,

				struct page *freelist)

{

	switch (mode) {

	case PAGE_MODE_NONE:

	case PAGE_MODE_7_LEVEL:

		break;

	case PAGE_MODE_1_LEVEL:

		freelist = free_pt_page(root, freelist);

		break;

	case PAGE_MODE_2_LEVEL:

		freelist = free_pt_l2(root, freelist);

		break;

	case PAGE_MODE_3_LEVEL:

		freelist = free_pt_l3(root, freelist);

		break;

	case PAGE_MODE_4_LEVEL:

		freelist = free_pt_l4(root, freelist);

		break;

	case PAGE_MODE_5_LEVEL:

		freelist = free_pt_l5(root, freelist);

		break;

	case PAGE_MODE_6_LEVEL:

		freelist = free_pt_l6(root, freelist);

		break;

	default:

		BUG();

	}



	return freelist;

}



static void free_pagetable(struct domain_pgtable *pgtable)

{

	struct page *freelist = NULL;

	unsigned long root;



	if (pgtable->mode == PAGE_MODE_NONE)

		return;



	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||

	       pgtable->mode > PAGE_MODE_6_LEVEL);



	root = (unsigned long)pgtable->root;

	freelist = free_sub_pt(root, pgtable->mode, freelist);



	free_page_list(freelist);

}



/*

 * This function is used to add another level to an IO page table. Adding

 * another level increases the size of the address space by 9 bits to a size up

 * to 64 bits.

 */

static bool increase_address_space(struct protection_domain *domain,

				   unsigned long address,

				   gfp_t gfp)

{

	struct domain_pgtable pgtable;

	unsigned long flags;

	bool ret = true;

	u64 *pte;



	spin_lock_irqsave(&domain->lock, flags);



	amd_iommu_domain_get_pgtable(domain, &pgtable);



	if (address <= PM_LEVEL_SIZE(pgtable.mode))

		goto out;



	ret = false;

	if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))

		goto out;



	pte = (void *)get_zeroed_page(gfp);

	if (!pte)

		goto out;



	*pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));



	pgtable.root  = pte;

	pgtable.mode += 1;

	amd_iommu_update_and_flush_device_table(domain);

	amd_iommu_domain_flush_complete(domain);



	/*

	 * Device Table needs to be updated and flushed before the new root can

	 * be published.

	 */

	amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);



	ret = true;



out:

	spin_unlock_irqrestore(&domain->lock, flags);



	return ret;

}



static u64 *alloc_pte(struct protection_domain *domain,

		      unsigned long address,

		      unsigned long page_size,

		      u64 **pte_page,

		      gfp_t gfp,

		      bool *updated)

{

	struct domain_pgtable pgtable;

	int level, end_lvl;

	u64 *pte, *page;



	BUG_ON(!is_power_of_2(page_size));



	amd_iommu_domain_get_pgtable(domain, &pgtable);



	while (address > PM_LEVEL_SIZE(pgtable.mode)) {

		/*

		 * Return an error if there is no memory to update the

		 * page-table.

		 */

		if (!increase_address_space(domain, address, gfp))

			return NULL;



		/* Read new values to check if update was successful */

		amd_iommu_domain_get_pgtable(domain, &pgtable);

	}





	level   = pgtable.mode - 1;

	pte     = &pgtable.root[PM_LEVEL_INDEX(level, address)];

	address = PAGE_SIZE_ALIGN(address, page_size);

	end_lvl = PAGE_SIZE_LEVEL(page_size);



	while (level > end_lvl) {

		u64 __pte, __npte;

		int pte_level;



		__pte     = *pte;

		pte_level = PM_PTE_LEVEL(__pte);



		/*

		 * If we replace a series of large PTEs, we need

		 * to tear down all of them.

		 */

		if (IOMMU_PTE_PRESENT(__pte) &&

		    pte_level == PAGE_MODE_7_LEVEL) {

			unsigned long count, i;

			u64 *lpte;



			lpte = first_pte_l7(pte, NULL, &count);



			/*

			 * Unmap the replicated PTEs that still match the

			 * original large mapping

			 */

			for (i = 0; i < count; ++i)

				cmpxchg64(&lpte[i], __pte, 0ULL);



			*updated = true;

			continue;

		}



		if (!IOMMU_PTE_PRESENT(__pte) ||

		    pte_level == PAGE_MODE_NONE) {

			page = (u64 *)get_zeroed_page(gfp);



			if (!page)

				return NULL;



			__npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));



			/* pte could have been changed somewhere. */

			if (cmpxchg64(pte, __pte, __npte) != __pte)

				free_page((unsigned long)page);

			else if (IOMMU_PTE_PRESENT(__pte))

				*updated = true;



			continue;

		}



		/* No level skipping support yet */

		if (pte_level != level)

			return NULL;



		level -= 1;



		pte = IOMMU_PTE_PAGE(__pte);



		if (pte_page && level == end_lvl)

			*pte_page = pte;



		pte = &pte[PM_LEVEL_INDEX(level, address)];

	}



	return pte;

}



/*

 * This function checks if there is a PTE for a given dma address. If

 * there is one, it returns the pointer to it.

 */

static u64 *fetch_pte(struct protection_domain *domain,

		      unsigned long address,

		      unsigned long *page_size)

{

	struct domain_pgtable pgtable;

	int level;

	u64 *pte;



	*page_size = 0;



	amd_iommu_domain_get_pgtable(domain, &pgtable);



	if (address > PM_LEVEL_SIZE(pgtable.mode))

		return NULL;



	level	   =  pgtable.mode - 1;

	pte	   = &pgtable.root[PM_LEVEL_INDEX(level, address)];

	*page_size =  PTE_LEVEL_PAGE_SIZE(level);



	while (level > 0) {



		/* Not Present */

		if (!IOMMU_PTE_PRESENT(*pte))

			return NULL;



		/* Large PTE */

		if (PM_PTE_LEVEL(*pte) == 7 ||

		    PM_PTE_LEVEL(*pte) == 0)

			break;



		/* No level skipping support yet */

		if (PM_PTE_LEVEL(*pte) != level)

			return NULL;



		level -= 1;



		/* Walk to the next level */

		pte	   = IOMMU_PTE_PAGE(*pte);

		pte	   = &pte[PM_LEVEL_INDEX(level, address)];

		*page_size = PTE_LEVEL_PAGE_SIZE(level);

	}



	/*

	 * If we have a series of large PTEs, make

	 * sure to return a pointer to the first one.

	 */

	if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)

		pte = first_pte_l7(pte, page_size, NULL);



	return pte;

}



static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)

{

	unsigned long pt;

	int mode;



	while (cmpxchg64(pte, pteval, 0) != pteval) {

		pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");

		pteval = *pte;

	}



	if (!IOMMU_PTE_PRESENT(pteval))

		return freelist;



	pt   = (unsigned long)IOMMU_PTE_PAGE(pteval);

	mode = IOMMU_PTE_MODE(pteval);



	return free_sub_pt(pt, mode, freelist);

}



/*

 * Generic mapping functions. It maps a physical address into a DMA

 * address space. It allocates the page table pages if necessary.

 * In the future it can be extended to a generic mapping function

 * supporting all features of AMD IOMMU page tables like level skipping

 * and full 64 bit address spaces.

 */

static int iommu_map_page(struct protection_domain *dom,

			  unsigned long bus_addr,

			  unsigned long phys_addr,

			  unsigned long page_size,

			  int prot,

			  gfp_t gfp)

{

	struct page *freelist = NULL;

	bool updated = false;

	u64 __pte, *pte;

	int ret, i, count;



	BUG_ON(!IS_ALIGNED(bus_addr, page_size));

	BUG_ON(!IS_ALIGNED(phys_addr, page_size));



	ret = -EINVAL;

	if (!(prot & IOMMU_PROT_MASK))

		goto out;



	count = PAGE_SIZE_PTE_COUNT(page_size);

	pte   = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);



	ret = -ENOMEM;

	if (!pte)

		goto out;



	for (i = 0; i < count; ++i)

		freelist = free_clear_pte(&pte[i], pte[i], freelist);



	if (freelist != NULL)

		updated = true;



	if (count > 1) {

		__pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);

		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;

	} else

		__pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;



	if (prot & IOMMU_PROT_IR)

		__pte |= IOMMU_PTE_IR;

	if (prot & IOMMU_PROT_IW)

		__pte |= IOMMU_PTE_IW;



	for (i = 0; i < count; ++i)

		pte[i] = __pte;



	ret = 0;



out:

	if (updated) {

		unsigned long flags;



		spin_lock_irqsave(&dom->lock, flags);

		/*

		 * Flush domain TLB(s) and wait for completion. Any Device-Table

		 * Updates and flushing already happened in

		 * increase_address_space().

		 */

		amd_iommu_domain_flush_tlb_pde(dom);

		amd_iommu_domain_flush_complete(dom);

		spin_unlock_irqrestore(&dom->lock, flags);

	}



	/* Everything flushed out, free pages now */

	free_page_list(freelist);



	return ret;

}



static unsigned long iommu_unmap_page(struct protection_domain *dom,

				      unsigned long bus_addr,

				      unsigned long page_size)

{

	unsigned long long unmapped;

	unsigned long unmap_size;

	u64 *pte;



	BUG_ON(!is_power_of_2(page_size));



	unmapped = 0;



	while (unmapped < page_size) {



		pte = fetch_pte(dom, bus_addr, &unmap_size);



		if (pte) {

			int i, count;



			count = PAGE_SIZE_PTE_COUNT(unmap_size);

			for (i = 0; i < count; i++)

				pte[i] = 0ULL;

		}



		bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;

		unmapped += unmap_size;

	}



	BUG_ON(unmapped && !is_power_of_2(unmapped));



	return unmapped;

}



/****************************************************************************

 *

 * The next functions belong to the domain allocation. A domain is