Commit f5af4a4f authored by Jason Gunthorpe's avatar Jason Gunthorpe Committed by Joerg Roedel
Browse files

iommu/pages: De-inline the substantial functions 



These are called in a lot of places and are not trivial. Move them to the
core module.

Tidy some of the comments and function arguments, fold
__iommu_alloc_account() into its only caller, change
__iommu_free_account() into __iommu_free_page() to remove some
duplication.

Reviewed-by: default avatarLu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: default avatarMostafa Saleh <smostafa@google.com>
Tested-by: default avatarNicolin Chen <nicolinc@nvidia.com>
Tested-by: default avatarAlejandro Jimenez <alejandro.j.jimenez@oracle.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/7-v4-c8663abbb606+3f7-iommu_pages_jgg@nvidia.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 3e8e986c

drivers/iommu/Makefile

+1 −0
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0

obj-y += amd/ intel/ arm/ iommufd/ riscv/

obj-$(CONFIG_IOMMU_API) += iommu.o

obj-$(CONFIG_IOMMU_SUPPORT) += iommu-pages.o

obj-$(CONFIG_IOMMU_API) += iommu-traces.o

obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o

obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o

drivers/iommu/iommu-pages.c

0 → 100644
+84 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2024, Google LLC.

 * Pasha Tatashin <pasha.tatashin@soleen.com>

 */

#include "iommu-pages.h"

#include <linux/gfp.h>

#include <linux/mm.h>



/**

 * iommu_alloc_pages_node - Allocate a zeroed page of a given order from

 *                          specific NUMA node

 * @nid: memory NUMA node id

 * @gfp: buddy allocator flags

 * @order: page order

 *

 * Returns the virtual address of the allocated page. The page must be

 * freed either by calling iommu_free_pages() or via iommu_put_pages_list().

 */

void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order)

{

	const unsigned long pgcnt = 1UL << order;

	struct page *page;



	page = alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order);

	if (unlikely(!page))

		return NULL;



	/*

	 * All page allocations that should be reported to as "iommu-pagetables"

	 * to userspace must use one of the functions below. This includes

	 * allocations of page-tables and other per-iommu_domain configuration

	 * structures.

	 *

	 * This is necessary for the proper accounting as IOMMU state can be

	 * rather large, i.e. multiple gigabytes in size.

	 */

	mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);

	mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);



	return page_address(page);

}

EXPORT_SYMBOL_GPL(iommu_alloc_pages_node);



static void __iommu_free_page(struct page *page)

{

	unsigned int order = folio_order(page_folio(page));

	const unsigned long pgcnt = 1UL << order;



	mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);

	mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);

	put_page(page);

}



/**

 * iommu_free_pages - free pages

 * @virt: virtual address of the page to be freed.

 *

 * The page must have have been allocated by iommu_alloc_pages_node()

 */

void iommu_free_pages(void *virt)

{

	if (!virt)

		return;

	__iommu_free_page(virt_to_page(virt));

}

EXPORT_SYMBOL_GPL(iommu_free_pages);



/**

 * iommu_put_pages_list - free a list of pages.

 * @head: the head of the lru list to be freed.

 *

 * Frees a list of pages allocated by iommu_alloc_pages_node().

 */

void iommu_put_pages_list(struct list_head *head)

{

	while (!list_empty(head)) {

		struct page *p = list_entry(head->prev, struct page, lru);



		list_del(&p->lru);

		__iommu_free_page(p);

	}

}

EXPORT_SYMBOL_GPL(iommu_put_pages_list);

drivers/iommu/iommu-pages.h

+5 −98
Original line number Diff line number Diff line
@@ -7,67 +7,12 @@ 
#ifndef __IOMMU_PAGES_H

#define __IOMMU_PAGES_H



#include <linux/vmstat.h>

#include <linux/gfp.h>

#include <linux/mm.h>

#include <linux/types.h>

#include <linux/topology.h>



/*

 * All page allocations that should be reported to as "iommu-pagetables" to

 * userspace must use one of the functions below.  This includes allocations of

 * page-tables and other per-iommu_domain configuration structures.

 *

 * This is necessary for the proper accounting as IOMMU state can be rather

 * large, i.e. multiple gigabytes in size.

 */



/**

 * __iommu_alloc_account - account for newly allocated page.

 * @page: head struct page of the page.

 * @order: order of the page

 */

static inline void __iommu_alloc_account(struct page *page, int order)

{

	const long pgcnt = 1l << order;



	mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);

	mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);

}



/**

 * __iommu_free_account - account a page that is about to be freed.

 * @page: head struct page of the page.

 * @order: order of the page

 */

static inline void __iommu_free_account(struct page *page)

{

	unsigned int order = folio_order(page_folio(page));

	const long pgcnt = 1l << order;



	mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);

	mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);

}



/**

 * iommu_alloc_pages_node - allocate a zeroed page of a given order from

 * specific NUMA node.

 * @nid: memory NUMA node id

 * @gfp: buddy allocator flags

 * @order: page order

 *

 * returns the virtual address of the allocated page

 */

static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order)

{

	struct page *page =

		alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order);



	if (unlikely(!page))

		return NULL;



	__iommu_alloc_account(page, order);



	return page_address(page);

}

void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order);

void iommu_free_pages(void *virt);

void iommu_put_pages_list(struct list_head *head);



/**

 * iommu_alloc_pages - allocate a zeroed page of a given order
@@ -104,42 +49,4 @@ static inline void *iommu_alloc_page(gfp_t gfp) 
	return iommu_alloc_pages_node(numa_node_id(), gfp, 0);

}



/**

 * iommu_free_pages - free pages

 * @virt: virtual address of the page to be freed.

 *

 * The page must have have been allocated by iommu_alloc_pages_node()

 */

static inline void iommu_free_pages(void *virt)

{

	struct page *page;



	if (!virt)

		return;



	page = virt_to_page(virt);

	__iommu_free_account(page);

	put_page(page);

}



/**

 * iommu_put_pages_list - free a list of pages.

 * @page: the head of the lru list to be freed.

 *

 * There are no locking requirement for these pages, as they are going to be

 * put on a free list as soon as refcount reaches 0. Pages are put on this LRU

 * list once they are removed from the IOMMU page tables. However, they can

 * still be access through debugfs.

 */

static inline void iommu_put_pages_list(struct list_head *page)

{

	while (!list_empty(page)) {

		struct page *p = list_entry(page->prev, struct page, lru);



		list_del(&p->lru);

		__iommu_free_account(p);

		put_page(p);

	}

}



#endif	/* __IOMMU_PAGES_H */