Commit 98c3ca00 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Andrew Morton
Browse files

ksm: convert cmp_and_merge_page() to use a folio 



By making try_to_merge_two_pages() and stable_tree_search() return a
folio, we can replace kpage with kfolio.  This replaces 7 calls to
compound_head() with one.

[cuigaosheng1@huawei.com: add IS_ERR_OR_NULL check for stable_tree_search()]
  Signed-off-by: default avatarGaosheng Cui <cuigaosheng1@huawei.com>
Link: https://lkml.kernel.org/r/20241002152533.1350629-3-willy@infradead.org


Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Cc: Alex Shi <alexs@kernel.org>
Cc: Gaosheng Cui <cuigaosheng1@huawei.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 9c0a1b99

mm/ksm.c

+26 −25
Original line number Diff line number Diff line
@@ -1582,7 +1582,7 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, 
 * Note that this function upgrades page to ksm page: if one of the pages

 * is already a ksm page, try_to_merge_with_ksm_page should be used.

 */

static struct page *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item,

static struct folio *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item,

					   struct page *page,

					   struct ksm_rmap_item *tree_rmap_item,

					   struct page *tree_page)
@@ -1600,7 +1600,7 @@ static struct page *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item, 
		if (err)

			break_cow(rmap_item);

	}

	return err ? NULL : page;

	return err ? NULL : page_folio(page);

}



static __always_inline
@@ -1787,9 +1787,9 @@ static __always_inline struct folio *chain(struct ksm_stable_node **s_n_d, 
 * with identical content to the page that we are scanning right now.

 *

 * This function returns the stable tree node of identical content if found,

 * NULL otherwise.

 * -EBUSY if the stable node's page is being migrated, NULL otherwise.

 */

static struct page *stable_tree_search(struct page *page)

static struct folio *stable_tree_search(struct page *page)

{

	int nid;

	struct rb_root *root;
@@ -1804,7 +1804,7 @@ static struct page *stable_tree_search(struct page *page) 
	if (page_node && page_node->head != &migrate_nodes) {

		/* ksm page forked */

		folio_get(folio);

		return &folio->page;

		return folio;

	}



	nid = get_kpfn_nid(folio_pfn(folio));
@@ -1899,7 +1899,7 @@ static struct page *stable_tree_search(struct page *page) 
				folio_put(tree_folio);

				goto replace;

			}

			return &tree_folio->page;

			return tree_folio;

		}

	}
@@ -1913,7 +1913,7 @@ static struct page *stable_tree_search(struct page *page) 
out:

	if (is_page_sharing_candidate(page_node)) {

		folio_get(folio);

		return &folio->page;

		return folio;

	} else

		return NULL;
@@ -1963,7 +1963,7 @@ static struct page *stable_tree_search(struct page *page) 
	}

	stable_node_dup->head = &migrate_nodes;

	list_add(&stable_node_dup->list, stable_node_dup->head);

	return &folio->page;

	return folio;



chain_append:

	/*
@@ -2217,7 +2217,7 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite 
	struct ksm_rmap_item *tree_rmap_item;

	struct page *tree_page = NULL;

	struct ksm_stable_node *stable_node;

	struct page *kpage;

	struct folio *kfolio;

	unsigned int checksum;

	int err;

	bool max_page_sharing_bypass = false;
@@ -2259,31 +2259,32 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite 
			return;

	}



	/* We first start with searching the page inside the stable tree */

	kpage = stable_tree_search(page);

	if (kpage == page && rmap_item->head == stable_node) {

		put_page(kpage);

	/* Start by searching for the folio in the stable tree */

	kfolio = stable_tree_search(page);

	if (!IS_ERR_OR_NULL(kfolio) && &kfolio->page == page &&

	    rmap_item->head == stable_node) {

		folio_put(kfolio);

		return;

	}



	remove_rmap_item_from_tree(rmap_item);



	if (kpage) {

		if (PTR_ERR(kpage) == -EBUSY)

	if (kfolio) {

		if (kfolio == ERR_PTR(-EBUSY))

			return;



		err = try_to_merge_with_ksm_page(rmap_item, page, kpage);

		err = try_to_merge_with_ksm_page(rmap_item, page, &kfolio->page);

		if (!err) {

			/*

			 * The page was successfully merged:

			 * add its rmap_item to the stable tree.

			 */

			lock_page(kpage);

			stable_tree_append(rmap_item, page_stable_node(kpage),

			folio_lock(kfolio);

			stable_tree_append(rmap_item, folio_stable_node(kfolio),

					   max_page_sharing_bypass);

			unlock_page(kpage);

			folio_unlock(kfolio);

		}

		put_page(kpage);

		folio_put(kfolio);

		return;

	}
@@ -2292,7 +2293,7 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite 
	if (tree_rmap_item) {

		bool split;



		kpage = try_to_merge_two_pages(rmap_item, page,

		kfolio = try_to_merge_two_pages(rmap_item, page,

						tree_rmap_item, tree_page);

		/*

		 * If both pages we tried to merge belong to the same compound
@@ -2307,20 +2308,20 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite 
		split = PageTransCompound(page)

			&& compound_head(page) == compound_head(tree_page);

		put_page(tree_page);

		if (kpage) {

		if (kfolio) {

			/*

			 * The pages were successfully merged: insert new

			 * node in the stable tree and add both rmap_items.

			 */

			lock_page(kpage);

			stable_node = stable_tree_insert(page_folio(kpage));

			folio_lock(kfolio);

			stable_node = stable_tree_insert(kfolio);

			if (stable_node) {

				stable_tree_append(tree_rmap_item, stable_node,

						   false);

				stable_tree_append(rmap_item, stable_node,

						   false);

			}

			unlock_page(kpage);

			folio_unlock(kfolio);



			/*

			 * If we fail to insert the page into the stable tree,