Commit 96fd7495 authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm/rmap: factor out adding folio mappings into __folio_add_rmap() 

Let's factor it out to prepare for reuse as we convert
page_add_anon_rmap() to folio_add_anon_rmap_[pte|ptes|pmd]().

Make the compiler always special-case on the granularity by using
__always_inline.

Link: https://lkml.kernel.org/r/20231220224504.646757-14-david@redhat.com


Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarYin Fengwei <fengwei.yin@intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Muchun Song <songmuchun@bytedance.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent be6e57cf

mm/rmap.c

+44 −34
Original line number Diff line number Diff line
@@ -1157,6 +1157,48 @@ int folio_total_mapcount(struct folio *folio) 
	return mapcount;

}



static __always_inline unsigned int __folio_add_rmap(struct folio *folio,

		struct page *page, int nr_pages, enum rmap_level level,

		int *nr_pmdmapped)

{

	atomic_t *mapped = &folio->_nr_pages_mapped;

	int first, nr = 0;



	__folio_rmap_sanity_checks(folio, page, nr_pages, level);



	switch (level) {

	case RMAP_LEVEL_PTE:

		do {

			first = atomic_inc_and_test(&page->_mapcount);

			if (first && folio_test_large(folio)) {

				first = atomic_inc_return_relaxed(mapped);

				first = (first < COMPOUND_MAPPED);

			}



			if (first)

				nr++;

		} while (page++, --nr_pages > 0);

		break;

	case RMAP_LEVEL_PMD:

		first = atomic_inc_and_test(&folio->_entire_mapcount);

		if (first) {

			nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);

			if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {

				*nr_pmdmapped = folio_nr_pages(folio);

				nr = *nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);

				/* Raced ahead of a remove and another add? */

				if (unlikely(nr < 0))

					nr = 0;

			} else {

				/* Raced ahead of a remove of COMPOUND_MAPPED */

				nr = 0;

			}

		}

		break;

	}

	return nr;

}



/**

 * folio_move_anon_rmap - move a folio to our anon_vma

 * @folio:	The folio to move to our anon_vma
@@ -1382,43 +1424,11 @@ static __always_inline void __folio_add_file_rmap(struct folio *folio, 
		struct page *page, int nr_pages, struct vm_area_struct *vma,

		enum rmap_level level)

{

	atomic_t *mapped = &folio->_nr_pages_mapped;

	int nr = 0, nr_pmdmapped = 0, first;

	int nr, nr_pmdmapped = 0;



	VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);

	__folio_rmap_sanity_checks(folio, page, nr_pages, level);



	switch (level) {

	case RMAP_LEVEL_PTE:

		do {

			first = atomic_inc_and_test(&page->_mapcount);

			if (first && folio_test_large(folio)) {

				first = atomic_inc_return_relaxed(mapped);

				first = (first < COMPOUND_MAPPED);

			}



			if (first)

				nr++;

		} while (page++, --nr_pages > 0);

		break;

	case RMAP_LEVEL_PMD:

		first = atomic_inc_and_test(&folio->_entire_mapcount);

		if (first) {

			nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);

			if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {

				nr_pmdmapped = folio_nr_pages(folio);

				nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);

				/* Raced ahead of a remove and another add? */

				if (unlikely(nr < 0))

					nr = 0;

			} else {

				/* Raced ahead of a remove of COMPOUND_MAPPED */

				nr = 0;

			}

		}

		break;

	}



	nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);

	if (nr_pmdmapped)

		__lruvec_stat_mod_folio(folio, folio_test_swapbacked(folio) ?

			NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped);