Commit 8bd51300 authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm/rmap: introduce folio_add_anon_rmap_[pte|ptes|pmd]() 

Let's mimic what we did with folio_add_file_rmap_*() so we can similarly
replace page_add_anon_rmap() next.

Make the compiler always special-case on the granularity by using
__always_inline.

For the PageAnonExclusive sanity checks, when adding a PMD mapping, we're
now also checking each individual subpage covered by that PMD, instead of
only the head page.

Note that the new functions ignore the RMAP_COMPOUND flag, which we will
remove as soon as page_add_anon_rmap() is gone.

Link: https://lkml.kernel.org/r/20231220224504.646757-15-david@redhat.com


Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarYin Fengwei <fengwei.yin@intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Muchun Song <songmuchun@bytedance.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 96fd7495

include/linux/rmap.h

+6 −0
Original line number Diff line number Diff line
@@ -233,6 +233,12 @@ static inline void __folio_rmap_sanity_checks(struct folio *folio, 
 * rmap interfaces called when adding or removing pte of page

 */

void folio_move_anon_rmap(struct folio *, struct vm_area_struct *);

void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,

		struct vm_area_struct *, unsigned long address, rmap_t flags);

#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \

	folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags)

void folio_add_anon_rmap_pmd(struct folio *, struct page *,

		struct vm_area_struct *, unsigned long address, rmap_t flags);

void page_add_anon_rmap(struct page *, struct vm_area_struct *,

		unsigned long address, rmap_t flags);

void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *,

mm/rmap.c

+82 −38
Original line number Diff line number Diff line
@@ -1299,40 +1299,20 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma, 
		unsigned long address, rmap_t flags)

{

	struct folio *folio = page_folio(page);

	atomic_t *mapped = &folio->_nr_pages_mapped;

	int nr = 0, nr_pmdmapped = 0;

	bool compound = flags & RMAP_COMPOUND;

	bool first;



	VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);



	/* Is page being mapped by PTE? Is this its first map to be added? */

	if (likely(!compound)) {

		first = atomic_inc_and_test(&page->_mapcount);

		nr = first;

		if (first && folio_test_large(folio)) {

			nr = atomic_inc_return_relaxed(mapped);

			nr = (nr < COMPOUND_MAPPED);

	if (likely(!(flags & RMAP_COMPOUND)))

		folio_add_anon_rmap_pte(folio, page, vma, address, flags);

	else

		folio_add_anon_rmap_pmd(folio, page, vma, address, flags);

}

	} else if (folio_test_pmd_mappable(folio)) {

		/* That test is redundant: it's for safety or to optimize out */



		first = atomic_inc_and_test(&folio->_entire_mapcount);

		if (first) {

			nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);

			if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {

				nr_pmdmapped = folio_nr_pages(folio);

				nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);

				/* Raced ahead of a remove and another add? */

				if (unlikely(nr < 0))

					nr = 0;

			} else {

				/* Raced ahead of a remove of COMPOUND_MAPPED */

				nr = 0;

			}

		}

	}

static __always_inline void __folio_add_anon_rmap(struct folio *folio,

		struct page *page, int nr_pages, struct vm_area_struct *vma,

		unsigned long address, rmap_t flags, enum rmap_level level)

{

	int i, nr, nr_pmdmapped = 0;



	nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);

	if (nr_pmdmapped)

		__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped);

	if (nr)
@@ -1346,18 +1326,34 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma, 
		 * folio->index right when not given the address of the head

		 * page.

		 */

		VM_WARN_ON_FOLIO(folio_test_large(folio) && !compound, folio);

		VM_WARN_ON_FOLIO(folio_test_large(folio) &&

				 level != RMAP_LEVEL_PMD, folio);

		__folio_set_anon(folio, vma, address,

				 !!(flags & RMAP_EXCLUSIVE));

	} else if (likely(!folio_test_ksm(folio))) {

		__page_check_anon_rmap(folio, page, vma, address);

	}

	if (flags & RMAP_EXCLUSIVE)



	if (flags & RMAP_EXCLUSIVE) {

		switch (level) {

		case RMAP_LEVEL_PTE:

			for (i = 0; i < nr_pages; i++)

				SetPageAnonExclusive(page + i);

			break;

		case RMAP_LEVEL_PMD:

			SetPageAnonExclusive(page);

			break;

		}

	}

	for (i = 0; i < nr_pages; i++) {

		struct page *cur_page = page + i;



		/* While PTE-mapping a THP we have a PMD and a PTE mapping. */

	VM_WARN_ON_FOLIO((atomic_read(&page->_mapcount) > 0 ||

			  (folio_test_large(folio) && folio_entire_mapcount(folio) > 1)) &&

			 PageAnonExclusive(page), folio);

		VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||

				  (folio_test_large(folio) &&

				   folio_entire_mapcount(folio) > 1)) &&

				 PageAnonExclusive(cur_page), folio);

	}



	/*

	 * For large folio, only mlock it if it's fully mapped to VMA. It's
@@ -1369,6 +1365,54 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma, 
		mlock_vma_folio(folio, vma);

}



/**

 * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio

 * @folio:	The folio to add the mappings to

 * @page:	The first page to add

 * @nr_pages:	The number of pages which will be mapped

 * @vma:	The vm area in which the mappings are added

 * @address:	The user virtual address of the first page to map

 * @flags:	The rmap flags

 *

 * The page range of folio is defined by [first_page, first_page + nr_pages)

 *

 * The caller needs to hold the page table lock, and the page must be locked in

 * the anon_vma case: to serialize mapping,index checking after setting,

 * and to ensure that an anon folio is not being upgraded racily to a KSM folio

 * (but KSM folios are never downgraded).

 */

void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,

		int nr_pages, struct vm_area_struct *vma, unsigned long address,

		rmap_t flags)

{

	__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,

			      RMAP_LEVEL_PTE);

}



/**

 * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio

 * @folio:	The folio to add the mapping to

 * @page:	The first page to add

 * @vma:	The vm area in which the mapping is added

 * @address:	The user virtual address of the first page to map

 * @flags:	The rmap flags

 *

 * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)

 *

 * The caller needs to hold the page table lock, and the page must be locked in

 * the anon_vma case: to serialize mapping,index checking after setting.

 */

void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,

		struct vm_area_struct *vma, unsigned long address, rmap_t flags)

{

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

	__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,

			      RMAP_LEVEL_PMD);

#else

	WARN_ON_ONCE(true);

#endif

}



/**

 * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.

 * @folio:	The folio to add the mapping to.