Commit 44885447 authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm/rmap: use folio_large_nr_pages() in add/remove functions 

Let's just use the "large" variant in code where we are sure that we have
a large folio in our hands: this way we are sure that we don't perform any
unnecessary "large" checks.

While at it, convert the VM_BUG_ON_VMA to a VM_WARN_ON_ONCE.

Maybe in the future there will not be a difference in that regard between
large and small folios; in that case, unifying the handling again will be
easy.  E.g., folio_large_nr_pages() will simply translate to
folio_nr_pages() until we replace all instances.

Link: https://lkml.kernel.org/r/20250303163014.1128035-12-david@redhat.com


Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andy Lutomirks^H^Hski <luto@kernel.org>
Cc: Borislav Betkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcow (Oracle) <willy@infradead.org>
Cc: Michal Koutn <mkoutny@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: tejun heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Zefan Li <lizefan.x@bytedance.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent b85aa9b1

mm/rmap.c

+8 −6
Original line number Diff line number Diff line
@@ -1274,7 +1274,7 @@ static __always_inline unsigned int __folio_add_rmap(struct folio *folio, 
		if (first) {

			nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);

			if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {

				nr_pages = folio_nr_pages(folio);

				nr_pages = folio_large_nr_pages(folio);

				/*

				 * We only track PMD mappings of PMD-sized

				 * folios separately.
@@ -1522,14 +1522,11 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, 
void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,

		unsigned long address, rmap_t flags)

{

	const int nr = folio_nr_pages(folio);

	const bool exclusive = flags & RMAP_EXCLUSIVE;

	int nr_pmdmapped = 0;

	int nr = 1, nr_pmdmapped = 0;



	VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);

	VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);

	VM_BUG_ON_VMA(address < vma->vm_start ||

			address + (nr << PAGE_SHIFT) > vma->vm_end, vma);



	/*

	 * VM_DROPPABLE mappings don't swap; instead they're just dropped when
@@ -1547,6 +1544,7 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 
	} else if (!folio_test_pmd_mappable(folio)) {

		int i;



		nr = folio_large_nr_pages(folio);

		for (i = 0; i < nr; i++) {

			struct page *page = folio_page(folio, i);
@@ -1559,6 +1557,7 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 
		folio_set_large_mapcount(folio, nr, vma);

		atomic_set(&folio->_nr_pages_mapped, nr);

	} else {

		nr = folio_large_nr_pages(folio);

		/* increment count (starts at -1) */

		atomic_set(&folio->_entire_mapcount, 0);

		folio_set_large_mapcount(folio, 1, vma);
@@ -1568,6 +1567,9 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 
		nr_pmdmapped = nr;

	}



	VM_WARN_ON_ONCE(address < vma->vm_start ||

			address + (nr << PAGE_SHIFT) > vma->vm_end);



	__folio_mod_stat(folio, nr, nr_pmdmapped);

	mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);

}
@@ -1681,7 +1683,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, 
		if (last) {

			nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);

			if (likely(nr < ENTIRELY_MAPPED)) {

				nr_pages = folio_nr_pages(folio);

				nr_pages = folio_large_nr_pages(folio);

				if (level == RMAP_LEVEL_PMD)

					nr_pmdmapped = nr_pages;

				nr = nr_pages - (nr & FOLIO_PAGES_MAPPED);