Commit c4297465 authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm/huge_memory: don't mark refcounted folios special in vmf_insert_folio_pmd() 

Marking PMDs that map a "normal" refcounted folios as special is against
our rules documented for vm_normal_page(): normal (refcounted) folios
shall never have the page table mapping marked as special.

Fortunately, there are not that many pmd_special() check that can be
mislead, and most vm_normal_page_pmd()/vm_normal_folio_pmd() users that
would get this wrong right now are rather harmless: e.g., none so far
bases decisions whether to grab a folio reference on that decision.

Well, and GUP-fast will fallback to GUP-slow.  All in all, so far no big
implications as it seems.

Getting this right will get more important as we use
folio_normal_page_pmd() in more places.

Fix it by teaching insert_pfn_pmd() to properly handle folios and pfns --
moving refcount/mapcount/etc handling in there, renaming it to
insert_pmd(), and distinguishing between both cases using a new simple
"struct folio_or_pfn" structure.

Use folio_mk_pmd() to create a pmd for a folio cleanly.

Link: https://lkml.kernel.org/r/20250613092702.1943533-3-david@redhat.com


Fixes: 6c88f726 ("mm/huge_memory: add vmf_insert_folio_pmd()")
Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Reviewed-by: default avatarLorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: default avatarDan Williams <dan.j.williams@intel.com>
Tested-by: default avatarDan Williams <dan.j.williams@intel.com>
Reviewed-by: default avatarOscar Salvador <osalvador@suse.de>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Mariano Pache <npache@redhat.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 09fefdca

mm/huge_memory.c

+40 −19
Original line number Diff line number Diff line
@@ -1372,9 +1372,17 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 
	return __do_huge_pmd_anonymous_page(vmf);

}



static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,

		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,

		pgtable_t pgtable)

struct folio_or_pfn {

	union {

		struct folio *folio;

		pfn_t pfn;

	};

	bool is_folio;

};



static int insert_pmd(struct vm_area_struct *vma, unsigned long addr,

		pmd_t *pmd, struct folio_or_pfn fop, pgprot_t prot,

		bool write, pgtable_t pgtable)

{

	struct mm_struct *mm = vma->vm_mm;

	pmd_t entry;
@@ -1382,8 +1390,11 @@ static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 
	lockdep_assert_held(pmd_lockptr(mm, pmd));



	if (!pmd_none(*pmd)) {

		const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) :

					  pfn_t_to_pfn(fop.pfn);



		if (write) {

			if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {

			if (pmd_pfn(*pmd) != pfn) {

				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));

				return -EEXIST;

			}
@@ -1396,11 +1407,20 @@ static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 
		return -EEXIST;

	}



	entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));

	if (pfn_t_devmap(pfn))

	if (fop.is_folio) {

		entry = folio_mk_pmd(fop.folio, vma->vm_page_prot);



		folio_get(fop.folio);

		folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma);

		add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR);

	} else {

		entry = pmd_mkhuge(pfn_t_pmd(fop.pfn, prot));



		if (pfn_t_devmap(fop.pfn))

			entry = pmd_mkdevmap(entry);

		else

			entry = pmd_mkspecial(entry);

	}

	if (write) {

		entry = pmd_mkyoung(pmd_mkdirty(entry));

		entry = maybe_pmd_mkwrite(entry, vma);
@@ -1431,6 +1451,9 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) 
	unsigned long addr = vmf->address & PMD_MASK;

	struct vm_area_struct *vma = vmf->vma;

	pgprot_t pgprot = vma->vm_page_prot;

	struct folio_or_pfn fop = {

		.pfn = pfn,

	};

	pgtable_t pgtable = NULL;

	spinlock_t *ptl;

	int error;
@@ -1458,7 +1481,7 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) 
	pfnmap_setup_cachemode_pfn(pfn_t_to_pfn(pfn), &pgprot);



	ptl = pmd_lock(vma->vm_mm, vmf->pmd);

	error = insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write,

	error = insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write,

			   pgtable);

	spin_unlock(ptl);

	if (error && pgtable)
@@ -1474,6 +1497,10 @@ vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 
	struct vm_area_struct *vma = vmf->vma;

	unsigned long addr = vmf->address & PMD_MASK;

	struct mm_struct *mm = vma->vm_mm;

	struct folio_or_pfn fop = {

		.folio = folio,

		.is_folio = true,

	};

	spinlock_t *ptl;

	pgtable_t pgtable = NULL;

	int error;
@@ -1491,13 +1518,7 @@ vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 
	}



	ptl = pmd_lock(mm, vmf->pmd);

	if (pmd_none(*vmf->pmd)) {

		folio_get(folio);

		folio_add_file_rmap_pmd(folio, &folio->page, vma);

		add_mm_counter(mm, mm_counter_file(folio), HPAGE_PMD_NR);

	}

	error = insert_pfn_pmd(vma, addr, vmf->pmd,

			pfn_to_pfn_t(folio_pfn(folio)), vma->vm_page_prot,

	error = insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot,

			   write, pgtable);

	spin_unlock(ptl);

	if (error && pgtable)