Commit d3f7922b authored by Alistair Popple's avatar Alistair Popple Committed by Andrew Morton
Browse files

mm/gup: remove dead pgmap refcounting code 

Prior to commit aed877c2 ("device/dax: properly refcount device dax
pages when mapping") ZONE_DEVICE pages were not fully reference counted
when mapped into user page tables.  Instead GUP would take a reference on
the associated pgmap to ensure the results of pfn_to_page() remained
valid.

This is no longer required and most of the code was removed by commit
fd2825b0 ("mm/gup: remove pXX_devmap usage from get_user_pages()"). 
Finish cleaning this up by removing the dead calls to put_dev_pagemap()
and the temporary context struct.

Link: https://lkml.kernel.org/r/20250903225926.34702-1-apopple@nvidia.com


Signed-off-by: default avatarAlistair Popple <apopple@nvidia.com>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Reviewed-by: default avatarDan Williams <dan.j.williams@intel.com>
Reviewed-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 4805ef37

mm/gup.c

+26 −41
Original line number Diff line number Diff line
@@ -28,11 +28,6 @@ 
#include "internal.h"

#include "swap.h"



struct follow_page_context {

	struct dev_pagemap *pgmap;

	unsigned int page_mask;

};



static inline void sanity_check_pinned_pages(struct page **pages,

					     unsigned long npages)

{
@@ -653,7 +648,7 @@ static inline bool can_follow_write_pud(pud_t pud, struct page *page, 


static struct page *follow_huge_pud(struct vm_area_struct *vma,

				    unsigned long addr, pud_t *pudp,

				    int flags, struct follow_page_context *ctx)

				    int flags, unsigned long *page_mask)

{

	struct mm_struct *mm = vma->vm_mm;

	struct page *page;
@@ -680,7 +675,7 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma, 
	if (ret)

		page = ERR_PTR(ret);

	else

		ctx->page_mask = HPAGE_PUD_NR - 1;

		*page_mask = HPAGE_PUD_NR - 1;



	return page;

}
@@ -706,7 +701,7 @@ static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page, 
static struct page *follow_huge_pmd(struct vm_area_struct *vma,

				    unsigned long addr, pmd_t *pmd,

				    unsigned int flags,

				    struct follow_page_context *ctx)

				    unsigned long *page_mask)

{

	struct mm_struct *mm = vma->vm_mm;

	pmd_t pmdval = *pmd;
@@ -743,7 +738,7 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma, 
#endif	/* CONFIG_TRANSPARENT_HUGEPAGE */



	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;

	ctx->page_mask = HPAGE_PMD_NR - 1;

	*page_mask = HPAGE_PMD_NR - 1;



	return page;

}
@@ -751,7 +746,7 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma, 
#else  /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */

static struct page *follow_huge_pud(struct vm_area_struct *vma,

				    unsigned long addr, pud_t *pudp,

				    int flags, struct follow_page_context *ctx)

				    int flags, unsigned long *page_mask)

{

	return NULL;

}
@@ -759,7 +754,7 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma, 
static struct page *follow_huge_pmd(struct vm_area_struct *vma,

				    unsigned long addr, pmd_t *pmd,

				    unsigned int flags,

				    struct follow_page_context *ctx)

				    unsigned long *page_mask)

{

	return NULL;

}
@@ -805,8 +800,7 @@ static inline bool can_follow_write_pte(pte_t pte, struct page *page, 
}



static struct page *follow_page_pte(struct vm_area_struct *vma,

		unsigned long address, pmd_t *pmd, unsigned int flags,

		struct dev_pagemap **pgmap)

		unsigned long address, pmd_t *pmd, unsigned int flags)

{

	struct mm_struct *mm = vma->vm_mm;

	struct folio *folio;
@@ -904,7 +898,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, 
static struct page *follow_pmd_mask(struct vm_area_struct *vma,

				    unsigned long address, pud_t *pudp,

				    unsigned int flags,

				    struct follow_page_context *ctx)

				    unsigned long *page_mask)

{

	pmd_t *pmd, pmdval;

	spinlock_t *ptl;
@@ -918,7 +912,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, 
	if (!pmd_present(pmdval))

		return no_page_table(vma, flags, address);

	if (likely(!pmd_leaf(pmdval)))

		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);

		return follow_page_pte(vma, address, pmd, flags);



	if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags))

		return no_page_table(vma, flags, address);
@@ -931,16 +925,16 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, 
	}

	if (unlikely(!pmd_leaf(pmdval))) {

		spin_unlock(ptl);

		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);

		return follow_page_pte(vma, address, pmd, flags);

	}

	if (pmd_trans_huge(pmdval) && (flags & FOLL_SPLIT_PMD)) {

		spin_unlock(ptl);

		split_huge_pmd(vma, pmd, address);

		/* If pmd was left empty, stuff a page table in there quickly */

		return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) :

			follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);

			follow_page_pte(vma, address, pmd, flags);

	}

	page = follow_huge_pmd(vma, address, pmd, flags, ctx);

	page = follow_huge_pmd(vma, address, pmd, flags, page_mask);

	spin_unlock(ptl);

	return page;

}
@@ -948,7 +942,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, 
static struct page *follow_pud_mask(struct vm_area_struct *vma,

				    unsigned long address, p4d_t *p4dp,

				    unsigned int flags,

				    struct follow_page_context *ctx)

				    unsigned long *page_mask)

{

	pud_t *pudp, pud;

	spinlock_t *ptl;
@@ -961,7 +955,7 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, 
		return no_page_table(vma, flags, address);

	if (pud_leaf(pud)) {

		ptl = pud_lock(mm, pudp);

		page = follow_huge_pud(vma, address, pudp, flags, ctx);

		page = follow_huge_pud(vma, address, pudp, flags, page_mask);

		spin_unlock(ptl);

		if (page)

			return page;
@@ -970,13 +964,13 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, 
	if (unlikely(pud_bad(pud)))

		return no_page_table(vma, flags, address);



	return follow_pmd_mask(vma, address, pudp, flags, ctx);

	return follow_pmd_mask(vma, address, pudp, flags, page_mask);

}



static struct page *follow_p4d_mask(struct vm_area_struct *vma,

				    unsigned long address, pgd_t *pgdp,

				    unsigned int flags,

				    struct follow_page_context *ctx)

				    unsigned long *page_mask)

{

	p4d_t *p4dp, p4d;
@@ -987,7 +981,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, 
	if (!p4d_present(p4d) || p4d_bad(p4d))

		return no_page_table(vma, flags, address);



	return follow_pud_mask(vma, address, p4dp, flags, ctx);

	return follow_pud_mask(vma, address, p4dp, flags, page_mask);

}



/**
@@ -995,20 +989,16 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, 
 * @vma: vm_area_struct mapping @address

 * @address: virtual address to look up

 * @flags: flags modifying lookup behaviour

 * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a

 *       pointer to output page_mask

 * @page_mask: a pointer to output page_mask

 *

 * @flags can have FOLL_ flags set, defined in <linux/mm.h>

 *

 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches

 * the device's dev_pagemap metadata to avoid repeating expensive lookups.

 *

 * When getting an anonymous page and the caller has to trigger unsharing

 * of a shared anonymous page first, -EMLINK is returned. The caller should

 * trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only

 * relevant with FOLL_PIN and !FOLL_WRITE.

 *

 * On output, the @ctx->page_mask is set according to the size of the page.

 * On output, @page_mask is set according to the size of the page.

 *

 * Return: the mapped (struct page *), %NULL if no mapping exists, or

 * an error pointer if there is a mapping to something not represented
@@ -1016,7 +1006,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, 
 */

static struct page *follow_page_mask(struct vm_area_struct *vma,

			      unsigned long address, unsigned int flags,

			      struct follow_page_context *ctx)

			      unsigned long *page_mask)

{

	pgd_t *pgd;

	struct mm_struct *mm = vma->vm_mm;
@@ -1024,13 +1014,13 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, 


	vma_pgtable_walk_begin(vma);



	ctx->page_mask = 0;

	*page_mask = 0;

	pgd = pgd_offset(mm, address);



	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))

		page = no_page_table(vma, flags, address);

	else

		page = follow_p4d_mask(vma, address, pgd, flags, ctx);

		page = follow_p4d_mask(vma, address, pgd, flags, page_mask);



	vma_pgtable_walk_end(vma);
@@ -1368,7 +1358,7 @@ static long __get_user_pages(struct mm_struct *mm, 
{

	long ret = 0, i = 0;

	struct vm_area_struct *vma = NULL;

	struct follow_page_context ctx = { NULL };

	unsigned long page_mask = 0;



	if (!nr_pages)

		return 0;
@@ -1410,7 +1400,7 @@ static long __get_user_pages(struct mm_struct *mm, 
						pages ? &page : NULL);

				if (ret)

					goto out;

				ctx.page_mask = 0;

				page_mask = 0;

				goto next_page;

			}
@@ -1433,7 +1423,7 @@ static long __get_user_pages(struct mm_struct *mm, 
		}

		cond_resched();



		page = follow_page_mask(vma, start, gup_flags, &ctx);

		page = follow_page_mask(vma, start, gup_flags, &page_mask);

		if (!page || PTR_ERR(page) == -EMLINK) {

			ret = faultin_page(vma, start, gup_flags,

					   PTR_ERR(page) == -EMLINK, locked);
@@ -1466,7 +1456,7 @@ static long __get_user_pages(struct mm_struct *mm, 
			goto out;

		}

next_page:

		page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);

		page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);

		if (page_increm > nr_pages)

			page_increm = nr_pages;
@@ -1516,8 +1506,6 @@ static long __get_user_pages(struct mm_struct *mm, 
		nr_pages -= page_increm;

	} while (nr_pages);

out:

	if (ctx.pgmap)

		put_dev_pagemap(ctx.pgmap);

	return i ? i : ret;

}
@@ -2853,7 +2841,6 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, 
		unsigned long end, unsigned int flags, struct page **pages,

		int *nr)

{

	struct dev_pagemap *pgmap = NULL;

	int ret = 0;

	pte_t *ptep, *ptem;
@@ -2926,8 +2913,6 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, 
	ret = 1;



pte_unmap:

	if (pgmap)

		put_dev_pagemap(pgmap);

	pte_unmap(ptem);

	return ret;

}