Commit 7b6ec181 authored by Vishal Moola (Oracle)'s avatar Vishal Moola (Oracle) Committed by Andrew Morton
Browse files

hugetlb: convert hugetlb_no_page() to use struct vm_fault 

hugetlb_no_page() can use the struct vm_fault passed in from
hugetlb_fault(). This alleviates the stack by consolidating 7
variables into a single struct.

[vishal.moola@gmail.com:  simplify hugetlb_no_page() arguments]
  Link: https://lkml.kernel.org/r/ZhQtN8y5zud8iI1u@fedora
Link: https://lkml.kernel.org/r/20240401202651.31440-3-vishal.moola@gmail.com


Signed-off-by: default avatarVishal Moola (Oracle) <vishal.moola@gmail.com>
Reviewed-by: default avatarOscar Salvador <osalvador@suse.de>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 9b42fa16

mm/hugetlb.c

+31 −32
Original line number Diff line number Diff line
@@ -6188,23 +6188,19 @@ static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, 
	return same;

}



static vm_fault_t hugetlb_no_page(struct mm_struct *mm,

			struct vm_area_struct *vma,

			struct address_space *mapping, pgoff_t idx,

			unsigned long address, pte_t *ptep,

			pte_t old_pte, unsigned int flags,

static vm_fault_t hugetlb_no_page(struct address_space *mapping,

			struct vm_fault *vmf)

{

	struct vm_area_struct *vma = vmf->vma;

	struct mm_struct *mm = vma->vm_mm;

	struct hstate *h = hstate_vma(vma);

	vm_fault_t ret = VM_FAULT_SIGBUS;

	int anon_rmap = 0;

	unsigned long size;

	struct folio *folio;

	pte_t new_pte;

	spinlock_t *ptl;

	unsigned long haddr = address & huge_page_mask(h);

	bool new_folio, new_pagecache_folio = false;

	u32 hash = hugetlb_fault_mutex_hash(mapping, idx);

	u32 hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);



	/*

	 * Currently, we are forced to kill the process in the event the
@@ -6223,10 +6219,10 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
	 * before we get page_table_lock.

	 */

	new_folio = false;

	folio = filemap_lock_hugetlb_folio(h, mapping, idx);

	folio = filemap_lock_hugetlb_folio(h, mapping, vmf->pgoff);

	if (IS_ERR(folio)) {

		size = i_size_read(mapping->host) >> huge_page_shift(h);

		if (idx >= size)

		if (vmf->pgoff >= size)

			goto out;

		/* Check for page in userfault range */

		if (userfaultfd_missing(vma)) {
@@ -6247,7 +6243,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
			 * never happen on the page after UFFDIO_COPY has

			 * correctly installed the page and returned.

			 */

			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {

			if (!hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) {

				ret = 0;

				goto out;

			}
@@ -6262,7 +6258,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
				goto out;

		}



		folio = alloc_hugetlb_folio(vma, haddr, 0);

		folio = alloc_hugetlb_folio(vma, vmf->address, 0);

		if (IS_ERR(folio)) {

			/*

			 * Returning error will result in faulting task being
@@ -6276,18 +6272,20 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
			 * here.  Before returning error, get ptl and make

			 * sure there really is no pte entry.

			 */

			if (hugetlb_pte_stable(h, mm, ptep, old_pte))

			if (hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte))

				ret = vmf_error(PTR_ERR(folio));

			else

				ret = 0;

			goto out;

		}

		clear_huge_page(&folio->page, address, pages_per_huge_page(h));

		clear_huge_page(&folio->page, vmf->real_address,

				pages_per_huge_page(h));

		__folio_mark_uptodate(folio);

		new_folio = true;



		if (vma->vm_flags & VM_MAYSHARE) {

			int err = hugetlb_add_to_page_cache(folio, mapping, idx);

			int err = hugetlb_add_to_page_cache(folio, mapping,

							vmf->pgoff);

			if (err) {

				/*

				 * err can't be -EEXIST which implies someone
@@ -6296,7 +6294,8 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
				 * to the page cache. So it's safe to call

				 * restore_reserve_on_error() here.

				 */

				restore_reserve_on_error(h, vma, haddr, folio);

				restore_reserve_on_error(h, vma, vmf->address,

							folio);

				folio_put(folio);

				ret = VM_FAULT_SIGBUS;

				goto out;
@@ -6323,7 +6322,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
			folio_unlock(folio);

			folio_put(folio);

			/* See comment in userfaultfd_missing() block above */

			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {

			if (!hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) {

				ret = 0;

				goto out;

			}
@@ -6338,23 +6337,23 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
	 * any allocations necessary to record that reservation occur outside

	 * the spinlock.

	 */

	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {

		if (vma_needs_reservation(h, vma, haddr) < 0) {

	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {

		if (vma_needs_reservation(h, vma, vmf->address) < 0) {

			ret = VM_FAULT_OOM;

			goto backout_unlocked;

		}

		/* Just decrements count, does not deallocate */

		vma_end_reservation(h, vma, haddr);

		vma_end_reservation(h, vma, vmf->address);

	}



	ptl = huge_pte_lock(h, mm, ptep);

	vmf->ptl = huge_pte_lock(h, mm, vmf->pte);

	ret = 0;

	/* If pte changed from under us, retry */

	if (!pte_same(huge_ptep_get(ptep), old_pte))

	if (!pte_same(huge_ptep_get(vmf->pte), vmf->orig_pte))

		goto backout;



	if (anon_rmap)

		hugetlb_add_new_anon_rmap(folio, vma, haddr);

		hugetlb_add_new_anon_rmap(folio, vma, vmf->address);

	else

		hugetlb_add_file_rmap(folio);

	new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE)
@@ -6363,17 +6362,18 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
	 * If this pte was previously wr-protected, keep it wr-protected even

	 * if populated.

	 */

	if (unlikely(pte_marker_uffd_wp(old_pte)))

	if (unlikely(pte_marker_uffd_wp(vmf->orig_pte)))

		new_pte = huge_pte_mkuffd_wp(new_pte);

	set_huge_pte_at(mm, haddr, ptep, new_pte, huge_page_size(h));

	set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h));



	hugetlb_count_add(pages_per_huge_page(h), mm);

	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {

	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {

		/* Optimization, do the COW without a second fault */

		ret = hugetlb_wp(mm, vma, address, ptep, flags, folio, ptl, vmf);

		ret = hugetlb_wp(mm, vma, vmf->real_address, vmf->pte,

				vmf->flags, folio, vmf->ptl, vmf);

	}



	spin_unlock(ptl);

	spin_unlock(vmf->ptl);



	/*

	 * Only set hugetlb_migratable in newly allocated pages.  Existing pages
@@ -6390,10 +6390,10 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 
	return ret;



backout:

	spin_unlock(ptl);

	spin_unlock(vmf->ptl);

backout_unlocked:

	if (new_folio && !new_pagecache_folio)

		restore_reserve_on_error(h, vma, haddr, folio);

		restore_reserve_on_error(h, vma, vmf->address, folio);



	folio_unlock(folio);

	folio_put(folio);
@@ -6489,8 +6489,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 
		 * hugetlb_no_page will drop vma lock and hugetlb fault

		 * mutex internally, which make us return immediately.

		 */

		return hugetlb_no_page(mm, vma, mapping, vmf.pgoff, address,

					vmf.pte, vmf.orig_pte, flags, &vmf);

		return hugetlb_no_page(mapping, &vmf);

	}



	ret = 0;