Commit f74991b4 authored by Mike Rapoport (Microsoft)'s avatar Mike Rapoport (Microsoft) Committed by Andrew Morton
Browse files

shmem, userfaultfd: implement shmem uffd operations using vm_uffd_ops 

Add filemap_add() and filemap_remove() methods to vm_uffd_ops and use them
in __mfill_atomic_pte() to add shmem folios to page cache and remove them
in case of error.

Implement these methods in shmem along with vm_uffd_ops->alloc_folio() and
drop shmem_mfill_atomic_pte().

Since userfaultfd now does not reference any functions from shmem, drop
include if linux/shmem_fs.h from mm/userfaultfd.c

mfill_atomic_install_pte() is not used anywhere outside of mm/userfaultfd,
make it static.

Link: https://lore.kernel.org/20260402041156.1377214-11-rppt@kernel.org


Signed-off-by: default avatarMike Rapoport (Microsoft) <rppt@kernel.org>
Reviewed-by: default avatarJames Houghton <jthoughton@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrei Vagin <avagin@google.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Harry Yoo (Oracle) <harry@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nikita Kalyazin <kalyazin@amazon.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: David Carlier <devnexen@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent ad9ac308

include/linux/shmem_fs.h

+0 −14
Original line number Diff line number Diff line
@@ -221,20 +221,6 @@ static inline pgoff_t shmem_fallocend(struct inode *inode, pgoff_t eof) 


extern bool shmem_charge(struct inode *inode, long pages);



#ifdef CONFIG_USERFAULTFD

#ifdef CONFIG_SHMEM

extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd,

				  struct vm_area_struct *dst_vma,

				  unsigned long dst_addr,

				  unsigned long src_addr,

				  uffd_flags_t flags,

				  struct folio **foliop);

#else /* !CONFIG_SHMEM */

#define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \

			       src_addr, flags, foliop) ({ BUG(); 0; })

#endif /* CONFIG_SHMEM */

#endif /* CONFIG_USERFAULTFD */



/*

 * Used space is stored as unsigned 64-bit value in bytes but

 * quota core supports only signed 64-bit values so use that

include/linux/userfaultfd_k.h

+14 −5
Original line number Diff line number Diff line
@@ -100,6 +100,20 @@ struct vm_uffd_ops { 
	 */

	struct folio *(*alloc_folio)(struct vm_area_struct *vma,

				     unsigned long addr);

	/*

	 * Called during resolution of UFFDIO_COPY request.

	 * Should only be called with a folio returned by alloc_folio() above.

	 * The folio will be set to locked.

	 * Returns 0 on success, error code on failure.

	 */

	int (*filemap_add)(struct folio *folio, struct vm_area_struct *vma,

			 unsigned long addr);

	/*

	 * Called during resolution of UFFDIO_COPY request on the error

	 * handling path.

	 * Should revert the operation of ->filemap_add().

	 */

	void (*filemap_remove)(struct folio *folio, struct vm_area_struct *vma);

};



/* A combined operation mode + behavior flags. */
@@ -133,11 +147,6 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at 
/* Flags controlling behavior. These behavior changes are mode-independent. */

#define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0)



extern int mfill_atomic_install_pte(pmd_t *dst_pmd,

				    struct vm_area_struct *dst_vma,

				    unsigned long dst_addr, struct page *page,

				    bool newly_allocated, uffd_flags_t flags);



extern ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start,

				 unsigned long src_start, unsigned long len,

				 uffd_flags_t flags);

mm/shmem.c

+53 −95
Original line number Diff line number Diff line
@@ -3175,118 +3175,73 @@ static struct inode *shmem_get_inode(struct mnt_idmap *idmap, 
#endif /* CONFIG_TMPFS_QUOTA */



#ifdef CONFIG_USERFAULTFD

int shmem_mfill_atomic_pte(pmd_t *dst_pmd,

			   struct vm_area_struct *dst_vma,

			   unsigned long dst_addr,

			   unsigned long src_addr,

			   uffd_flags_t flags,

			   struct folio **foliop)

{

	struct inode *inode = file_inode(dst_vma->vm_file);

	struct shmem_inode_info *info = SHMEM_I(inode);

static struct folio *shmem_mfill_folio_alloc(struct vm_area_struct *vma,

					     unsigned long addr)

{

	struct inode *inode = file_inode(vma->vm_file);

	struct address_space *mapping = inode->i_mapping;

	struct shmem_inode_info *info = SHMEM_I(inode);

	pgoff_t pgoff = linear_page_index(vma, addr);

	gfp_t gfp = mapping_gfp_mask(mapping);

	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);

	void *page_kaddr;

	struct folio *folio;

	int ret;

	pgoff_t max_off;



	if (shmem_inode_acct_blocks(inode, 1)) {

		/*

		 * We may have got a page, returned -ENOENT triggering a retry,

		 * and now we find ourselves with -ENOMEM. Release the page, to

		 * avoid a BUG_ON in our caller.

		 */

		if (unlikely(*foliop)) {

			folio_put(*foliop);

			*foliop = NULL;

		}

		return -ENOMEM;

	}

	if (unlikely(pgoff >= DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE)))

		return NULL;



	if (!*foliop) {

		ret = -ENOMEM;

	folio = shmem_alloc_folio(gfp, 0, info, pgoff);

	if (!folio)

			goto out_unacct_blocks;

		return NULL;



		if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) {

			page_kaddr = kmap_local_folio(folio, 0);

			/*

			 * The read mmap_lock is held here.  Despite the

			 * mmap_lock being read recursive a deadlock is still

			 * possible if a writer has taken a lock.  For example:

			 *

			 * process A thread 1 takes read lock on own mmap_lock

			 * process A thread 2 calls mmap, blocks taking write lock

			 * process B thread 1 takes page fault, read lock on own mmap lock

			 * process B thread 2 calls mmap, blocks taking write lock

			 * process A thread 1 blocks taking read lock on process B

			 * process B thread 1 blocks taking read lock on process A

			 *

			 * Disable page faults to prevent potential deadlock

			 * and retry the copy outside the mmap_lock.

			 */

			pagefault_disable();

			ret = copy_from_user(page_kaddr,

					     (const void __user *)src_addr,

					     PAGE_SIZE);

			pagefault_enable();

			kunmap_local(page_kaddr);



			/* fallback to copy_from_user outside mmap_lock */

			if (unlikely(ret)) {

				*foliop = folio;

				ret = -ENOENT;

				/* don't free the page */

				goto out_unacct_blocks;

	if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL)) {

		folio_put(folio);

		return NULL;

	}



			flush_dcache_folio(folio);

		} else {		/* ZEROPAGE */

			clear_user_highpage(&folio->page, dst_addr);

		}

	} else {

		folio = *foliop;

		VM_BUG_ON_FOLIO(folio_test_large(folio), folio);

		*foliop = NULL;

	return folio;

}



	VM_BUG_ON(folio_test_locked(folio));

	VM_BUG_ON(folio_test_swapbacked(folio));

static int shmem_mfill_filemap_add(struct folio *folio,

				   struct vm_area_struct *vma,

				   unsigned long addr)

{

	struct inode *inode = file_inode(vma->vm_file);

	struct address_space *mapping = inode->i_mapping;

	pgoff_t pgoff = linear_page_index(vma, addr);

	gfp_t gfp = mapping_gfp_mask(mapping);

	int err;



	__folio_set_locked(folio);

	__folio_set_swapbacked(folio);

	__folio_mark_uptodate(folio);



	ret = -EFAULT;

	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);

	if (unlikely(pgoff >= max_off))

		goto out_release;



	ret = mem_cgroup_charge(folio, dst_vma->vm_mm, gfp);

	if (ret)

		goto out_release;

	ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, gfp);

	if (ret)

		goto out_release;

	err = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, gfp);

	if (err)

		goto err_unlock;



	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,

				       &folio->page, true, flags);

	if (ret)

		goto out_delete_from_cache;

	if (shmem_inode_acct_blocks(inode, 1)) {

		err = -ENOMEM;

		goto err_delete_from_cache;

	}



	folio_add_lru(folio);

	shmem_recalc_inode(inode, 1, 0);

	folio_unlock(folio);



	return 0;

out_delete_from_cache:



err_delete_from_cache:

	filemap_remove_folio(folio);

out_release:

err_unlock:

	folio_unlock(folio);

	return err;

}



static void shmem_mfill_filemap_remove(struct folio *folio,

				       struct vm_area_struct *vma)

{

	struct inode *inode = file_inode(vma->vm_file);



	filemap_remove_folio(folio);

	shmem_recalc_inode(inode, 0, 0);

	folio_unlock(folio);

	folio_put(folio);

out_unacct_blocks:

	shmem_inode_unacct_blocks(inode, 1);

	return ret;

}



static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff)
@@ -3309,6 +3264,9 @@ static bool shmem_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) 
static const struct vm_uffd_ops shmem_uffd_ops = {

	.can_userfault		= shmem_can_userfault,

	.get_folio_noalloc	= shmem_get_folio_noalloc,

	.alloc_folio		= shmem_mfill_folio_alloc,

	.filemap_add		= shmem_mfill_filemap_add,

	.filemap_remove		= shmem_mfill_filemap_remove,

};

#endif /* CONFIG_USERFAULTFD */

mm/userfaultfd.c

+39 −41
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ 
#include <linux/userfaultfd_k.h>

#include <linux/mmu_notifier.h>

#include <linux/hugetlb.h>

#include <linux/shmem_fs.h>

#include <asm/tlbflush.h>

#include <asm/tlb.h>

#include "internal.h"
@@ -338,10 +337,10 @@ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, 
 * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem

 * and anon, and for both shared and private VMAs.

 */

int mfill_atomic_install_pte(pmd_t *dst_pmd,

static int mfill_atomic_install_pte(pmd_t *dst_pmd,

				    struct vm_area_struct *dst_vma,

				    unsigned long dst_addr, struct page *page,

			     bool newly_allocated, uffd_flags_t flags)

				    uffd_flags_t flags)

{

	int ret;

	struct mm_struct *dst_mm = dst_vma->vm_mm;
@@ -385,9 +384,6 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, 
		goto out_unlock;



	if (page_in_cache) {

		/* Usually, cache pages are already added to LRU */

		if (newly_allocated)

			folio_add_lru(folio);

		folio_add_file_rmap_pte(folio, page, dst_vma);

	} else {

		folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE);
@@ -402,6 +398,9 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, 


	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);



	if (page_in_cache)

		folio_unlock(folio);



	/* No need to invalidate - it was non-present before */

	update_mmu_cache(dst_vma, dst_addr, dst_pte);

	ret = 0;
@@ -514,13 +513,22 @@ static int __mfill_atomic_pte(struct mfill_state *state, 
	 */

	__folio_mark_uptodate(folio);



	ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr,

				       &folio->page, true, flags);

	if (ops->filemap_add) {

		ret = ops->filemap_add(folio, state->vma, state->dst_addr);

		if (ret)

			goto err_folio_put;

	}



	ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr,

				       &folio->page, flags);

	if (ret)

		goto err_filemap_remove;



	return 0;



err_filemap_remove:

	if (ops->filemap_remove)

		ops->filemap_remove(folio, state->vma);

err_folio_put:

	folio_put(folio);

	/* Don't return -ENOENT so that our caller won't retry */
@@ -533,6 +541,18 @@ static int mfill_atomic_pte_copy(struct mfill_state *state) 
{

	const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma);



	/*

	 * The normal page fault path for a MAP_PRIVATE mapping in a

	 * file-backed VMA will invoke the fault, fill the hole in the file and

	 * COW it right away. The result generates plain anonymous memory.

	 * So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll

	 * generate anonymous memory directly without actually filling the

	 * hole. For the MAP_PRIVATE case the robustness check only happens in

	 * the pagetable (to verify it's still none) and not in the page cache.

	 */

	if (!(state->vma->vm_flags & VM_SHARED))

		ops = &anon_uffd_ops;



	return __mfill_atomic_pte(state, ops);

}
@@ -552,7 +572,8 @@ static int mfill_atomic_pte_zeropage(struct mfill_state *state) 
	spinlock_t *ptl;

	int ret;



	if (mm_forbids_zeropage(dst_vma->vm_mm))

	if (mm_forbids_zeropage(dst_vma->vm_mm) ||

	    (dst_vma->vm_flags & VM_SHARED))

		return mfill_atomic_pte_zeroed_folio(state);



	_dst_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr),
@@ -609,11 +630,10 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) 
	}



	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,

				       page, false, flags);

				       page, flags);

	if (ret)

		goto out_release;



	folio_unlock(folio);

	return 0;



out_release:
@@ -836,41 +856,19 @@ extern ssize_t mfill_atomic_hugetlb(struct userfaultfd_ctx *ctx, 


static __always_inline ssize_t mfill_atomic_pte(struct mfill_state *state)

{

	struct vm_area_struct *dst_vma = state->vma;

	unsigned long src_addr = state->src_addr;

	unsigned long dst_addr = state->dst_addr;

	struct folio **foliop = &state->folio;

	uffd_flags_t flags = state->flags;

	pmd_t *dst_pmd = state->pmd;

	ssize_t err;



	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))

		return mfill_atomic_pte_continue(state);

	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON))

		return mfill_atomic_pte_poison(state);



	/*

	 * The normal page fault path for a shmem will invoke the

	 * fault, fill the hole in the file and COW it right away. The

	 * result generates plain anonymous memory. So when we are

	 * asked to fill an hole in a MAP_PRIVATE shmem mapping, we'll

	 * generate anonymous memory directly without actually filling

	 * the hole. For the MAP_PRIVATE case the robustness check

	 * only happens in the pagetable (to verify it's still none)

	 * and not in the radix tree.

	 */

	if (!(dst_vma->vm_flags & VM_SHARED)) {

	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY))

			err = mfill_atomic_pte_copy(state);

		else

			err = mfill_atomic_pte_zeropage(state);

	} else {

		err = shmem_mfill_atomic_pte(dst_pmd, dst_vma,

					     dst_addr, src_addr,

					     flags, foliop);

	}

		return mfill_atomic_pte_copy(state);

	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE))

		return mfill_atomic_pte_zeropage(state);



	return err;

	VM_WARN_ONCE(1, "Unknown UFFDIO operation, flags: %x", flags);

	return -EOPNOTSUPP;

}



static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,