Commit 2f1c6611 authored by Lorenzo Stoakes's avatar Lorenzo Stoakes Committed by Andrew Morton
Browse files

mm: introduce vma_merge_struct and abstract vma_merge(),vma_modify() 

Rather than passing around huge numbers of parameters to numerous helper
functions, abstract them into a single struct that we thread through the
operation, the vma_merge_struct ('vmg').

Adjust vma_merge() and vma_modify() to accept this parameter, as well as
predicate functions can_vma_merge_before(), can_vma_merge_after(), and the
vma_modify_...() helper functions.

Also introduce VMG_STATE() and VMG_VMA_STATE() helper macros to allow for
easy vmg declaration.

We additionally remove the requirement that vma_merge() is passed a VMA
object representing the candidate new VMA.  Previously it used this to
obtain the mm_struct, file and anon_vma properties of the proposed range
(a rather confusing state of affairs), which are now provided by the vmg
directly.

We also remove the pgoff calculation previously performed vma_modify(),
and instead calculate this in VMG_VMA_STATE() via the vma_pgoff_offset()
helper.

Link: https://lkml.kernel.org/r/a955aad09d81329f6fbeb636b2dd10cde7b73dab.1725040657.git.lorenzo.stoakes@oracle.com


Signed-off-by: default avatarLorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: default avatarLiam R. Howlett <Liam.Howlett@oracle.com>
Cc: Mark Brown <broonie@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Bert Karwatzki <spasswolf@web.de>
Cc: Jeff Xu <jeffxu@chromium.org>
Cc: Jiri Olsa <olsajiri@gmail.com>
Cc: Kees Cook <kees@kernel.org>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 955db396

mm/mmap.c

+42 −34
Original line number Diff line number Diff line
@@ -1373,10 +1373,11 @@ unsigned long mmap_region(struct file *file, unsigned long addr, 
	unsigned long end = addr + len;

	unsigned long merge_start = addr, merge_end = end;

	bool writable_file_mapping = false;

	pgoff_t vm_pgoff;

	int error = -ENOMEM;

	VMA_ITERATOR(vmi, mm, addr);

	VMG_STATE(vmg, mm, &vmi, addr, end, vm_flags, pgoff);



	vmg.file = file;

	/* Find the first overlapping VMA */

	vma = vma_find(&vmi, end);

	init_vma_munmap(&vms, &vmi, vma, addr, end, uf, /* unlock = */ false);
@@ -1389,12 +1390,12 @@ unsigned long mmap_region(struct file *file, unsigned long addr, 
		if (error)

			goto gather_failed;



		next = vms.next;

		prev = vms.prev;

		next = vmg.next = vms.next;

		prev = vmg.prev = vms.prev;

		vma = NULL;

	} else {

		next = vma_next(&vmi);

		prev = vma_prev(&vmi);

		next = vmg.next = vma_next(&vmi);

		prev = vmg.prev = vma_prev(&vmi);

		if (prev)

			vma_iter_next_range(&vmi);

	}
@@ -1414,6 +1415,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, 


		vms.nr_accounted = 0;

		vm_flags |= VM_ACCOUNT;

		vmg.flags = vm_flags;

	}



	if (vm_flags & VM_SPECIAL)
@@ -1422,28 +1424,31 @@ unsigned long mmap_region(struct file *file, unsigned long addr, 
	/* Attempt to expand an old mapping */

	/* Check next */

	if (next && next->vm_start == end && !vma_policy(next) &&

	    can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen,

				 NULL_VM_UFFD_CTX, NULL)) {

	    can_vma_merge_before(&vmg)) {

		merge_end = next->vm_end;

		vma = next;

		vm_pgoff = next->vm_pgoff - pglen;

		vmg.pgoff = next->vm_pgoff - pglen;

		/*

		 * We set this here so if we will merge with the previous VMA in

		 * the code below, can_vma_merge_after() ensures anon_vma

		 * compatibility between prev and next.

		 */

		vmg.anon_vma = vma->anon_vma;

		vmg.uffd_ctx = vma->vm_userfaultfd_ctx;

	}



	/* Check prev */

	if (prev && prev->vm_end == addr && !vma_policy(prev) &&

	    (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file,

				       pgoff, vma->vm_userfaultfd_ctx, NULL) :

		   can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,

				       NULL_VM_UFFD_CTX, NULL))) {

	    can_vma_merge_after(&vmg)) {

		merge_start = prev->vm_start;

		vma = prev;

		vm_pgoff = prev->vm_pgoff;

		vmg.pgoff = prev->vm_pgoff;

		vma_prev(&vmi); /* Equivalent to going to the previous range */

	}



	if (vma) {

		/* Actually expand, if possible */

		if (!vma_expand(&vmi, vma, merge_start, merge_end, vm_pgoff, next)) {

		if (!vma_expand(&vmi, vma, merge_start, merge_end, vmg.pgoff, next)) {

			khugepaged_enter_vma(vma, vm_flags);

			goto expanded;

		}
@@ -1774,9 +1779,11 @@ static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, 
	 * Expand the existing vma if possible; Note that singular lists do not

	 * occur after forking, so the expand will only happen on new VMAs.

	 */

	if (vma && vma->vm_end == addr && !vma_policy(vma) &&

	    can_vma_merge_after(vma, flags, NULL, NULL,

				addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) {

	if (vma && vma->vm_end == addr && !vma_policy(vma)) {

		VMG_STATE(vmg, mm, vmi, addr, addr + len, flags, PHYS_PFN(addr));



		vmg.prev = vma;

		if (can_vma_merge_after(&vmg)) {

			vma_iter_config(vmi, vma->vm_start, addr + len);

			if (vma_iter_prealloc(vmi, vma))

				goto unacct_fail;
@@ -1795,6 +1802,7 @@ static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, 
			khugepaged_enter_vma(vma, flags);

			goto out;

		}

	}



	if (vma)

		vma_iter_next_range(vmi);

mm/vma.c

+126 −81
Original line number Diff line number Diff line
@@ -7,16 +7,18 @@ 
#include "vma_internal.h"

#include "vma.h"



static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)

{

	struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;

	/*

 * If the vma has a ->close operation then the driver probably needs to release

 * per-vma resources, so we don't attempt to merge those if the caller indicates

 * the current vma may be removed as part of the merge.

	 * If the vma has a ->close operation then the driver probably needs to

	 * release per-vma resources, so we don't attempt to merge those if the

	 * caller indicates the current vma may be removed as part of the merge,

	 * which is the case if we are attempting to merge the next VMA into

	 * this one.

	 */

static inline bool is_mergeable_vma(struct vm_area_struct *vma,

		struct file *file, unsigned long vm_flags,

		struct vm_userfaultfd_ctx vm_userfaultfd_ctx,

		struct anon_vma_name *anon_name, bool may_remove_vma)

{

	bool may_remove_vma = merge_next;



	/*

	 * VM_SOFTDIRTY should not prevent from VMA merging, if we

	 * match the flags but dirty bit -- the caller should mark
@@ -25,15 +27,15 @@ static inline bool is_mergeable_vma(struct vm_area_struct *vma, 
	 * the kernel to generate new VMAs when old one could be

	 * extended instead.

	 */

	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)

	if ((vma->vm_flags ^ vmg->flags) & ~VM_SOFTDIRTY)

		return false;

	if (vma->vm_file != file)

	if (vma->vm_file != vmg->file)

		return false;

	if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)

		return false;

	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))

	if (!is_mergeable_vm_userfaultfd_ctx(vma, vmg->uffd_ctx))

		return false;

	if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))

	if (!anon_vma_name_eq(anon_vma_name(vma), vmg->anon_name))

		return false;

	return true;

}
@@ -94,16 +96,16 @@ static void init_multi_vma_prep(struct vma_prepare *vp, 
 * We assume the vma may be removed as part of the merge.

 */

bool

can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,

		struct anon_vma *anon_vma, struct file *file,

		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,

		struct anon_vma_name *anon_name)

can_vma_merge_before(struct vma_merge_struct *vmg)

{

	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&

	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {

		if (vma->vm_pgoff == vm_pgoff)

	pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start);



	if (is_mergeable_vma(vmg, /* merge_next = */ true) &&

	    is_mergeable_anon_vma(vmg->anon_vma, vmg->next->anon_vma, vmg->next)) {

		if (vmg->next->vm_pgoff == vmg->pgoff + pglen)

			return true;

	}



	return false;

}
@@ -116,18 +118,11 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 
 *

 * We assume that vma is not removed as part of the merge.

 */

bool

can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,

		struct anon_vma *anon_vma, struct file *file,

		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,

		struct anon_vma_name *anon_name)

bool can_vma_merge_after(struct vma_merge_struct *vmg)

{

	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&

	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {

		pgoff_t vm_pglen;



		vm_pglen = vma_pages(vma);

		if (vma->vm_pgoff + vm_pglen == vm_pgoff)

	if (is_mergeable_vma(vmg, /* merge_next = */ false) &&

	    is_mergeable_anon_vma(vmg->anon_vma, vmg->prev->anon_vma, vmg->prev)) {

		if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff)

			return true;

	}

	return false;
@@ -1017,16 +1012,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 
 * **** is not represented - it will be merged and the vma containing the

 *      area is returned, or the function will return NULL

 */

static struct vm_area_struct

*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,

	   struct vm_area_struct *src, unsigned long addr, unsigned long end,

	   unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,

	   struct vm_userfaultfd_ctx vm_userfaultfd_ctx,

	   struct anon_vma_name *anon_name)

static struct vm_area_struct *vma_merge(struct vma_merge_struct *vmg)

{

	struct mm_struct *mm = src->vm_mm;

	struct anon_vma *anon_vma = src->anon_vma;

	struct file *file = src->vm_file;

	struct mm_struct *mm = vmg->mm;

	struct vm_area_struct *prev = vmg->prev;

	struct vm_area_struct *curr, *next, *res;

	struct vm_area_struct *vma, *adjust, *remove, *remove2;

	struct vm_area_struct *anon_dup = NULL;
@@ -1036,16 +1025,18 @@ static struct vm_area_struct 
	bool merge_prev = false;

	bool merge_next = false;

	bool vma_expanded = false;

	unsigned long addr = vmg->start;

	unsigned long end = vmg->end;

	unsigned long vma_start = addr;

	unsigned long vma_end = end;

	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;

	pgoff_t pglen = PHYS_PFN(end - addr);

	long adj_start = 0;



	/*

	 * We later require that vma->vm_flags == vm_flags,

	 * so this tests vma->vm_flags & VM_SPECIAL, too.

	 */

	if (vm_flags & VM_SPECIAL)

	if (vmg->flags & VM_SPECIAL)

		return NULL;



	/* Does the input range span an existing VMA? (cases 5 - 8) */
@@ -1053,27 +1044,26 @@ static struct vm_area_struct 


	if (!curr ||			/* cases 1 - 4 */

	    end == curr->vm_end)	/* cases 6 - 8, adjacent VMA */

		next = vma_lookup(mm, end);

		next = vmg->next = vma_lookup(mm, end);

	else

		next = NULL;		/* case 5 */

		next = vmg->next = NULL;	/* case 5 */



	if (prev) {

		vma_start = prev->vm_start;

		vma_pgoff = prev->vm_pgoff;



		/* Can we merge the predecessor? */

		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)

		    && can_vma_merge_after(prev, vm_flags, anon_vma, file,

					   pgoff, vm_userfaultfd_ctx, anon_name)) {

		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), vmg->policy)

		    && can_vma_merge_after(vmg)) {



			merge_prev = true;

			vma_prev(vmi);

			vma_prev(vmg->vmi);

		}

	}



	/* Can we merge the successor? */

	if (next && mpol_equal(policy, vma_policy(next)) &&

	    can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,

				 vm_userfaultfd_ctx, anon_name)) {

	if (next && mpol_equal(vmg->policy, vma_policy(next)) &&

	    can_vma_merge_before(vmg)) {

		merge_next = true;

	}
@@ -1164,13 +1154,13 @@ static struct vm_area_struct 
		vma_expanded = true;



	if (vma_expanded) {

		vma_iter_config(vmi, vma_start, vma_end);

		vma_iter_config(vmg->vmi, vma_start, vma_end);

	} else {

		vma_iter_config(vmi, adjust->vm_start + adj_start,

		vma_iter_config(vmg->vmi, adjust->vm_start + adj_start,

				adjust->vm_end);

	}



	if (vma_iter_prealloc(vmi, vma))

	if (vma_iter_prealloc(vmg->vmi, vma))

		goto prealloc_fail;



	init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
@@ -1182,20 +1172,20 @@ static struct vm_area_struct 
	vma_set_range(vma, vma_start, vma_end, vma_pgoff);



	if (vma_expanded)

		vma_iter_store(vmi, vma);

		vma_iter_store(vmg->vmi, vma);



	if (adj_start) {

		adjust->vm_start += adj_start;

		adjust->vm_pgoff += adj_start >> PAGE_SHIFT;

		if (adj_start < 0) {

			WARN_ON(vma_expanded);

			vma_iter_store(vmi, next);

			vma_iter_store(vmg->vmi, next);

		}

	}



	vma_complete(&vp, vmi, mm);

	vma_complete(&vp, vmg->vmi, mm);

	validate_mm(mm);

	khugepaged_enter_vma(res, vm_flags);

	khugepaged_enter_vma(res, vmg->flags);

	return res;



prealloc_fail:
@@ -1203,8 +1193,8 @@ static struct vm_area_struct 
		unlink_anon_vmas(anon_dup);



anon_vma_fail:

	vma_iter_set(vmi, addr);

	vma_iter_load(vmi);

	vma_iter_set(vmg->vmi, addr);

	vma_iter_load(vmg->vmi);

	return NULL;

}
@@ -1221,32 +1211,27 @@ static struct vm_area_struct 
 * The function returns either the merged VMA, the original VMA if a split was

 * required instead, or an error if the split failed.

 */

struct vm_area_struct *vma_modify(struct vma_iterator *vmi,

				  struct vm_area_struct *prev,

				  struct vm_area_struct *vma,

				  unsigned long start, unsigned long end,

				  unsigned long vm_flags,

				  struct mempolicy *policy,

				  struct vm_userfaultfd_ctx uffd_ctx,

				  struct anon_vma_name *anon_name)

static struct vm_area_struct *vma_modify(struct vma_merge_struct *vmg)

{

	pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);

	struct vm_area_struct *vma = vmg->vma;

	struct vm_area_struct *merged;



	merged = vma_merge(vmi, prev, vma, start, end, vm_flags,

			   pgoff, policy, uffd_ctx, anon_name);

	/* First, try to merge. */

	merged = vma_merge(vmg);

	if (merged)

		return merged;



	if (vma->vm_start < start) {

		int err = split_vma(vmi, vma, start, 1);

	/* Split any preceding portion of the VMA. */

	if (vma->vm_start < vmg->start) {

		int err = split_vma(vmg->vmi, vma, vmg->start, 1);



		if (err)

			return ERR_PTR(err);

	}



	if (vma->vm_end > end) {

		int err = split_vma(vmi, vma, end, 0);

	/* Split any trailing portion of the VMA. */

	if (vma->vm_end > vmg->end) {

		int err = split_vma(vmg->vmi, vma, vmg->end, 0);



		if (err)

			return ERR_PTR(err);
@@ -1255,6 +1240,65 @@ struct vm_area_struct *vma_modify(struct vma_iterator *vmi, 
	return vma;

}



struct vm_area_struct *vma_modify_flags(

	struct vma_iterator *vmi, struct vm_area_struct *prev,

	struct vm_area_struct *vma, unsigned long start, unsigned long end,

	unsigned long new_flags)

{

	VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);



	vmg.flags = new_flags;



	return vma_modify(&vmg);

}



struct vm_area_struct

*vma_modify_flags_name(struct vma_iterator *vmi,

		       struct vm_area_struct *prev,

		       struct vm_area_struct *vma,

		       unsigned long start,

		       unsigned long end,

		       unsigned long new_flags,

		       struct anon_vma_name *new_name)

{

	VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);



	vmg.flags = new_flags;

	vmg.anon_name = new_name;



	return vma_modify(&vmg);

}



struct vm_area_struct

*vma_modify_policy(struct vma_iterator *vmi,

		   struct vm_area_struct *prev,

		   struct vm_area_struct *vma,

		   unsigned long start, unsigned long end,

		   struct mempolicy *new_pol)

{

	VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);



	vmg.policy = new_pol;



	return vma_modify(&vmg);

}



struct vm_area_struct

*vma_modify_flags_uffd(struct vma_iterator *vmi,

		       struct vm_area_struct *prev,

		       struct vm_area_struct *vma,

		       unsigned long start, unsigned long end,

		       unsigned long new_flags,

		       struct vm_userfaultfd_ctx new_ctx)

{

	VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);



	vmg.flags = new_flags;

	vmg.uffd_ctx = new_ctx;



	return vma_modify(&vmg);

}



/*

 * Attempt to merge a newly mapped VMA with those adjacent to it. The caller

 * must ensure that [start, end) does not overlap any existing VMA.
@@ -1264,8 +1308,11 @@ struct vm_area_struct 
		   struct vm_area_struct *vma, unsigned long start,

		   unsigned long end, pgoff_t pgoff)

{

	return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,

			 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));

	VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);



	vmg.pgoff = pgoff;



	return vma_merge(&vmg);

}



/*
@@ -1276,12 +1323,10 @@ struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 
					struct vm_area_struct *vma,

					unsigned long delta)

{

	pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);

	VMG_VMA_STATE(vmg, vmi, vma, vma, vma->vm_end, vma->vm_end + delta);



	/* vma is specified as prev, so case 1 or 2 will apply. */

	return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,

			 vma->vm_flags, pgoff, vma_policy(vma),

			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));

	return vma_merge(&vmg);

}



void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)

mm/vma.h

+76 −51
Original line number Diff line number Diff line
@@ -52,6 +52,59 @@ struct vma_munmap_struct { 
	unsigned long data_vm;

};



/* Represents a VMA merge operation. */

struct vma_merge_struct {

	struct mm_struct *mm;

	struct vma_iterator *vmi;

	pgoff_t pgoff;

	struct vm_area_struct *prev;

	struct vm_area_struct *next; /* Modified by vma_merge(). */

	struct vm_area_struct *vma; /* Either a new VMA or the one being modified. */

	unsigned long start;

	unsigned long end;

	unsigned long flags;

	struct file *file;

	struct anon_vma *anon_vma;

	struct mempolicy *policy;

	struct vm_userfaultfd_ctx uffd_ctx;

	struct anon_vma_name *anon_name;

};



/* Assumes addr >= vma->vm_start. */

static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma,

				       unsigned long addr)

{

	return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start);

}



#define VMG_STATE(name, mm_, vmi_, start_, end_, flags_, pgoff_)	\

	struct vma_merge_struct name = {				\

		.mm = mm_,						\

		.vmi = vmi_,						\

		.start = start_,					\

		.end = end_,						\

		.flags = flags_,					\

		.pgoff = pgoff_,					\

	}



#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_)	\

	struct vma_merge_struct name = {			\

		.mm = vma_->vm_mm,				\

		.vmi = vmi_,					\

		.prev = prev_,					\

		.next = NULL,					\

		.vma = vma_,					\

		.start = start_,				\

		.end = end_,					\

		.flags = vma_->vm_flags,			\

		.pgoff = vma_pgoff_offset(vma_, start_),	\

		.file = vma_->vm_file,				\

		.anon_vma = vma_->anon_vma,			\

		.policy = vma_policy(vma_),			\

		.uffd_ctx = vma_->vm_userfaultfd_ctx,		\

		.anon_name = anon_vma_name(vma_),		\

	}



#ifdef CONFIG_DEBUG_VM_MAPLE_TREE

void validate_mm(struct mm_struct *mm);

#else
@@ -212,80 +265,52 @@ void remove_vma(struct vm_area_struct *vma, bool unreachable, bool closed); 
void unmap_region(struct ma_state *mas, struct vm_area_struct *vma,

		struct vm_area_struct *prev, struct vm_area_struct *next);



/* Required by mmap_region(). */

bool

can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,

		struct anon_vma *anon_vma, struct file *file,

		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,

		struct anon_vma_name *anon_name);



/* Required by mmap_region() and do_brk_flags(). */

bool

can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,

		struct anon_vma *anon_vma, struct file *file,

		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,

		struct anon_vma_name *anon_name);



struct vm_area_struct *vma_modify(struct vma_iterator *vmi,

				  struct vm_area_struct *prev,

				  struct vm_area_struct *vma,

				  unsigned long start, unsigned long end,

				  unsigned long vm_flags,

				  struct mempolicy *policy,

				  struct vm_userfaultfd_ctx uffd_ctx,

				  struct anon_vma_name *anon_name);

/*

 * Can we merge the VMA described by vmg into the following VMA vmg->next?

 *

 * Required by mmap_region().

 */

bool can_vma_merge_before(struct vma_merge_struct *vmg);



/*

 * Can we merge the VMA described by vmg into the preceding VMA vmg->prev?

 *

 * Required by mmap_region() and do_brk_flags().

 */

bool can_vma_merge_after(struct vma_merge_struct *vmg);



/* We are about to modify the VMA's flags. */

static inline struct vm_area_struct

*vma_modify_flags(struct vma_iterator *vmi,

		  struct vm_area_struct *prev,

		  struct vm_area_struct *vma,

struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,

		struct vm_area_struct *prev, struct vm_area_struct *vma,

		unsigned long start, unsigned long end,

		  unsigned long new_flags)

{

	return vma_modify(vmi, prev, vma, start, end, new_flags,

			  vma_policy(vma), vma->vm_userfaultfd_ctx,

			  anon_vma_name(vma));

}

		unsigned long new_flags);



/* We are about to modify the VMA's flags and/or anon_name. */

static inline struct vm_area_struct

struct vm_area_struct

*vma_modify_flags_name(struct vma_iterator *vmi,

		       struct vm_area_struct *prev,

		       struct vm_area_struct *vma,

		       unsigned long start,

		       unsigned long end,

		       unsigned long new_flags,

		       struct anon_vma_name *new_name)

{

	return vma_modify(vmi, prev, vma, start, end, new_flags,

			  vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);

}

		       struct anon_vma_name *new_name);



/* We are about to modify the VMA's memory policy. */

static inline struct vm_area_struct

struct vm_area_struct

*vma_modify_policy(struct vma_iterator *vmi,

		   struct vm_area_struct *prev,

		   struct vm_area_struct *vma,

		   unsigned long start, unsigned long end,

		   struct mempolicy *new_pol)

{

	return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,

			  new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));

}

		   struct mempolicy *new_pol);



/* We are about to modify the VMA's flags and/or uffd context. */

static inline struct vm_area_struct

struct vm_area_struct

*vma_modify_flags_uffd(struct vma_iterator *vmi,

		       struct vm_area_struct *prev,

		       struct vm_area_struct *vma,

		       unsigned long start, unsigned long end,

		       unsigned long new_flags,

		       struct vm_userfaultfd_ctx new_ctx)

{

	return vma_modify(vmi, prev, vma, start, end, new_flags,

			  vma_policy(vma), new_ctx, anon_vma_name(vma));

}

		       struct vm_userfaultfd_ctx new_ctx);



struct vm_area_struct

*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,

tools/testing/vma/vma.c

+2 −41
Original line number Diff line number Diff line
@@ -22,26 +22,6 @@ static bool fail_prealloc; 
 */

#include "../../../mm/vma.c"



/*

 * Temporarily forward-ported from a future in which vmg's are used for merging.

 */

struct vma_merge_struct {

	struct mm_struct *mm;

	struct vma_iterator *vmi;

	pgoff_t pgoff;

	struct vm_area_struct *prev;

	struct vm_area_struct *next; /* Modified by vma_merge(). */

	struct vm_area_struct *vma; /* Either a new VMA or the one being modified. */

	unsigned long start;

	unsigned long end;

	unsigned long flags;

	struct file *file;

	struct anon_vma *anon_vma;

	struct mempolicy *policy;

	struct vm_userfaultfd_ctx uffd_ctx;

	struct anon_vma_name *anon_name;

};



const struct vm_operations_struct vma_dummy_vm_ops;

static struct anon_vma dummy_anon_vma;
@@ -115,14 +95,6 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm, 
/* Helper function which provides a wrapper around a merge new VMA operation. */

static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg)

{

	/* vma_merge() needs a VMA to determine mm, anon_vma, and file. */

	struct vm_area_struct dummy = {

		.vm_mm = vmg->mm,

		.vm_flags = vmg->flags,

		.anon_vma = vmg->anon_vma,

		.vm_file = vmg->file,

	};



	/*

	 * For convenience, get prev and next VMAs. Which the new VMA operation

	 * requires.
@@ -131,8 +103,7 @@ static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg) 
	vmg->prev = vma_prev(vmg->vmi);



	vma_iter_set(vmg->vmi, vmg->start);

	return vma_merge_new_vma(vmg->vmi, vmg->prev, &dummy, vmg->start,

				 vmg->end, vmg->pgoff);

	return vma_merge(vmg);

}



/*
@@ -141,17 +112,7 @@ static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg) 
 */

static struct vm_area_struct *merge_existing(struct vma_merge_struct *vmg)

{

	/* vma_merge() needs a VMA to determine mm, anon_vma, and file. */

	struct vm_area_struct dummy = {

		.vm_mm = vmg->mm,

		.vm_flags = vmg->flags,

		.anon_vma = vmg->anon_vma,

		.vm_file = vmg->file,

	};



	return vma_merge(vmg->vmi, vmg->prev, &dummy, vmg->start, vmg->end,

			 vmg->flags, vmg->pgoff, vmg->policy, vmg->uffd_ctx,

			 vmg->anon_name);

	return vma_merge(vmg);

}



/*