Commit 529ce23a authored by Rick Edgecombe's avatar Rick Edgecombe Committed by Andrew Morton
Browse files

mm: switch mm->get_unmapped_area() to a flag 

The mm_struct contains a function pointer *get_unmapped_area(), which is
set to either arch_get_unmapped_area() or arch_get_unmapped_area_topdown()
during the initialization of the mm.

Since the function pointer only ever points to two functions that are
named the same across all arch's, a function pointer is not really
required.  In addition future changes will want to add versions of the
functions that take additional arguments.  So to save a pointers worth of
bytes in mm_struct, and prevent adding additional function pointers to
mm_struct in future changes, remove it and keep the information about
which get_unmapped_area() to use in a flag.

Add the new flag to MMF_INIT_MASK so it doesn't get clobbered on fork by
mmf_init_flags().  Most MM flags get clobbered on fork.  In the
pre-existing behavior mm->get_unmapped_area() would get copied to the new
mm in dup_mm(), so not clobbering the flag preserves the existing behavior
around inheriting the topdown-ness.

Introduce a helper, mm_get_unmapped_area(), to easily convert code that
refers to the old function pointer to instead select and call either
arch_get_unmapped_area() or arch_get_unmapped_area_topdown() based on the
flag.  Then drop the mm->get_unmapped_area() function pointer.  Leave the
get_unmapped_area() pointer in struct file_operations alone.  The main
purpose of this change is to reorganize in preparation for future changes,
but it also converts the calls of mm->get_unmapped_area() from indirect
branches into a direct ones.

The stress-ng bigheap benchmark calls realloc a lot, which calls through
get_unmapped_area() in the kernel.  On x86, the change yielded a ~1%
improvement there on a retpoline config.

In testing a few x86 configs, removing the pointer unfortunately didn't
result in any actual size reductions in the compiled layout of mm_struct. 
But depending on compiler or arch alignment requirements, the change could
shrink the size of mm_struct.

Link: https://lkml.kernel.org/r/20240326021656.202649-3-rick.p.edgecombe@intel.com


Signed-off-by: default avatarRick Edgecombe <rick.p.edgecombe@intel.com>
Acked-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Acked-by: default avatarLiam R. Howlett <Liam.Howlett@oracle.com>
Reviewed-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@kernel.org>
Cc: Borislav Petkov (AMD) <bp@alien8.de>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Deepak Gupta <debug@rivosinc.com>
Cc: Guo Ren <guoren@kernel.org>
Cc: Helge Deller <deller@gmx.de>
Cc: H. Peter Anvin (Intel) <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mark Brown <broonie@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Naveen N. Rao <naveen.n.rao@linux.ibm.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 5def1e0f

arch/s390/mm/hugetlbpage.c

+1 −1
Original line number Diff line number Diff line
@@ -318,7 +318,7 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, 
			goto check_asce_limit;

	}



	if (mm->get_unmapped_area == arch_get_unmapped_area)

	if (!test_bit(MMF_TOPDOWN, &mm->flags))

		addr = hugetlb_get_unmapped_area_bottomup(file, addr, len,

				pgoff, flags);

	else

arch/s390/mm/mmap.c

+2 −2
Original line number Diff line number Diff line
@@ -185,10 +185,10 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) 
	 */

	if (mmap_is_legacy(rlim_stack)) {

		mm->mmap_base = mmap_base_legacy(random_factor);

		mm->get_unmapped_area = arch_get_unmapped_area;

		clear_bit(MMF_TOPDOWN, &mm->flags);

	} else {

		mm->mmap_base = mmap_base(random_factor, rlim_stack);

		mm->get_unmapped_area = arch_get_unmapped_area_topdown;

		set_bit(MMF_TOPDOWN, &mm->flags);

	}

}

arch/sparc/kernel/sys_sparc_64.c

+6 −9
Original line number Diff line number Diff line
@@ -218,14 +218,10 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, 
unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, unsigned long len, unsigned long pgoff, unsigned long flags)

{

	unsigned long align_goal, addr = -ENOMEM;

	unsigned long (*get_area)(struct file *, unsigned long,

				  unsigned long, unsigned long, unsigned long);



	get_area = current->mm->get_unmapped_area;



	if (flags & MAP_FIXED) {

		/* Ok, don't mess with it. */

		return get_area(NULL, orig_addr, len, pgoff, flags);

		return mm_get_unmapped_area(current->mm, NULL, orig_addr, len, pgoff, flags);

	}

	flags &= ~MAP_SHARED;
@@ -238,7 +234,8 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u 
		align_goal = (64UL * 1024);



	do {

		addr = get_area(NULL, orig_addr, len + (align_goal - PAGE_SIZE), pgoff, flags);

		addr = mm_get_unmapped_area(current->mm, NULL, orig_addr,

					    len + (align_goal - PAGE_SIZE), pgoff, flags);

		if (!(addr & ~PAGE_MASK)) {

			addr = (addr + (align_goal - 1UL)) & ~(align_goal - 1UL);

			break;
@@ -256,7 +253,7 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u 
	 * be obtained.

	 */

	if (addr & ~PAGE_MASK)

		addr = get_area(NULL, orig_addr, len, pgoff, flags);

		addr = mm_get_unmapped_area(current->mm, NULL, orig_addr, len, pgoff, flags);



	return addr;

}
@@ -292,7 +289,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) 
	    gap == RLIM_INFINITY ||

	    sysctl_legacy_va_layout) {

		mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;

		mm->get_unmapped_area = arch_get_unmapped_area;

		clear_bit(MMF_TOPDOWN, &mm->flags);

	} else {

		/* We know it's 32-bit */

		unsigned long task_size = STACK_TOP32;
@@ -303,7 +300,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) 
			gap = (task_size / 6 * 5);



		mm->mmap_base = PAGE_ALIGN(task_size - gap - random_factor);

		mm->get_unmapped_area = arch_get_unmapped_area_topdown;

		set_bit(MMF_TOPDOWN, &mm->flags);

	}

}

arch/sparc/mm/hugetlbpage.c

+1 −1
Original line number Diff line number Diff line
@@ -123,7 +123,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, 
		    (!vma || addr + len <= vm_start_gap(vma)))

			return addr;

	}

	if (mm->get_unmapped_area == arch_get_unmapped_area)

	if (!test_bit(MMF_TOPDOWN, &mm->flags))

		return hugetlb_get_unmapped_area_bottomup(file, addr, len,

				pgoff, flags);

	else

arch/x86/kernel/cpu/sgx/driver.c

+1 −1
Original line number Diff line number Diff line
@@ -113,7 +113,7 @@ static unsigned long sgx_get_unmapped_area(struct file *file, 
	if (flags & MAP_FIXED)

		return addr;



	return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);

	return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags);

}



#ifdef CONFIG_COMPAT