Commit a642b27b authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

alloc_tag: uninline code gated by mem_alloc_profiling_key in slab allocator 

When a sizable code section is protected by a disabled static key, that
code gets into the instruction cache even though it's not executed and
consumes the cache, increasing cache misses.  This can be remedied by
moving such code into a separate uninlined function.  On a Pixel6 phone,
slab allocation profiling overhead measured with
CONFIG_MEM_ALLOC_PROFILING=y and profiling disabled is:

             baseline             modified
Big core     3.31%                0.17%
Medium core  3.79%                0.57%
Little core  6.68%                1.28%

This improvement comes at the expense of the configuration when profiling
gets enabled, since there is now an additional function call.  The
overhead from this additional call on Pixel6 is:

Big core     0.66%
Middle core  1.23%
Little core  2.42%

However this is negligible when compared with the overall overhead of the
memory allocation profiling when it is enabled.

On x86 this patch does not make noticeable difference because the overhead
with mem_alloc_profiling_key disabled is much lower (under 1%) to start
with, so any improvement is less visible and hard to distinguish from the
noise.  The overhead from additional call when profiling is enabled is
also within noise levels.

Link: https://lkml.kernel.org/r/20250201231803.2661189-2-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarShakeel Butt <shakeel.butt@linux.dev>
Cc: David Wang <00107082@163.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Minchan Kim <minchan@google.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zhenhua Huang <quic_zhenhuah@quicinc.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 51ff4d74

mm/slub.c

+32 −19
Original line number Diff line number Diff line
@@ -2000,7 +2000,8 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 
	return 0;

}



static inline void free_slab_obj_exts(struct slab *slab)

/* Should be called only if mem_alloc_profiling_enabled() */

static noinline void free_slab_obj_exts(struct slab *slab)

{

	struct slabobj_ext *obj_exts;
@@ -2077,10 +2078,10 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) 
	return slab_obj_exts(slab) + obj_to_index(s, slab, p);

}



static inline void

alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)

/* Should be called only if mem_alloc_profiling_enabled() */

static noinline void

__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)

{

	if (need_slab_obj_ext()) {

	struct slabobj_ext *obj_exts;



	obj_exts = prepare_slab_obj_exts_hook(s, flags, object);
@@ -2092,18 +2093,22 @@ alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 
	if (likely(obj_exts))

		alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);

}

}



static inline void

alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)

{

	if (need_slab_obj_ext())

		__alloc_tagging_slab_alloc_hook(s, object, flags);

}



/* Should be called only if mem_alloc_profiling_enabled() */

static noinline void

__alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

			       int objects)

{

	struct slabobj_ext *obj_exts;

	int i;



	if (!mem_alloc_profiling_enabled())

		return;



	/* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */

	if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))

		return;
@@ -2119,6 +2124,14 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 
	}

}



static inline void

alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

			     int objects)

{

	if (mem_alloc_profiling_enabled())

		__alloc_tagging_slab_free_hook(s, slab, p, objects);

}



#else /* CONFIG_MEM_ALLOC_PROFILING */



static inline void