Commit 51ff4d74 authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

mm: avoid extra mem_alloc_profiling_enabled() checks 

Refactor code to avoid extra mem_alloc_profiling_enabled() checks inside
pgalloc_tag_get() function which is often called after that check was
already done.

Link: https://lkml.kernel.org/r/20250201231803.2661189-1-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Reviewed-by: default avatarShakeel Butt <shakeel.butt@linux.dev>
Cc: David Wang <00107082@163.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Minchan Kim <minchan@google.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zhenhua Huang <quic_zhenhuah@quicinc.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent c372473a

include/linux/pgalloc_tag.h

+19 −16
Original line number Diff line number Diff line
@@ -205,11 +205,10 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) 
	}

}



static inline struct alloc_tag *pgalloc_tag_get(struct page *page)

/* Should be called only if mem_alloc_profiling_enabled() */

static inline struct alloc_tag *__pgalloc_tag_get(struct page *page)

{

	struct alloc_tag *tag = NULL;



	if (mem_alloc_profiling_enabled()) {

	union pgtag_ref_handle handle;

	union codetag_ref ref;
@@ -219,14 +218,19 @@ static inline struct alloc_tag *pgalloc_tag_get(struct page *page) 
			tag = ct_to_alloc_tag(ref.ct);

		put_page_tag_ref(handle);

	}

	}



	return tag;

}



static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)

static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr)

{

	if (mem_alloc_profiling_enabled() && tag)

	struct alloc_tag *tag;



	if (!mem_alloc_profiling_enabled())

		return;



	tag = __pgalloc_tag_get(page);

	if (tag)

		this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr);

}
@@ -241,8 +245,7 @@ static inline void clear_page_tag_ref(struct page *page) {} 
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,

				   unsigned int nr) {}

static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}

static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; }

static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}

static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr) {}

static inline void alloc_tag_sec_init(void) {}

static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) {}

static inline void pgalloc_tag_swap(struct folio *new, struct folio *old) {}

lib/alloc_tag.c

+3 −3
Original line number Diff line number Diff line
@@ -174,7 +174,7 @@ void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) 
	if (!mem_alloc_profiling_enabled())

		return;



	tag = pgalloc_tag_get(&folio->page);

	tag = __pgalloc_tag_get(&folio->page);

	if (!tag)

		return;
@@ -200,10 +200,10 @@ void pgalloc_tag_swap(struct folio *new, struct folio *old) 
	if (!mem_alloc_profiling_enabled())

		return;



	tag_old = pgalloc_tag_get(&old->page);

	tag_old = __pgalloc_tag_get(&old->page);

	if (!tag_old)

		return;

	tag_new = pgalloc_tag_get(&new->page);

	tag_new = __pgalloc_tag_get(&new->page);

	if (!tag_new)

		return;

mm/page_alloc.c

+1 −2
Original line number Diff line number Diff line
@@ -4833,12 +4833,11 @@ void __free_pages(struct page *page, unsigned int order) 
{

	/* get PageHead before we drop reference */

	int head = PageHead(page);

	struct alloc_tag *tag = pgalloc_tag_get(page);



	if (put_page_testzero(page))

		free_frozen_pages(page, order);

	else if (!head) {

		pgalloc_tag_sub_pages(tag, (1 << order) - 1);

		pgalloc_tag_sub_pages(page, (1 << order) - 1);

		while (order-- > 0)

			free_frozen_pages(page + (1 << order), order);

	}