Commit dcfe378c authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

lib: introduce support for page allocation tagging 

Introduce helper functions to easily instrument page allocators by storing
a pointer to the allocation tag associated with the code that allocated
the page in a page_ext field.

Link: https://lkml.kernel.org/r/20240321163705.3067592-15-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Co-developed-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Tested-by: default avatarKees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 22d407b1

include/linux/page_ext.h

+0 −1
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ 


#include <linux/types.h>

#include <linux/stacktrace.h>

#include <linux/stackdepot.h>



struct pglist_data;

include/linux/pgalloc_tag.h

0 → 100644
+78 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * page allocation tagging

 */

#ifndef _LINUX_PGALLOC_TAG_H

#define _LINUX_PGALLOC_TAG_H



#include <linux/alloc_tag.h>



#ifdef CONFIG_MEM_ALLOC_PROFILING



#include <linux/page_ext.h>



extern struct page_ext_operations page_alloc_tagging_ops;

extern struct page_ext *page_ext_get(struct page *page);

extern void page_ext_put(struct page_ext *page_ext);



static inline union codetag_ref *codetag_ref_from_page_ext(struct page_ext *page_ext)

{

	return (void *)page_ext + page_alloc_tagging_ops.offset;

}



static inline struct page_ext *page_ext_from_codetag_ref(union codetag_ref *ref)

{

	return (void *)ref - page_alloc_tagging_ops.offset;

}



/* Should be called only if mem_alloc_profiling_enabled() */

static inline union codetag_ref *get_page_tag_ref(struct page *page)

{

	if (page) {

		struct page_ext *page_ext = page_ext_get(page);



		if (page_ext)

			return codetag_ref_from_page_ext(page_ext);

	}

	return NULL;

}



static inline void put_page_tag_ref(union codetag_ref *ref)

{

	page_ext_put(page_ext_from_codetag_ref(ref));

}



static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,

				   unsigned int nr)

{

	if (mem_alloc_profiling_enabled()) {

		union codetag_ref *ref = get_page_tag_ref(page);



		if (ref) {

			alloc_tag_add(ref, task->alloc_tag, PAGE_SIZE * nr);

			put_page_tag_ref(ref);

		}

	}

}



static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)

{

	if (mem_alloc_profiling_enabled()) {

		union codetag_ref *ref = get_page_tag_ref(page);



		if (ref) {

			alloc_tag_sub(ref, PAGE_SIZE * nr);

			put_page_tag_ref(ref);

		}

	}

}



#else /* CONFIG_MEM_ALLOC_PROFILING */



static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,

				   unsigned int nr) {}

static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}



#endif /* CONFIG_MEM_ALLOC_PROFILING */



#endif /* _LINUX_PGALLOC_TAG_H */

lib/Kconfig.debug

+1 −0
Original line number Diff line number Diff line
@@ -978,6 +978,7 @@ config MEM_ALLOC_PROFILING 
	depends on PROC_FS

	depends on !DEBUG_FORCE_WEAK_PER_CPU

	select CODE_TAGGING

	select PAGE_EXTENSION

	help

	  Track allocation source code and record total allocation size

	  initiated at that code location. The mechanism can be used to track

lib/alloc_tag.c

+17 −0
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#include <linux/fs.h>

#include <linux/gfp.h>

#include <linux/module.h>

#include <linux/page_ext.h>

#include <linux/proc_fs.h>

#include <linux/seq_buf.h>

#include <linux/seq_file.h>
@@ -118,6 +119,22 @@ static bool alloc_tag_module_unload(struct codetag_type *cttype, 
	return module_unused;

}



static __init bool need_page_alloc_tagging(void)

{

	return true;

}



static __init void init_page_alloc_tagging(void)

{

}



struct page_ext_operations page_alloc_tagging_ops = {

	.size = sizeof(union codetag_ref),

	.need = need_page_alloc_tagging,

	.init = init_page_alloc_tagging,

};

EXPORT_SYMBOL(page_alloc_tagging_ops);



static struct ctl_table memory_allocation_profiling_sysctls[] = {

	{

		.procname	= "mem_profiling",

mm/mm_init.c

+1 −0
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <linux/page_ext.h>

#include <linux/pti.h>

#include <linux/pgtable.h>

#include <linux/stackdepot.h>

#include <linux/swap.h>

#include <linux/cma.h>

#include <linux/crash_dump.h>