Commit b951aaff authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

mm: enable page allocation tagging 

Redefine page allocators to record allocation tags upon their invocation. 
Instrument post_alloc_hook and free_pages_prepare to modify current
allocation tag.

[surenb@google.com: undo _noprof additions in the documentation]
  Link: https://lkml.kernel.org/r/20240326231453.1206227-3-surenb@google.com
Link: https://lkml.kernel.org/r/20240321163705.3067592-19-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Co-developed-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Reviewed-by: default avatarKees Cook <keescook@chromium.org>
Tested-by: default avatarKees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 8a2f1187

include/linux/alloc_tag.h

+14 −0
Original line number Diff line number Diff line
@@ -153,4 +153,18 @@ static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {} 


#endif /* CONFIG_MEM_ALLOC_PROFILING */



#define alloc_hooks_tag(_tag, _do_alloc)				\

({									\

	struct alloc_tag * __maybe_unused _old = alloc_tag_save(_tag);	\

	typeof(_do_alloc) _res = _do_alloc;				\

	alloc_tag_restore(_tag, _old);					\

	_res;								\

})



#define alloc_hooks(_do_alloc)						\

({									\

	DEFINE_ALLOC_TAG(_alloc_tag);					\

	alloc_hooks_tag(&_alloc_tag, _do_alloc);			\

})



#endif /* _LINUX_ALLOC_TAG_H */

include/linux/gfp.h

+79 −47
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ 


#include <linux/mmzone.h>

#include <linux/topology.h>

#include <linux/alloc_tag.h>

#include <linux/sched.h>



struct vm_area_struct;

struct mempolicy;
@@ -175,42 +177,46 @@ static inline void arch_free_page(struct page *page, int order) { } 
static inline void arch_alloc_page(struct page *page, int order) { }

#endif



struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,

struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid,

		nodemask_t *nodemask);

struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid,

#define __alloc_pages(...)			alloc_hooks(__alloc_pages_noprof(__VA_ARGS__))



struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid,

		nodemask_t *nodemask);

#define __folio_alloc(...)			alloc_hooks(__folio_alloc_noprof(__VA_ARGS__))



unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,

unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,

				nodemask_t *nodemask, int nr_pages,

				struct list_head *page_list,

				struct page **page_array);

#define __alloc_pages_bulk(...)			alloc_hooks(alloc_pages_bulk_noprof(__VA_ARGS__))



unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,

unsigned long alloc_pages_bulk_array_mempolicy_noprof(gfp_t gfp,

				unsigned long nr_pages,

				struct page **page_array);

#define  alloc_pages_bulk_array_mempolicy(...)				\

	alloc_hooks(alloc_pages_bulk_array_mempolicy_noprof(__VA_ARGS__))



/* Bulk allocate order-0 pages */

static inline unsigned long

alloc_pages_bulk_list(gfp_t gfp, unsigned long nr_pages, struct list_head *list)

{

	return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, list, NULL);

}

#define alloc_pages_bulk_list(_gfp, _nr_pages, _list)			\

	__alloc_pages_bulk(_gfp, numa_mem_id(), NULL, _nr_pages, _list, NULL)



static inline unsigned long

alloc_pages_bulk_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array)

{

	return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, NULL, page_array);

}

#define alloc_pages_bulk_array(_gfp, _nr_pages, _page_array)		\

	__alloc_pages_bulk(_gfp, numa_mem_id(), NULL, _nr_pages, NULL, _page_array)



static inline unsigned long

alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct page **page_array)

alloc_pages_bulk_array_node_noprof(gfp_t gfp, int nid, unsigned long nr_pages,

				   struct page **page_array)

{

	if (nid == NUMA_NO_NODE)

		nid = numa_mem_id();



	return __alloc_pages_bulk(gfp, nid, NULL, nr_pages, NULL, page_array);

	return alloc_pages_bulk_noprof(gfp, nid, NULL, nr_pages, NULL, page_array);

}



#define alloc_pages_bulk_array_node(...)				\

	alloc_hooks(alloc_pages_bulk_array_node_noprof(__VA_ARGS__))



static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask)

{

	gfp_t warn_gfp = gfp_mask & (__GFP_THISNODE|__GFP_NOWARN);
@@ -230,76 +236,98 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) 
 * online. For more general interface, see alloc_pages_node().

 */

static inline struct page *

__alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order)

__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order)

{

	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);

	warn_if_node_offline(nid, gfp_mask);



	return __alloc_pages(gfp_mask, order, nid, NULL);

	return __alloc_pages_noprof(gfp_mask, order, nid, NULL);

}



#define  __alloc_pages_node(...)		alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__))



static inline

struct folio *__folio_alloc_node(gfp_t gfp, unsigned int order, int nid)

struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid)

{

	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);

	warn_if_node_offline(nid, gfp);



	return __folio_alloc(gfp, order, nid, NULL);

	return __folio_alloc_noprof(gfp, order, nid, NULL);

}



#define  __folio_alloc_node(...)		alloc_hooks(__folio_alloc_node_noprof(__VA_ARGS__))



/*

 * Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE,

 * prefer the current CPU's closest node. Otherwise node must be valid and

 * online.

 */

static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask,

static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask,

						   unsigned int order)

{

	if (nid == NUMA_NO_NODE)

		nid = numa_mem_id();



	return __alloc_pages_node(nid, gfp_mask, order);

	return __alloc_pages_node_noprof(nid, gfp_mask, order);

}



#define  alloc_pages_node(...)			alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__))



#ifdef CONFIG_NUMA

struct page *alloc_pages(gfp_t gfp, unsigned int order);

struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,

struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order);

struct page *alloc_pages_mpol_noprof(gfp_t gfp, unsigned int order,

		struct mempolicy *mpol, pgoff_t ilx, int nid);

struct folio *folio_alloc(gfp_t gfp, unsigned int order);

struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma,

struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order);

struct folio *vma_alloc_folio_noprof(gfp_t gfp, int order, struct vm_area_struct *vma,

		unsigned long addr, bool hugepage);

#else

static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order)

static inline struct page *alloc_pages_noprof(gfp_t gfp_mask, unsigned int order)

{

	return alloc_pages_node(numa_node_id(), gfp_mask, order);

	return alloc_pages_node_noprof(numa_node_id(), gfp_mask, order);

}

static inline struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,

static inline struct page *alloc_pages_mpol_noprof(gfp_t gfp, unsigned int order,

		struct mempolicy *mpol, pgoff_t ilx, int nid)

{

	return alloc_pages(gfp, order);

	return alloc_pages_noprof(gfp, order);

}

static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order)

static inline struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order)

{

	return __folio_alloc_node(gfp, order, numa_node_id());

}

#define vma_alloc_folio(gfp, order, vma, addr, hugepage)		\

	folio_alloc(gfp, order)

#define vma_alloc_folio_noprof(gfp, order, vma, addr, hugepage)		\

	folio_alloc_noprof(gfp, order)

#endif



#define alloc_pages(...)			alloc_hooks(alloc_pages_noprof(__VA_ARGS__))

#define alloc_pages_mpol(...)			alloc_hooks(alloc_pages_mpol_noprof(__VA_ARGS__))

#define folio_alloc(...)			alloc_hooks(folio_alloc_noprof(__VA_ARGS__))

#define vma_alloc_folio(...)			alloc_hooks(vma_alloc_folio_noprof(__VA_ARGS__))



#define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0)

static inline struct page *alloc_page_vma(gfp_t gfp,



static inline struct page *alloc_page_vma_noprof(gfp_t gfp,

		struct vm_area_struct *vma, unsigned long addr)

{

	struct folio *folio = vma_alloc_folio(gfp, 0, vma, addr, false);

	struct folio *folio = vma_alloc_folio_noprof(gfp, 0, vma, addr, false);



	return &folio->page;

}

#define alloc_page_vma(...)			alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__))



extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order);

#define __get_free_pages(...)			alloc_hooks(get_free_pages_noprof(__VA_ARGS__))



extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);

extern unsigned long get_zeroed_page(gfp_t gfp_mask);

extern unsigned long get_zeroed_page_noprof(gfp_t gfp_mask);

#define get_zeroed_page(...)			alloc_hooks(get_zeroed_page_noprof(__VA_ARGS__))



void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1);

#define alloc_pages_exact(...)			alloc_hooks(alloc_pages_exact_noprof(__VA_ARGS__))



void *alloc_pages_exact(size_t size, gfp_t gfp_mask) __alloc_size(1);

void free_pages_exact(void *virt, size_t size);

__meminit void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);



__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);

#define alloc_pages_exact_nid(...)					\

	alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__))



#define __get_free_page(gfp_mask)					\

	__get_free_pages((gfp_mask), 0)
@@ -374,10 +402,14 @@ extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma); 


#ifdef CONFIG_CONTIG_ALLOC

/* The below functions must be run on a range from a single zone. */

extern int alloc_contig_range(unsigned long start, unsigned long end,

extern int alloc_contig_range_noprof(unsigned long start, unsigned long end,

			      unsigned migratetype, gfp_t gfp_mask);

extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,

#define alloc_contig_range(...)			alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__))



extern struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,

					      int nid, nodemask_t *nodemask);

#define alloc_contig_pages(...)			alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__))



#endif

void free_contig_range(unsigned long pfn, unsigned long nr_pages);

include/linux/pagemap.h

+6 −3
Original line number Diff line number Diff line
@@ -542,14 +542,17 @@ static inline void *detach_page_private(struct page *page) 
#endif



#ifdef CONFIG_NUMA

struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order);

struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order);

#else

static inline struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order)

static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)

{

	return folio_alloc(gfp, order);

	return folio_alloc_noprof(gfp, order);

}

#endif



#define filemap_alloc_folio(...)				\

	alloc_hooks(filemap_alloc_folio_noprof(__VA_ARGS__))



static inline struct page *__page_cache_alloc(gfp_t gfp)

{

	return &filemap_alloc_folio(gfp, 0)->page;

mm/compaction.c

+6 −1
Original line number Diff line number Diff line
@@ -1851,7 +1851,7 @@ static void isolate_freepages(struct compact_control *cc) 
 * This is a migrate-callback that "allocates" freepages by taking pages

 * from the isolated freelists in the block we are migrating to.

 */

static struct folio *compaction_alloc(struct folio *src, unsigned long data)

static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data)

{

	struct compact_control *cc = (struct compact_control *)data;

	struct folio *dst;
@@ -1898,6 +1898,11 @@ static struct folio *compaction_alloc(struct folio *src, unsigned long data) 
	return page_rmappable_folio(&dst->page);

}



static struct folio *compaction_alloc(struct folio *src, unsigned long data)

{

	return alloc_hooks(compaction_alloc_noprof(src, data));

}



/*

 * This is a migrate-callback that "frees" freepages back to the isolated

 * freelist.  All pages on the freelist are from the same zone, so there is no

mm/filemap.c

+3 −3
Original line number Diff line number Diff line
@@ -966,7 +966,7 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio, 
EXPORT_SYMBOL_GPL(filemap_add_folio);



#ifdef CONFIG_NUMA

struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order)

struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)

{

	int n;

	struct folio *folio;
@@ -981,9 +981,9 @@ struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) 


		return folio;

	}

	return folio_alloc(gfp, order);

	return folio_alloc_noprof(gfp, order);

}

EXPORT_SYMBOL(filemap_alloc_folio);

EXPORT_SYMBOL(filemap_alloc_folio_noprof);

#endif



/*