Commit 2c321f3f authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

mm: change inlined allocation helpers to account at the call site 

Main goal of memory allocation profiling patchset is to provide accounting
that is cheap enough to run in production.  To achieve that we inject
counters using codetags at the allocation call sites to account every time
allocation is made.  This injection allows us to perform accounting
efficiently because injected counters are immediately available as opposed
to the alternative methods, such as using _RET_IP_, which would require
counter lookup and appropriate locking that makes accounting much more
expensive.  This method requires all allocation functions to inject
separate counters at their call sites so that their callers can be
individually accounted.  Counter injection is implemented by allocation
hooks which should wrap all allocation functions.

Inlined functions which perform allocations but do not use allocation
hooks are directly charged for the allocations they perform.  In most
cases these functions are just specialized allocation wrappers used from
multiple places to allocate objects of a specific type.  It would be more
useful to do the accounting at their call sites instead.  Instrument these
helpers to do accounting at the call site.  Simple inlined allocation
wrappers are converted directly into macros.  More complex allocators or
allocators with documentation are converted into _noprof versions and
allocation hooks are added.  This allows memory allocation profiling
mechanism to charge allocations to the callers of these functions.

Link: https://lkml.kernel.org/r/20240415020731.1152108-1-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Acked-by: Jan Kara <jack@suse.cz>		[jbd2]
Cc: Anna Schumaker <anna@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Tissoires <benjamin.tissoires@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Jakub Kicinski <kuba@kernel.org>
Cc: Jakub Sitnicki <jakub@cloudflare.com>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Paolo Abeni <pabeni@redhat.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Theodore Ts'o <tytso@mit.edu>
Cc: Trond Myklebust <trond.myklebust@hammerspace.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent ebdf9ad4

drivers/iommu/amd/amd_iommu.h

+3 −2
Original line number Diff line number Diff line
@@ -134,13 +134,14 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) 
	return PCI_SEG_DEVID_TO_SBDF(seg, devid);

}



static inline void *alloc_pgtable_page(int nid, gfp_t gfp)

static inline void *alloc_pgtable_page_noprof(int nid, gfp_t gfp)

{

	struct page *page;



	page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0);

	page = alloc_pages_node_noprof(nid, gfp | __GFP_ZERO, 0);

	return page ? page_address(page) : NULL;

}

#define alloc_pgtable_page(...)	alloc_hooks(alloc_pgtable_page_noprof(__VA_ARGS__))



/*

 * This must be called after device probe completes. During probe

fs/nfs/iostat.h

+1 −4
Original line number Diff line number Diff line
@@ -46,10 +46,7 @@ static inline void nfs_add_stats(const struct inode *inode, 
	nfs_add_server_stats(NFS_SERVER(inode), stat, addend);

}



static inline struct nfs_iostats __percpu *nfs_alloc_iostats(void)

{

	return alloc_percpu(struct nfs_iostats);

}

#define nfs_alloc_iostats()	alloc_percpu(struct nfs_iostats)



static inline void nfs_free_iostats(struct nfs_iostats __percpu *stats)

{

include/acpi/platform/aclinuxex.h

+6 −13
Original line number Diff line number Diff line
@@ -47,26 +47,19 @@ acpi_status acpi_os_terminate(void); 
 * However, boot has  (system_state != SYSTEM_RUNNING)

 * to quiet __might_sleep() in kmalloc() and resume does not.

 */

static inline void *acpi_os_allocate(acpi_size size)

{

	return kmalloc(size, irqs_disabled()? GFP_ATOMIC : GFP_KERNEL);

}

#define acpi_os_allocate(_size)	\

		kmalloc(_size, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL)



static inline void *acpi_os_allocate_zeroed(acpi_size size)

{

	return kzalloc(size, irqs_disabled()? GFP_ATOMIC : GFP_KERNEL);

}

#define acpi_os_allocate_zeroed(_size)	\

		kzalloc(_size, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL)



static inline void acpi_os_free(void *memory)

{

	kfree(memory);

}



static inline void *acpi_os_acquire_object(acpi_cache_t * cache)

{

	return kmem_cache_zalloc(cache,

				 irqs_disabled()? GFP_ATOMIC : GFP_KERNEL);

}

#define acpi_os_acquire_object(_cache)	\

		kmem_cache_zalloc(_cache, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL)



static inline acpi_thread_id acpi_os_get_thread_id(void)

{

include/asm-generic/pgalloc.h

+21 −14
Original line number Diff line number Diff line
@@ -16,15 +16,16 @@ 
 *

 * Return: pointer to the allocated memory or %NULL on error

 */

static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm)

static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm)

{

	struct ptdesc *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL &

	struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL &

			~__GFP_HIGHMEM, 0);



	if (!ptdesc)

		return NULL;

	return ptdesc_address(ptdesc);

}

#define __pte_alloc_one_kernel(...)	alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__))



#ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL

/**
@@ -33,10 +34,11 @@ static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) 
 *

 * Return: pointer to the allocated memory or %NULL on error

 */

static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)

static inline pte_t *pte_alloc_one_kernel_noprof(struct mm_struct *mm)

{

	return __pte_alloc_one_kernel(mm);

	return __pte_alloc_one_kernel_noprof(mm);

}

#define pte_alloc_one_kernel(...)	alloc_hooks(pte_alloc_one_kernel_noprof(__VA_ARGS__))

#endif



/**
@@ -61,11 +63,11 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 
 *

 * Return: `struct page` referencing the ptdesc or %NULL on error

 */

static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)

static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp)

{

	struct ptdesc *ptdesc;



	ptdesc = pagetable_alloc(gfp, 0);

	ptdesc = pagetable_alloc_noprof(gfp, 0);

	if (!ptdesc)

		return NULL;

	if (!pagetable_pte_ctor(ptdesc)) {
@@ -75,6 +77,7 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) 


	return ptdesc_page(ptdesc);

}

#define __pte_alloc_one(...)	alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__))



#ifndef __HAVE_ARCH_PTE_ALLOC_ONE

/**
@@ -85,10 +88,11 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) 
 *

 * Return: `struct page` referencing the ptdesc or %NULL on error

 */

static inline pgtable_t pte_alloc_one(struct mm_struct *mm)

static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm)

{

	return __pte_alloc_one(mm, GFP_PGTABLE_USER);

	return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER);

}

#define pte_alloc_one(...)	alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__))

#endif



/*
@@ -124,14 +128,14 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) 
 *

 * Return: pointer to the allocated memory or %NULL on error

 */

static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)

static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr)

{

	struct ptdesc *ptdesc;

	gfp_t gfp = GFP_PGTABLE_USER;



	if (mm == &init_mm)

		gfp = GFP_PGTABLE_KERNEL;

	ptdesc = pagetable_alloc(gfp, 0);

	ptdesc = pagetable_alloc_noprof(gfp, 0);

	if (!ptdesc)

		return NULL;

	if (!pagetable_pmd_ctor(ptdesc)) {
@@ -140,6 +144,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 
	}

	return ptdesc_address(ptdesc);

}

#define pmd_alloc_one(...)	alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__))

#endif



#ifndef __HAVE_ARCH_PMD_FREE
@@ -157,7 +162,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 


#if CONFIG_PGTABLE_LEVELS > 3



static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr)

static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr)

{

	gfp_t gfp = GFP_PGTABLE_USER;

	struct ptdesc *ptdesc;
@@ -166,13 +171,14 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) 
		gfp = GFP_PGTABLE_KERNEL;

	gfp &= ~__GFP_HIGHMEM;



	ptdesc = pagetable_alloc(gfp, 0);

	ptdesc = pagetable_alloc_noprof(gfp, 0);

	if (!ptdesc)

		return NULL;



	pagetable_pud_ctor(ptdesc);

	return ptdesc_address(ptdesc);

}

#define __pud_alloc_one(...)	alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__))



#ifndef __HAVE_ARCH_PUD_ALLOC_ONE

/**
@@ -184,10 +190,11 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) 
 *

 * Return: pointer to the allocated memory or %NULL on error

 */

static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)

static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr)

{

	return __pud_alloc_one(mm, addr);

	return __pud_alloc_one_noprof(mm, addr);

}

#define pud_alloc_one(...)	alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__))

#endif



static inline void __pud_free(struct mm_struct *mm, pud_t *pud)

include/crypto/hash.h

+4 −3
Original line number Diff line number Diff line
@@ -578,12 +578,12 @@ static inline void ahash_request_set_tfm(struct ahash_request *req, 
 *

 * Return: allocated request handle in case of success, or NULL if out of memory

 */

static inline struct ahash_request *ahash_request_alloc(

static inline struct ahash_request *ahash_request_alloc_noprof(

	struct crypto_ahash *tfm, gfp_t gfp)

{

	struct ahash_request *req;



	req = kmalloc(sizeof(struct ahash_request) +

	req = kmalloc_noprof(sizeof(struct ahash_request) +

			     crypto_ahash_reqsize(tfm), gfp);



	if (likely(req))
@@ -591,6 +591,7 @@ static inline struct ahash_request *ahash_request_alloc( 


	return req;

}

#define ahash_request_alloc(...)	alloc_hooks(ahash_request_alloc_noprof(__VA_ARGS__))



/**

 * ahash_request_free() - zeroize and free the request data structure