Commit 088bb10e authored by Heiko Carstens's avatar Heiko Carstens Committed by Alexander Gordeev
Browse files

s390/mm: Add memory allocation profiling hooks



Similar to common code changes [1] add alloc_hook() wrappers to page table
allocation functions to allow for memory allocation profiling.

If CONFIG_MEM_ALLOC_PROFILING is enabled call sites of page table
allocations are accounted, instead of e.g. only crst_table_alloc() and
page_table_alloc(). This allows for slightly better profiling data, and the
output of /proc/allocinfo is similar to other architectures.

Without alloc_hook() wrappers the output of /proc/allocinfo looks like
this:

    17096704     4174 mm/memory.c:1061 func:folio_prealloc
    17809408     4348 mm/memory.c:1063 func:folio_prealloc
           0        0 mm/memory.c:4422 func:alloc_swap_folio
           0        0 mm/memory.c:4286 func:__alloc_swap_folio
           0        0 mm/memory.c:4971 func:alloc_anon_folio
         ...
     1589248       97 arch/s390/mm/pgalloc.c:25 func:crst_table_alloc
           0        0 arch/s390/mm/pgalloc.c:124 func:page_table_alloc_pgste
     4280320     1045 arch/s390/mm/pgalloc.c:149 func:page_table_alloc

With alloc_hook() wrappers:

     1097728      268 mm/memory.c:5147 func:__do_fault
    20119552     4912 mm/memory.c:1061 func:folio_prealloc
    17534976     4281 mm/memory.c:1063 func:folio_prealloc
           0        0 mm/memory.c:4422 func:alloc_swap_folio
           0        0 mm/memory.c:4286 func:__alloc_swap_folio
      786432      192 mm/memory.c:452 func:__pte_alloc
      405504       99 mm/memory.c:464 func:__pte_alloc_kernel
     1880064      459 mm/memory.c:5525 func:do_fault_around
           0        0 mm/memory.c:6403 func:__p4d_alloc
           0        0 mm/memory.c:6426 func:__pud_alloc
     1064960       65 mm/memory.c:6450 func:__pmd_alloc
           0        0 mm/memory.c:4971 func:alloc_anon_folio
           0        0 mm/memory.c:5233 func:do_set_pmd

[1] commit 2c321f3f ("mm: change inlined allocation helpers to account at the call site")

Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
Acked-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
parent 72105fc1
Loading
Loading
Loading
Loading
+19 −11
Original line number Diff line number Diff line
@@ -19,12 +19,16 @@

#define CRST_ALLOC_ORDER 2

unsigned long *crst_table_alloc(struct mm_struct *);
unsigned long *crst_table_alloc_noprof(struct mm_struct *);
#define crst_table_alloc(...)	alloc_hooks(crst_table_alloc_noprof(__VA_ARGS__))
void crst_table_free(struct mm_struct *, unsigned long *);

unsigned long *page_table_alloc(struct mm_struct *);
struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm);
unsigned long *page_table_alloc_noprof(struct mm_struct *);
#define page_table_alloc(...)	alloc_hooks(page_table_alloc_noprof(__VA_ARGS__))
void page_table_free(struct mm_struct *, unsigned long *);

struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm);
#define page_table_alloc_pgste(...)	alloc_hooks(page_table_alloc_pgste_noprof(__VA_ARGS__))
void page_table_free_pgste(struct ptdesc *ptdesc);

static inline void crst_table_init(unsigned long *crst, unsigned long entry)
@@ -48,9 +52,9 @@ static inline unsigned long check_asce_limit(struct mm_struct *mm, unsigned long
	return addr;
}

static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long address)
static inline p4d_t *p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long address)
{
	unsigned long *table = crst_table_alloc(mm);
	unsigned long *table = crst_table_alloc_noprof(mm);

	if (!table)
		return NULL;
@@ -59,6 +63,7 @@ static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long address)

	return (p4d_t *) table;
}
#define p4d_alloc_one(...)	alloc_hooks(p4d_alloc_one_noprof(__VA_ARGS__))

static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
{
@@ -69,9 +74,9 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
	crst_table_free(mm, (unsigned long *) p4d);
}

static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long address)
{
	unsigned long *table = crst_table_alloc(mm);
	unsigned long *table = crst_table_alloc_noprof(mm);

	if (!table)
		return NULL;
@@ -80,6 +85,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)

	return (pud_t *) table;
}
#define pud_alloc_one(...)	alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__))

static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
@@ -90,9 +96,9 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
	crst_table_free(mm, (unsigned long *) pud);
}

static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr)
static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long vmaddr)
{
	unsigned long *table = crst_table_alloc(mm);
	unsigned long *table = crst_table_alloc_noprof(mm);

	if (!table)
		return NULL;
@@ -103,6 +109,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr)
	}
	return (pmd_t *) table;
}
#define pmd_alloc_one(...)	alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__))

static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
@@ -127,9 +134,9 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
	set_pud(pud, __pud(_REGION3_ENTRY | __pa(pmd)));
}

static inline pgd_t *pgd_alloc(struct mm_struct *mm)
static inline pgd_t *pgd_alloc_noprof(struct mm_struct *mm)
{
	unsigned long *table = crst_table_alloc(mm);
	unsigned long *table = crst_table_alloc_noprof(mm);

	if (!table)
		return NULL;
@@ -137,6 +144,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)

	return (pgd_t *) table;
}
#define pgd_alloc(...)	alloc_hooks(pgd_alloc_noprof(__VA_ARGS__))

static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
+6 −6
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>

unsigned long *crst_table_alloc(struct mm_struct *mm)
unsigned long *crst_table_alloc_noprof(struct mm_struct *mm)
{
	gfp_t gfp = GFP_KERNEL_ACCOUNT;
	struct ptdesc *ptdesc;
@@ -22,7 +22,7 @@ unsigned long *crst_table_alloc(struct mm_struct *mm)

	if (mm == &init_mm)
		gfp &= ~__GFP_ACCOUNT;
	ptdesc = pagetable_alloc(gfp, CRST_ALLOC_ORDER);
	ptdesc = pagetable_alloc_noprof(gfp, CRST_ALLOC_ORDER);
	if (!ptdesc)
		return NULL;
	table = ptdesc_to_virt(ptdesc);
@@ -116,12 +116,12 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end)

#ifdef CONFIG_PGSTE

struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm)
struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm)
{
	struct ptdesc *ptdesc;
	u64 *table;

	ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, 0);
	ptdesc = pagetable_alloc_noprof(GFP_KERNEL_ACCOUNT, 0);
	if (ptdesc) {
		table = (u64 *)ptdesc_to_virt(ptdesc);
		__arch_set_page_dat(table, 1);
@@ -138,7 +138,7 @@ void page_table_free_pgste(struct ptdesc *ptdesc)

#endif /* CONFIG_PGSTE */

unsigned long *page_table_alloc(struct mm_struct *mm)
unsigned long *page_table_alloc_noprof(struct mm_struct *mm)
{
	gfp_t gfp = GFP_KERNEL_ACCOUNT;
	struct ptdesc *ptdesc;
@@ -146,7 +146,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm)

	if (mm == &init_mm)
		gfp &= ~__GFP_ACCOUNT;
	ptdesc = pagetable_alloc(gfp, 0);
	ptdesc = pagetable_alloc_noprof(gfp, 0);
	if (!ptdesc)
		return NULL;
	if (!pagetable_pte_ctor(mm, ptdesc)) {