Commit 57e9cc50 authored by Johannes Weiner's avatar Johannes Weiner Committed by Andrew Morton
Browse files

mm: vmscan: split khugepaged stats from direct reclaim stats 

Direct reclaim stats are useful for identifying a potential source for
application latency, as well as spotting issues with kswapd.  However,
khugepaged currently distorts the picture: as a kernel thread it doesn't
impose allocation latencies on userspace, and it explicitly opts out of
kswapd reclaim.  Its activity showing up in the direct reclaim stats is
misleading.  Counting it as kswapd reclaim could also cause confusion when
trying to understand actual kswapd behavior.

Break out khugepaged from the direct reclaim counters into new
pgsteal_khugepaged, pgdemote_khugepaged, pgscan_khugepaged counters.

Test with a huge executable (CONFIG_READ_ONLY_THP_FOR_FS):

pgsteal_kswapd 1342185
pgsteal_direct 0
pgsteal_khugepaged 3623
pgscan_kswapd 1345025
pgscan_direct 0
pgscan_khugepaged 3623

Link: https://lkml.kernel.org/r/20221026180133.377671-1-hannes@cmpxchg.org


Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Reported-by: default avatarEric Bergen <ebergen@meta.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 1b016638

Documentation/admin-guide/cgroup-v2.rst

+6 −0
Original line number Diff line number Diff line
@@ -1488,12 +1488,18 @@ PAGE_SIZE multiple when read back. 
	  pgscan_direct (npn)

		Amount of scanned pages directly  (in an inactive LRU list)



	  pgscan_khugepaged (npn)

		Amount of scanned pages by khugepaged  (in an inactive LRU list)



	  pgsteal_kswapd (npn)

		Amount of reclaimed pages by kswapd



	  pgsteal_direct (npn)

		Amount of reclaimed pages directly



	  pgsteal_khugepaged (npn)

		Amount of reclaimed pages by khugepaged



	  pgfault (npn)

		Total number of page faults incurred

include/linux/khugepaged.h

+6 −0
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ extern void __khugepaged_exit(struct mm_struct *mm); 
extern void khugepaged_enter_vma(struct vm_area_struct *vma,

				 unsigned long vm_flags);

extern void khugepaged_min_free_kbytes_update(void);

extern bool current_is_khugepaged(void);

#ifdef CONFIG_SHMEM

extern int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr,

				   bool install_pmd);
@@ -57,6 +58,11 @@ static inline int collapse_pte_mapped_thp(struct mm_struct *mm, 
static inline void khugepaged_min_free_kbytes_update(void)

{

}



static inline bool current_is_khugepaged(void)

{

	return false;

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



#endif /* _LINUX_KHUGEPAGED_H */

include/linux/vm_event_item.h

+3 −0
Original line number Diff line number Diff line
@@ -40,10 +40,13 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, 
		PGREUSE,

		PGSTEAL_KSWAPD,

		PGSTEAL_DIRECT,

		PGSTEAL_KHUGEPAGED,

		PGDEMOTE_KSWAPD,

		PGDEMOTE_DIRECT,

		PGDEMOTE_KHUGEPAGED,

		PGSCAN_KSWAPD,

		PGSCAN_DIRECT,

		PGSCAN_KHUGEPAGED,

		PGSCAN_DIRECT_THROTTLE,

		PGSCAN_ANON,

		PGSCAN_FILE,

mm/khugepaged.c

+5 −0
Original line number Diff line number Diff line
@@ -2577,6 +2577,11 @@ void khugepaged_min_free_kbytes_update(void) 
	mutex_unlock(&khugepaged_mutex);

}



bool current_is_khugepaged(void)

{

	return kthread_func(current) == khugepaged;

}



static int madvise_collapse_errno(enum scan_result r)

{

	/*

mm/memcontrol.c

+6 −2
Original line number Diff line number Diff line
@@ -661,8 +661,10 @@ static const unsigned int memcg_vm_event_stat[] = { 
	PGPGOUT,

	PGSCAN_KSWAPD,

	PGSCAN_DIRECT,

	PGSCAN_KHUGEPAGED,

	PGSTEAL_KSWAPD,

	PGSTEAL_DIRECT,

	PGSTEAL_KHUGEPAGED,

	PGFAULT,

	PGMAJFAULT,

	PGREFILL,
@@ -1574,10 +1576,12 @@ static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize) 
	/* Accumulated memory events */

	seq_buf_printf(&s, "pgscan %lu\n",

		       memcg_events(memcg, PGSCAN_KSWAPD) +

		       memcg_events(memcg, PGSCAN_DIRECT));

		       memcg_events(memcg, PGSCAN_DIRECT) +

		       memcg_events(memcg, PGSCAN_KHUGEPAGED));

	seq_buf_printf(&s, "pgsteal %lu\n",

		       memcg_events(memcg, PGSTEAL_KSWAPD) +

		       memcg_events(memcg, PGSTEAL_DIRECT));

		       memcg_events(memcg, PGSTEAL_DIRECT) +

		       memcg_events(memcg, PGSTEAL_KHUGEPAGED));



	for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {

		if (memcg_vm_event_stat[i] == PGPGIN ||