Commit e452872b authored by Hao Jia's avatar Hao Jia Committed by Andrew Morton
Browse files

mm: vmscan: split proactive reclaim statistics from direct reclaim statistics 

Patch series "Adding Proactive Memory Reclaim Statistics".

These two patches are related to proactive memory reclaim.

Patch 1 Split proactive reclaim statistics from direct reclaim counters
and introduces new counters: pgsteal_proactive, pgdemote_proactive,
and pgscan_proactive.

Patch 2 Adds pswpin and pswpout items to the cgroup-v2 documentation.


This patch (of 2):

In proactive memory reclaim scenarios, it is necessary to accurately track
proactive reclaim statistics to dynamically adjust the frequency and
amount of memory being reclaimed proactively.  Currently, proactive
reclaim is included in direct reclaim statistics, which can make these
direct reclaim statistics misleading.

Therefore, separate proactive reclaim memory from the direct reclaim
counters by introducing new counters: pgsteal_proactive,
pgdemote_proactive, and pgscan_proactive, to avoid confusion with direct
reclaim.

Link: https://lkml.kernel.org/r/20250318075833.90615-1-jiahao.kernel@gmail.com
Link: https://lkml.kernel.org/r/20250318075833.90615-2-jiahao.kernel@gmail.com


Signed-off-by: default avatarHao Jia <jiahao1@lixiang.com>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent a2c6f9c3

Documentation/admin-guide/cgroup-v2.rst

+9 −0
Original line number Diff line number Diff line
@@ -1576,6 +1576,9 @@ The following nested keys are defined. 
	  pgscan_khugepaged (npn)

		Amount of scanned pages by khugepaged  (in an inactive LRU list)



	  pgscan_proactive (npn)

		Amount of scanned pages proactively (in an inactive LRU list)



	  pgsteal_kswapd (npn)

		Amount of reclaimed pages by kswapd
@@ -1585,6 +1588,9 @@ The following nested keys are defined. 
	  pgsteal_khugepaged (npn)

		Amount of reclaimed pages by khugepaged



	  pgsteal_proactive (npn)

		Amount of reclaimed pages proactively



	  pgfault (npn)

		Total number of page faults incurred
@@ -1662,6 +1668,9 @@ The following nested keys are defined. 
	  pgdemote_khugepaged

		Number of pages demoted by khugepaged.



	  pgdemote_proactive

		Number of pages demoted by proactively.



	  hugetlb

		Amount of memory used by hugetlb pages. This metric only shows

		up if hugetlb usage is accounted for in memory.current (i.e.

include/linux/mmzone.h

+1 −0
Original line number Diff line number Diff line
@@ -221,6 +221,7 @@ enum node_stat_item { 
	PGDEMOTE_KSWAPD,

	PGDEMOTE_DIRECT,

	PGDEMOTE_KHUGEPAGED,

	PGDEMOTE_PROACTIVE,

#ifdef CONFIG_HUGETLB_PAGE

	NR_HUGETLB,

#endif

include/linux/vm_event_item.h

+2 −0
Original line number Diff line number Diff line
@@ -41,9 +41,11 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, 
		PGSTEAL_KSWAPD,

		PGSTEAL_DIRECT,

		PGSTEAL_KHUGEPAGED,

		PGSTEAL_PROACTIVE,

		PGSCAN_KSWAPD,

		PGSCAN_DIRECT,

		PGSCAN_KHUGEPAGED,

		PGSCAN_PROACTIVE,

		PGSCAN_DIRECT_THROTTLE,

		PGSCAN_ANON,

		PGSCAN_FILE,

mm/memcontrol.c

+7 −0
Original line number Diff line number Diff line
@@ -315,6 +315,7 @@ static const unsigned int memcg_node_stat_items[] = { 
	PGDEMOTE_KSWAPD,

	PGDEMOTE_DIRECT,

	PGDEMOTE_KHUGEPAGED,

	PGDEMOTE_PROACTIVE,

#ifdef CONFIG_HUGETLB_PAGE

	NR_HUGETLB,

#endif
@@ -431,9 +432,11 @@ static const unsigned int memcg_vm_event_stat[] = { 
	PGSCAN_KSWAPD,

	PGSCAN_DIRECT,

	PGSCAN_KHUGEPAGED,

	PGSCAN_PROACTIVE,

	PGSTEAL_KSWAPD,

	PGSTEAL_DIRECT,

	PGSTEAL_KHUGEPAGED,

	PGSTEAL_PROACTIVE,

	PGFAULT,

	PGMAJFAULT,

	PGREFILL,
@@ -1394,6 +1397,7 @@ static const struct memory_stat memory_stats[] = { 
	{ "pgdemote_kswapd",		PGDEMOTE_KSWAPD		},

	{ "pgdemote_direct",		PGDEMOTE_DIRECT		},

	{ "pgdemote_khugepaged",	PGDEMOTE_KHUGEPAGED	},

	{ "pgdemote_proactive",		PGDEMOTE_PROACTIVE	},

#ifdef CONFIG_NUMA_BALANCING

	{ "pgpromote_success",		PGPROMOTE_SUCCESS	},

#endif
@@ -1436,6 +1440,7 @@ static int memcg_page_state_output_unit(int item) 
	case PGDEMOTE_KSWAPD:

	case PGDEMOTE_DIRECT:

	case PGDEMOTE_KHUGEPAGED:

	case PGDEMOTE_PROACTIVE:

#ifdef CONFIG_NUMA_BALANCING

	case PGPROMOTE_SUCCESS:

#endif
@@ -1509,10 +1514,12 @@ static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) 
	seq_buf_printf(s, "pgscan %lu\n",

		       memcg_events(memcg, PGSCAN_KSWAPD) +

		       memcg_events(memcg, PGSCAN_DIRECT) +

		       memcg_events(memcg, PGSCAN_PROACTIVE) +

		       memcg_events(memcg, PGSCAN_KHUGEPAGED));

	seq_buf_printf(s, "pgsteal %lu\n",

		       memcg_events(memcg, PGSTEAL_KSWAPD) +

		       memcg_events(memcg, PGSTEAL_DIRECT) +

		       memcg_events(memcg, PGSTEAL_PROACTIVE) +

		       memcg_events(memcg, PGSTEAL_KHUGEPAGED));



	for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {

mm/vmscan.c

+20 −15
Original line number Diff line number Diff line
@@ -456,21 +456,26 @@ void drop_slab(void) 
	} while ((freed >> shift++) > 1);

}



static int reclaimer_offset(void)

#define CHECK_RECLAIMER_OFFSET(type)					\

	do {								\

		BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD !=		\

			     PGDEMOTE_##type - PGDEMOTE_KSWAPD);	\

		BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD !=		\

			     PGSCAN_##type - PGSCAN_KSWAPD);		\

	} while (0)



static int reclaimer_offset(struct scan_control *sc)

{

	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=

			PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD);

	BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=

			PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD);

	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=

			PGSCAN_DIRECT - PGSCAN_KSWAPD);

	BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=

			PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD);

	CHECK_RECLAIMER_OFFSET(DIRECT);

	CHECK_RECLAIMER_OFFSET(KHUGEPAGED);

	CHECK_RECLAIMER_OFFSET(PROACTIVE);



	if (current_is_kswapd())

		return 0;

	if (current_is_khugepaged())

		return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD;

	if (sc->proactive)

		return PGSTEAL_PROACTIVE - PGSTEAL_KSWAPD;

	return PGSTEAL_DIRECT - PGSTEAL_KSWAPD;

}
@@ -2008,7 +2013,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 
				     &nr_scanned, sc, lru);



	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);

	item = PGSCAN_KSWAPD + reclaimer_offset();

	item = PGSCAN_KSWAPD + reclaimer_offset(sc);

	if (!cgroup_reclaim(sc))

		__count_vm_events(item, nr_scanned);

	__count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
@@ -2024,10 +2029,10 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 
	spin_lock_irq(&lruvec->lru_lock);

	move_folios_to_lru(lruvec, &folio_list);



	__mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(),

	__mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),

					stat.nr_demoted);

	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);

	item = PGSTEAL_KSWAPD + reclaimer_offset();

	item = PGSTEAL_KSWAPD + reclaimer_offset(sc);

	if (!cgroup_reclaim(sc))

		__count_vm_events(item, nr_reclaimed);

	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
@@ -4571,7 +4576,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, 
			break;

	}



	item = PGSCAN_KSWAPD + reclaimer_offset();

	item = PGSCAN_KSWAPD + reclaimer_offset(sc);

	if (!cgroup_reclaim(sc)) {

		__count_vm_events(item, isolated);

		__count_vm_events(PGREFILL, sorted);
@@ -4721,10 +4726,10 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap 
		reset_batch_size(walk);

	}



	__mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(),

	__mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),

					stat.nr_demoted);



	item = PGSTEAL_KSWAPD + reclaimer_offset();

	item = PGSTEAL_KSWAPD + reclaimer_offset(sc);

	if (!cgroup_reclaim(sc))

		__count_vm_events(item, reclaimed);

	__count_memcg_events(memcg, item, reclaimed);