Commit b51820eb authored by Honggyu Kim's avatar Honggyu Kim Committed by Andrew Morton
Browse files

mm/damon/paddr: introduce DAMOS_MIGRATE_COLD action for demotion 

This patch introduces DAMOS_MIGRATE_COLD action, which is similar to
DAMOS_PAGEOUT, but migrate folios to the given 'target_nid' in the sysfs
instead of swapping them out.

The 'target_nid' sysfs knob informs the migration target node ID.

Here is one of the example usage of this 'migrate_cold' action.

  $ cd /sys/kernel/mm/damon/admin/kdamonds/<N>
  $ cat contexts/<N>/schemes/<N>/action
  migrate_cold
  $ echo 2 > contexts/<N>/schemes/<N>/target_nid
  $ echo commit > state
  $ numactl -p 0 ./hot_cold 500M 600M &
  $ numastat -c -p hot_cold

  Per-node process memory usage (in MBs)
  PID             Node 0 Node 1 Node 2 Total
  --------------  ------ ------ ------ -----
  701 (hot_cold)     501      0    601  1101

Since there are some common routines with pageout, many functions have
similar logics between pageout and migrate cold.

damon_pa_migrate_folio_list() is a minimized version of
shrink_folio_list().

Link: https://lkml.kernel.org/r/20240614030010.751-6-honggyu.kim@sk.com


Signed-off-by: default avatarHonggyu Kim <honggyu.kim@sk.com>
Signed-off-by: default avatarHyeongtak Ji <hyeongtak.ji@sk.com>
Signed-off-by: default avatarSeongJae Park <sj@kernel.org>
Cc: Gregory Price <gregory.price@memverge.com>
Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Cc: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Steven Rostedt (Google) <rostedt@goodmis.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent ced816a7

include/linux/damon.h

+2 −0
Original line number Diff line number Diff line
@@ -105,6 +105,7 @@ struct damon_target { 
 * @DAMOS_NOHUGEPAGE:	Call ``madvise()`` for the region with MADV_NOHUGEPAGE.

 * @DAMOS_LRU_PRIO:	Prioritize the region on its LRU lists.

 * @DAMOS_LRU_DEPRIO:	Deprioritize the region on its LRU lists.

 * @DAMOS_MIGRATE_COLD:	Migrate the regions prioritizing colder regions.

 * @DAMOS_STAT:		Do nothing but count the stat.

 * @NR_DAMOS_ACTIONS:	Total number of DAMOS actions

 *
@@ -122,6 +123,7 @@ enum damos_action { 
	DAMOS_NOHUGEPAGE,

	DAMOS_LRU_PRIO,

	DAMOS_LRU_DEPRIO,

	DAMOS_MIGRATE_COLD,

	DAMOS_STAT,		/* Do nothing but only record the stat */

	NR_DAMOS_ACTIONS,

};

mm/damon/paddr.c

+154 −0
Original line number Diff line number Diff line
@@ -12,6 +12,9 @@ 
#include <linux/pagemap.h>

#include <linux/rmap.h>

#include <linux/swap.h>

#include <linux/memory-tiers.h>

#include <linux/migrate.h>

#include <linux/mm_inline.h>



#include "../internal.h"

#include "ops-common.h"
@@ -325,6 +328,153 @@ static unsigned long damon_pa_deactivate_pages(struct damon_region *r, 
	return damon_pa_mark_accessed_or_deactivate(r, s, false);

}



static unsigned int __damon_pa_migrate_folio_list(

		struct list_head *migrate_folios, struct pglist_data *pgdat,

		int target_nid)

{

	unsigned int nr_succeeded;

	nodemask_t allowed_mask = NODE_MASK_NONE;

	struct migration_target_control mtc = {

		/*

		 * Allocate from 'node', or fail quickly and quietly.

		 * When this happens, 'page' will likely just be discarded

		 * instead of migrated.

		 */

		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |

			__GFP_NOWARN | __GFP_NOMEMALLOC | GFP_NOWAIT,

		.nid = target_nid,

		.nmask = &allowed_mask

	};



	if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE)

		return 0;



	if (list_empty(migrate_folios))

		return 0;



	/* Migration ignores all cpuset and mempolicy settings */

	migrate_pages(migrate_folios, alloc_migrate_folio, NULL,

		      (unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON,

		      &nr_succeeded);



	return nr_succeeded;

}



static unsigned int damon_pa_migrate_folio_list(struct list_head *folio_list,

						struct pglist_data *pgdat,

						int target_nid)

{

	unsigned int nr_migrated = 0;

	struct folio *folio;

	LIST_HEAD(ret_folios);

	LIST_HEAD(migrate_folios);



	while (!list_empty(folio_list)) {

		struct folio *folio;



		cond_resched();



		folio = lru_to_folio(folio_list);

		list_del(&folio->lru);



		if (!folio_trylock(folio))

			goto keep;



		/* Relocate its contents to another node. */

		list_add(&folio->lru, &migrate_folios);

		folio_unlock(folio);

		continue;

keep:

		list_add(&folio->lru, &ret_folios);

	}

	/* 'folio_list' is always empty here */



	/* Migrate folios selected for migration */

	nr_migrated += __damon_pa_migrate_folio_list(

			&migrate_folios, pgdat, target_nid);

	/*

	 * Folios that could not be migrated are still in @migrate_folios.  Add

	 * those back on @folio_list

	 */

	if (!list_empty(&migrate_folios))

		list_splice_init(&migrate_folios, folio_list);



	try_to_unmap_flush();



	list_splice(&ret_folios, folio_list);



	while (!list_empty(folio_list)) {

		folio = lru_to_folio(folio_list);

		list_del(&folio->lru);

		folio_putback_lru(folio);

	}



	return nr_migrated;

}



static unsigned long damon_pa_migrate_pages(struct list_head *folio_list,

					    int target_nid)

{

	int nid;

	unsigned long nr_migrated = 0;

	LIST_HEAD(node_folio_list);

	unsigned int noreclaim_flag;



	if (list_empty(folio_list))

		return nr_migrated;



	noreclaim_flag = memalloc_noreclaim_save();



	nid = folio_nid(lru_to_folio(folio_list));

	do {

		struct folio *folio = lru_to_folio(folio_list);



		if (nid == folio_nid(folio)) {

			list_move(&folio->lru, &node_folio_list);

			continue;

		}



		nr_migrated += damon_pa_migrate_folio_list(&node_folio_list,

							   NODE_DATA(nid),

							   target_nid);

		nid = folio_nid(lru_to_folio(folio_list));

	} while (!list_empty(folio_list));



	nr_migrated += damon_pa_migrate_folio_list(&node_folio_list,

						   NODE_DATA(nid),

						   target_nid);



	memalloc_noreclaim_restore(noreclaim_flag);



	return nr_migrated;

}



static unsigned long damon_pa_migrate(struct damon_region *r, struct damos *s)

{

	unsigned long addr, applied;

	LIST_HEAD(folio_list);



	for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {

		struct folio *folio = damon_get_folio(PHYS_PFN(addr));



		if (!folio)

			continue;



		if (damos_pa_filter_out(s, folio))

			goto put_folio;



		if (!folio_isolate_lru(folio))

			goto put_folio;

		list_add(&folio->lru, &folio_list);

put_folio:

		folio_put(folio);

	}

	applied = damon_pa_migrate_pages(&folio_list, s->target_nid);

	cond_resched();

	return applied * PAGE_SIZE;

}





static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,

		struct damon_target *t, struct damon_region *r,

		struct damos *scheme)
@@ -336,6 +486,8 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, 
		return damon_pa_mark_accessed(r, scheme);

	case DAMOS_LRU_DEPRIO:

		return damon_pa_deactivate_pages(r, scheme);

	case DAMOS_MIGRATE_COLD:

		return damon_pa_migrate(r, scheme);

	case DAMOS_STAT:

		break;

	default:
@@ -356,6 +508,8 @@ static int damon_pa_scheme_score(struct damon_ctx *context, 
		return damon_hot_score(context, r, scheme);

	case DAMOS_LRU_DEPRIO:

		return damon_cold_score(context, r, scheme);

	case DAMOS_MIGRATE_COLD:

		return damon_cold_score(context, r, scheme);

	default:

		break;

	}

mm/damon/sysfs-schemes.c

+1 −0
Original line number Diff line number Diff line
@@ -1458,6 +1458,7 @@ static const char * const damon_sysfs_damos_action_strs[] = { 
	"nohugepage",

	"lru_prio",

	"lru_deprio",

	"migrate_cold",

	"stat",

};