Commit 8c9ae56d authored by Kefeng Wang's avatar Kefeng Wang Committed by Andrew Morton
Browse files

sched/numa, mm: make numa migrate functions to take a folio 

The cpupid (or access time) is stored in the head page for THP, so it is
safely to make should_numa_migrate_memory() and numa_hint_fault_latency()
to take a folio.  This is in preparation for large folio numa balancing.

Link: https://lkml.kernel.org/r/20230921074417.24004-7-wangkefeng.wang@huawei.com


Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 75c70128

include/linux/sched/numa_balancing.h

+3 −3
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ extern void task_numa_fault(int last_node, int node, int pages, int flags); 
extern pid_t task_numa_group_id(struct task_struct *p);

extern void set_numabalancing_state(bool enabled);

extern void task_numa_free(struct task_struct *p, bool final);

extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,

bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,

				int src_nid, int dst_cpu);

#else

static inline void task_numa_fault(int last_node, int node, int pages,
@@ -38,7 +38,7 @@ static inline void task_numa_free(struct task_struct *p, bool final) 
{

}

static inline bool should_numa_migrate_memory(struct task_struct *p,

				struct page *page, int src_nid, int dst_cpu)

				struct folio *folio, int src_nid, int dst_cpu)

{

	return true;

}

kernel/sched/fair.c

+6 −6
Original line number Diff line number Diff line
@@ -1722,12 +1722,12 @@ static bool pgdat_free_space_enough(struct pglist_data *pgdat) 
 * The smaller the hint page fault latency, the higher the possibility

 * for the page to be hot.

 */

static int numa_hint_fault_latency(struct page *page)

static int numa_hint_fault_latency(struct folio *folio)

{

	int last_time, time;



	time = jiffies_to_msecs(jiffies);

	last_time = xchg_page_access_time(page, time);

	last_time = xchg_page_access_time(&folio->page, time);



	return (time - last_time) & PAGE_ACCESS_TIME_MASK;

}
@@ -1784,7 +1784,7 @@ static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 
	}

}



bool should_numa_migrate_memory(struct task_struct *p, struct page * page,

bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,

				int src_nid, int dst_cpu)

{

	struct numa_group *ng = deref_curr_numa_group(p);
@@ -1814,16 +1814,16 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 
		numa_promotion_adjust_threshold(pgdat, rate_limit, def_th);



		th = pgdat->nbp_threshold ? : def_th;

		latency = numa_hint_fault_latency(page);

		latency = numa_hint_fault_latency(folio);

		if (latency >= th)

			return false;



		return !numa_promotion_rate_limit(pgdat, rate_limit,

						  thp_nr_pages(page));

						  folio_nr_pages(folio));

	}



	this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);

	last_cpupid = page_cpupid_xchg_last(page, this_cpupid);

	last_cpupid = page_cpupid_xchg_last(&folio->page, this_cpupid);



	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&

	    !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid))

mm/mempolicy.c

+1 −1
Original line number Diff line number Diff line
@@ -2642,7 +2642,7 @@ int mpol_misplaced(struct folio *folio, struct vm_area_struct *vma, 
	if (pol->flags & MPOL_F_MORON) {

		polnid = thisnid;



		if (!should_numa_migrate_memory(current, &folio->page, curnid,

		if (!should_numa_migrate_memory(current, folio, curnid,

						thiscpu))

			goto out;

	}