Commit cc25bbe1 authored by Kinsey Ho's avatar Kinsey Ho Committed by Andrew Morton
Browse files

mm/mglru: improve struct lru_gen_mm_walk 

Rename max_seq to seq in struct lru_gen_mm_walk to keep consistent with
struct lru_gen_mm_state.  Note that seq is not always up to date with
max_seq from lru_gen_folio.

No functional changes.

Link: https://lkml.kernel.org/r/20240214060538.3524462-5-kinseyho@google.com


Signed-off-by: default avatarKinsey Ho <kinseyho@google.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Donet Tom <donettom@linux.vnet.ibm.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 2d823764

include/linux/mmzone.h

+3 −3
Original line number Diff line number Diff line
@@ -464,7 +464,7 @@ enum { 
#define NR_BLOOM_FILTERS	2



struct lru_gen_mm_state {

	/* set to max_seq after each iteration */

	/* synced with max_seq after each iteration */

	unsigned long seq;

	/* where the current iteration continues after */

	struct list_head *head;
@@ -479,8 +479,8 @@ struct lru_gen_mm_state { 
struct lru_gen_mm_walk {

	/* the lruvec under reclaim */

	struct lruvec *lruvec;

	/* unstable max_seq from lru_gen_folio */

	unsigned long max_seq;

	/* max_seq from lru_gen_folio: can be out of date */

	unsigned long seq;

	/* the next address within an mm to scan */

	unsigned long next_addr;

	/* to batch promoted pages */

mm/vmscan.c

+26 −24
Original line number Diff line number Diff line
@@ -2888,7 +2888,7 @@ static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) 


	lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);



	hist = lru_hist_from_seq(walk->max_seq);

	hist = lru_hist_from_seq(walk->seq);



	for (i = 0; i < NR_MM_STATS; i++) {

		WRITE_ONCE(mm_state->stats[hist][i],
@@ -2897,7 +2897,7 @@ static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) 
	}



	if (NR_HIST_GENS > 1 && last) {

		hist = lru_hist_from_seq(walk->max_seq + 1);

		hist = lru_hist_from_seq(walk->seq + 1);



		for (i = 0; i < NR_MM_STATS; i++)

			WRITE_ONCE(mm_state->stats[hist][i], 0);
@@ -2926,9 +2926,9 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite 
	 */

	spin_lock(&mm_list->lock);



	VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq);

	VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq);



	if (walk->max_seq <= mm_state->seq)

	if (walk->seq <= mm_state->seq)

		goto done;



	if (!mm_state->head)
@@ -2958,7 +2958,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite 
	spin_unlock(&mm_list->lock);



	if (mm && first)

		reset_bloom_filter(mm_state, walk->max_seq + 1);

		reset_bloom_filter(mm_state, walk->seq + 1);



	if (*iter)

		mmput_async(*iter);
@@ -2968,7 +2968,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite 
	return last;

}



static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)

static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq)

{

	bool success = false;

	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
@@ -2977,9 +2977,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) 


	spin_lock(&mm_list->lock);



	VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);

	VM_WARN_ON_ONCE(mm_state->seq + 1 < seq);



	if (max_seq > mm_state->seq) {

	if (seq > mm_state->seq) {

		mm_state->head = NULL;

		mm_state->tail = NULL;

		WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
@@ -3330,7 +3330,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 
	struct lru_gen_mm_walk *walk = args->private;

	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);

	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);

	int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);

	DEFINE_MAX_SEQ(walk->lruvec);

	int old_gen, new_gen = lru_gen_from_seq(max_seq);



	pte = pte_offset_map_nolock(args->mm, pmd, start & PMD_MASK, &ptl);

	if (!pte)
@@ -3397,7 +3398,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area 
	struct lru_gen_mm_walk *walk = args->private;

	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);

	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);

	int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);

	DEFINE_MAX_SEQ(walk->lruvec);

	int old_gen, new_gen = lru_gen_from_seq(max_seq);



	VM_WARN_ON_ONCE(pud_leaf(*pud));
@@ -3528,7 +3530,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 
			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);

		}



		if (!walk->force_scan && !test_bloom_filter(mm_state, walk->max_seq, pmd + i))

		if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))

			continue;



		walk->mm_stats[MM_NONLEAF_FOUND]++;
@@ -3539,7 +3541,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 
		walk->mm_stats[MM_NONLEAF_ADDED]++;



		/* carry over to the next generation */

		update_bloom_filter(mm_state, walk->max_seq + 1, pmd + i);

		update_bloom_filter(mm_state, walk->seq + 1, pmd + i);

	}



	walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
@@ -3610,7 +3612,7 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) 
		err = -EBUSY;



		/* another thread might have called inc_max_seq() */

		if (walk->max_seq != max_seq)

		if (walk->seq != max_seq)

			break;



		/* folio_update_gen() requires stable folio_memcg() */
@@ -3747,7 +3749,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) 
	return success;

}



static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,

static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq,

			bool can_swap, bool force_scan)

{

	bool success;
@@ -3755,14 +3757,14 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 
	int type, zone;

	struct lru_gen_folio *lrugen = &lruvec->lrugen;

restart:

	if (max_seq < READ_ONCE(lrugen->max_seq))

	if (seq < READ_ONCE(lrugen->max_seq))

		return false;



	spin_lock_irq(&lruvec->lru_lock);



	VM_WARN_ON_ONCE(!seq_is_valid(lruvec));



	success = max_seq == lrugen->max_seq;

	success = seq == lrugen->max_seq;

	if (!success)

		goto unlock;
@@ -3815,7 +3817,7 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 
	return success;

}



static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,

static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,

			       bool can_swap, bool force_scan)

{

	bool success;
@@ -3824,13 +3826,13 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 
	struct lru_gen_folio *lrugen = &lruvec->lrugen;

	struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);



	VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));

	VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq));



	if (!mm_state)

		return inc_max_seq(lruvec, max_seq, can_swap, force_scan);

		return inc_max_seq(lruvec, seq, can_swap, force_scan);



	/* see the comment in iterate_mm_list() */

	if (max_seq <= READ_ONCE(mm_state->seq))

	if (seq <= READ_ONCE(mm_state->seq))

		return false;



	/*
@@ -3840,18 +3842,18 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 
	 * is less efficient, but it avoids bursty page faults.

	 */

	if (!should_walk_mmu()) {

		success = iterate_mm_list_nowalk(lruvec, max_seq);

		success = iterate_mm_list_nowalk(lruvec, seq);

		goto done;

	}



	walk = set_mm_walk(NULL, true);

	if (!walk) {

		success = iterate_mm_list_nowalk(lruvec, max_seq);

		success = iterate_mm_list_nowalk(lruvec, seq);

		goto done;

	}



	walk->lruvec = lruvec;

	walk->max_seq = max_seq;

	walk->seq = seq;

	walk->can_swap = can_swap;

	walk->force_scan = force_scan;
@@ -3862,7 +3864,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 
	} while (mm);

done:

	if (success) {

		success = inc_max_seq(lruvec, max_seq, can_swap, force_scan);

		success = inc_max_seq(lruvec, seq, can_swap, force_scan);

		WARN_ON_ONCE(!success);

	}