Commit 3865301d authored by Hugh Dickins's avatar Hugh Dickins Committed by Andrew Morton
Browse files

mm: optimize lru_note_cost() by adding lru_note_cost_unlock_irq() 

Dropping a lock, just to demand it again for an afterthought, cannot be
good if contended: convert lru_note_cost() to lru_note_cost_unlock_irq().

[hughd@google.com: delete unneeded comment]
  Link: https://lkml.kernel.org/r/dbf9352a-1ed9-a021-c0c7-9309ac73e174@google.com
Link: https://lkml.kernel.org/r/21100102-51b6-79d5-03db-1bb7f97fa94c@google.com


Signed-off-by: default avatarHugh Dickins <hughd@google.com>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Reviewed-by: default avatarRoman Gushchin <roman.gushchin@linux.dev>
Tested-by: default avatarRoman Gushchin <roman.gushchin@linux.dev>
Reviewed-by: default avatarShakeel Butt <shakeel.butt@linux.dev>
Cc: David Hildenbrand <david@redhat.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 526660b9

include/linux/swap.h

+3 −2
Original line number Diff line number Diff line
@@ -376,8 +376,9 @@ extern unsigned long totalreserve_pages; 




/* linux/mm/swap.c */

void lru_note_cost(struct lruvec *lruvec, bool file,

		   unsigned int nr_io, unsigned int nr_rotated);

void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,

		unsigned int nr_io, unsigned int nr_rotated)

		__releases(lruvec->lru_lock);

void lru_note_cost_refault(struct folio *);

void folio_add_lru(struct folio *);

void folio_add_lru_vma(struct folio *, struct vm_area_struct *);

mm/swap.c

+19 −14
Original line number Diff line number Diff line
@@ -237,8 +237,9 @@ void folio_rotate_reclaimable(struct folio *folio) 
	folio_batch_add_and_move(folio, lru_move_tail, true);

}



void lru_note_cost(struct lruvec *lruvec, bool file,

void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,

		unsigned int nr_io, unsigned int nr_rotated)

		__releases(lruvec->lru_lock)

{

	unsigned long cost;
@@ -250,18 +251,14 @@ void lru_note_cost(struct lruvec *lruvec, bool file, 
	 * different between them, adjust scan balance for CPU work.

	 */

	cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated;

	if (!cost) {

		spin_unlock_irq(&lruvec->lru_lock);

		return;

	}



	do {

	for (;;) {

		unsigned long lrusize;



		/*

		 * Hold lruvec->lru_lock is safe here, since

		 * 1) The pinned lruvec in reclaim, or

		 * 2) From a pre-LRU page during refault (which also holds the

		 *    rcu lock, so would be safe even if the page was on the LRU

		 *    and could move simultaneously to a new lruvec).

		 */

		spin_lock_irq(&lruvec->lru_lock);

		/* Record cost event */

		if (file)

			lruvec->file_cost += cost;
@@ -285,13 +282,21 @@ void lru_note_cost(struct lruvec *lruvec, bool file, 
			lruvec->file_cost /= 2;

			lruvec->anon_cost /= 2;

		}



		spin_unlock_irq(&lruvec->lru_lock);

	} while ((lruvec = parent_lruvec(lruvec)));

		lruvec = parent_lruvec(lruvec);

		if (!lruvec)

			break;

		spin_lock_irq(&lruvec->lru_lock);

	}

}



void lru_note_cost_refault(struct folio *folio)

{

	lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio),

	struct lruvec *lruvec;



	lruvec = folio_lruvec_lock_irq(folio);

	lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio),

				folio_nr_pages(folio), 0);

}

mm/vmscan.c

+3 −5
Original line number Diff line number Diff line
@@ -2053,9 +2053,9 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 
		__count_vm_events(item, nr_reclaimed);

	count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);

	__count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);

	spin_unlock_irq(&lruvec->lru_lock);



	lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed);

	lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout,

					nr_scanned - nr_reclaimed);



	/*

	 * If dirty folios are scanned that are not queued for IO, it
@@ -2201,10 +2201,8 @@ static void shrink_active_list(unsigned long nr_to_scan, 
	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);



	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);

	spin_unlock_irq(&lruvec->lru_lock);



	if (nr_rotated)

		lru_note_cost(lruvec, file, 0, nr_rotated);

	lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated);

	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,

			nr_deactivate, nr_rotated, sc->priority, file);

}