Commit 0a97c01c authored by Nhat Pham's avatar Nhat Pham Committed by Andrew Morton
Browse files

list_lru: allow explicit memcg and NUMA node selection 

Patch series "workload-specific and memory pressure-driven zswap
writeback", v8.

There are currently several issues with zswap writeback:

1. There is only a single global LRU for zswap, making it impossible to
   perform worload-specific shrinking - an memcg under memory pressure
   cannot determine which pages in the pool it owns, and often ends up
   writing pages from other memcgs. This issue has been previously
   observed in practice and mitigated by simply disabling
   memcg-initiated shrinking:

   https://lore.kernel.org/all/20230530232435.3097106-1-nphamcs@gmail.com/T/#u

   But this solution leaves a lot to be desired, as we still do not
   have an avenue for an memcg to free up its own memory locked up in
   the zswap pool.

2. We only shrink the zswap pool when the user-defined limit is hit.
   This means that if we set the limit too high, cold data that are
   unlikely to be used again will reside in the pool, wasting precious
   memory. It is hard to predict how much zswap space will be needed
   ahead of time, as this depends on the workload (specifically, on
   factors such as memory access patterns and compressibility of the
   memory pages).

This patch series solves these issues by separating the global zswap LRU
into per-memcg and per-NUMA LRUs, and performs workload-specific (i.e
memcg- and NUMA-aware) zswap writeback under memory pressure.  The new
shrinker does not have any parameter that must be tuned by the user, and
can be opted in or out on a per-memcg basis.

As a proof of concept, we ran the following synthetic benchmark: build the
linux kernel in a memory-limited cgroup, and allocate some cold data in
tmpfs to see if the shrinker could write them out and improved the overall
performance.  Depending on the amount of cold data generated, we observe
from 14% to 35% reduction in kernel CPU time used in the kernel builds.


This patch (of 6):

The interface of list_lru is based on the assumption that the list node
and the data it represents belong to the same allocated on the correct
node/memcg.  While this assumption is valid for existing slab objects LRU
such as dentries and inodes, it is undocumented, and rather inflexible for
certain potential list_lru users (such as the upcoming zswap shrinker and
the THP shrinker).  It has caused us a lot of issues during our
development.

This patch changes list_lru interface so that the caller must explicitly
specify numa node and memcg when adding and removing objects.  The old
list_lru_add() and list_lru_del() are renamed to list_lru_add_obj() and
list_lru_del_obj(), respectively.

It also extends the list_lru API with a new function, list_lru_putback,
which undoes a previous list_lru_isolate call.  Unlike list_lru_add, it
does not increment the LRU node count (as list_lru_isolate does not
decrement the node count).  list_lru_putback also allows for explicit
memcg and NUMA node selection.

Link: https://lkml.kernel.org/r/20231130194023.4102148-1-nphamcs@gmail.com
Link: https://lkml.kernel.org/r/20231130194023.4102148-2-nphamcs@gmail.com


Signed-off-by: default avatarNhat Pham <nphamcs@gmail.com>
Suggested-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Tested-by: default avatarBagas Sanjaya <bagasdotme@gmail.com>
Cc: Chris Li <chrisl@kernel.org>
Cc: Dan Streetman <ddstreet@ieee.org>
Cc: Domenico Cerasuolo <cerasuolodomenico@gmail.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Seth Jennings <sjenning@redhat.com>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Vitaly Wool <vitaly.wool@konsulko.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 330018fe

drivers/android/binder_alloc.c

+3 −4
Original line number Diff line number Diff line
@@ -234,7 +234,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, 
		if (page->page_ptr) {

			trace_binder_alloc_lru_start(alloc, index);



			on_lru = list_lru_del(&binder_alloc_lru, &page->lru);

			on_lru = list_lru_del_obj(&binder_alloc_lru, &page->lru);

			WARN_ON(!on_lru);



			trace_binder_alloc_lru_end(alloc, index);
@@ -285,7 +285,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, 


		trace_binder_free_lru_start(alloc, index);



		ret = list_lru_add(&binder_alloc_lru, &page->lru);

		ret = list_lru_add_obj(&binder_alloc_lru, &page->lru);

		WARN_ON(!ret);



		trace_binder_free_lru_end(alloc, index);
@@ -848,7 +848,7 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) 
			if (!alloc->pages[i].page_ptr)

				continue;



			on_lru = list_lru_del(&binder_alloc_lru,

			on_lru = list_lru_del_obj(&binder_alloc_lru,

					      &alloc->pages[i].lru);

			page_addr = alloc->buffer + i * PAGE_SIZE;

			binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
@@ -1287,4 +1287,3 @@ int binder_alloc_copy_from_buffer(struct binder_alloc *alloc, 
	return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,

					   dest, bytes);

}

fs/dcache.c

+5 −3
Original line number Diff line number Diff line
@@ -428,7 +428,8 @@ static void d_lru_add(struct dentry *dentry) 
	this_cpu_inc(nr_dentry_unused);

	if (d_is_negative(dentry))

		this_cpu_inc(nr_dentry_negative);

	WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));

	WARN_ON_ONCE(!list_lru_add_obj(

			&dentry->d_sb->s_dentry_lru, &dentry->d_lru));

}



static void d_lru_del(struct dentry *dentry)
@@ -438,7 +439,8 @@ static void d_lru_del(struct dentry *dentry) 
	this_cpu_dec(nr_dentry_unused);

	if (d_is_negative(dentry))

		this_cpu_dec(nr_dentry_negative);

	WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));

	WARN_ON_ONCE(!list_lru_del_obj(

			&dentry->d_sb->s_dentry_lru, &dentry->d_lru));

}



static void d_shrink_del(struct dentry *dentry)
@@ -1240,7 +1242,7 @@ static enum lru_status dentry_lru_isolate(struct list_head *item, 
		 *

		 * This is guaranteed by the fact that all LRU management

		 * functions are intermediated by the LRU API calls like

		 * list_lru_add and list_lru_del. List movement in this file

		 * list_lru_add_obj and list_lru_del_obj. List movement in this file

		 * only ever occur through this functions or through callbacks

		 * like this one, that are called from the LRU API.

		 *

fs/gfs2/quota.c

+3 −3
Original line number Diff line number Diff line
@@ -271,7 +271,7 @@ static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash, 
		if (qd->qd_sbd != sdp)

			continue;

		if (lockref_get_not_dead(&qd->qd_lockref)) {

			list_lru_del(&gfs2_qd_lru, &qd->qd_lru);

			list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru);

			return qd;

		}

	}
@@ -344,7 +344,7 @@ static void qd_put(struct gfs2_quota_data *qd) 
	}



	qd->qd_lockref.count = 0;

	list_lru_add(&gfs2_qd_lru, &qd->qd_lru);

	list_lru_add_obj(&gfs2_qd_lru, &qd->qd_lru);

	spin_unlock(&qd->qd_lockref.lock);

}
@@ -1517,7 +1517,7 @@ void gfs2_quota_cleanup(struct gfs2_sbd *sdp) 
		lockref_mark_dead(&qd->qd_lockref);

		spin_unlock(&qd->qd_lockref.lock);



		list_lru_del(&gfs2_qd_lru, &qd->qd_lru);

		list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru);

		list_add(&qd->qd_lru, &dispose);

	}

	spin_unlock(&qd_lock);

fs/inode.c

+2 −2
Original line number Diff line number Diff line
@@ -464,7 +464,7 @@ static void __inode_add_lru(struct inode *inode, bool rotate) 
	if (!mapping_shrinkable(&inode->i_data))

		return;



	if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))

	if (list_lru_add_obj(&inode->i_sb->s_inode_lru, &inode->i_lru))

		this_cpu_inc(nr_unused);

	else if (rotate)

		inode->i_state |= I_REFERENCED;
@@ -482,7 +482,7 @@ void inode_add_lru(struct inode *inode) 


static void inode_lru_list_del(struct inode *inode)

{

	if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))

	if (list_lru_del_obj(&inode->i_sb->s_inode_lru, &inode->i_lru))

		this_cpu_dec(nr_unused);

}

fs/nfs/nfs42xattr.c

+4 −4
Original line number Diff line number Diff line
@@ -132,7 +132,7 @@ nfs4_xattr_entry_lru_add(struct nfs4_xattr_entry *entry) 
	lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ?

	    &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;



	return list_lru_add(lru, &entry->lru);

	return list_lru_add_obj(lru, &entry->lru);

}



static bool
@@ -143,7 +143,7 @@ nfs4_xattr_entry_lru_del(struct nfs4_xattr_entry *entry) 
	lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ?

	    &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;



	return list_lru_del(lru, &entry->lru);

	return list_lru_del_obj(lru, &entry->lru);

}



/*
@@ -349,7 +349,7 @@ nfs4_xattr_cache_unlink(struct inode *inode) 


	oldcache = nfsi->xattr_cache;

	if (oldcache != NULL) {

		list_lru_del(&nfs4_xattr_cache_lru, &oldcache->lru);

		list_lru_del_obj(&nfs4_xattr_cache_lru, &oldcache->lru);

		oldcache->inode = NULL;

	}

	nfsi->xattr_cache = NULL;
@@ -474,7 +474,7 @@ nfs4_xattr_get_cache(struct inode *inode, int add) 
			kref_get(&cache->ref);

			nfsi->xattr_cache = cache;

			cache->inode = inode;

			list_lru_add(&nfs4_xattr_cache_lru, &cache->lru);

			list_lru_add_obj(&nfs4_xattr_cache_lru, &cache->lru);

		}



		spin_unlock(&inode->i_lock);