Commit 45164322 authored by Dave Airlie's avatar Dave Airlie
Browse files

ttm/pool: track allocated_pages per numa node. 



This gets the memory sizes from the nodes and stores the limit
as 50% of those. I think eventually we should drop the limits
once we have memcg aware shrinking, but this should be more NUMA
friendly, and I think seems like what people would prefer to
happen on NUMA aware systems.

Cc: Christian Koenig <christian.koenig@amd.com>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent 0180d6ff

drivers/gpu/drm/ttm/ttm_pool.c

+47 −15
Original line number Diff line number Diff line
@@ -116,10 +116,11 @@ struct ttm_pool_tt_restore { 


static unsigned long page_pool_size;



MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool");

MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool per NUMA node");

module_param(page_pool_size, ulong, 0644);



static atomic_long_t allocated_pages;

static unsigned long pool_node_limit[MAX_NUMNODES];

static atomic_long_t allocated_pages[MAX_NUMNODES];



static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS];

static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS];
@@ -299,6 +300,7 @@ static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, 
static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p)

{

	unsigned int i, num_pages = 1 << pt->order;

	int nid = page_to_nid(p);



	for (i = 0; i < num_pages; ++i) {

		if (PageHighMem(p))
@@ -309,10 +311,10 @@ static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) 


	INIT_LIST_HEAD(&p->lru);

	rcu_read_lock();

	list_lru_add(&pt->pages, &p->lru, page_to_nid(p), NULL);

	list_lru_add(&pt->pages, &p->lru, nid, NULL);

	rcu_read_unlock();

	atomic_long_add(1 << pt->order, &allocated_pages);



	atomic_long_add(num_pages, &allocated_pages[nid]);

	mod_lruvec_page_state(p, NR_GPU_ACTIVE, -num_pages);

	mod_lruvec_page_state(p, NR_GPU_RECLAIM, num_pages);

}
@@ -338,7 +340,7 @@ static struct page *ttm_pool_type_take(struct ttm_pool_type *pt, int nid) 


	ret = list_lru_walk_node(&pt->pages, nid, take_one_from_lru, (void *)&p, &nr_to_walk);

	if (ret == 1 && p) {

		atomic_long_sub(1 << pt->order, &allocated_pages);

		atomic_long_sub(1 << pt->order, &allocated_pages[nid]);

		mod_lruvec_page_state(p, NR_GPU_ACTIVE, (1 << pt->order));

		mod_lruvec_page_state(p, NR_GPU_RECLAIM, -(1 << pt->order));

	}
@@ -377,7 +379,7 @@ static void ttm_pool_dispose_list(struct ttm_pool_type *pt, 
		struct page *p;

		p = list_first_entry(dispose, struct page, lru);

		list_del_init(&p->lru);

		atomic_long_sub(1 << pt->order, &allocated_pages);

		atomic_long_sub(1 << pt->order, &allocated_pages[page_to_nid(p)]);

		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p, true);

	}

}
@@ -940,11 +942,13 @@ int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, 
 */

void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)

{

	int nid = ttm_pool_nid(pool);



	ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages);



	while (atomic_long_read(&allocated_pages) > page_pool_size) {

		unsigned long diff = atomic_long_read(&allocated_pages) - page_pool_size;

		ttm_pool_shrink(ttm_pool_nid(pool), diff);

	while (atomic_long_read(&allocated_pages[nid]) > pool_node_limit[nid]) {

		unsigned long diff = atomic_long_read(&allocated_pages[nid]) - pool_node_limit[nid];

		ttm_pool_shrink(nid, diff);

	}

}

EXPORT_SYMBOL(ttm_pool_free);
@@ -1202,7 +1206,7 @@ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, 
	do

		num_freed += ttm_pool_shrink(sc->nid, sc->nr_to_scan);

	while (num_freed < sc->nr_to_scan &&

	       atomic_long_read(&allocated_pages));

	       atomic_long_read(&allocated_pages[sc->nid]));



	sc->nr_scanned = num_freed;
@@ -1213,7 +1217,7 @@ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, 
static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink,

					     struct shrink_control *sc)

{

	unsigned long num_pages = atomic_long_read(&allocated_pages);

	unsigned long num_pages = atomic_long_read(&allocated_pages[sc->nid]);



	return num_pages ? num_pages : SHRINK_EMPTY;

}
@@ -1250,8 +1254,12 @@ static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, 
/* Dump the total amount of allocated pages */

static void ttm_pool_debugfs_footer(struct seq_file *m)

{

	seq_printf(m, "\ntotal\t: %8lu of %8lu\n",

		   atomic_long_read(&allocated_pages), page_pool_size);

	int nid;



	for_each_node(nid) {

		seq_printf(m, "\ntotal node%d\t: %8lu of %8lu\n", nid,

			   atomic_long_read(&allocated_pages[nid]), pool_node_limit[nid]);

	}

}



/* Dump the information for the global pools */
@@ -1345,6 +1353,23 @@ DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); 


#endif



static inline u64 ttm_get_node_memory_size(int nid)

{

	/*

	 * This is directly using si_meminfo_node implementation as the

	 * function is not exported.

	 */

	int zone_type;

	u64 managed_pages = 0;



	pg_data_t *pgdat = NODE_DATA(nid);



	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)

		managed_pages +=

			zone_managed_pages(&pgdat->node_zones[zone_type]);

	return managed_pages * PAGE_SIZE;

}



/**

 * ttm_pool_mgr_init - Initialize globals

 *
@@ -1356,8 +1381,15 @@ int ttm_pool_mgr_init(unsigned long num_pages) 
{

	unsigned int i;



	if (!page_pool_size)

		page_pool_size = num_pages;

	int nid;

	for_each_node(nid) {

		if (!page_pool_size) {

			u64 node_size = ttm_get_node_memory_size(nid);

			pool_node_limit[nid] = (node_size >> PAGE_SHIFT) / 2;

		} else {

			pool_node_limit[nid] = page_pool_size;

		}

	}



	spin_lock_init(&shrinker_lock);

	INIT_LIST_HEAD(&shrinker_list);