Unverified Commit 2b624a2c authored by Maarten Lankhorst's avatar Maarten Lankhorst Committed by Maxime Ripard
Browse files

drm/ttm: Handle cgroup based eviction in TTM 



cgroup resource allocation has to be handled in TTM, so -EAGAIN from
cgroups can be converted into -ENOSPC, and the limitcg can be properly
evicted in ttm code.

When hitting a resource limit through -EAGAIN, the cgroup for which the
limit is hit is also returned. This allows eviction to delete only from
cgroups which are a subgroup of the current cgroup.

The returned CSS is used to determine if eviction is valuable for a
given resource, and allows TTM to only target specific resources to
lower memory usage.

Co-developed-by: default avatarFriedrich Vock <friedrich.vock@gmx.de>
Signed-off-by: default avatarFriedrich Vock <friedrich.vock@gmx.de>
Co-developed-by: default avatarMaxime Ripard <mripard@kernel.org>
Signed-off-by: default avatarMaarten Lankhorst <dev@lankhorst.se>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Link: https://lore.kernel.org/r/20241204134410.1161769-4-dev@lankhorst.se


Signed-off-by: default avatarMaxime Ripard <mripard@kernel.org>
parent 7b0af165

drivers/gpu/drm/ttm/tests/ttm_bo_test.c

+9 −9
Original line number Diff line number Diff line
@@ -258,13 +258,13 @@ static void ttm_bo_unreserve_basic(struct kunit *test) 
	bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL);

	bo->priority = bo_prio;



	err = ttm_resource_alloc(bo, place, &res1);

	err = ttm_resource_alloc(bo, place, &res1, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);



	bo->resource = res1;



	/* Add a dummy resource to populate LRU */

	ttm_resource_alloc(bo, place, &res2);

	ttm_resource_alloc(bo, place, &res2, NULL);



	dma_resv_lock(bo->base.resv, NULL);

	ttm_bo_unreserve(bo);
@@ -300,12 +300,12 @@ static void ttm_bo_unreserve_pinned(struct kunit *test) 
	dma_resv_lock(bo->base.resv, NULL);

	ttm_bo_pin(bo);



	err = ttm_resource_alloc(bo, place, &res1);

	err = ttm_resource_alloc(bo, place, &res1, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	bo->resource = res1;



	/* Add a dummy resource to the pinned list */

	err = ttm_resource_alloc(bo, place, &res2);

	err = ttm_resource_alloc(bo, place, &res2, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	KUNIT_ASSERT_EQ(test,

			list_is_last(&res2->lru.link, &priv->ttm_dev->unevictable), 1);
@@ -355,7 +355,7 @@ static void ttm_bo_unreserve_bulk(struct kunit *test) 
	ttm_bo_set_bulk_move(bo1, &lru_bulk_move);

	dma_resv_unlock(bo1->base.resv);



	err = ttm_resource_alloc(bo1, place, &res1);

	err = ttm_resource_alloc(bo1, place, &res1, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	bo1->resource = res1;
@@ -363,7 +363,7 @@ static void ttm_bo_unreserve_bulk(struct kunit *test) 
	ttm_bo_set_bulk_move(bo2, &lru_bulk_move);

	dma_resv_unlock(bo2->base.resv);



	err = ttm_resource_alloc(bo2, place, &res2);

	err = ttm_resource_alloc(bo2, place, &res2, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	bo2->resource = res2;
@@ -401,7 +401,7 @@ static void ttm_bo_put_basic(struct kunit *test) 
	bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL);

	bo->type = ttm_bo_type_device;



	err = ttm_resource_alloc(bo, place, &res);

	err = ttm_resource_alloc(bo, place, &res, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	bo->resource = res;
@@ -518,7 +518,7 @@ static void ttm_bo_pin_unpin_resource(struct kunit *test) 


	bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL);



	err = ttm_resource_alloc(bo, place, &res);

	err = ttm_resource_alloc(bo, place, &res, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	bo->resource = res;
@@ -569,7 +569,7 @@ static void ttm_bo_multiple_pin_one_unpin(struct kunit *test) 


	bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL);



	err = ttm_resource_alloc(bo, place, &res);

	err = ttm_resource_alloc(bo, place, &res, NULL);

	KUNIT_ASSERT_EQ(test, err, 0);

	bo->resource = res;

drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c

+2 −2
Original line number Diff line number Diff line
@@ -542,7 +542,7 @@ static void ttm_bo_validate_no_placement_signaled(struct kunit *test) 
		bo->ttm = old_tt;

	}



	err = ttm_resource_alloc(bo, place, &bo->resource);

	err = ttm_resource_alloc(bo, place, &bo->resource, NULL);

	KUNIT_EXPECT_EQ(test, err, 0);

	KUNIT_ASSERT_EQ(test, man->usage, size);
@@ -603,7 +603,7 @@ static void ttm_bo_validate_no_placement_not_signaled(struct kunit *test) 
	bo = ttm_bo_kunit_init(test, test->priv, size, NULL);

	bo->type = params->bo_type;



	err = ttm_resource_alloc(bo, place, &bo->resource);

	err = ttm_resource_alloc(bo, place, &bo->resource, NULL);

	KUNIT_EXPECT_EQ(test, err, 0);



	placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL);

drivers/gpu/drm/ttm/tests/ttm_resource_test.c

+1 −1
Original line number Diff line number Diff line
@@ -302,7 +302,7 @@ static void ttm_sys_man_free_basic(struct kunit *test) 
	res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL);

	KUNIT_ASSERT_NOT_NULL(test, res);



	ttm_resource_alloc(bo, place, &res);

	ttm_resource_alloc(bo, place, &res, NULL);



	man = ttm_manager_type(priv->devs->ttm_dev, mem_type);

	man->func->free(man, res);

drivers/gpu/drm/ttm/ttm_bo.c

+45 −7
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ 
#include <linux/file.h>

#include <linux/module.h>

#include <linux/atomic.h>

#include <linux/cgroup_dmem.h>

#include <linux/dma-resv.h>



#include "ttm_module.h"
@@ -499,6 +500,13 @@ struct ttm_bo_evict_walk { 
	struct ttm_resource **res;

	/** @evicted: Number of successful evictions. */

	unsigned long evicted;



	/** @limit_pool: Which pool limit we should test against */

	struct dmem_cgroup_pool_state *limit_pool;

	/** @try_low: Whether we should attempt to evict BO's with low watermark threshold */

	bool try_low;

	/** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */

	bool hit_low;

};



static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
@@ -507,6 +515,10 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * 
		container_of(walk, typeof(*evict_walk), walk);

	s64 lret;



	if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css,

					      evict_walk->try_low, &evict_walk->hit_low))

		return 0;



	if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place))

		return 0;
@@ -524,7 +536,7 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * 
	evict_walk->evicted++;

	if (evict_walk->res)

		lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place,

					  evict_walk->res);

					  evict_walk->res, NULL);

	if (lret == 0)

		return 1;

out:
@@ -545,7 +557,8 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, 
			      struct ttm_buffer_object *evictor,

			      struct ttm_operation_ctx *ctx,

			      struct ww_acquire_ctx *ticket,

			      struct ttm_resource **res)

			      struct ttm_resource **res,

			      struct dmem_cgroup_pool_state *limit_pool)

{

	struct ttm_bo_evict_walk evict_walk = {

		.walk = {
@@ -556,22 +569,39 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, 
		.place = place,

		.evictor = evictor,

		.res = res,

		.limit_pool = limit_pool,

	};

	s64 lret;



	evict_walk.walk.trylock_only = true;

	lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1);



	/* One more attempt if we hit low limit? */

	if (!lret && evict_walk.hit_low) {

		evict_walk.try_low = true;

		lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1);

	}

	if (lret || !ticket)

		goto out;



	/* Reset low limit */

	evict_walk.try_low = evict_walk.hit_low = false;

	/* If ticket-locking, repeat while making progress. */

	evict_walk.walk.trylock_only = false;



retry:

	do {

		/* The walk may clear the evict_walk.walk.ticket field */

		evict_walk.walk.ticket = ticket;

		evict_walk.evicted = 0;

		lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1);

	} while (!lret && evict_walk.evicted);



	/* We hit the low limit? Try once more */

	if (!lret && evict_walk.hit_low && !evict_walk.try_low) {

		evict_walk.try_low = true;

		goto retry;

	}

out:

	if (lret < 0)

		return lret;
@@ -689,6 +719,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, 


	for (i = 0; i < placement->num_placement; ++i) {

		const struct ttm_place *place = &placement->placement[i];

		struct dmem_cgroup_pool_state *limit_pool = NULL;

		struct ttm_resource_manager *man;

		bool may_evict;
@@ -701,15 +732,20 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, 
			continue;



		may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM);

		ret = ttm_resource_alloc(bo, place, res);

		ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL);

		if (ret) {

			if (ret != -ENOSPC)

			if (ret != -ENOSPC && ret != -EAGAIN) {

				dmem_cgroup_pool_state_put(limit_pool);

				return ret;

			if (!may_evict)

			}

			if (!may_evict) {

				dmem_cgroup_pool_state_put(limit_pool);

				continue;

			}



			ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx,

						 ticket, res);

						 ticket, res, limit_pool);

			dmem_cgroup_pool_state_put(limit_pool);

			if (ret == -EBUSY)

				continue;

			if (ret)
@@ -1056,6 +1092,8 @@ struct ttm_bo_swapout_walk { 
	struct ttm_lru_walk walk;

	/** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */

	gfp_t gfp_flags;



	bool hit_low, evict_low;

};



static s64
@@ -1106,7 +1144,7 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) 


		memset(&hop, 0, sizeof(hop));

		place.mem_type = TTM_PL_SYSTEM;

		ret = ttm_resource_alloc(bo, &place, &evict_mem);

		ret = ttm_resource_alloc(bo, &place, &evict_mem, NULL);

		if (ret)

			goto out;

drivers/gpu/drm/ttm/ttm_resource.c

+21 −2
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ 
#include <linux/io-mapping.h>

#include <linux/iosys-map.h>

#include <linux/scatterlist.h>

#include <linux/cgroup_dmem.h>



#include <drm/ttm/ttm_bo.h>

#include <drm/ttm/ttm_placement.h>
@@ -350,15 +351,28 @@ EXPORT_SYMBOL(ttm_resource_fini); 


int ttm_resource_alloc(struct ttm_buffer_object *bo,

		       const struct ttm_place *place,

		       struct ttm_resource **res_ptr)

		       struct ttm_resource **res_ptr,

		       struct dmem_cgroup_pool_state **ret_limit_pool)

{

	struct ttm_resource_manager *man =

		ttm_manager_type(bo->bdev, place->mem_type);

	struct dmem_cgroup_pool_state *pool = NULL;

	int ret;



	ret = man->func->alloc(man, bo, place, res_ptr);

	if (man->cg) {

		ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool);

		if (ret)

			return ret;

	}



	ret = man->func->alloc(man, bo, place, res_ptr);

	if (ret) {

		if (pool)

			dmem_cgroup_uncharge(pool, bo->base.size);

		return ret;

	}



	(*res_ptr)->css = pool;



	spin_lock(&bo->bdev->lru_lock);

	ttm_resource_add_bulk_move(*res_ptr, bo);
@@ -370,6 +384,7 @@ EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc); 
void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)

{

	struct ttm_resource_manager *man;

	struct dmem_cgroup_pool_state *pool;



	if (!*res)

		return;
@@ -377,9 +392,13 @@ void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) 
	spin_lock(&bo->bdev->lru_lock);

	ttm_resource_del_bulk_move(*res, bo);

	spin_unlock(&bo->bdev->lru_lock);



	pool = (*res)->css;

	man = ttm_manager_type(bo->bdev, (*res)->mem_type);

	man->func->free(man, *res);

	*res = NULL;

	if (man->cg)

		dmem_cgroup_uncharge(pool, bo->base.size);

}

EXPORT_SYMBOL(ttm_resource_free);