Commit 3756310e authored by Thomas Hellström's avatar Thomas Hellström Committed by Christian König
Browse files

drm/ttm: Use the LRU walker for eviction 



Use the LRU walker for eviction. This helps
removing a lot of code with weird locking
semantics.

The functionality is slightly changed so that
when trylocked buffer objects are exhausted, we
continue to interleave walks with ticket-locks while
there is still progress made. The list walks are
not restarted in-between evictions.

Also provide a separate ttm_bo_evict_first()
function for its single user. The context of that
user allows sleeping dma_resv locks.

v6:
- Various cleanups suggested by Matthew Brost.
- Fix error return code of ttm_bo_evict_first(). (Matthew Brost)
- Fix an error check that was inverted. (Matthew Brost)
v7:
- Use s64 rather than long (Christian König)
- Early ttm_resource_cursor_fini() in ttm_bo_evict_first().
- Simplify check for bo_moved in ttm_bo_evict_first().
  (Christian König)
- Don't evict pinned bos.

Cc: Christian König <christian.koenig@amd.com>
Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: <dri-devel@lists.freedesktop.org>
Signed-off-by: default avatarThomas Hellström <thomas.hellstrom@linux.intel.com>
Reviewed-by: Matthew Brost <matthew.brost@intel.com> #v6
Link: https://patchwork.freedesktop.org/patch/msgid/20240705153206.68526-8-thomas.hellstrom@linux.intel.com


Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
parent 10efe34d

drivers/gpu/drm/ttm/ttm_bo.c

+136 −209
Original line number Diff line number Diff line
@@ -224,80 +224,6 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 
	dma_resv_iter_end(&cursor);

}



/**

 * ttm_bo_cleanup_refs

 * If bo idle, remove from lru lists, and unref.

 * If not idle, block if possible.

 *

 * Must be called with lru_lock and reservation held, this function

 * will drop the lru lock and optionally the reservation lock before returning.

 *

 * @bo:                    The buffer object to clean-up

 * @interruptible:         Any sleeps should occur interruptibly.

 * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.

 * @unlock_resv:           Unlock the reservation lock as well.

 */



static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,

			       bool interruptible, bool no_wait_gpu,

			       bool unlock_resv)

{

	struct dma_resv *resv = &bo->base._resv;

	int ret;



	if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))

		ret = 0;

	else

		ret = -EBUSY;



	if (ret && !no_wait_gpu) {

		long lret;



		if (unlock_resv)

			dma_resv_unlock(bo->base.resv);

		spin_unlock(&bo->bdev->lru_lock);



		lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,

					     interruptible,

					     30 * HZ);



		if (lret < 0)

			return lret;

		else if (lret == 0)

			return -EBUSY;



		spin_lock(&bo->bdev->lru_lock);

		if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {

			/*

			 * We raced, and lost, someone else holds the reservation now,

			 * and is probably busy in ttm_bo_cleanup_memtype_use.

			 *

			 * Even if it's not the case, because we finished waiting any

			 * delayed destruction would succeed, so just return success

			 * here.

			 */

			spin_unlock(&bo->bdev->lru_lock);

			return 0;

		}

		ret = 0;

	}



	if (ret) {

		if (unlock_resv)

			dma_resv_unlock(bo->base.resv);

		spin_unlock(&bo->bdev->lru_lock);

		return ret;

	}



	spin_unlock(&bo->bdev->lru_lock);

	ttm_bo_cleanup_memtype_use(bo);



	if (unlock_resv)

		dma_resv_unlock(bo->base.resv);



	return 0;

}



/*

 * Block for the dma_resv object to become idle, lock the buffer and clean up

 * the resource and tt object.
@@ -505,151 +431,152 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 
}

EXPORT_SYMBOL(ttm_bo_eviction_valuable);



/*

 * Check the target bo is allowable to be evicted or swapout, including cases:

 *

 * a. if share same reservation object with ctx->resv, have assumption

 * reservation objects should already be locked, so not lock again and

 * return true directly when either the opreation allow_reserved_eviction

 * or the target bo already is in delayed free list;

/**

 * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list.

 * @bdev: The ttm device.

 * @man: The manager whose bo to evict.

 * @ctx: The TTM operation ctx governing the eviction.

 *

 * b. Otherwise, trylock it.

 * Return: 0 if successful or the resource disappeared. Negative error code on error.

 */

static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,

					   struct ttm_operation_ctx *ctx,

					   const struct ttm_place *place,

					   bool *locked, bool *busy)

int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man,

		       struct ttm_operation_ctx *ctx)

{

	bool ret = false;

	struct ttm_resource_cursor cursor;

	struct ttm_buffer_object *bo;

	struct ttm_resource *res;

	unsigned int mem_type;

	int ret = 0;



	if (bo->pin_count) {

		*locked = false;

		if (busy)

			*busy = false;

		return false;

	spin_lock(&bdev->lru_lock);

	res = ttm_resource_manager_first(man, &cursor);

	ttm_resource_cursor_fini(&cursor);

	if (!res) {

		ret = -ENOENT;

		goto out_no_ref;

	}

	bo = res->bo;

	if (!ttm_bo_get_unless_zero(bo))

		goto out_no_ref;

	mem_type = res->mem_type;

	spin_unlock(&bdev->lru_lock);

	ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL);

	if (ret)

		goto out_no_lock;

	if (!bo->resource || bo->resource->mem_type != mem_type)

		goto out_bo_moved;



	if (bo->base.resv == ctx->resv) {

		dma_resv_assert_held(bo->base.resv);

		if (ctx->allow_res_evict)

			ret = true;

		*locked = false;

		if (busy)

			*busy = false;

	if (bo->deleted) {

		ret = ttm_bo_wait_ctx(bo, ctx);

		if (!ret)

			ttm_bo_cleanup_memtype_use(bo);

	} else {

		ret = dma_resv_trylock(bo->base.resv);

		*locked = ret;

		if (busy)

			*busy = !ret;

		ret = ttm_bo_evict(bo, ctx);

	}



	if (ret && place && (bo->resource->mem_type != place->mem_type ||

		!bo->bdev->funcs->eviction_valuable(bo, place))) {

		ret = false;

		if (*locked) {

out_bo_moved:

	dma_resv_unlock(bo->base.resv);

			*locked = false;

		}

	}

out_no_lock:

	ttm_bo_put(bo);

	return ret;



out_no_ref:

	spin_unlock(&bdev->lru_lock);

	return ret;

}



/**

 * ttm_mem_evict_wait_busy - wait for a busy BO to become available

 *

 * @busy_bo: BO which couldn't be locked with trylock

 * @ctx: operation context

 * @ticket: acquire ticket

 *

 * Try to lock a busy buffer object to avoid failing eviction.

 * struct ttm_bo_evict_walk - Parameters for the evict walk.

 */

static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,

				   struct ttm_operation_ctx *ctx,

				   struct ww_acquire_ctx *ticket)

struct ttm_bo_evict_walk {

	/** @walk: The walk base parameters. */

	struct ttm_lru_walk walk;

	/** @place: The place passed to the resource allocation. */

	const struct ttm_place *place;

	/** @evictor: The buffer object we're trying to make room for. */

	struct ttm_buffer_object *evictor;

	/** @res: The allocated resource if any. */

	struct ttm_resource **res;

	/** @evicted: Number of successful evictions. */

	unsigned long evicted;

};



static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)

{

	int r;

	struct ttm_bo_evict_walk *evict_walk =

		container_of(walk, typeof(*evict_walk), walk);

	s64 lret;



	if (!busy_bo || !ticket)

		return -EBUSY;

	if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place))

		return 0;



	if (ctx->interruptible)

		r = dma_resv_lock_interruptible(busy_bo->base.resv,

							  ticket);

	else

		r = dma_resv_lock(busy_bo->base.resv, ticket);

	if (bo->deleted) {

		lret = ttm_bo_wait_ctx(bo, walk->ctx);

		if (!lret)

			ttm_bo_cleanup_memtype_use(bo);

	} else {

		lret = ttm_bo_evict(bo, walk->ctx);

	}



	/*

	 * TODO: It would be better to keep the BO locked until allocation is at

	 * least tried one more time, but that would mean a much larger rework

	 * of TTM.

	 */

	if (!r)

		dma_resv_unlock(busy_bo->base.resv);

	if (lret)

		goto out;



	evict_walk->evicted++;

	if (evict_walk->res)

		lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place,

					  evict_walk->res);

	if (lret == 0)

		return 1;

out:

	/* Errors that should terminate the walk. */

	if (lret == -ENOSPC)

		return -EBUSY;



	return r == -EDEADLK ? -EBUSY : r;

	return lret;

}



int ttm_mem_evict_first(struct ttm_device *bdev,

static const struct ttm_lru_walk_ops ttm_evict_walk_ops = {

	.process_bo = ttm_bo_evict_cb,

};



static int ttm_bo_evict_alloc(struct ttm_device *bdev,

			      struct ttm_resource_manager *man,

			      const struct ttm_place *place,

			      struct ttm_buffer_object *evictor,

			      struct ttm_operation_ctx *ctx,

			struct ww_acquire_ctx *ticket)

			      struct ww_acquire_ctx *ticket,

			      struct ttm_resource **res)

{

	struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;

	struct ttm_resource_cursor cursor;

	struct ttm_resource *res;

	bool locked = false;

	int ret;



	spin_lock(&bdev->lru_lock);

	ttm_resource_manager_for_each_res(man, &cursor, res) {

		bool busy;



		if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,

						    &locked, &busy)) {

			if (busy && !busy_bo && ticket !=

			    dma_resv_locking_ctx(res->bo->base.resv))

				busy_bo = res->bo;

			continue;

		}



		if (ttm_bo_get_unless_zero(res->bo)) {

			bo = res->bo;

			break;

		}

		if (locked)

			dma_resv_unlock(res->bo->base.resv);

	}

	ttm_resource_cursor_fini(&cursor);



	if (!bo) {

		if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))

			busy_bo = NULL;

		spin_unlock(&bdev->lru_lock);

		ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);

		if (busy_bo)

			ttm_bo_put(busy_bo);

		return ret;

	}



	if (bo->deleted) {

		ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,

					  ctx->no_wait_gpu, locked);

		ttm_bo_put(bo);

		return ret;

	}



	spin_unlock(&bdev->lru_lock);

	struct ttm_bo_evict_walk evict_walk = {

		.walk = {

			.ops = &ttm_evict_walk_ops,

			.ctx = ctx,

			.ticket = ticket,

		},

		.place = place,

		.evictor = evictor,

		.res = res,

	};

	s64 lret;



	ret = ttm_bo_evict(bo, ctx);

	if (locked)

		ttm_bo_unreserve(bo);

	else

		ttm_bo_move_to_lru_tail_unlocked(bo);

	evict_walk.walk.trylock_only = true;

	lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1);

	if (lret || !ticket)

		goto out;



	ttm_bo_put(bo);

	return ret;

	/* If ticket-locking, repeat while making progress. */

	evict_walk.walk.trylock_only = false;

	do {

		/* The walk may clear the evict_walk.walk.ticket field */

		evict_walk.walk.ticket = ticket;

		evict_walk.evicted = 0;

		lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1);

	} while (!lret && evict_walk.evicted);

out:

	if (lret < 0)

		return lret;

	if (lret == 0)

		return -EBUSY;

	return 0;

}



/**
@@ -760,6 +687,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, 
	for (i = 0; i < placement->num_placement; ++i) {

		const struct ttm_place *place = &placement->placement[i];

		struct ttm_resource_manager *man;

		bool may_evict;



		man = ttm_manager_type(bdev, place->mem_type);

		if (!man || !ttm_resource_manager_used(man))
@@ -769,22 +697,21 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, 
				    TTM_PL_FLAG_FALLBACK))

			continue;



		do {

		may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM);

		ret = ttm_resource_alloc(bo, place, res);

			if (unlikely(ret && ret != -ENOSPC))

		if (ret) {

			if (ret != -ENOSPC)

				return ret;

			if (likely(!ret) || !force_space)

				break;

			if (!may_evict)

				continue;



			ret = ttm_mem_evict_first(bdev, man, place, ctx,

						  ticket);

			if (unlikely(ret == -EBUSY))

				break;

			if (unlikely(ret))

				return ret;

		} while (1);

		if (ret)

			ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx,

						 ticket, res);

			if (ret == -EBUSY)

				continue;

			if (ret)

				return ret;

		}



		ret = ttm_bo_add_move_fence(bo, man, ctx->no_wait_gpu);

		if (unlikely(ret)) {

drivers/gpu/drm/ttm/ttm_resource.c

+4 −17
Original line number Diff line number Diff line
@@ -495,24 +495,11 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, 
	};

	struct dma_fence *fence;

	int ret;

	unsigned i;



	/*

	 * Can't use standard list traversal since we're unlocking.

	 */



	spin_lock(&bdev->lru_lock);

	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {

		while (!list_empty(&man->lru[i])) {

			spin_unlock(&bdev->lru_lock);

			ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,

						  NULL);

			if (ret)

				return ret;

			spin_lock(&bdev->lru_lock);

		}

	}

	spin_unlock(&bdev->lru_lock);

	do {

		ret = ttm_bo_evict_first(bdev, man, &ctx);

		cond_resched();

	} while (!ret);



	spin_lock(&man->move_lock);

	fence = dma_fence_get(man->move);

include/drm/ttm/ttm_bo.h

+3 −5
Original line number Diff line number Diff line
@@ -422,11 +422,9 @@ s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, 
		   s64 target);

void ttm_bo_pin(struct ttm_buffer_object *bo);

void ttm_bo_unpin(struct ttm_buffer_object *bo);

int ttm_mem_evict_first(struct ttm_device *bdev,

int ttm_bo_evict_first(struct ttm_device *bdev,

		       struct ttm_resource_manager *man,

			const struct ttm_place *place,

			struct ttm_operation_ctx *ctx,

			struct ww_acquire_ctx *ticket);

		       struct ttm_operation_ctx *ctx);

vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,

			     struct vm_fault *vmf);

vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,