Commit 00c8efc3 authored by Thomas Hellström's avatar Thomas Hellström
Browse files

drm/xe: Add a shrinker for xe bos 



Rather than relying on the TTM watermark accounting add a shrinker
for xe_bos in TT or system memory.

Leverage the newly added TTM per-page shrinking and shmem backup
support.

Although xe doesn't fully support WONTNEED (purgeable) bos yet,
introduce and add shrinker support for purgeable ttm_tts.

v2:
- Cleanups bugfixes and a KUNIT shrinker test.
- Add writeback support, and activate if kswapd.
v3:
- Move the try_shrink() helper to core TTM.
- Minor cleanups.
v4:
- Add runtime pm for the shrinker. Shrinking may require an active
  device for CCS metadata copying.
v5:
- Separately purge ghost- and zombie objects in the shrinker.
- Fix a format specifier - type inconsistency. (Kernel test robot).
v7:
- s/long/s64/ (Christian König)
- s/sofar/progress/ (Matt Brost)
v8:
- Rebase on Xe KUNIT update.
- Add content verifying to the shrinker kunit test.
- Split out TTM changes to a separate patch.
- Get rid of multiple bool arguments for clarity (Matt Brost)
- Avoid an error pointer dereference (Matt Brost)
- Avoid an integer overflow (Matt Auld)
- Address misc review comments by Matt Brost.
v9:
- Fix a compliation error.
- Rebase.
v10:
- Update to new LRU walk interface.
- Rework ghost-, zombie and purged object shrinking.
- Rebase.
v11:
- Use additional TTM helpers.
- Honor __GFP_FS and __GFP_IO
- Rebase.
v13:
- Use ttm_tt_setup_backup().
v14:
- Don't set up backup on imported bos.
v15:
- Rebase on backup interface changes.

Cc: Christian König <christian.koenig@amd.com>
Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: <dri-devel@lists.freedesktop.org>
Signed-off-by: default avatarThomas Hellström <thomas.hellstrom@linux.intel.com>
Reviewed-by: default avatarMatthew Brost <matthew.brost@intel.com>
Acked-by: default avatarChristian König <christian.koenig@amd.com>
Link: https://lore.kernel.org/intel-xe/20250305092220.123405-7-thomas.hellstrom@linux.intel.com
parent 70d645de

drivers/gpu/drm/xe/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -94,6 +94,7 @@ xe-y += xe_bb.o \ 
	xe_ring_ops.o \

	xe_sa.o \

	xe_sched_job.o \

	xe_shrinker.o \

	xe_step.o \

	xe_sync.o \

	xe_tile.o \

drivers/gpu/drm/xe/tests/xe_bo.c

+5 −1
Original line number Diff line number Diff line
@@ -514,8 +514,13 @@ static int shrink_test_run_device(struct xe_device *xe) 
		 * other way around, they may not be subject to swapping...

		 */

		if (alloced < purgeable) {

			xe_ttm_tt_account_subtract(&xe_tt->ttm);

			xe_tt->purgeable = true;

			xe_ttm_tt_account_add(&xe_tt->ttm);

			bo->ttm.priority = 0;

			spin_lock(&bo->ttm.bdev->lru_lock);

			ttm_bo_move_to_lru_tail(&bo->ttm);

			spin_unlock(&bo->ttm.bdev->lru_lock);

		} else {

			int ret = shrink_test_fill_random(bo, &prng, link);
@@ -570,7 +575,6 @@ static int shrink_test_run_device(struct xe_device *xe) 
				if (ret == -EINTR)

					intr = true;

			} while (ret == -EINTR && !signal_pending(current));



			if (!ret && !purgeable)

				failed = shrink_test_verify(test, bo, count, &prng, link);

drivers/gpu/drm/xe/xe_bo.c

+185 −17
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#include <drm/drm_drv.h>

#include <drm/drm_gem_ttm_helper.h>

#include <drm/drm_managed.h>

#include <drm/ttm/ttm_backup.h>

#include <drm/ttm/ttm_device.h>

#include <drm/ttm/ttm_placement.h>

#include <drm/ttm/ttm_tt.h>
@@ -25,6 +26,7 @@ 
#include "xe_pm.h"

#include "xe_preempt_fence.h"

#include "xe_res_cursor.h"

#include "xe_shrinker.h"

#include "xe_trace_bo.h"

#include "xe_ttm_stolen_mgr.h"

#include "xe_vm.h"
@@ -281,9 +283,11 @@ static void xe_evict_flags(struct ttm_buffer_object *tbo, 
	}

}



/* struct xe_ttm_tt - Subclassed ttm_tt for xe */

struct xe_ttm_tt {

	struct ttm_tt ttm;

	struct device *dev;

	/** @xe - The xe device */

	struct xe_device *xe;

	struct sg_table sgt;

	struct sg_table *sg;

	/** @purgeable: Whether the content of the pages of @ttm is purgeable. */
@@ -296,7 +300,8 @@ static int xe_tt_map_sg(struct ttm_tt *tt) 
	unsigned long num_pages = tt->num_pages;

	int ret;



	XE_WARN_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL);

	XE_WARN_ON((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&

		   !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE));



	if (xe_tt->sg)

		return 0;
@@ -304,13 +309,13 @@ static int xe_tt_map_sg(struct ttm_tt *tt) 
	ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages,

						num_pages, 0,

						(u64)num_pages << PAGE_SHIFT,

						xe_sg_segment_size(xe_tt->dev),

						xe_sg_segment_size(xe_tt->xe->drm.dev),

						GFP_KERNEL);

	if (ret)

		return ret;



	xe_tt->sg = &xe_tt->sgt;

	ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL,

	ret = dma_map_sgtable(xe_tt->xe->drm.dev, xe_tt->sg, DMA_BIDIRECTIONAL,

			      DMA_ATTR_SKIP_CPU_SYNC);

	if (ret) {

		sg_free_table(xe_tt->sg);
@@ -326,7 +331,7 @@ static void xe_tt_unmap_sg(struct ttm_tt *tt) 
	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);



	if (xe_tt->sg) {

		dma_unmap_sgtable(xe_tt->dev, xe_tt->sg,

		dma_unmap_sgtable(xe_tt->xe->drm.dev, xe_tt->sg,

				  DMA_BIDIRECTIONAL, 0);

		sg_free_table(xe_tt->sg);

		xe_tt->sg = NULL;
@@ -341,21 +346,47 @@ struct sg_table *xe_bo_sg(struct xe_bo *bo) 
	return xe_tt->sg;

}



/*

 * Account ttm pages against the device shrinker's shrinkable and

 * purgeable counts.

 */

static void xe_ttm_tt_account_add(struct ttm_tt *tt)

{

	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);



	if (xe_tt->purgeable)

		xe_shrinker_mod_pages(xe_tt->xe->mem.shrinker, 0, tt->num_pages);

	else

		xe_shrinker_mod_pages(xe_tt->xe->mem.shrinker, tt->num_pages, 0);

}



static void xe_ttm_tt_account_subtract(struct ttm_tt *tt)

{

	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);



	if (xe_tt->purgeable)

		xe_shrinker_mod_pages(xe_tt->xe->mem.shrinker, 0, -(long)tt->num_pages);

	else

		xe_shrinker_mod_pages(xe_tt->xe->mem.shrinker, -(long)tt->num_pages, 0);

}



static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,

				       u32 page_flags)

{

	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);

	struct xe_device *xe = xe_bo_device(bo);

	struct xe_ttm_tt *tt;

	struct xe_ttm_tt *xe_tt;

	struct ttm_tt *tt;

	unsigned long extra_pages;

	enum ttm_caching caching = ttm_cached;

	int err;



	tt = kzalloc(sizeof(*tt), GFP_KERNEL);

	if (!tt)

	xe_tt = kzalloc(sizeof(*xe_tt), GFP_KERNEL);

	if (!xe_tt)

		return NULL;



	tt->dev = xe->drm.dev;

	tt = &xe_tt->ttm;

	xe_tt->xe = xe;



	extra_pages = 0;

	if (xe_bo_needs_ccs_pages(bo))
@@ -401,42 +432,66 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, 
		caching = ttm_uncached;

	}



	err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages);

	if (ttm_bo->type != ttm_bo_type_sg)

		page_flags |= TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE;



	err = ttm_tt_init(tt, &bo->ttm, page_flags, caching, extra_pages);

	if (err) {

		kfree(tt);

		kfree(xe_tt);

		return NULL;

	}



	if (ttm_bo->type != ttm_bo_type_sg) {

		err = ttm_tt_setup_backup(tt);

		if (err) {

			ttm_tt_fini(tt);

			kfree(xe_tt);

			return NULL;

		}

	}



	return &tt->ttm;

	return tt;

}



static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,

			      struct ttm_operation_ctx *ctx)

{

	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);

	int err;



	/*

	 * dma-bufs are not populated with pages, and the dma-

	 * addresses are set up when moved to XE_PL_TT.

	 */

	if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)

	if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&

	    !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))

		return 0;



	if (ttm_tt_is_backed_up(tt) && !xe_tt->purgeable) {

		err = ttm_tt_restore(ttm_dev, tt, ctx);

	} else {

		ttm_tt_clear_backed_up(tt);

		err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx);

	}

	if (err)

		return err;



	return err;

	xe_tt->purgeable = false;

	xe_ttm_tt_account_add(tt);



	return 0;

}



static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt)

{

	if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)

	if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&

	    !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))

		return;



	xe_tt_unmap_sg(tt);



	return ttm_pool_free(&ttm_dev->pool, tt);

	ttm_pool_free(&ttm_dev->pool, tt);

	xe_ttm_tt_account_subtract(tt);

}



static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)
@@ -871,6 +926,111 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, 
	return ret;

}



static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx,

			       struct ttm_buffer_object *bo,

			       unsigned long *scanned)

{

	long lret;



	/* Fake move to system, without copying data. */

	if (bo->resource->mem_type != XE_PL_SYSTEM) {

		struct ttm_resource *new_resource;



		lret = ttm_bo_wait_ctx(bo, ctx);

		if (lret)

			return lret;



		lret = ttm_bo_mem_space(bo, &sys_placement, &new_resource, ctx);

		if (lret)

			return lret;



		xe_tt_unmap_sg(bo->ttm);

		ttm_bo_move_null(bo, new_resource);

	}



	*scanned += bo->ttm->num_pages;

	lret = ttm_bo_shrink(ctx, bo, (struct ttm_bo_shrink_flags)

			     {.purge = true,

			      .writeback = false,

			      .allow_move = false});



	if (lret > 0)

		xe_ttm_tt_account_subtract(bo->ttm);



	return lret;

}



/**

 * xe_bo_shrink() - Try to shrink an xe bo.

 * @ctx: The struct ttm_operation_ctx used for shrinking.

 * @bo: The TTM buffer object whose pages to shrink.

 * @flags: Flags governing the shrink behaviour.

 * @scanned: Pointer to a counter of the number of pages

 * attempted to shrink.

 *

 * Try to shrink- or purge a bo, and if it succeeds, unmap dma.

 * Note that we need to be able to handle also non xe bos

 * (ghost bos), but only if the struct ttm_tt is embedded in

 * a struct xe_ttm_tt. When the function attempts to shrink

 * the pages of a buffer object, The value pointed to by @scanned

 * is updated.

 *

 * Return: The number of pages shrunken or purged, or negative error

 * code on failure.

 */

long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,

		  const struct xe_bo_shrink_flags flags,

		  unsigned long *scanned)

{

	struct ttm_tt *tt = bo->ttm;

	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);

	struct ttm_place place = {.mem_type = bo->resource->mem_type};

	struct xe_bo *xe_bo = ttm_to_xe_bo(bo);

	struct xe_device *xe = xe_tt->xe;

	bool needs_rpm;

	long lret = 0L;



	if (!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE) ||

	    (flags.purge && !xe_tt->purgeable))

		return -EBUSY;



	if (!ttm_bo_eviction_valuable(bo, &place))

		return -EBUSY;



	if (!xe_bo_is_xe_bo(bo) || !xe_bo_get_unless_zero(xe_bo))

		return xe_bo_shrink_purge(ctx, bo, scanned);



	if (xe_tt->purgeable) {

		if (bo->resource->mem_type != XE_PL_SYSTEM)

			lret = xe_bo_move_notify(xe_bo, ctx);

		if (!lret)

			lret = xe_bo_shrink_purge(ctx, bo, scanned);

		goto out_unref;

	}



	/* System CCS needs gpu copy when moving PL_TT -> PL_SYSTEM */

	needs_rpm = (!IS_DGFX(xe) && bo->resource->mem_type != XE_PL_SYSTEM &&

		     xe_bo_needs_ccs_pages(xe_bo));

	if (needs_rpm && !xe_pm_runtime_get_if_active(xe))

		goto out_unref;



	*scanned += tt->num_pages;

	lret = ttm_bo_shrink(ctx, bo, (struct ttm_bo_shrink_flags)

			     {.purge = false,

			      .writeback = flags.writeback,

			      .allow_move = true});

	if (needs_rpm)

		xe_pm_runtime_put(xe);



	if (lret > 0)

		xe_ttm_tt_account_subtract(tt);



out_unref:

	xe_bo_put(xe_bo);



	return lret;

}



/**

 * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory

 * @bo: The buffer object to move.
@@ -1885,6 +2045,8 @@ int xe_bo_pin_external(struct xe_bo *bo) 
	}



	ttm_bo_pin(&bo->ttm);

	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))

		xe_ttm_tt_account_subtract(bo->ttm.ttm);



	/*

	 * FIXME: If we always use the reserve / unreserve functions for locking
@@ -1944,6 +2106,8 @@ int xe_bo_pin(struct xe_bo *bo) 
	}



	ttm_bo_pin(&bo->ttm);

	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))

		xe_ttm_tt_account_subtract(bo->ttm.ttm);



	/*

	 * FIXME: If we always use the reserve / unreserve functions for locking
@@ -1978,6 +2142,8 @@ void xe_bo_unpin_external(struct xe_bo *bo) 
	spin_unlock(&xe->pinned.lock);



	ttm_bo_unpin(&bo->ttm);

	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))

		xe_ttm_tt_account_add(bo->ttm.ttm);



	/*

	 * FIXME: If we always use the reserve / unreserve functions for locking
@@ -2001,6 +2167,8 @@ void xe_bo_unpin(struct xe_bo *bo) 
		spin_unlock(&xe->pinned.lock);

	}

	ttm_bo_unpin(&bo->ttm);

	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))

		xe_ttm_tt_account_add(bo->ttm.ttm);

}



/**

drivers/gpu/drm/xe/xe_bo.h

+36 −0
Original line number Diff line number Diff line
@@ -146,6 +146,28 @@ static inline struct xe_bo *xe_bo_get(struct xe_bo *bo) 


void xe_bo_put(struct xe_bo *bo);



/*

 * xe_bo_get_unless_zero() - Conditionally obtain a GEM object refcount on an

 * xe bo

 * @bo: The bo for which we want to obtain a refcount.

 *

 * There is a short window between where the bo's GEM object refcount reaches

 * zero and where we put the final ttm_bo reference. Code in the eviction- and

 * shrinking path should therefore attempt to grab a gem object reference before

 * trying to use members outside of the base class ttm object. This function is

 * intended for that purpose. On successful return, this function must be paired

 * with an xe_bo_put().

 *

 * Return: @bo on success, NULL on failure.

 */

static inline __must_check struct xe_bo *xe_bo_get_unless_zero(struct xe_bo *bo)

{

	if (!bo || !kref_get_unless_zero(&bo->ttm.base.refcount))

		return NULL;



	return bo;

}



static inline void __xe_bo_unset_bulk_move(struct xe_bo *bo)

{

	if (bo)
@@ -341,6 +363,20 @@ static inline unsigned int xe_sg_segment_size(struct device *dev) 
	return round_down(max / 2, PAGE_SIZE);

}



/**

 * struct xe_bo_shrink_flags - flags governing the shrink behaviour.

 * @purge: Only purging allowed. Don't shrink if bo not purgeable.

 * @writeback: Attempt to immediately move content to swap.

 */

struct xe_bo_shrink_flags {

	u32 purge : 1;

	u32 writeback : 1;

};



long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,

		  const struct xe_bo_shrink_flags flags,

		  unsigned long *scanned);



#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)

/**

 * xe_bo_is_mem_type - Whether the bo currently resides in the given

drivers/gpu/drm/xe/xe_device.c

+8 −0
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ 
#include "xe_pcode.h"

#include "xe_pm.h"

#include "xe_query.h"

#include "xe_shrinker.h"

#include "xe_sriov.h"

#include "xe_tile.h"

#include "xe_ttm_stolen_mgr.h"
@@ -289,6 +290,9 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy) 
	if (xe->unordered_wq)

		destroy_workqueue(xe->unordered_wq);



	if (!IS_ERR_OR_NULL(xe->mem.shrinker))

		xe_shrinker_destroy(xe->mem.shrinker);



	if (xe->destroy_wq)

		destroy_workqueue(xe->destroy_wq);
@@ -321,6 +325,10 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, 
	if (err)

		goto err;



	xe->mem.shrinker = xe_shrinker_create(xe);

	if (IS_ERR(xe->mem.shrinker))

		return ERR_CAST(xe->mem.shrinker);



	xe->info.devid = pdev->device;

	xe->info.revid = pdev->revision;

	xe->info.force_execlist = xe_modparam.force_execlist;