Commit 7f387e60 authored by Matthew Auld's avatar Matthew Auld
Browse files

drm/xe: add XE_BO_FLAG_PINNED_LATE_RESTORE 



With the idea of having more pinned objects using the blitter engine
where possible, during suspend/resume, mark the pinned objects which
can be done during the late phase once submission/migration has been
setup. Start out simple with lrc and page-tables from userspace.

v2:
 - s/early_restore/late_restore; early restore was way too bold with too
   many places being impacted at once.
v3:
 - Split late vs early into separate lists, to align with newly added
   apply-to-pinned infra.
v4:
 - Rebase.
v5:
 - Make sure we restore the late phase kernel_bo_present in igpu.

Signed-off-by: default avatarMatthew Auld <matthew.auld@intel.com>
Cc: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Reviewed-by: default avatarSatyanarayana K V P <satyanarayana.k.v.p@intel.com>
Link: https://lore.kernel.org/r/20250403102440.266113-13-matthew.auld@intel.com
parent 58fa61ce

drivers/gpu/drm/xe/tests/xe_bo.c

+2 −2
Original line number Diff line number Diff line
@@ -252,7 +252,7 @@ static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struc 


		for_each_gt(__gt, xe, id)

			xe_gt_sanitize(__gt);

		err = xe_bo_restore_kernel(xe);

		err = xe_bo_restore_early(xe);

		/*

		 * Snapshotting the CTB and copying back a potentially old

		 * version seems risky, depending on what might have been
@@ -273,7 +273,7 @@ static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struc 
			goto cleanup_all;

		}



		err = xe_bo_restore_user(xe);

		err = xe_bo_restore_late(xe);

		if (err) {

			KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err));

			goto cleanup_all;

drivers/gpu/drm/xe/xe_bo.c

+7 −4
Original line number Diff line number Diff line
@@ -1121,7 +1121,7 @@ int xe_bo_evict_pinned(struct xe_bo *bo) 
		goto out_unlock_bo;

	}



	if (xe_bo_is_user(bo)) {

	if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) {

		struct xe_migrate *migrate;

		struct dma_fence *fence;
@@ -1216,7 +1216,7 @@ int xe_bo_restore_pinned(struct xe_bo *bo) 
		goto out_backup;

	}



	if (xe_bo_is_user(bo)) {

	if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) {

		struct xe_migrate *migrate;

		struct dma_fence *fence;
@@ -2187,7 +2187,7 @@ int xe_bo_pin_external(struct xe_bo *bo) 
			return err;



		spin_lock(&xe->pinned.lock);

		list_add_tail(&bo->pinned_link, &xe->pinned.external);

		list_add_tail(&bo->pinned_link, &xe->pinned.late.external);

		spin_unlock(&xe->pinned.lock);

	}
@@ -2232,7 +2232,10 @@ int xe_bo_pin(struct xe_bo *bo) 


	if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {

		spin_lock(&xe->pinned.lock);

		list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present);

		if (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)

			list_add_tail(&bo->pinned_link, &xe->pinned.late.kernel_bo_present);

		else

			list_add_tail(&bo->pinned_link, &xe->pinned.early.kernel_bo_present);

		spin_unlock(&xe->pinned.lock);

	}

drivers/gpu/drm/xe/xe_bo.h

+5 −4
Original line number Diff line number Diff line
@@ -40,10 +40,11 @@ 
#define XE_BO_FLAG_NEEDS_2M		BIT(16)

#define XE_BO_FLAG_GGTT_INVALIDATE	BIT(17)

#define XE_BO_FLAG_PINNED_NORESTORE	BIT(18)

#define XE_BO_FLAG_GGTT0                BIT(19)

#define XE_BO_FLAG_GGTT1                BIT(20)

#define XE_BO_FLAG_GGTT2                BIT(21)

#define XE_BO_FLAG_GGTT3                BIT(22)

#define XE_BO_FLAG_PINNED_LATE_RESTORE BIT(19)

#define XE_BO_FLAG_GGTT0                BIT(20)

#define XE_BO_FLAG_GGTT1                BIT(21)

#define XE_BO_FLAG_GGTT2                BIT(22)

#define XE_BO_FLAG_GGTT3                BIT(23)

#define XE_BO_FLAG_GGTT_ALL             (XE_BO_FLAG_GGTT0 | \

					 XE_BO_FLAG_GGTT1 | \

					 XE_BO_FLAG_GGTT2 | \

drivers/gpu/drm/xe/xe_bo_evict.c

+42 −22
Original line number Diff line number Diff line
@@ -91,10 +91,14 @@ int xe_bo_evict_all(struct xe_device *xe) 
		}

	}



	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.external,

				    &xe->pinned.external,

	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,

				    &xe->pinned.late.external,

				    xe_bo_evict_pinned);



	if (!ret)

		ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,

					    &xe->pinned.late.evicted, xe_bo_evict_pinned);



	/*

	 * Wait for all user BO to be evicted as those evictions depend on the

	 * memory moved below.
@@ -105,8 +109,8 @@ int xe_bo_evict_all(struct xe_device *xe) 
	if (ret)

		return ret;



	return xe_bo_apply_to_pinned(xe, &xe->pinned.kernel_bo_present,

				     &xe->pinned.evicted,

	return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,

				     &xe->pinned.early.evicted,

				     xe_bo_evict_pinned);

}
@@ -137,13 +141,14 @@ static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo) 
	 * We expect validate to trigger a move VRAM and our move code

	 * should setup the iosys map.

	 */

	xe_assert(xe, !iosys_map_is_null(&bo->vmap));

	xe_assert(xe, !(bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE) ||

		  !iosys_map_is_null(&bo->vmap));



	return 0;

}



/**

 * xe_bo_restore_kernel - restore kernel BOs to VRAM

 * xe_bo_restore_early - restore early phase kernel BOs to VRAM

 *

 * @xe: xe device

 *
@@ -153,34 +158,44 @@ static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo) 
 * This function should be called early, before trying to init the GT, on device

 * resume.

 */

int xe_bo_restore_kernel(struct xe_device *xe)

int xe_bo_restore_early(struct xe_device *xe)

{

	return xe_bo_apply_to_pinned(xe, &xe->pinned.evicted,

				     &xe->pinned.kernel_bo_present,

	return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted,

				     &xe->pinned.early.kernel_bo_present,

				     xe_bo_restore_and_map_ggtt);

}



/**

 * xe_bo_restore_user - restore pinned user BOs to VRAM

 * xe_bo_restore_late - restore pinned late phase BOs

 *

 * @xe: xe device

 *

 * Move pinned user BOs from temporary (typically system) memory to VRAM via

 * CPU. All moves done via TTM calls.

 * Move pinned user and kernel BOs which can use blitter from temporary

 * (typically system) memory to VRAM. All moves done via TTM calls.

 *

 * This function should be called late, after GT init, on device resume.

 */

int xe_bo_restore_user(struct xe_device *xe)

int xe_bo_restore_late(struct xe_device *xe)

{

	struct xe_tile *tile;

	int ret, id;



	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted,

				    &xe->pinned.late.kernel_bo_present,

				    xe_bo_restore_and_map_ggtt);



	for_each_tile(tile, xe, id)

		xe_tile_migrate_wait(tile);



	if (ret)

		return ret;



	if (!IS_DGFX(xe))

		return 0;



	/* Pinned user memory in VRAM should be validated on resume */

	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.external,

				    &xe->pinned.external,

	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,

				    &xe->pinned.late.external,

				    xe_bo_restore_pinned);



	/* Wait for restore to complete */
@@ -195,8 +210,8 @@ static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe) 
	struct xe_tile *tile;

	unsigned int id;



	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.external,

				    &xe->pinned.external,

	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,

				    &xe->pinned.late.external,

				    xe_bo_dma_unmap_pinned);

	for_each_tile(tile, xe, id)

		xe_tile_migrate_wait(tile);
@@ -241,8 +256,11 @@ static void xe_bo_pinned_fini(void *arg) 
{

	struct xe_device *xe = arg;



	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.kernel_bo_present,

				    &xe->pinned.kernel_bo_present,

	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,

				    &xe->pinned.late.kernel_bo_present,

				    xe_bo_dma_unmap_pinned);

	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,

				    &xe->pinned.early.kernel_bo_present,

				    xe_bo_dma_unmap_pinned);

}
@@ -259,9 +277,11 @@ static void xe_bo_pinned_fini(void *arg) 
int xe_bo_pinned_init(struct xe_device *xe)

{

	spin_lock_init(&xe->pinned.lock);

	INIT_LIST_HEAD(&xe->pinned.kernel_bo_present);

	INIT_LIST_HEAD(&xe->pinned.external);

	INIT_LIST_HEAD(&xe->pinned.evicted);

	INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present);

	INIT_LIST_HEAD(&xe->pinned.early.evicted);

	INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present);

	INIT_LIST_HEAD(&xe->pinned.late.evicted);

	INIT_LIST_HEAD(&xe->pinned.late.external);



	return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe);

}

drivers/gpu/drm/xe/xe_bo_evict.h

+2 −2
Original line number Diff line number Diff line
@@ -9,8 +9,8 @@ 
struct xe_device;



int xe_bo_evict_all(struct xe_device *xe);

int xe_bo_restore_kernel(struct xe_device *xe);

int xe_bo_restore_user(struct xe_device *xe);

int xe_bo_restore_early(struct xe_device *xe);

int xe_bo_restore_late(struct xe_device *xe);



void xe_bo_pci_dev_remove_all(struct xe_device *xe);