Unverified Commit 92f3403a authored by Arvind Yadav's avatar Arvind Yadav Committed by Rodrigo Vivi
Browse files

drm/xe/madvise: Track purgeability with BO-local counters 



xe_bo_recompute_purgeable_state() walks all VMAs of a BO to determine
whether the BO can be made purgeable. This makes VMA create/destroy and
madvise updates O(n) in the number of mappings.

Replace the walk with BO-local counters protected by the BO dma-resv
lock:

  - vma_count tracks the number of VMAs mapping the BO.
  - willneed_count tracks active WILLNEED holders, including WILLNEED
    VMAs and active dma-buf exports for non-imported BOs.

A DONTNEED BO is promoted back to WILLNEED on a 0->1 transition of
willneed_count. A BO is demoted to DONTNEED on a 1->0 transition only
when it still has VMAs, preserving the previous behaviour where a BO
with no mappings keeps its current madvise state.

PURGED remains terminal, preserving the existing "once purged, always
purged" rule.

Fixes: 4f44961e ("drm/xe/vm: Prevent binding of purged buffer objects")

v2:
  - Use early return for imported BOs in all four helpers to avoid
    nesting (Matt B).
  - Group purgeability state into a purgeable sub-struct on struct
    xe_bo (Matt B).
  - Reword xe_bo_willneed_put_locked() kernel-doc to explain that a 1->0
    transition means all remaining active VMAs are DONTNEED (Matt B).

v3:
  - Move DONTNEED/PURGED reject from vma_lock_and_validate() into
    xe_vma_create(), gated on attr->purgeable_state == WILLNEED.
    Fixes vm_bind bypass and partial-unbind rejection on DONTNEED
    BOs (Matt B).
  - Drop .check_purged from MAP and REMAP; keep it for PREFETCH and
    add a comment why (Matt B).
  - Skip BO validation in vma_lock_and_validate() for non-WILLNEED
    VMA remnants so cleanup/remap paths do not repopulate
    DONTNEED/PURGED BOs.

Suggested-by: default avatarThomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Himal Prasad Ghimiray <himal.prasad.ghimiray@intel.com>
Signed-off-by: default avatarArvind Yadav <arvind.yadav@intel.com>
Reviewed-by: default avatarMatthew Brost <matthew.brost@intel.com>
Link: https://patch.msgid.link/20260506132027.2556046-1-arvind.yadav@intel.com


Signed-off-by: default avatarHimal Prasad Ghimiray <himal.prasad.ghimiray@intel.com>
(cherry picked from commit 23fb2ea56cb4fa2587bc072b04e4e698687a48e4)
Signed-off-by: default avatarRodrigo Vivi <rodrigo.vivi@intel.com>
parent 5d691905

drivers/gpu/drm/xe/xe_bo.c

+3 −3
Original line number Diff line number Diff line
@@ -897,10 +897,10 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo, 
		  new_state == XE_MADV_PURGEABLE_PURGED);



	/* Once purged, always purged - cannot transition out */

	xe_assert(xe, !(bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED &&

	xe_assert(xe, !(bo->purgeable.state == XE_MADV_PURGEABLE_PURGED &&

			new_state != XE_MADV_PURGEABLE_PURGED));



	bo->madv_purgeable = new_state;

	bo->purgeable.state = new_state;

	xe_bo_set_purgeable_shrinker(bo, new_state);

}
@@ -2368,7 +2368,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, 
	INIT_LIST_HEAD(&bo->vram_userfault_link);



	/* Initialize purge advisory state */

	bo->madv_purgeable = XE_MADV_PURGEABLE_WILLNEED;

	bo->purgeable.state = XE_MADV_PURGEABLE_WILLNEED;



	drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);

drivers/gpu/drm/xe/xe_bo.h

+86 −2
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ static inline bool xe_bo_is_protected(const struct xe_bo *bo) 
static inline bool xe_bo_is_purged(struct xe_bo *bo)

{

	xe_bo_assert_held(bo);

	return bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED;

	return bo->purgeable.state == XE_MADV_PURGEABLE_PURGED;

}



/**
@@ -268,11 +268,95 @@ static inline bool xe_bo_is_purged(struct xe_bo *bo) 
static inline bool xe_bo_madv_is_dontneed(struct xe_bo *bo)

{

	xe_bo_assert_held(bo);

	return bo->madv_purgeable == XE_MADV_PURGEABLE_DONTNEED;

	return bo->purgeable.state == XE_MADV_PURGEABLE_DONTNEED;

}



void xe_bo_set_purgeable_state(struct xe_bo *bo, enum xe_madv_purgeable_state new_state);



/**

 * xe_bo_willneed_get_locked() - Acquire a WILLNEED holder on a BO

 * @bo: Buffer object

 *

 * Increments willneed_count and, on a 0->1 transition, promotes the BO

 * from DONTNEED to WILLNEED. PURGED is terminal and is never modified.

 *

 * Caller must hold the BO's dma-resv lock.

 */

static inline void xe_bo_willneed_get_locked(struct xe_bo *bo)

{

	xe_bo_assert_held(bo);



	/* Imported BOs are owned externally; do not track purgeability. */

	if (drm_gem_is_imported(&bo->ttm.base))

		return;



	if (bo->purgeable.willneed_count++ == 0 && xe_bo_madv_is_dontneed(bo))

		xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_WILLNEED);

}



/**

 * xe_bo_willneed_put_locked() - Release a WILLNEED holder on a BO

 * @bo: Buffer object

 *

 * Decrements willneed_count and, on a 1->0 transition, marks the BO

 * DONTNEED only if it still has VMAs (implying all active VMAs are

 * DONTNEED). If the last VMA is being removed, preserve the current BO

 * state to match the previous VMA-walk semantics.

 *

 * PURGED is terminal and the BO state is never modified.

 *

 * Caller must hold the BO's dma-resv lock.

 */

static inline void xe_bo_willneed_put_locked(struct xe_bo *bo)

{

	xe_bo_assert_held(bo);



	if (drm_gem_is_imported(&bo->ttm.base))

		return;



	xe_assert(xe_bo_device(bo), bo->purgeable.willneed_count > 0);

	if (--bo->purgeable.willneed_count == 0 && bo->purgeable.vma_count > 0 &&

	    !xe_bo_is_purged(bo))

		xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED);

}



/**

 * xe_bo_vma_count_inc_locked() - Account a new VMA on a BO

 * @bo: Buffer object

 *

 * Increments vma_count.

 *

 * Caller must hold the BO's dma-resv lock.

 */

static inline void xe_bo_vma_count_inc_locked(struct xe_bo *bo)

{

	xe_bo_assert_held(bo);



	if (drm_gem_is_imported(&bo->ttm.base))

		return;



	bo->purgeable.vma_count++;

}



/**

 * xe_bo_vma_count_dec_locked() - Account a VMA removal on a BO

 * @bo: Buffer object

 *

 * Decrements vma_count.

 *

 * Caller must hold the BO's dma-resv lock.

 */

static inline void xe_bo_vma_count_dec_locked(struct xe_bo *bo)

{

	xe_bo_assert_held(bo);



	if (drm_gem_is_imported(&bo->ttm.base))

		return;



	xe_assert(xe_bo_device(bo), bo->purgeable.vma_count > 0);

	bo->purgeable.vma_count--;

}



static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo)

{

	if (likely(bo)) {

drivers/gpu/drm/xe/xe_bo_types.h

+25 −3
Original line number Diff line number Diff line
@@ -111,10 +111,32 @@ struct xe_bo { 
	u64 min_align;



	/**

	 * @madv_purgeable: user space advise on BO purgeability, protected

	 * by BO's dma-resv lock.

	 * @purgeable: Purgeability state and accounting.

	 *

	 * All fields are protected by the BO's dma-resv lock.

	 */

	u32 madv_purgeable;

	struct {

		/**

		 * @purgeable.state: BO purgeability state

		 *                   (WILLNEED/DONTNEED/PURGED).

		 */

		u32 state;



		/**

		 * @purgeable.vma_count: Number of VMAs currently mapping this BO.

		 */

		u32 vma_count;



		/**

		 * @purgeable.willneed_count: Number of active WILLNEED holders.

		 *

		 * Counts WILLNEED VMAs plus active dma-buf exports for

		 * non-imported BOs. The BO flips to DONTNEED on a 1->0

		 * transition only when VMAs still exist; if the last VMA is

		 * removed, the previous BO state is preserved.

		 */

		u32 willneed_count;

	} purgeable;

};



#endif

drivers/gpu/drm/xe/xe_dma_buf.c

+24 −4
Original line number Diff line number Diff line
@@ -193,6 +193,18 @@ static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, 
	return 0;

}



static void xe_dma_buf_release(struct dma_buf *dmabuf)

{

	struct drm_gem_object *obj = dmabuf->priv;

	struct xe_bo *bo = gem_to_xe_bo(obj);



	xe_bo_lock(bo, false);

	xe_bo_willneed_put_locked(bo);

	xe_bo_unlock(bo);



	drm_gem_dmabuf_release(dmabuf);

}



static const struct dma_buf_ops xe_dmabuf_ops = {

	.attach = xe_dma_buf_attach,

	.detach = xe_dma_buf_detach,
@@ -200,7 +212,7 @@ static const struct dma_buf_ops xe_dmabuf_ops = { 
	.unpin = xe_dma_buf_unpin,

	.map_dma_buf = xe_dma_buf_map,

	.unmap_dma_buf = xe_dma_buf_unmap,

	.release = drm_gem_dmabuf_release,

	.release = xe_dma_buf_release,

	.begin_cpu_access = xe_dma_buf_begin_cpu_access,

	.mmap = drm_gem_dmabuf_mmap,

	.vmap = drm_gem_dmabuf_vmap,
@@ -241,18 +253,26 @@ struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) 
		ret = -EINVAL;

		goto out_unlock;

	}



	xe_bo_willneed_get_locked(bo);

	xe_bo_unlock(bo);



	ret = ttm_bo_setup_export(&bo->ttm, &ctx);

	if (ret)

		return ERR_PTR(ret);

		goto out_put;



	buf = drm_gem_prime_export(obj, flags);

	if (!IS_ERR(buf))

		buf->ops = &xe_dmabuf_ops;

	if (IS_ERR(buf)) {

		ret = PTR_ERR(buf);

		goto out_put;

	}



	buf->ops = &xe_dmabuf_ops;

	return buf;



out_put:

	xe_bo_lock(bo, false);

	xe_bo_willneed_put_locked(bo);

out_unlock:

	xe_bo_unlock(bo);

	return ERR_PTR(ret);

drivers/gpu/drm/xe/xe_vm.c

+42 −9
Original line number Diff line number Diff line
@@ -1120,6 +1120,25 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, 


		xe_bo_assert_held(bo);



		/*

		 * Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This

		 * gates new vm_bind ioctls (user supplies WILLNEED) while

		 * still allowing partial-unbind / remap splits whose new VMAs

		 * inherit the parent's DONTNEED attr. It must also run before

		 * xe_bo_willneed_get_locked() below so a 0->1 holder bump

		 * cannot silently promote DONTNEED back to WILLNEED.

		 */

		if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) {

			if (xe_bo_madv_is_dontneed(bo)) {

				xe_vma_free(vma);

				return ERR_PTR(-EBUSY);

			}

			if (xe_bo_is_purged(bo)) {

				xe_vma_free(vma);

				return ERR_PTR(-EINVAL);

			}

		}



		vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base);

		if (IS_ERR(vm_bo)) {

			xe_vma_free(vma);
@@ -1131,6 +1150,10 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, 
		vma->gpuva.gem.offset = bo_offset_or_userptr;

		drm_gpuva_link(&vma->gpuva, vm_bo);

		drm_gpuvm_bo_put(vm_bo);



		xe_bo_vma_count_inc_locked(bo);

		if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED)

			xe_bo_willneed_get_locked(bo);

	} else /* userptr or null */ {

		if (!is_null && !is_cpu_addr_mirror) {

			struct xe_userptr_vma *uvma = to_userptr_vma(vma);
@@ -1208,7 +1231,10 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) 
		xe_bo_assert_held(bo);



		drm_gpuva_unlink(&vma->gpuva);

		xe_bo_recompute_purgeable_state(bo);



		xe_bo_vma_count_dec_locked(bo);

		if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED)

			xe_bo_willneed_put_locked(bo);

	}



	xe_vm_assert_held(vm);
@@ -3016,7 +3042,7 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, 
 * @res_evict: Allow evicting resources during validation

 * @validate: Perform BO validation

 * @request_decompress: Request BO decompression

 * @check_purged: Reject operation if BO is purged

 * @check_purged: Reject operation if BO is DONTNEED or PURGED

 */

struct xe_vma_lock_and_validate_flags {

	u32 res_evict : 1;
@@ -3030,6 +3056,7 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, 
{

	struct xe_bo *bo = xe_vma_bo(vma);

	struct xe_vm *vm = xe_vma_vm(vma);

	bool validate_bo = flags.validate;

	int err = 0;



	if (bo) {
@@ -3044,7 +3071,11 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, 
				err = -EINVAL; /* BO already purged */

		}



		if (!err && flags.validate)

		/* Don't validate the BO for DONTNEED/PURGED remap remnants. */

		if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED)

			validate_bo = false;



		if (!err && validate_bo)

			err = xe_bo_validate(bo, vm,

					     xe_vm_allow_vm_eviction(vm) &&

					     flags.res_evict, exec);
@@ -3152,7 +3183,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, 
								    op->map.immediate,

							.request_decompress =

							op->map.request_decompress,

							.check_purged = true,

							.check_purged = false,

						    });

		break;

	case DRM_GPUVA_OP_REMAP:
@@ -3174,7 +3205,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, 
							    .res_evict = res_evict,

							    .validate = true,

							    .request_decompress = false,

							    .check_purged = true,

							    .check_purged = false,

						    });

		if (!err && op->remap.next)

			err = vma_lock_and_validate(exec, op->remap.next,
@@ -3182,7 +3213,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, 
							    .res_evict = res_evict,

							    .validate = true,

							    .request_decompress = false,

							    .check_purged = true,

							    .check_purged = false,

						    });

		break;

	case DRM_GPUVA_OP_UNMAP:
@@ -3211,9 +3242,11 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, 
		}



		/*

		 * Prefetch attempts to migrate BO's backing store without

		 * repopulating it first. Purged BOs have no backing store

		 * to migrate, so reject the operation.

		 * PREFETCH is the only op that still gates on BO purge state.

		 * MAP/REMAP handle this inside xe_vma_create() so partial

		 * unbind on a DONTNEED BO still works. PREFETCH skips

		 * xe_vma_create() and would migrate a BO with no backing

		 * store, so reject DONTNEED/PURGED here.

		 */

		err = vma_lock_and_validate(exec,

					    gpuva_to_vma(op->base.prefetch.va),