Commit 3d420e9f authored by Matthew Brost's avatar Matthew Brost
Browse files

drm/xe: Rework GPU page fault handling 



Add helper function to implement VMA (user binding) page faults, remove
unnecessary userptr.invalidate_link list del operation, retry on memory
pressure, remove unnecessary xe_vma_userptr_check_repin after rebinding,
remove unnecessary TLB invalidation, and always use vm->lock in write
mode. Changes help facilitate SVM page faults.

Signed-off-by: default avatarMatthew Brost <matthew.brost@intel.com>
Reviewed-by: default avatarNirmoy Das <nirmoy.das@intel.com>
Reviewed-by: default avatarJagmeet Randhawa <jagmeet.randhawa@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240607015212.2190106-1-matthew.brost@intel.com
parent 78247e48

drivers/gpu/drm/xe/xe_gt_pagefault.c

+62 −80
Original line number Diff line number Diff line
@@ -125,126 +125,108 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, 
	return 0;

}



static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)

static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,

				struct xe_vma *vma)

{

	struct xe_device *xe = gt_to_xe(gt);

	struct xe_tile *tile = gt_to_tile(gt);

	struct xe_vm *vm = xe_vma_vm(vma);

	struct drm_exec exec;

	struct xe_vm *vm;

	struct xe_vma *vma = NULL;

	struct dma_fence *fence;

	bool write_locked;

	int ret = 0;

	ktime_t end = 0;

	int err;

	bool atomic;



	/* SW isn't expected to handle TRTT faults */

	if (pf->trva_fault)

		return -EFAULT;



	/* ASID to VM */

	mutex_lock(&xe->usm.lock);

	vm = xa_load(&xe->usm.asid_to_vm, pf->asid);

	if (vm && xe_vm_in_fault_mode(vm))

		xe_vm_get(vm);

	else

		vm = NULL;

	mutex_unlock(&xe->usm.lock);

	if (!vm)

		return -EINVAL;



retry_userptr:

	/*

	 * TODO: Avoid exclusive lock if VM doesn't have userptrs, or

	 * start out read-locked?

	 */

	down_write(&vm->lock);

	write_locked = true;

	vma = lookup_vma(vm, pf->page_addr);

	if (!vma) {

		ret = -EINVAL;

		goto unlock_vm;

	}



	if (!xe_vma_is_userptr(vma) ||

	    !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {

		downgrade_write(&vm->lock);

		write_locked = false;

	}



	trace_xe_vma_pagefault(vma);



	atomic = access_is_atomic(pf->access_type);



	/* Check if VMA is valid */

	if (vma_is_valid(tile, vma) && !atomic)

		goto unlock_vm;



	/* TODO: Validate fault */

		return 0;



	if (xe_vma_is_userptr(vma) && write_locked) {

retry_userptr:

	if (xe_vma_is_userptr(vma) &&

	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {

		struct xe_userptr_vma *uvma = to_userptr_vma(vma);



		spin_lock(&vm->userptr.invalidated_lock);

		list_del_init(&uvma->userptr.invalidate_link);

		spin_unlock(&vm->userptr.invalidated_lock);



		ret = xe_vma_userptr_pin_pages(uvma);

		if (ret)

			goto unlock_vm;



		downgrade_write(&vm->lock);

		write_locked = false;

		err = xe_vma_userptr_pin_pages(uvma);

		if (err)

			return err;

	}



	/* Lock VM and BOs dma-resv */

	drm_exec_init(&exec, 0, 0);

	drm_exec_until_all_locked(&exec) {

		ret = xe_pf_begin(&exec, vma, atomic, tile->id);

		err = xe_pf_begin(&exec, vma, atomic, tile->id);

		drm_exec_retry_on_contention(&exec);

		if (ret)

		if (xe_vm_validate_should_retry(&exec, err, &end))

			err = -EAGAIN;

		if (err)

			goto unlock_dma_resv;



		/* Bind VMA only to the GT that has faulted */

		trace_xe_vma_pf_bind(vma);

		fence = xe_vma_rebind(vm, vma, BIT(tile->id));

		if (IS_ERR(fence)) {

			ret = PTR_ERR(fence);

			err = PTR_ERR(fence);

			if (xe_vm_validate_should_retry(&exec, err, &end))

				err = -EAGAIN;

			goto unlock_dma_resv;

		}

	}



	/*

	 * XXX: Should we drop the lock before waiting? This only helps if doing

	 * GPU binds which is currently only done if we have to wait for more

	 * than 10ms on a move.

	 */

	dma_fence_wait(fence, false);

	dma_fence_put(fence);



	if (xe_vma_is_userptr(vma))

		ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));

	vma->tile_invalidated &= ~BIT(tile->id);



unlock_dma_resv:

	drm_exec_fini(&exec);

unlock_vm:

	if (!ret)

		vm->usm.last_fault_vma = vma;

	if (write_locked)

		up_write(&vm->lock);

	else

		up_read(&vm->lock);

	if (ret == -EAGAIN)

	if (err == -EAGAIN)

		goto retry_userptr;



	if (!ret) {

		ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);

		if (ret >= 0)

			ret = 0;

	return err;

}



static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)

{

	struct xe_device *xe = gt_to_xe(gt);

	struct xe_tile *tile = gt_to_tile(gt);

	struct xe_vm *vm;

	struct xe_vma *vma = NULL;

	int err;



	/* SW isn't expected to handle TRTT faults */

	if (pf->trva_fault)

		return -EFAULT;



	/* ASID to VM */

	mutex_lock(&xe->usm.lock);

	vm = xa_load(&xe->usm.asid_to_vm, pf->asid);

	if (vm && xe_vm_in_fault_mode(vm))

		xe_vm_get(vm);

	else

		vm = NULL;

	mutex_unlock(&xe->usm.lock);

	if (!vm)

		return -EINVAL;



	/*

	 * TODO: Change to read lock? Using write lock for simplicity.

	 */

	down_write(&vm->lock);

	vma = lookup_vma(vm, pf->page_addr);

	if (!vma) {

		err = -EINVAL;

		goto unlock_vm;

	}



	err = handle_vma_pagefault(tile, pf, vma);



unlock_vm:

	if (!err)

		vm->usm.last_fault_vma = vma;

	up_write(&vm->lock);

	xe_vm_put(vm);



	return ret;

	return err;

}



static int send_pagefault_reply(struct xe_guc *guc,