Commit 40520283 authored by Matthew Brost's avatar Matthew Brost
Browse files

drm/xe: Invalidate media_gt TLBs in PT code 



Testing on LNL has shown media GT's TLBs need to be invalidated via the
GuC, update PT code appropriately.

v2:
 - Do dma_fence_get before first call of invalidation_fence_init (Himal)
 - No need to check for valid chain fence (Himal)

Fixes: 33303615 ("drm/xe/lnl: Add LNL platform definition")
Signed-off-by: default avatarMatthew Brost <matthew.brost@intel.com>
Reviewed-by: default avatarHimal Prasad Ghimiray <himal.prasad.ghimiray@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240820161632.987369-1-matthew.brost@intel.com
parent 77cc3f6c

drivers/gpu/drm/xe/xe_pt.c

+80 −19
Original line number Diff line number Diff line
@@ -3,6 +3,8 @@ 
 * Copyright © 2022 Intel Corporation

 */



#include <linux/dma-fence-chain.h>



#include "xe_pt.h"



#include "regs/xe_gtt_defs.h"
@@ -1849,13 +1851,20 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) 


static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,

			   struct xe_vm_pgtable_update_ops *pt_update_ops,

			   struct xe_vma *vma, struct dma_fence *fence)

			   struct xe_vma *vma, struct dma_fence *fence,

			   struct dma_fence *fence2)

{

	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)

	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {

		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,

				   pt_update_ops->wait_vm_bookkeep ?

				   DMA_RESV_USAGE_KERNEL :

				   DMA_RESV_USAGE_BOOKKEEP);

		if (fence2)

			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,

					   pt_update_ops->wait_vm_bookkeep ?

					   DMA_RESV_USAGE_KERNEL :

					   DMA_RESV_USAGE_BOOKKEEP);

	}

	vma->tile_present |= BIT(tile->id);

	vma->tile_staged &= ~BIT(tile->id);

	if (xe_vma_is_userptr(vma)) {
@@ -1875,13 +1884,20 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, 


static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,

			     struct xe_vm_pgtable_update_ops *pt_update_ops,

			     struct xe_vma *vma, struct dma_fence *fence)

			     struct xe_vma *vma, struct dma_fence *fence,

			     struct dma_fence *fence2)

{

	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)

	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {

		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,

				   pt_update_ops->wait_vm_bookkeep ?

				   DMA_RESV_USAGE_KERNEL :

				   DMA_RESV_USAGE_BOOKKEEP);

		if (fence2)

			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,

					   pt_update_ops->wait_vm_bookkeep ?

					   DMA_RESV_USAGE_KERNEL :

					   DMA_RESV_USAGE_BOOKKEEP);

	}

	vma->tile_present &= ~BIT(tile->id);

	if (!vma->tile_present) {

		list_del_init(&vma->combined_links.rebind);
@@ -1898,7 +1914,8 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, 
static void op_commit(struct xe_vm *vm,

		      struct xe_tile *tile,

		      struct xe_vm_pgtable_update_ops *pt_update_ops,

		      struct xe_vma_op *op, struct dma_fence *fence)

		      struct xe_vma_op *op, struct dma_fence *fence,

		      struct dma_fence *fence2)

{

	xe_vm_assert_held(vm);
@@ -1907,26 +1924,28 @@ static void op_commit(struct xe_vm *vm, 
		if (!op->map.immediate && xe_vm_in_fault_mode(vm))

			break;



		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence);

		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,

			       fence2);

		break;

	case DRM_GPUVA_OP_REMAP:

		unbind_op_commit(vm, tile, pt_update_ops,

				 gpuva_to_vma(op->base.remap.unmap->va), fence);

				 gpuva_to_vma(op->base.remap.unmap->va), fence,

				 fence2);



		if (op->remap.prev)

			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,

				       fence);

				       fence, fence2);

		if (op->remap.next)

			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,

				       fence);

				       fence, fence2);

		break;

	case DRM_GPUVA_OP_UNMAP:

		unbind_op_commit(vm, tile, pt_update_ops,

				 gpuva_to_vma(op->base.unmap.va), fence);

				 gpuva_to_vma(op->base.unmap.va), fence, fence2);

		break;

	case DRM_GPUVA_OP_PREFETCH:

		bind_op_commit(vm, tile, pt_update_ops,

			       gpuva_to_vma(op->base.prefetch.va), fence);

			       gpuva_to_vma(op->base.prefetch.va), fence, fence2);

		break;

	default:

		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
@@ -1963,7 +1982,8 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) 
	struct xe_vm_pgtable_update_ops *pt_update_ops =

		&vops->pt_update_ops[tile->id];

	struct dma_fence *fence;

	struct invalidation_fence *ifence = NULL;

	struct invalidation_fence *ifence = NULL, *mfence = NULL;

	struct dma_fence_chain *chain_fence = NULL;

	struct xe_range_fence *rfence;

	struct xe_vma_op *op;

	int err = 0, i;
@@ -1996,6 +2016,18 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) 
			err = -ENOMEM;

			goto kill_vm_tile1;

		}

		if (tile->media_gt) {

			mfence = kzalloc(sizeof(*ifence), GFP_KERNEL);

			if (!mfence) {

				err = -ENOMEM;

				goto free_ifence;

			}

			chain_fence = dma_fence_chain_alloc();

			if (!chain_fence) {

				err = -ENOMEM;

				goto free_ifence;

			}

		}

	}



	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
@@ -2027,19 +2059,46 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) 


	/* tlb invalidation must be done before signaling rebind */

	if (ifence) {

		if (mfence)

			dma_fence_get(fence);

		invalidation_fence_init(tile->primary_gt, ifence, fence,

					pt_update_ops->start,

					pt_update_ops->last, vm->usm.asid);

		if (mfence) {

			invalidation_fence_init(tile->media_gt, mfence, fence,

						pt_update_ops->start,

						pt_update_ops->last, vm->usm.asid);

			dma_fence_chain_init(chain_fence, &ifence->base.base,

					     &mfence->base.base, 0);

			fence = &chain_fence->base;

		} else {

			fence = &ifence->base.base;

		}

	}



	if (!mfence) {

		dma_resv_add_fence(xe_vm_resv(vm), fence,

				   pt_update_ops->wait_vm_bookkeep ?

				   DMA_RESV_USAGE_KERNEL :

				   DMA_RESV_USAGE_BOOKKEEP);



		list_for_each_entry(op, &vops->list, link)

		op_commit(vops->vm, tile, pt_update_ops, op, fence);

			op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);

	} else {

		dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base,

				   pt_update_ops->wait_vm_bookkeep ?

				   DMA_RESV_USAGE_KERNEL :

				   DMA_RESV_USAGE_BOOKKEEP);



		dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base,

				   pt_update_ops->wait_vm_bookkeep ?

				   DMA_RESV_USAGE_KERNEL :

				   DMA_RESV_USAGE_BOOKKEEP);



		list_for_each_entry(op, &vops->list, link)

			op_commit(vops->vm, tile, pt_update_ops, op,

				  &ifence->base.base, &mfence->base.base);

	}



	if (pt_update_ops->needs_userptr_lock)

		up_read(&vm->userptr.notifier_lock);
@@ -2049,6 +2108,8 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) 
free_rfence:

	kfree(rfence);

free_ifence:

	dma_fence_chain_free(chain_fence);

	kfree(mfence);

	kfree(ifence);

kill_vm_tile1:

	if (err != -EAGAIN && tile->id)