Commit 08abccc9 authored by Christian König's avatar Christian König Committed by Alex Deucher
Browse files

drm/amdgpu: further move TLB hw workarounds a layer up 



For the PASID flushing we already handled that at a higher layer, apply
those workarounds to the standard flush as well.

Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
Reviewed-by: default avatarAlex Deucher <alexander.deucher@amd.com>
Reviewed-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent e2e37888

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c

+19 −0
Original line number Diff line number Diff line
@@ -604,6 +604,14 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 
		if (!down_read_trylock(&adev->reset_domain->sem))

			return;



		if (adev->gmc.flush_tlb_needs_extra_type_2)

			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,

							   vmhub, 2);



		if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)

			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,

							   vmhub, 0);



		adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub,

						   flush_type);

		up_read(&adev->reset_domain->sem);
@@ -654,6 +662,17 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, 


	if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready ||

	    !down_read_trylock(&adev->reset_domain->sem)) {



		if (adev->gmc.flush_tlb_needs_extra_type_2)

			adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,

								 2, all_hub,

								 inst);



		if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)

			adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,

								 0, all_hub,

								 inst);



		adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,

							 flush_type, all_hub,

							 inst);

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c

+23 −51
Original line number Diff line number Diff line
@@ -814,37 +814,18 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 
					uint32_t vmhub, uint32_t flush_type)

{

	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);

	u32 j, inv_req, inv_req2, tmp, sem, req, ack;

	u32 j, inv_req, tmp, sem, req, ack;

	const unsigned int eng = 17;

	struct amdgpu_vmhub *hub;



	BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);



	hub = &adev->vmhub[vmhub];

	inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);

	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;

	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;

	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;



	if (adev->gmc.xgmi.num_physical_nodes &&

	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0)) {

		/* Vega20+XGMI caches PTEs in TC and TLB. Add a

		 * heavy-weight TLB flush (type 2), which flushes

		 * both. Due to a race condition with concurrent

		 * memory accesses using the same TLB cache line, we

		 * still need a second TLB flush after this.

		 */

		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);

		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);

	} else if (flush_type == 2 &&

		   amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&

		   adev->rev_id == 0) {

		inv_req = gmc_v9_0_get_invalidate_req(vmid, 0);

		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);

	} else {

		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);

		inv_req2 = 0;

	}



	/* This is necessary for a HW workaround under SRIOV as well

	 * as GFXOFF under bare metal

	 */
@@ -855,10 +836,6 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 


		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,

						   1 << vmid);

		if (inv_req2)

			amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack,

							   inv_req2, 1 << vmid);



		return;

	}
@@ -888,7 +865,6 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 
			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");

	}



	do {

	if (vmhub >= AMDGPU_MMHUB0(0))

		WREG32_SOC15_IP_NO_KIQ(MMHUB, req, inv_req);

	else
@@ -900,8 +876,8 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 
	 * GRBM interface.

	 */

	if ((vmhub == AMDGPU_GFXHUB(0)) &&

		    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 4, 2)))

			RREG32_SOC15_IP_NO_KIQ(GC, req);

	    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))

		RREG32_NO_KIQ(req);



	for (j = 0; j < adev->usec_timeout; j++) {

		if (vmhub >= AMDGPU_MMHUB0(0))
@@ -913,10 +889,6 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 
		udelay(1);

	}



		inv_req = inv_req2;

		inv_req2 = 0;

	} while (inv_req);



	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */

	if (use_semaphore) {

		/*