Commit fb910658 authored by Philip Yang's avatar Philip Yang Committed by Alex Deucher
Browse files

drm/amdkfd: Refactor queue wptr_bo GART mapping 



Add helper function kfd_queue_acquire_buffers to get queue wptr_bo
reference from queue write_ptr if it is mapped to the KFD node with
expected size.

Add wptr_bo to structure queue_properties because structure queue is
allocated after queue buffers are validated, then we can remove wptr_bo
parameter from pqm_create_queue.

Rename structure queue wptr_bo_gart to hold wptr_bo reference for GART
mapping and umapping. Move MES wptr_bo_gart mapping to init_user_queue,
the same location with queue ctx_bo GART mapping.

Signed-off-by: default avatarPhilip Yang <Philip.Yang@amd.com>
Reviewed-by: default avatarFelix Kuehling <felix.kuehling@amd.com>
Acked-by: default avatarChristian König <christian.koenig@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent db54a725

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h

+1 −1
Original line number Diff line number Diff line
@@ -322,7 +322,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, 
					     void **kptr, uint64_t *size);

void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem);



int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo);

int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart);



int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,

					    struct dma_fence __rcu **ef);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c

+3 −2
Original line number Diff line number Diff line
@@ -2226,11 +2226,12 @@ int amdgpu_amdkfd_gpuvm_sync_memory( 
/**

 * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count

 * @bo: Buffer object to be mapped

 * @bo_gart: Return bo reference

 *

 * Before return, bo reference count is incremented. To release the reference and unpin/

 * unmap the BO, call amdgpu_amdkfd_free_gtt_mem.

 */

int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo)

int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart)

{

	int ret;
@@ -2257,7 +2258,7 @@ int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo) 


	amdgpu_bo_unreserve(bo);



	bo = amdgpu_bo_ref(bo);

	*bo_gart = amdgpu_bo_ref(bo);



	return 0;

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c

+9 −47
Original line number Diff line number Diff line
@@ -247,8 +247,8 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties, 
	q_properties->priority = args->queue_priority;

	q_properties->queue_address = args->ring_base_address;

	q_properties->queue_size = args->ring_size;

	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;

	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;

	q_properties->read_ptr = (void __user *)args->read_pointer_address;

	q_properties->write_ptr = (void __user *)args->write_pointer_address;

	q_properties->eop_ring_buffer_address = args->eop_buffer_address;

	q_properties->eop_ring_buffer_size = args->eop_buffer_size;

	q_properties->ctx_save_restore_area_address =
@@ -306,7 +306,6 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 
	struct kfd_process_device *pdd;

	struct queue_properties q_properties;

	uint32_t doorbell_offset_in_process = 0;

	struct amdgpu_bo *wptr_bo = NULL;



	memset(&q_properties, 0, sizeof(struct queue_properties));
@@ -342,53 +341,17 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 
		}

	}



	/* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work

	 * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)

	 */

	if (dev->kfd->shared_resources.enable_mes &&

			((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)

			>> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {

		struct amdgpu_bo_va_mapping *wptr_mapping;

		struct amdgpu_vm *wptr_vm;



		wptr_vm = drm_priv_to_vm(pdd->drm_priv);

		err = amdgpu_bo_reserve(wptr_vm->root.bo, false);

		if (err)

			goto err_wptr_map_gart;



		wptr_mapping = amdgpu_vm_bo_lookup_mapping(

				wptr_vm, args->write_pointer_address >> PAGE_SHIFT);

		amdgpu_bo_unreserve(wptr_vm->root.bo);

		if (!wptr_mapping) {

			pr_err("Failed to lookup wptr bo\n");

			err = -EINVAL;

			goto err_wptr_map_gart;

		}



		wptr_bo = wptr_mapping->bo_va->base.bo;

		if (wptr_bo->tbo.base.size > PAGE_SIZE) {

			pr_err("Requested GART mapping for wptr bo larger than one page\n");

			err = -EINVAL;

			goto err_wptr_map_gart;

		}

		if (dev->adev != amdgpu_ttm_adev(wptr_bo->tbo.bdev)) {

			pr_err("Queue memory allocated to wrong device\n");

			err = -EINVAL;

			goto err_wptr_map_gart;

		}



		err = amdgpu_amdkfd_map_gtt_bo_to_gart(wptr_bo);

	err = kfd_queue_acquire_buffers(pdd, &q_properties);

	if (err) {

			pr_err("Failed to map wptr bo to GART\n");

			goto err_wptr_map_gart;

		}

		pr_debug("failed to acquire user queue buffers\n");

		goto err_acquire_queue_buf;

	}



	pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",

			p->pasid,

			dev->id);



	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, wptr_bo,

	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id,

			NULL, NULL, NULL, &doorbell_offset_in_process);

	if (err != 0)

		goto err_create_queue;
@@ -422,9 +385,8 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 
	return 0;



err_create_queue:

	if (wptr_bo)

		amdgpu_amdkfd_free_gtt_mem(dev->adev, (void **)&wptr_bo);

err_wptr_map_gart:

	kfd_queue_release_buffers(pdd, &q_properties);

err_acquire_queue_buf:

err_bind_process:

err_pdd:

	mutex_unlock(&p->mutex);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

+2 −4
Original line number Diff line number Diff line
@@ -208,10 +208,8 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 
	queue_input.mqd_addr = q->gart_mqd_addr;

	queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;



	if (q->wptr_bo) {

	wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);

		queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->wptr_bo) + wptr_addr_off;

	}

	queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off;



	queue_input.is_kfd_process = 1;

	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);

drivers/gpu/drm/amd/amdkfd/kfd_priv.h

+9 −4
Original line number Diff line number Diff line
@@ -494,8 +494,8 @@ struct queue_properties { 
	uint64_t  queue_size;

	uint32_t priority;

	uint32_t queue_percent;

	uint32_t *read_ptr;

	uint32_t *write_ptr;

	void __user *read_ptr;

	void __user *write_ptr;

	void __iomem *doorbell_ptr;

	uint32_t doorbell_off;

	bool is_interop;
@@ -522,6 +522,8 @@ struct queue_properties { 
	uint64_t tba_addr;

	uint64_t tma_addr;

	uint64_t exception_status;



	struct amdgpu_bo *wptr_bo;

};



#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 &&	\
@@ -604,7 +606,7 @@ struct queue { 
	uint64_t gang_ctx_gpu_addr;

	void *gang_ctx_cpu_ptr;



	struct amdgpu_bo *wptr_bo;

	struct amdgpu_bo *wptr_bo_gart;

};



enum KFD_MQD_TYPE {
@@ -1284,6 +1286,10 @@ int init_queue(struct queue **q, const struct queue_properties *properties); 
void uninit_queue(struct queue *q);

void print_queue_properties(struct queue_properties *q);

void print_queue(struct queue *q);

int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct amdgpu_bo **pbo,

			 u64 expected_size);

int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties);

int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties);



struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,

		struct kfd_node *dev);
@@ -1320,7 +1326,6 @@ int pqm_create_queue(struct process_queue_manager *pqm, 
			    struct file *f,

			    struct queue_properties *properties,

			    unsigned int *qid,

			    struct amdgpu_bo *wptr_bo,

			    const struct kfd_criu_queue_priv_data *q_data,

			    const void *restore_mqd,

			    const void *restore_ctl_stack,