Commit aac24309 authored by Lizhi Hou's avatar Lizhi Hou Committed by Jeff Hugo
Browse files

accel/amdxdna: Add command execution 



Add interfaces for user application to submit command and wait for its
completion.

Co-developed-by: default avatarMin Ma <min.ma@amd.com>
Signed-off-by: default avatarMin Ma <min.ma@amd.com>
Reviewed-by: default avatarJeffrey Hugo <quic_jhugo@quicinc.com>
Signed-off-by: default avatarLizhi Hou <lizhi.hou@amd.com>
Signed-off-by: default avatarJeffrey Hugo <quic_jhugo@quicinc.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20241118172942.2014541-8-lizhi.hou@amd.com
parent ac49797c

drivers/accel/amdxdna/aie2_ctx.c

+604 −3
Original line number Diff line number Diff line
@@ -8,8 +8,12 @@ 
#include <drm/drm_gem.h>

#include <drm/drm_gem_shmem_helper.h>

#include <drm/drm_print.h>

#include <drm/drm_syncobj.h>

#include <linux/hmm.h>

#include <linux/types.h>

#include <trace/events/amdxdna.h>



#include "aie2_msg_priv.h"

#include "aie2_pci.h"

#include "aie2_solver.h"

#include "amdxdna_ctx.h"
@@ -17,6 +21,321 @@ 
#include "amdxdna_mailbox.h"

#include "amdxdna_pci_drv.h"



bool force_cmdlist;

module_param(force_cmdlist, bool, 0600);

MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default false)");



#define HWCTX_MAX_TIMEOUT	60000 /* milliseconds */



static void aie2_job_release(struct kref *ref)

{

	struct amdxdna_sched_job *job;



	job = container_of(ref, struct amdxdna_sched_job, refcnt);

	amdxdna_sched_job_cleanup(job);

	if (job->out_fence)

		dma_fence_put(job->out_fence);

	kfree(job);

}



static void aie2_job_put(struct amdxdna_sched_job *job)

{

	kref_put(&job->refcnt, aie2_job_release);

}



/* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */

static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx,

			    struct drm_sched_job *bad_job)

{

	drm_sched_stop(&hwctx->priv->sched, bad_job);

	aie2_destroy_context(xdna->dev_handle, hwctx);

}



static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx)

{

	struct amdxdna_gem_obj *heap = hwctx->priv->heap;

	int ret;



	ret = aie2_create_context(xdna->dev_handle, hwctx);

	if (ret) {

		XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret);

		goto out;

	}



	ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,

				heap->mem.userptr, heap->mem.size);

	if (ret) {

		XDNA_ERR(xdna, "Map host buf failed, ret %d", ret);

		goto out;

	}



	if (hwctx->status != HWCTX_STAT_READY) {

		XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status);

		goto out;

	}



	ret = aie2_config_cu(hwctx);

	if (ret) {

		XDNA_ERR(xdna, "Config cu failed, ret %d", ret);

		goto out;

	}



out:

	drm_sched_start(&hwctx->priv->sched, 0);

	XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret);

	return ret;

}



void aie2_restart_ctx(struct amdxdna_client *client)

{

	struct amdxdna_dev *xdna = client->xdna;

	struct amdxdna_hwctx *hwctx;

	int next = 0;



	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));

	mutex_lock(&client->hwctx_lock);

	idr_for_each_entry_continue(&client->hwctx_idr, hwctx, next) {

		if (hwctx->status != HWCTX_STAT_STOP)

			continue;



		hwctx->status = hwctx->old_status;

		XDNA_DBG(xdna, "Resetting %s", hwctx->name);

		aie2_hwctx_restart(xdna, hwctx);

	}

	mutex_unlock(&client->hwctx_lock);

}



static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq)

{

	struct dma_fence *fence, *out_fence = NULL;

	int ret;



	fence = drm_syncobj_fence_get(hwctx->priv->syncobj);

	if (!fence)

		return NULL;



	ret = dma_fence_chain_find_seqno(&fence,  seq);

	if (ret)

		goto out;



	out_fence = dma_fence_get(dma_fence_chain_contained(fence));



out:

	dma_fence_put(fence);

	return out_fence;

}



static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx)

{

	struct dma_fence *fence;



	fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1);

	if (!fence)

		return;



	dma_fence_wait(fence, false);

	dma_fence_put(fence);

}



static void

aie2_sched_notify(struct amdxdna_sched_job *job)

{

	struct dma_fence *fence = job->fence;



	trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq);

	job->hwctx->priv->completed++;

	dma_fence_signal(fence);



	up(&job->hwctx->priv->job_sem);

	job->job_done = true;

	dma_fence_put(fence);

	mmput(job->mm);

	aie2_job_put(job);

}



static int

aie2_sched_resp_handler(void *handle, const u32 *data, size_t size)

{

	struct amdxdna_sched_job *job = handle;

	struct amdxdna_gem_obj *cmd_abo;

	u32 ret = 0;

	u32 status;



	cmd_abo = job->cmd_bo;



	if (unlikely(!data))

		goto out;



	if (unlikely(size != sizeof(u32))) {

		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT);

		ret = -EINVAL;

		goto out;

	}



	status = *data;

	XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status);

	if (status == AIE2_STATUS_SUCCESS)

		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED);

	else

		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR);



out:

	aie2_sched_notify(job);

	return ret;

}



static int

aie2_sched_nocmd_resp_handler(void *handle, const u32 *data, size_t size)

{

	struct amdxdna_sched_job *job = handle;

	u32 ret = 0;

	u32 status;



	if (unlikely(!data))

		goto out;



	if (unlikely(size != sizeof(u32))) {

		ret = -EINVAL;

		goto out;

	}



	status = *data;

	XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status);



out:

	aie2_sched_notify(job);

	return ret;

}



static int

aie2_sched_cmdlist_resp_handler(void *handle, const u32 *data, size_t size)

{

	struct amdxdna_sched_job *job = handle;

	struct amdxdna_gem_obj *cmd_abo;

	struct cmd_chain_resp *resp;

	struct amdxdna_dev *xdna;

	u32 fail_cmd_status;

	u32 fail_cmd_idx;

	u32 ret = 0;



	cmd_abo = job->cmd_bo;

	if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) {

		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT);

		ret = -EINVAL;

		goto out;

	}



	resp = (struct cmd_chain_resp *)data;

	xdna = job->hwctx->client->xdna;

	XDNA_DBG(xdna, "Status 0x%x", resp->status);

	if (resp->status == AIE2_STATUS_SUCCESS) {

		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED);

		goto out;

	}



	/* Slow path to handle error, read from ringbuf on BAR */

	fail_cmd_idx = resp->fail_cmd_idx;

	fail_cmd_status = resp->fail_cmd_status;

	XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x",

		 fail_cmd_idx, fail_cmd_status);



	if (fail_cmd_status == AIE2_STATUS_SUCCESS) {

		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT);

		ret = -EINVAL;

		goto out;

	}

	amdxdna_cmd_set_state(cmd_abo, fail_cmd_status);



	if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) {

		struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL);



		cc->error_index = fail_cmd_idx;

		if (cc->error_index >= cc->command_count)

			cc->error_index = 0;

	}

out:

	aie2_sched_notify(job);

	return ret;

}



static struct dma_fence *

aie2_sched_job_run(struct drm_sched_job *sched_job)

{

	struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);

	struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;

	struct amdxdna_hwctx *hwctx = job->hwctx;

	struct dma_fence *fence;

	int ret;



	if (!mmget_not_zero(job->mm))

		return ERR_PTR(-ESRCH);



	kref_get(&job->refcnt);

	fence = dma_fence_get(job->fence);



	if (unlikely(!cmd_abo)) {

		ret = aie2_sync_bo(hwctx, job, aie2_sched_nocmd_resp_handler);

		goto out;

	}



	amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW);



	if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN)

		ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler);

	else if (force_cmdlist)

		ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler);

	else

		ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler);



out:

	if (ret) {

		dma_fence_put(job->fence);

		aie2_job_put(job);

		mmput(job->mm);

		fence = ERR_PTR(ret);

	}

	trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq);



	return fence;

}



static void aie2_sched_job_free(struct drm_sched_job *sched_job)

{

	struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);

	struct amdxdna_hwctx *hwctx = job->hwctx;



	trace_xdna_job(sched_job, hwctx->name, "job free", job->seq);

	if (!job->job_done)

		up(&hwctx->priv->job_sem);



	drm_sched_job_cleanup(sched_job);

	aie2_job_put(job);

}



static enum drm_gpu_sched_stat

aie2_sched_job_timedout(struct drm_sched_job *sched_job)

{

	struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);

	struct amdxdna_hwctx *hwctx = job->hwctx;

	struct amdxdna_dev *xdna;



	xdna = hwctx->client->xdna;

	trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq);

	mutex_lock(&xdna->dev_lock);

	aie2_hwctx_stop(xdna, hwctx, sched_job);



	aie2_hwctx_restart(xdna, hwctx);

	mutex_unlock(&xdna->dev_lock);



	return DRM_GPU_SCHED_STAT_NOMINAL;

}



const struct drm_sched_backend_ops sched_ops = {

	.run_job = aie2_sched_job_run,

	.free_job = aie2_sched_job_free,

	.timedout_job = aie2_sched_job_timedout,

};



static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx)

{

	struct amdxdna_dev *xdna = hwctx->client->xdna;
@@ -126,13 +445,50 @@ static void aie2_release_resource(struct amdxdna_hwctx *hwctx) 
		XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);

}



static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx)

{

	struct amdxdna_dev *xdna = hwctx->client->xdna;

	struct drm_file *filp = hwctx->client->filp;

	struct drm_syncobj *syncobj;

	u32 hdl;

	int ret;



	hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE;



	ret = drm_syncobj_create(&syncobj, 0, NULL);

	if (ret) {

		XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret);

		return ret;

	}

	ret = drm_syncobj_get_handle(filp, syncobj, &hdl);

	if (ret) {

		drm_syncobj_put(syncobj);

		XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret);

		return ret;

	}

	hwctx->priv->syncobj = syncobj;

	hwctx->syncobj_hdl = hdl;



	return 0;

}



static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx)

{

	/*

	 * The syncobj_hdl is owned by user space and will be cleaned up

	 * separately.

	 */

	drm_syncobj_put(hwctx->priv->syncobj);

}



int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)

{

	struct amdxdna_client *client = hwctx->client;

	struct amdxdna_dev *xdna = client->xdna;

	struct drm_gpu_scheduler *sched;

	struct amdxdna_hwctx_priv *priv;

	struct amdxdna_gem_obj *heap;

	int ret;

	int i, ret;



	priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL);

	if (!priv)
@@ -150,6 +506,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 
	drm_gem_object_get(to_gobj(heap));

	mutex_unlock(&client->mm_lock);

	priv->heap = heap;

	sema_init(&priv->job_sem, HWCTX_MAX_CMDS);



	ret = amdxdna_gem_pin(heap);

	if (ret) {
@@ -157,10 +514,52 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 
		goto put_heap;

	}



	for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {

		struct amdxdna_gem_obj *abo;

		struct amdxdna_drm_create_bo args = {

			.flags = 0,

			.type = AMDXDNA_BO_DEV,

			.vaddr = 0,

			.size = MAX_CHAIN_CMDBUF_SIZE,

		};



		abo = amdxdna_drm_alloc_dev_bo(&xdna->ddev, &args, client->filp, true);

		if (IS_ERR(abo)) {

			ret = PTR_ERR(abo);

			goto free_cmd_bufs;

		}



		XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx",

			 i, abo->mem.dev_addr, abo->mem.size);

		priv->cmd_buf[i] = abo;

	}



	sched = &priv->sched;

	mutex_init(&priv->io_lock);



	fs_reclaim_acquire(GFP_KERNEL);

	might_lock(&priv->io_lock);

	fs_reclaim_release(GFP_KERNEL);



	ret = drm_sched_init(sched, &sched_ops, NULL, DRM_SCHED_PRIORITY_COUNT,

			     HWCTX_MAX_CMDS, 0, msecs_to_jiffies(HWCTX_MAX_TIMEOUT),

			     NULL, NULL, hwctx->name, xdna->ddev.dev);

	if (ret) {

		XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);

		goto free_cmd_bufs;

	}



	ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL,

				    &sched, 1, NULL);

	if (ret) {

		XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret);

		goto free_sched;

	}



	ret = aie2_hwctx_col_list(hwctx);

	if (ret) {

		XDNA_ERR(xdna, "Create col list failed, ret %d", ret);

		goto unpin;

		goto free_entity;

	}



	ret = aie2_alloc_resource(hwctx);
@@ -175,6 +574,13 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 
		XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret);

		goto release_resource;

	}



	ret = aie2_ctx_syncobj_create(hwctx);

	if (ret) {

		XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret);

		goto release_resource;

	}



	hwctx->status = HWCTX_STAT_INIT;



	XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name);
@@ -185,7 +591,16 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 
	aie2_release_resource(hwctx);

free_col_list:

	kfree(hwctx->col_list);

unpin:

free_entity:

	drm_sched_entity_destroy(&priv->entity);

free_sched:

	drm_sched_fini(&priv->sched);

free_cmd_bufs:

	for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {

		if (!priv->cmd_buf[i])

			continue;

		drm_gem_object_put(to_gobj(priv->cmd_buf[i]));

	}

	amdxdna_gem_unpin(heap);

put_heap:

	drm_gem_object_put(to_gobj(heap));
@@ -196,11 +611,35 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 


void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)

{

	struct amdxdna_dev *xdna;

	int idx;



	xdna = hwctx->client->xdna;

	drm_sched_wqueue_stop(&hwctx->priv->sched);



	/* Now, scheduler will not send command to device. */

	aie2_release_resource(hwctx);



	/*

	 * All submitted commands are aborted.

	 * Restart scheduler queues to cleanup jobs. The amdxdna_sched_job_run()

	 * will return NODEV if it is called.

	 */

	drm_sched_wqueue_start(&hwctx->priv->sched);



	aie2_hwctx_wait_for_idle(hwctx);

	drm_sched_entity_destroy(&hwctx->priv->entity);

	drm_sched_fini(&hwctx->priv->sched);

	aie2_ctx_syncobj_destroy(hwctx);



	XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq);



	for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++)

		drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx]));

	amdxdna_gem_unpin(hwctx->priv->heap);

	drm_gem_object_put(to_gobj(hwctx->priv->heap));



	mutex_destroy(&hwctx->priv->io_lock);

	kfree(hwctx->col_list);

	kfree(hwctx->priv);

	kfree(hwctx->cus);
@@ -267,3 +706,165 @@ int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *bu 
		return -EOPNOTSUPP;

	}

}



static int aie2_populate_range(struct amdxdna_gem_obj *abo)

{

	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);

	struct mm_struct *mm = abo->mem.notifier.mm;

	struct hmm_range range = { 0 };

	unsigned long timeout;

	int ret;



	XDNA_INFO_ONCE(xdna, "populate memory range %llx size %lx",

		       abo->mem.userptr, abo->mem.size);

	range.notifier = &abo->mem.notifier;

	range.start = abo->mem.userptr;

	range.end = abo->mem.userptr + abo->mem.size;

	range.hmm_pfns = abo->mem.pfns;

	range.default_flags = HMM_PFN_REQ_FAULT;



	if (!mmget_not_zero(mm))

		return -EFAULT;



	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);

again:

	range.notifier_seq = mmu_interval_read_begin(&abo->mem.notifier);

	mmap_read_lock(mm);

	ret = hmm_range_fault(&range);

	mmap_read_unlock(mm);

	if (ret) {

		if (time_after(jiffies, timeout)) {

			ret = -ETIME;

			goto put_mm;

		}



		if (ret == -EBUSY)

			goto again;



		goto put_mm;

	}



	down_read(&xdna->notifier_lock);

	if (mmu_interval_read_retry(&abo->mem.notifier, range.notifier_seq)) {

		up_read(&xdna->notifier_lock);

		goto again;

	}

	abo->mem.map_invalid = false;

	up_read(&xdna->notifier_lock);



put_mm:

	mmput(mm);

	return ret;

}



int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq)

{

	struct amdxdna_dev *xdna = hwctx->client->xdna;

	struct ww_acquire_ctx acquire_ctx;

	struct dma_fence_chain *chain;

	struct amdxdna_gem_obj *abo;

	unsigned long timeout = 0;

	int ret, i;



	ret = down_interruptible(&hwctx->priv->job_sem);

	if (ret) {

		XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret);

		return ret;

	}



	chain = dma_fence_chain_alloc();

	if (!chain) {

		XDNA_ERR(xdna, "Alloc fence chain failed");

		ret = -ENOMEM;

		goto up_sem;

	}



	ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx);

	if (ret) {

		XDNA_ERR(xdna, "DRM job init failed, ret %d", ret);

		goto free_chain;

	}



retry:

	ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx);

	if (ret) {

		XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret);

		goto cleanup_job;

	}



	for (i = 0; i < job->bo_cnt; i++) {

		ret = dma_resv_reserve_fences(job->bos[i]->resv, 1);

		if (ret) {

			XDNA_WARN(xdna, "Failed to reserve fences %d", ret);

			drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx);

			goto cleanup_job;

		}

	}



	down_read(&xdna->notifier_lock);

	for (i = 0; i < job->bo_cnt; i++) {

		abo = to_xdna_obj(job->bos[i]);

		if (abo->mem.map_invalid) {

			up_read(&xdna->notifier_lock);

			drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx);

			if (!timeout) {

				timeout = jiffies +

					msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);

			} else if (time_after(jiffies, timeout)) {

				ret = -ETIME;

				goto cleanup_job;

			}



			ret = aie2_populate_range(abo);

			if (ret)

				goto cleanup_job;

			goto retry;

		}

	}



	mutex_lock(&hwctx->priv->io_lock);

	drm_sched_job_arm(&job->base);

	job->out_fence = dma_fence_get(&job->base.s_fence->finished);

	for (i = 0; i < job->bo_cnt; i++)

		dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE);

	job->seq = hwctx->priv->seq++;

	kref_get(&job->refcnt);

	drm_sched_entity_push_job(&job->base);



	*seq = job->seq;

	drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq);

	mutex_unlock(&hwctx->priv->io_lock);



	up_read(&xdna->notifier_lock);

	drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx);



	aie2_job_put(job);



	return 0;



cleanup_job:

	drm_sched_job_cleanup(&job->base);

free_chain:

	dma_fence_chain_free(chain);

up_sem:

	up(&hwctx->priv->job_sem);

	job->job_done = true;

	return ret;

}



void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo,

			 unsigned long cur_seq)

{

	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);

	struct drm_gem_object *gobj = to_gobj(abo);

	long ret;



	down_write(&xdna->notifier_lock);

	abo->mem.map_invalid = true;

	mmu_interval_set_seq(&abo->mem.notifier, cur_seq);

	up_write(&xdna->notifier_lock);

	ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP,

				    true, MAX_SCHEDULE_TIMEOUT);

	if (!ret || ret == -ERESTARTSYS)

		XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret);

}

drivers/accel/amdxdna/aie2_message.c

+343 −0

File changed.

Preview size limit exceeded, changes collapsed.

drivers/accel/amdxdna/aie2_pci.c

+5 −0
Original line number Diff line number Diff line
@@ -5,8 +5,10 @@ 


#include <drm/amdxdna_accel.h>

#include <drm/drm_device.h>

#include <drm/drm_gem_shmem_helper.h>

#include <drm/drm_managed.h>

#include <drm/drm_print.h>

#include <drm/gpu_scheduler.h>

#include <linux/errno.h>

#include <linux/firmware.h>

#include <linux/iommu.h>
@@ -17,6 +19,7 @@ 
#include "aie2_pci.h"

#include "aie2_solver.h"

#include "amdxdna_ctx.h"

#include "amdxdna_gem.h"

#include "amdxdna_mailbox.h"

#include "amdxdna_pci_drv.h"
@@ -496,4 +499,6 @@ const struct amdxdna_dev_ops aie2_ops = { 
	.hwctx_init     = aie2_hwctx_init,

	.hwctx_fini     = aie2_hwctx_fini,

	.hwctx_config   = aie2_hwctx_config,

	.cmd_submit     = aie2_cmd_submit,

	.hmm_invalidate = aie2_hmm_invalidate,

};

drivers/accel/amdxdna/aie2_pci.h

+39 −0
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ 
#ifndef _AIE2_PCI_H_

#define _AIE2_PCI_H_



#include <linux/semaphore.h>



#include "amdxdna_mailbox.h"



#define AIE2_INTERVAL	20000	/* us */
@@ -76,8 +78,10 @@ enum psp_reg_idx { 
	PSP_MAX_REGS /* Keep this at the end */

};



struct amdxdna_client;

struct amdxdna_fw_ver;

struct amdxdna_hwctx;

struct amdxdna_sched_job;



struct psp_config {

	const void	*fw_buf;
@@ -118,9 +122,31 @@ struct rt_config { 
	u32	value;

};



/*

 * Define the maximum number of pending commands in a hardware context.

 * Must be power of 2!

 */

#define HWCTX_MAX_CMDS		4

#define get_job_idx(seq) ((seq) & (HWCTX_MAX_CMDS - 1))

struct amdxdna_hwctx_priv {

	struct amdxdna_gem_obj		*heap;

	void				*mbox_chann;



	struct drm_gpu_scheduler	sched;

	struct drm_sched_entity		entity;



	struct mutex			io_lock; /* protect seq and cmd order */

	struct wait_queue_head		job_free_wq;

	u32				num_pending;

	u64				seq;

	struct semaphore		job_sem;

	bool				job_done;



	/* Completed job counter */

	u64				completed;



	struct amdxdna_gem_obj		*cmd_buf[HWCTX_MAX_CMDS];

	struct drm_syncobj		*syncobj;

};



struct amdxdna_dev_hdl {
@@ -199,10 +225,23 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct 
int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);

int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size);

int aie2_config_cu(struct amdxdna_hwctx *hwctx);

int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,

		 int (*notify_cb)(void *, const u32 *, size_t));

int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,

				struct amdxdna_sched_job *job,

				int (*notify_cb)(void *, const u32 *, size_t));

int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,

			       struct amdxdna_sched_job *job,

			       int (*notify_cb)(void *, const u32 *, size_t));

int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,

		 int (*notify_cb)(void *, const u32 *, size_t));



/* aie2_hwctx.c */

int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);

void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx);

int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size);

int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq);

void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq);

void aie2_restart_ctx(struct amdxdna_client *client);



#endif /* _AIE2_PCI_H_ */

drivers/accel/amdxdna/aie2_psp.c

+2 −0
Original line number Diff line number Diff line
@@ -4,8 +4,10 @@ 
 */



#include <drm/drm_device.h>

#include <drm/drm_gem_shmem_helper.h>

#include <drm/drm_managed.h>

#include <drm/drm_print.h>

#include <drm/gpu_scheduler.h>

#include <linux/bitfield.h>

#include <linux/iopoll.h>