diff options
Diffstat (limited to 'drivers/accel/amdxdna/aie2_ctx.c')
-rw-r--r-- | drivers/accel/amdxdna/aie2_ctx.c | 130 |
1 files changed, 78 insertions, 52 deletions
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 5f43db02b240..e04549f64d69 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -34,6 +34,8 @@ static void aie2_job_release(struct kref *ref) job = container_of(ref, struct amdxdna_sched_job, refcnt); amdxdna_sched_job_cleanup(job); + atomic64_inc(&job->hwctx->job_free_cnt); + wake_up(&job->hwctx->priv->job_free_wq); if (job->out_fence) dma_fence_put(job->out_fence); kfree(job); @@ -134,7 +136,8 @@ static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) if (!fence) return; - dma_fence_wait(fence, false); + /* Wait up to 2 seconds for fw to finish all pending requests */ + dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); dma_fence_put(fence); } @@ -185,7 +188,7 @@ aie2_sched_notify(struct amdxdna_sched_job *job) } static int -aie2_sched_resp_handler(void *handle, const u32 *data, size_t size) +aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) { struct amdxdna_sched_job *job = handle; struct amdxdna_gem_obj *cmd_abo; @@ -203,7 +206,7 @@ aie2_sched_resp_handler(void *handle, const u32 *data, size_t size) goto out; } - status = *data; + status = readl(data); XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); if (status == AIE2_STATUS_SUCCESS) amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); @@ -216,7 +219,7 @@ out: } static int -aie2_sched_nocmd_resp_handler(void *handle, const u32 *data, size_t size) +aie2_sched_nocmd_resp_handler(void *handle, void __iomem *data, size_t size) { struct amdxdna_sched_job *job = handle; u32 ret = 0; @@ -230,7 +233,7 @@ aie2_sched_nocmd_resp_handler(void *handle, const u32 *data, size_t size) goto out; } - status = *data; + status = readl(data); XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); out: @@ -239,14 +242,14 @@ out: } static int -aie2_sched_cmdlist_resp_handler(void *handle, const u32 *data, size_t size) +aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) { struct amdxdna_sched_job *job = handle; struct amdxdna_gem_obj *cmd_abo; - struct cmd_chain_resp *resp; struct amdxdna_dev *xdna; u32 fail_cmd_status; u32 fail_cmd_idx; + u32 cmd_status; u32 ret = 0; cmd_abo = job->cmd_bo; @@ -256,17 +259,17 @@ aie2_sched_cmdlist_resp_handler(void *handle, const u32 *data, size_t size) goto out; } - resp = (struct cmd_chain_resp *)data; + cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); xdna = job->hwctx->client->xdna; - XDNA_DBG(xdna, "Status 0x%x", resp->status); - if (resp->status == AIE2_STATUS_SUCCESS) { + XDNA_DBG(xdna, "Status 0x%x", cmd_status); + if (cmd_status == AIE2_STATUS_SUCCESS) { amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); goto out; } /* Slow path to handle error, read from ringbuf on BAR */ - fail_cmd_idx = resp->fail_cmd_idx; - fail_cmd_status = resp->fail_cmd_status; + fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); + fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", fail_cmd_idx, fail_cmd_status); @@ -361,7 +364,7 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job) return DRM_GPU_SCHED_STAT_NOMINAL; } -const struct drm_sched_backend_ops sched_ops = { +static const struct drm_sched_backend_ops sched_ops = { .run_job = aie2_sched_job_run, .free_job = aie2_sched_job_free, .timedout_job = aie2_sched_job_timedout, @@ -516,6 +519,14 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) { struct amdxdna_client *client = hwctx->client; struct amdxdna_dev *xdna = client->xdna; + const struct drm_sched_init_args args = { + .ops = &sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, + .credit_limit = HWCTX_MAX_CMDS, + .timeout = msecs_to_jiffies(HWCTX_MAX_TIMEOUT), + .name = hwctx->name, + .dev = xdna->ddev.dev, + }; struct drm_gpu_scheduler *sched; struct amdxdna_hwctx_priv *priv; struct amdxdna_gem_obj *heap; @@ -573,9 +584,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) might_lock(&priv->io_lock); fs_reclaim_release(GFP_KERNEL); - ret = drm_sched_init(sched, &sched_ops, NULL, DRM_SCHED_PRIORITY_COUNT, - HWCTX_MAX_CMDS, 0, msecs_to_jiffies(HWCTX_MAX_TIMEOUT), - NULL, NULL, hwctx->name, xdna->ddev.dev); + ret = drm_sched_init(sched, &args); if (ret) { XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); goto free_cmd_bufs; @@ -616,6 +625,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) hwctx->status = HWCTX_STAT_INIT; ndev = xdna->dev_handle; ndev->hwctx_num++; + init_waitqueue_head(&priv->job_free_wq); XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); @@ -652,25 +662,23 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) xdna = hwctx->client->xdna; ndev = xdna->dev_handle; ndev->hwctx_num--; - drm_sched_wqueue_stop(&hwctx->priv->sched); - /* Now, scheduler will not send command to device. */ + XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); + drm_sched_entity_destroy(&hwctx->priv->entity); + + aie2_hwctx_wait_for_idle(hwctx); + + /* Request fw to destroy hwctx and cancel the rest pending requests */ aie2_release_resource(hwctx); - /* - * All submitted commands are aborted. - * Restart scheduler queues to cleanup jobs. The amdxdna_sched_job_run() - * will return NODEV if it is called. - */ - drm_sched_wqueue_start(&hwctx->priv->sched); + /* Wait for all submitted jobs to be completed or canceled */ + wait_event(hwctx->priv->job_free_wq, + atomic64_read(&hwctx->job_submit_cnt) == + atomic64_read(&hwctx->job_free_cnt)); - aie2_hwctx_wait_for_idle(hwctx); - drm_sched_entity_destroy(&hwctx->priv->entity); drm_sched_fini(&hwctx->priv->sched); aie2_ctx_syncobj_destroy(hwctx); - XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); - for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); amdxdna_gem_unpin(hwctx->priv->heap); @@ -750,27 +758,42 @@ int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *bu static int aie2_populate_range(struct amdxdna_gem_obj *abo) { struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); - struct mm_struct *mm = abo->mem.notifier.mm; - struct hmm_range range = { 0 }; + struct amdxdna_umap *mapp; unsigned long timeout; + struct mm_struct *mm; + bool found; int ret; - XDNA_INFO_ONCE(xdna, "populate memory range %llx size %lx", - abo->mem.userptr, abo->mem.size); - range.notifier = &abo->mem.notifier; - range.start = abo->mem.userptr; - range.end = abo->mem.userptr + abo->mem.size; - range.hmm_pfns = abo->mem.pfns; - range.default_flags = HMM_PFN_REQ_FAULT; + timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); +again: + found = false; + down_write(&xdna->notifier_lock); + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (mapp->invalid) { + found = true; + break; + } + } + + if (!found) { + abo->mem.map_invalid = false; + up_write(&xdna->notifier_lock); + return 0; + } + kref_get(&mapp->refcnt); + up_write(&xdna->notifier_lock); - if (!mmget_not_zero(mm)) + XDNA_DBG(xdna, "populate memory range %lx %lx", + mapp->vma->vm_start, mapp->vma->vm_end); + mm = mapp->notifier.mm; + if (!mmget_not_zero(mm)) { + amdxdna_umap_put(mapp); return -EFAULT; + } - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); -again: - range.notifier_seq = mmu_interval_read_begin(&abo->mem.notifier); + mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); mmap_read_lock(mm); - ret = hmm_range_fault(&range); + ret = hmm_range_fault(&mapp->range); mmap_read_unlock(mm); if (ret) { if (time_after(jiffies, timeout)) { @@ -778,21 +801,27 @@ again: goto put_mm; } - if (ret == -EBUSY) + if (ret == -EBUSY) { + amdxdna_umap_put(mapp); goto again; + } goto put_mm; } - down_read(&xdna->notifier_lock); - if (mmu_interval_read_retry(&abo->mem.notifier, range.notifier_seq)) { - up_read(&xdna->notifier_lock); + down_write(&xdna->notifier_lock); + if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { + up_write(&xdna->notifier_lock); + amdxdna_umap_put(mapp); goto again; } - abo->mem.map_invalid = false; - up_read(&xdna->notifier_lock); + mapp->invalid = false; + up_write(&xdna->notifier_lock); + amdxdna_umap_put(mapp); + goto again; put_mm: + amdxdna_umap_put(mapp); mmput(mm); return ret; } @@ -879,6 +908,7 @@ retry: drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); aie2_job_put(job); + atomic64_inc(&hwctx->job_submit_cnt); return 0; @@ -899,10 +929,6 @@ void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, struct drm_gem_object *gobj = to_gobj(abo); long ret; - down_write(&xdna->notifier_lock); - abo->mem.map_invalid = true; - mmu_interval_set_seq(&abo->mem.notifier, cur_seq); - up_write(&xdna->notifier_lock); ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, true, MAX_SCHEDULE_TIMEOUT); if (!ret || ret == -ERESTARTSYS) |