summaryrefslogtreecommitdiff
path: root/drivers/misc/habanalabs/common
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/misc/habanalabs/common')
-rw-r--r--drivers/misc/habanalabs/common/Makefile4
-rw-r--r--drivers/misc/habanalabs/common/command_buffer.c229
-rw-r--r--drivers/misc/habanalabs/common/command_submission.c107
-rw-r--r--drivers/misc/habanalabs/common/context.c38
-rw-r--r--drivers/misc/habanalabs/common/debugfs.c92
-rw-r--r--drivers/misc/habanalabs/common/device.c31
-rw-r--r--drivers/misc/habanalabs/common/firmware_if.c229
-rw-r--r--drivers/misc/habanalabs/common/habanalabs.h200
-rw-r--r--drivers/misc/habanalabs/common/habanalabs_drv.c76
-rw-r--r--drivers/misc/habanalabs/common/habanalabs_ioctl.c105
-rw-r--r--drivers/misc/habanalabs/common/hw_queue.c10
-rw-r--r--drivers/misc/habanalabs/common/hwmon.c60
-rw-r--r--drivers/misc/habanalabs/common/irq.c2
-rw-r--r--drivers/misc/habanalabs/common/memory.c90
-rw-r--r--drivers/misc/habanalabs/common/mmu.c812
-rw-r--r--drivers/misc/habanalabs/common/mmu_v1.c863
-rw-r--r--drivers/misc/habanalabs/common/pci.c17
-rw-r--r--drivers/misc/habanalabs/common/sysfs.c60
18 files changed, 1934 insertions, 1091 deletions
diff --git a/drivers/misc/habanalabs/common/Makefile b/drivers/misc/habanalabs/common/Makefile
index b984bfa4face..eccd8c7dc62d 100644
--- a/drivers/misc/habanalabs/common/Makefile
+++ b/drivers/misc/habanalabs/common/Makefile
@@ -3,5 +3,5 @@ HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \
common/asid.o common/habanalabs_ioctl.o \
common/command_buffer.o common/hw_queue.o common/irq.o \
common/sysfs.o common/hwmon.o common/memory.o \
- common/command_submission.o common/mmu.o common/firmware_if.o \
- common/pci.o
+ common/command_submission.o common/mmu.o common/mmu_v1.o \
+ common/firmware_if.o common/pci.o
diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c
index a8004911c977..901e213daf40 100644
--- a/drivers/misc/habanalabs/common/command_buffer.c
+++ b/drivers/misc/habanalabs/common/command_buffer.c
@@ -13,6 +13,131 @@
#include <linux/uaccess.h>
#include <linux/genalloc.h>
+static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_vm_va_block *va_block, *tmp;
+ dma_addr_t bus_addr;
+ u64 virt_addr;
+ u32 page_size = prop->pmmu.page_size;
+ s32 offset;
+ int rc;
+
+ if (!hdev->supports_cb_mapping) {
+ dev_err_ratelimited(hdev->dev,
+ "Cannot map CB because no VA range is allocated for CB mapping\n");
+ return -EINVAL;
+ }
+
+ if (!hdev->mmu_enable) {
+ dev_err_ratelimited(hdev->dev,
+ "Cannot map CB because MMU is disabled\n");
+ return -EINVAL;
+ }
+
+ INIT_LIST_HEAD(&cb->va_block_list);
+
+ for (bus_addr = cb->bus_address;
+ bus_addr < cb->bus_address + cb->size;
+ bus_addr += page_size) {
+
+ virt_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, page_size);
+ if (!virt_addr) {
+ dev_err(hdev->dev,
+ "Failed to allocate device virtual address for CB\n");
+ rc = -ENOMEM;
+ goto err_va_pool_free;
+ }
+
+ va_block = kzalloc(sizeof(*va_block), GFP_KERNEL);
+ if (!va_block) {
+ rc = -ENOMEM;
+ gen_pool_free(ctx->cb_va_pool, virt_addr, page_size);
+ goto err_va_pool_free;
+ }
+
+ va_block->start = virt_addr;
+ va_block->end = virt_addr + page_size;
+ va_block->size = page_size;
+ list_add_tail(&va_block->node, &cb->va_block_list);
+ }
+
+ mutex_lock(&ctx->mmu_lock);
+
+ bus_addr = cb->bus_address;
+ offset = 0;
+ list_for_each_entry(va_block, &cb->va_block_list, node) {
+ rc = hl_mmu_map(ctx, va_block->start, bus_addr, va_block->size,
+ list_is_last(&va_block->node,
+ &cb->va_block_list));
+ if (rc) {
+ dev_err(hdev->dev, "Failed to map VA %#llx to CB\n",
+ va_block->start);
+ goto err_va_umap;
+ }
+
+ bus_addr += va_block->size;
+ offset += va_block->size;
+ }
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, false, VM_TYPE_USERPTR);
+
+ mutex_unlock(&ctx->mmu_lock);
+
+ cb->is_mmu_mapped = true;
+
+ return 0;
+
+err_va_umap:
+ list_for_each_entry(va_block, &cb->va_block_list, node) {
+ if (offset <= 0)
+ break;
+ hl_mmu_unmap(ctx, va_block->start, va_block->size,
+ offset <= va_block->size);
+ offset -= va_block->size;
+ }
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
+
+ mutex_unlock(&ctx->mmu_lock);
+
+err_va_pool_free:
+ list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) {
+ gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size);
+ list_del(&va_block->node);
+ kfree(va_block);
+ }
+
+ return rc;
+}
+
+static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct hl_vm_va_block *va_block, *tmp;
+
+ mutex_lock(&ctx->mmu_lock);
+
+ list_for_each_entry(va_block, &cb->va_block_list, node)
+ if (hl_mmu_unmap(ctx, va_block->start, va_block->size,
+ list_is_last(&va_block->node,
+ &cb->va_block_list)))
+ dev_warn_ratelimited(hdev->dev,
+ "Failed to unmap CB's va 0x%llx\n",
+ va_block->start);
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
+
+ mutex_unlock(&ctx->mmu_lock);
+
+ list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) {
+ gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size);
+ list_del(&va_block->node);
+ kfree(va_block);
+ }
+}
+
static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
{
if (cb->is_internal)
@@ -47,6 +172,11 @@ static void cb_release(struct kref *ref)
hl_debugfs_remove_cb(cb);
+ if (cb->is_mmu_mapped)
+ cb_unmap_mem(cb->ctx, cb);
+
+ hl_ctx_put(cb->ctx);
+
cb_do_release(hdev, cb);
}
@@ -107,11 +237,12 @@ static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
}
int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
- u32 cb_size, u64 *handle, int ctx_id, bool internal_cb)
+ struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
+ bool map_cb, u64 *handle)
{
struct hl_cb *cb;
bool alloc_new_cb = true;
- int rc;
+ int rc, ctx_id = ctx->asid;
/*
* Can't use generic function to check this because of special case
@@ -163,7 +294,21 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
}
cb->hdev = hdev;
- cb->ctx_id = ctx_id;
+ cb->ctx = ctx;
+ hl_ctx_get(hdev, cb->ctx);
+
+ if (map_cb) {
+ if (ctx_id == HL_KERNEL_ASID_ID) {
+ dev_err(hdev->dev,
+ "CB mapping is not supported for kernel context\n");
+ rc = -EINVAL;
+ goto release_cb;
+ }
+
+ rc = cb_map_mem(ctx, cb);
+ if (rc)
+ goto release_cb;
+ }
spin_lock(&mgr->cb_lock);
rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
@@ -171,10 +316,10 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
if (rc < 0) {
dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
- goto release_cb;
+ goto unmap_mem;
}
- cb->id = rc;
+ cb->id = (u64) rc;
kref_init(&cb->refcount);
spin_lock_init(&cb->lock);
@@ -183,14 +328,18 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
* idr is 32-bit so we can safely OR it with a mask that is above
* 32 bit
*/
- *handle = cb->id | HL_MMAP_CB_MASK;
+ *handle = cb->id | HL_MMAP_TYPE_CB;
*handle <<= PAGE_SHIFT;
hl_debugfs_add_cb(cb);
return 0;
+unmap_mem:
+ if (cb->is_mmu_mapped)
+ cb_unmap_mem(cb->ctx, cb);
release_cb:
+ hl_ctx_put(cb->ctx);
cb_do_release(hdev, cb);
out_err:
*handle = 0;
@@ -250,9 +399,10 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
args->in.cb_size, HL_MAX_CB_SIZE);
rc = -EINVAL;
} else {
- rc = hl_cb_create(hdev, &hpriv->cb_mgr,
- args->in.cb_size, &handle,
- hpriv->ctx->asid, false);
+ rc = hl_cb_create(hdev, &hpriv->cb_mgr, hpriv->ctx,
+ args->in.cb_size, false,
+ !!(args->in.flags & HL_CB_FLAGS_MAP),
+ &handle);
}
memset(args, 0, sizeof(*args));
@@ -300,11 +450,14 @@ int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
{
struct hl_device *hdev = hpriv->hdev;
struct hl_cb *cb;
- phys_addr_t address;
u32 handle, user_cb_size;
int rc;
+ /* We use the page offset to hold the idr and thus we need to clear
+ * it before doing the mmap itself
+ */
handle = vma->vm_pgoff;
+ vma->vm_pgoff = 0;
/* reference was taken here */
cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
@@ -356,12 +509,8 @@ int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
vma->vm_private_data = cb;
- /* Calculate address for CB */
- address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
-
- rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
- address, cb->size);
-
+ rc = hdev->asic_funcs->cb_mmap(hdev, vma, (void *) cb->kernel_address,
+ cb->bus_address, cb->size);
if (rc) {
spin_lock(&cb->lock);
cb->mmap = false;
@@ -425,7 +574,7 @@ void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
if (kref_put(&cb->refcount, cb_release) != 1)
dev_err(hdev->dev,
"CB %d for CTX ID %d is still alive\n",
- id, cb->ctx_id);
+ id, cb->ctx->asid);
}
idr_destroy(&mgr->cb_handles);
@@ -438,8 +587,8 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
struct hl_cb *cb;
int rc;
- rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
- HL_KERNEL_ASID_ID, internal_cb);
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, cb_size,
+ internal_cb, false, &cb_handle);
if (rc) {
dev_err(hdev->dev,
"Failed to allocate CB for the kernel driver %d\n", rc);
@@ -495,3 +644,45 @@ int hl_cb_pool_fini(struct hl_device *hdev)
return 0;
}
+
+int hl_cb_va_pool_init(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ int rc;
+
+ if (!hdev->supports_cb_mapping)
+ return 0;
+
+ ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1);
+ if (!ctx->cb_va_pool) {
+ dev_err(hdev->dev,
+ "Failed to create VA gen pool for CB mapping\n");
+ return -ENOMEM;
+ }
+
+ rc = gen_pool_add(ctx->cb_va_pool, prop->cb_va_start_addr,
+ prop->cb_va_end_addr - prop->cb_va_start_addr, -1);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to add memory to VA gen pool for CB mapping\n");
+ goto err_pool_destroy;
+ }
+
+ return 0;
+
+err_pool_destroy:
+ gen_pool_destroy(ctx->cb_va_pool);
+
+ return rc;
+}
+
+void hl_cb_va_pool_fini(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (!hdev->supports_cb_mapping)
+ return;
+
+ gen_pool_destroy(ctx->cb_va_pool);
+}
diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c
index 2e3fcbc794db..b2b974ecc431 100644
--- a/drivers/misc/habanalabs/common/command_submission.c
+++ b/drivers/misc/habanalabs/common/command_submission.c
@@ -38,26 +38,10 @@ void hl_sob_reset_error(struct kref *ref)
hw_sob->q_idx, hw_sob->sob_id);
}
-static const char *hl_fence_get_driver_name(struct dma_fence *fence)
-{
- return "HabanaLabs";
-}
-
-static const char *hl_fence_get_timeline_name(struct dma_fence *fence)
-{
- struct hl_cs_compl *hl_cs_compl =
- container_of(fence, struct hl_cs_compl, base_fence);
-
- return dev_name(hl_cs_compl->hdev->dev);
-}
-
-static bool hl_fence_enable_signaling(struct dma_fence *fence)
-{
- return true;
-}
-
-static void hl_fence_release(struct dma_fence *fence)
+static void hl_fence_release(struct kref *kref)
{
+ struct hl_fence *fence =
+ container_of(kref, struct hl_fence, refcount);
struct hl_cs_compl *hl_cs_cmpl =
container_of(fence, struct hl_cs_compl, base_fence);
struct hl_device *hdev = hl_cs_cmpl->hdev;
@@ -99,15 +83,27 @@ static void hl_fence_release(struct dma_fence *fence)
}
free:
- kfree_rcu(hl_cs_cmpl, base_fence.rcu);
+ kfree(hl_cs_cmpl);
}
-static const struct dma_fence_ops hl_fence_ops = {
- .get_driver_name = hl_fence_get_driver_name,
- .get_timeline_name = hl_fence_get_timeline_name,
- .enable_signaling = hl_fence_enable_signaling,
- .release = hl_fence_release
-};
+void hl_fence_put(struct hl_fence *fence)
+{
+ if (fence)
+ kref_put(&fence->refcount, hl_fence_release);
+}
+
+void hl_fence_get(struct hl_fence *fence)
+{
+ if (fence)
+ kref_get(&fence->refcount);
+}
+
+static void hl_fence_init(struct hl_fence *fence)
+{
+ kref_init(&fence->refcount);
+ fence->error = 0;
+ init_completion(&fence->completion);
+}
static void cs_get(struct hl_cs *cs)
{
@@ -256,6 +252,8 @@ static void cs_counters_aggregate(struct hl_device *hdev, struct hl_ctx *ctx)
ctx->cs_counters.parsing_drop_cnt;
hdev->aggregated_cs_counters.queue_full_drop_cnt +=
ctx->cs_counters.queue_full_drop_cnt;
+ hdev->aggregated_cs_counters.max_cs_in_flight_drop_cnt +=
+ ctx->cs_counters.max_cs_in_flight_drop_cnt;
}
static void cs_do_release(struct kref *ref)
@@ -336,7 +334,7 @@ static void cs_do_release(struct kref *ref)
* In case the wait for signal CS was submitted, the put occurs
* in init_signal_wait_cs() right before hanging on the PQ.
*/
- dma_fence_put(cs->signal_fence);
+ hl_fence_put(cs->signal_fence);
}
/*
@@ -348,19 +346,18 @@ static void cs_do_release(struct kref *ref)
hl_ctx_put(cs->ctx);
/* We need to mark an error for not submitted because in that case
- * the dma fence release flow is different. Mainly, we don't need
+ * the hl fence release flow is different. Mainly, we don't need
* to handle hw_sob for signal/wait
*/
if (cs->timedout)
- dma_fence_set_error(cs->fence, -ETIMEDOUT);
+ cs->fence->error = -ETIMEDOUT;
else if (cs->aborted)
- dma_fence_set_error(cs->fence, -EIO);
+ cs->fence->error = -EIO;
else if (!cs->submitted)
- dma_fence_set_error(cs->fence, -EBUSY);
-
- dma_fence_signal(cs->fence);
- dma_fence_put(cs->fence);
+ cs->fence->error = -EBUSY;
+ complete_all(&cs->fence->completion);
+ hl_fence_put(cs->fence);
cs_counters_aggregate(hdev, cs->ctx);
kfree(cs->jobs_in_queue_cnt);
@@ -401,7 +398,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
enum hl_cs_type cs_type, struct hl_cs **cs_new)
{
struct hl_cs_compl *cs_cmpl;
- struct dma_fence *other = NULL;
+ struct hl_fence *other = NULL;
struct hl_cs *cs;
int rc;
@@ -434,9 +431,11 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
cs_cmpl->cs_seq = ctx->cs_sequence;
other = ctx->cs_pending[cs_cmpl->cs_seq &
(hdev->asic_prop.max_pending_cs - 1)];
- if ((other) && (!dma_fence_is_signaled(other))) {
- dev_dbg(hdev->dev,
+
+ if (other && !completion_done(&other->completion)) {
+ dev_dbg_ratelimited(hdev->dev,
"Rejecting CS because of too many in-flights CS\n");
+ ctx->cs_counters.max_cs_in_flight_drop_cnt++;
rc = -EAGAIN;
goto free_fence;
}
@@ -448,8 +447,8 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
goto free_fence;
}
- dma_fence_init(&cs_cmpl->base_fence, &hl_fence_ops, &cs_cmpl->lock,
- ctx->asid, ctx->cs_sequence);
+ /* init hl_fence */
+ hl_fence_init(&cs_cmpl->base_fence);
cs->sequence = cs_cmpl->cs_seq;
@@ -458,9 +457,9 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
&cs_cmpl->base_fence;
ctx->cs_sequence++;
- dma_fence_get(&cs_cmpl->base_fence);
+ hl_fence_get(&cs_cmpl->base_fence);
- dma_fence_put(other);
+ hl_fence_put(other);
spin_unlock(&ctx->cs_lock);
@@ -690,8 +689,8 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
rc = -ENOMEM;
if (is_kernel_allocated_cb)
goto release_cb;
- else
- goto free_cs_object;
+
+ goto free_cs_object;
}
job->id = i + 1;
@@ -773,7 +772,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
struct hl_ctx *ctx = hpriv->ctx;
struct hl_cs_chunk *cs_chunk_array, *chunk;
struct hw_queue_properties *hw_queue_prop;
- struct dma_fence *sig_fence = NULL;
+ struct hl_fence *sig_fence = NULL;
struct hl_cs_job *job;
struct hl_cs *cs;
struct hl_cb *cb;
@@ -883,14 +882,14 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
dev_err(hdev->dev,
"CS seq 0x%llx is not of a signal CS\n",
signal_seq);
- dma_fence_put(sig_fence);
+ hl_fence_put(sig_fence);
rc = -EINVAL;
goto free_signal_seq_array;
}
- if (dma_fence_is_signaled(sig_fence)) {
+ if (completion_done(&sig_fence->completion)) {
/* signal CS already finished */
- dma_fence_put(sig_fence);
+ hl_fence_put(sig_fence);
rc = 0;
goto free_signal_seq_array;
}
@@ -902,7 +901,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
rc = allocate_cs(hdev, ctx, cs_type, &cs);
if (rc) {
if (cs_type == CS_TYPE_WAIT)
- dma_fence_put(sig_fence);
+ hl_fence_put(sig_fence);
hl_ctx_put(ctx);
goto free_signal_seq_array;
}
@@ -1162,7 +1161,7 @@ out:
static long _hl_cs_wait_ioctl(struct hl_device *hdev,
struct hl_ctx *ctx, u64 timeout_us, u64 seq)
{
- struct dma_fence *fence;
+ struct hl_fence *fence;
unsigned long timeout;
long rc;
@@ -1181,12 +1180,18 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev,
"Can't wait on CS %llu because current CS is at seq %llu\n",
seq, ctx->cs_sequence);
} else if (fence) {
- rc = dma_fence_wait_timeout(fence, true, timeout);
+ if (!timeout_us)
+ rc = completion_done(&fence->completion);
+ else
+ rc = wait_for_completion_interruptible_timeout(
+ &fence->completion, timeout);
+
if (fence->error == -ETIMEDOUT)
rc = -ETIMEDOUT;
else if (fence->error == -EIO)
rc = -EIO;
- dma_fence_put(fence);
+
+ hl_fence_put(fence);
} else {
dev_dbg(hdev->dev,
"Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c
index 3e375958e73b..7a59dd7c6450 100644
--- a/drivers/misc/habanalabs/common/context.c
+++ b/drivers/misc/habanalabs/common/context.c
@@ -12,6 +12,7 @@
static void hl_ctx_fini(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
+ u64 idle_mask = 0;
int i;
/*
@@ -23,11 +24,13 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
*/
for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
- dma_fence_put(ctx->cs_pending[i]);
+ hl_fence_put(ctx->cs_pending[i]);
kfree(ctx->cs_pending);
if (ctx->asid != HL_KERNEL_ASID_ID) {
+ dev_dbg(hdev->dev, "closing user context %d\n", ctx->asid);
+
/* The engines are stopped as there is no executing CS, but the
* Coresight might be still working by accessing addresses
* related to the stopped engines. Hence stop it explicitly.
@@ -37,9 +40,18 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
if ((hdev->in_debug) && (hdev->compute_ctx == ctx))
hl_device_set_debug_mode(hdev, false);
+ hl_cb_va_pool_fini(ctx);
hl_vm_ctx_fini(ctx);
hl_asid_free(hdev, ctx->asid);
+
+ if ((!hdev->pldm) && (hdev->pdev) &&
+ (!hdev->asic_funcs->is_device_idle(hdev,
+ &idle_mask, NULL)))
+ dev_notice(hdev->dev,
+ "device not idle after user context is closed (0x%llx)\n",
+ idle_mask);
} else {
+ dev_dbg(hdev->dev, "closing kernel context\n");
hl_mmu_ctx_fini(ctx);
}
}
@@ -128,7 +140,7 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
atomic_set(&ctx->thread_ctx_switch_token, 1);
ctx->thread_ctx_switch_wait_token = 0;
ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
- sizeof(struct dma_fence *),
+ sizeof(struct hl_fence *),
GFP_KERNEL);
if (!ctx->cs_pending)
return -ENOMEM;
@@ -155,15 +167,26 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
goto err_asid_free;
}
+ rc = hl_cb_va_pool_init(ctx);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to init VA pool for mapped CB\n");
+ goto err_vm_ctx_fini;
+ }
+
rc = hdev->asic_funcs->ctx_init(ctx);
if (rc) {
dev_err(hdev->dev, "ctx_init failed\n");
- goto err_vm_ctx_fini;
+ goto err_cb_va_pool_fini;
}
+
+ dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
}
return 0;
+err_cb_va_pool_fini:
+ hl_cb_va_pool_fini(ctx);
err_vm_ctx_fini:
hl_vm_ctx_fini(ctx);
err_asid_free:
@@ -184,10 +207,10 @@ int hl_ctx_put(struct hl_ctx *ctx)
return kref_put(&ctx->refcount, hl_ctx_do_release);
}
-struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
+struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
{
struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
- struct dma_fence *fence;
+ struct hl_fence *fence;
spin_lock(&ctx->cs_lock);
@@ -201,8 +224,9 @@ struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
return NULL;
}
- fence = dma_fence_get(
- ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)]);
+ fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
+ hl_fence_get(fence);
+
spin_unlock(&ctx->cs_lock);
return fence;
diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c
index aa77771635d3..912ddfa360b1 100644
--- a/drivers/misc/habanalabs/common/debugfs.c
+++ b/drivers/misc/habanalabs/common/debugfs.c
@@ -21,7 +21,7 @@ static struct dentry *hl_debug_root;
static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
u8 i2c_reg, long *val)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
if (hl_device_disabled_or_in_reset(hdev))
@@ -29,8 +29,8 @@ static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_I2C_RD <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_I2C_RD <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.i2c_bus = i2c_bus;
pkt.i2c_addr = i2c_addr;
pkt.i2c_reg = i2c_reg;
@@ -47,7 +47,7 @@ static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
static int hl_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
u8 i2c_reg, u32 val)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
if (hl_device_disabled_or_in_reset(hdev))
@@ -55,8 +55,8 @@ static int hl_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_I2C_WR <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_I2C_WR <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.i2c_bus = i2c_bus;
pkt.i2c_addr = i2c_addr;
pkt.i2c_reg = i2c_reg;
@@ -73,7 +73,7 @@ static int hl_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
static void hl_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
if (hl_device_disabled_or_in_reset(hdev))
@@ -81,8 +81,8 @@ static void hl_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state)
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_LED_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_LED_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.led_index = cpu_to_le32(led);
pkt.value = cpu_to_le64(state);
@@ -110,8 +110,8 @@ static int command_buffers_show(struct seq_file *s, void *data)
seq_puts(s, "---------------------------------------------------------------\n");
}
seq_printf(s,
- " %03d %d 0x%08x %d %d %d\n",
- cb->id, cb->ctx_id, cb->size,
+ " %03llu %d 0x%08x %d %d %d\n",
+ cb->id, cb->ctx->asid, cb->size,
kref_read(&cb->refcount),
cb->mmap, cb->cs_cnt);
}
@@ -354,6 +354,14 @@ static inline u64 get_hop4_pte_addr(struct hl_ctx *ctx,
mmu_specs->hop4_shift);
}
+static inline u64 get_hop5_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_specs,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop5_mask,
+ mmu_specs->hop5_shift);
+}
+
static inline u64 get_next_hop_addr(u64 curr_pte)
{
if (curr_pte & PAGE_PRESENT_MASK)
@@ -377,6 +385,7 @@ static int mmu_show(struct seq_file *s, void *data)
hop2_addr = 0, hop2_pte_addr = 0, hop2_pte = 0,
hop3_addr = 0, hop3_pte_addr = 0, hop3_pte = 0,
hop4_addr = 0, hop4_pte_addr = 0, hop4_pte = 0,
+ hop5_addr = 0, hop5_pte_addr = 0, hop5_pte = 0,
virt_addr = dev_entry->mmu_addr;
if (!hdev->mmu_enable)
@@ -428,20 +437,49 @@ static int mmu_show(struct seq_file *s, void *data)
hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);
hop3_pte = hdev->asic_funcs->read_pte(hdev, hop3_pte_addr);
- if (!(hop3_pte & LAST_MASK)) {
+ if (mmu_prop->num_hops == MMU_ARCH_5_HOPS) {
+ if (!(hop3_pte & LAST_MASK)) {
+ hop4_addr = get_next_hop_addr(hop3_pte);
+
+ if (hop4_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop,
+ hop4_addr, virt_addr);
+ hop4_pte = hdev->asic_funcs->read_pte(hdev,
+ hop4_pte_addr);
+ if (!(hop4_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ } else {
+ if (!(hop3_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ }
+ } else {
hop4_addr = get_next_hop_addr(hop3_pte);
if (hop4_addr == ULLONG_MAX)
goto not_mapped;
- hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop4_addr,
- virt_addr);
- hop4_pte = hdev->asic_funcs->read_pte(hdev, hop4_pte_addr);
- if (!(hop4_pte & PAGE_PRESENT_MASK))
- goto not_mapped;
- } else {
- if (!(hop3_pte & PAGE_PRESENT_MASK))
- goto not_mapped;
+ hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop,
+ hop4_addr, virt_addr);
+ hop4_pte = hdev->asic_funcs->read_pte(hdev,
+ hop4_pte_addr);
+ if (!(hop4_pte & LAST_MASK)) {
+ hop5_addr = get_next_hop_addr(hop4_pte);
+
+ if (hop5_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop5_pte_addr = get_hop5_pte_addr(ctx, mmu_prop,
+ hop5_addr, virt_addr);
+ hop5_pte = hdev->asic_funcs->read_pte(hdev,
+ hop5_pte_addr);
+ if (!(hop5_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ } else {
+ if (!(hop4_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ }
}
seq_printf(s, "asid: %u, virt_addr: 0x%llx\n",
@@ -463,10 +501,22 @@ static int mmu_show(struct seq_file *s, void *data)
seq_printf(s, "hop3_pte_addr: 0x%llx\n", hop3_pte_addr);
seq_printf(s, "hop3_pte: 0x%llx\n", hop3_pte);
- if (!(hop3_pte & LAST_MASK)) {
+ if (mmu_prop->num_hops == MMU_ARCH_5_HOPS) {
+ if (!(hop3_pte & LAST_MASK)) {
+ seq_printf(s, "hop4_addr: 0x%llx\n", hop4_addr);
+ seq_printf(s, "hop4_pte_addr: 0x%llx\n", hop4_pte_addr);
+ seq_printf(s, "hop4_pte: 0x%llx\n", hop4_pte);
+ }
+ } else {
seq_printf(s, "hop4_addr: 0x%llx\n", hop4_addr);
seq_printf(s, "hop4_pte_addr: 0x%llx\n", hop4_pte_addr);
seq_printf(s, "hop4_pte: 0x%llx\n", hop4_pte);
+
+ if (!(hop4_pte & LAST_MASK)) {
+ seq_printf(s, "hop5_addr: 0x%llx\n", hop5_addr);
+ seq_printf(s, "hop5_pte_addr: 0x%llx\n", hop5_pte_addr);
+ seq_printf(s, "hop5_pte: 0x%llx\n", hop5_pte);
+ }
}
goto out;
diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c
index 24b01cce0a38..20572224099a 100644
--- a/drivers/misc/habanalabs/common/device.c
+++ b/drivers/misc/habanalabs/common/device.c
@@ -123,9 +123,13 @@ static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct hl_fpriv *hpriv = filp->private_data;
+ unsigned long vm_pgoff;
- if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
- vma->vm_pgoff ^= HL_MMAP_CB_MASK;
+ vm_pgoff = vma->vm_pgoff;
+ vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
+
+ switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
+ case HL_MMAP_TYPE_CB:
return hl_cb_mmap(hpriv, vma);
}
@@ -286,7 +290,7 @@ static int device_early_init(struct hl_device *hdev)
}
for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
- snprintf(workq_name, 32, "hl-free-jobs-%u", i);
+ snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
if (hdev->cq_wq[i] == NULL) {
dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
@@ -317,6 +321,10 @@ static int device_early_init(struct hl_device *hdev)
goto free_chip_info;
}
+ rc = hl_mmu_if_set_funcs(hdev);
+ if (rc)
+ goto free_idle_busy_ts_arr;
+
hl_cb_mgr_init(&hdev->kernel_cb_mgr);
mutex_init(&hdev->send_cpu_message_lock);
@@ -330,6 +338,8 @@ static int device_early_init(struct hl_device *hdev)
return 0;
+free_idle_busy_ts_arr:
+ kfree(hdev->idle_busy_ts_arr);
free_chip_info:
kfree(hdev->hl_chip_info);
free_eq_wq:
@@ -871,7 +881,7 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
* so this message won't be sent
*/
if (hl_fw_send_pci_access_msg(hdev,
- ARMCP_PACKET_DISABLE_PCI_ACCESS))
+ CPUCP_PACKET_DISABLE_PCI_ACCESS))
dev_warn(hdev->dev,
"Failed to disable PCI access by F/W\n");
}
@@ -957,14 +967,13 @@ again:
flush_workqueue(hdev->eq_wq);
}
- /* Release kernel context */
- if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
- hdev->kernel_ctx = NULL;
-
/* Reset the H/W. It will be in idle state after this returns */
hdev->asic_funcs->hw_fini(hdev, hard_reset);
if (hard_reset) {
+ /* Release kernel context */
+ if (hl_ctx_put(hdev->kernel_ctx) == 1)
+ hdev->kernel_ctx = NULL;
hl_vm_fini(hdev);
hl_mmu_fini(hdev);
hl_eq_reset(hdev, &hdev->event_queue);
@@ -1455,13 +1464,13 @@ void hl_device_fini(struct hl_device *hdev)
hl_cb_pool_fini(hdev);
+ /* Reset the H/W. It will be in idle state after this returns */
+ hdev->asic_funcs->hw_fini(hdev, true);
+
/* Release kernel context */
if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
dev_err(hdev->dev, "kernel ctx is still alive\n");
- /* Reset the H/W. It will be in idle state after this returns */
- hdev->asic_funcs->hw_fini(hdev, true);
-
hl_vm_fini(hdev);
hl_mmu_fini(hdev);
diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c
index f52bc690dfc5..cd41c7ceb0e7 100644
--- a/drivers/misc/habanalabs/common/firmware_if.c
+++ b/drivers/misc/habanalabs/common/firmware_if.c
@@ -68,9 +68,9 @@ out:
int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
{
- struct armcp_packet pkt = {};
+ struct cpucp_packet pkt = {};
- pkt.ctl = cpu_to_le32(opcode << ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
sizeof(pkt), 0, NULL);
@@ -79,7 +79,7 @@ int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
u16 len, u32 timeout, long *result)
{
- struct armcp_packet *pkt;
+ struct cpucp_packet *pkt;
dma_addr_t pkt_dma_addr;
u32 tmp;
int rc = 0;
@@ -111,7 +111,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
}
rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
- (tmp == ARMCP_PACKET_FENCE_VAL), 1000,
+ (tmp == CPUCP_PACKET_FENCE_VAL), 1000,
timeout, true);
hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
@@ -124,12 +124,12 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
tmp = le32_to_cpu(pkt->ctl);
- rc = (tmp & ARMCP_PKT_CTL_RC_MASK) >> ARMCP_PKT_CTL_RC_SHIFT;
+ rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
if (rc) {
dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
rc,
- (tmp & ARMCP_PKT_CTL_OPCODE_MASK)
- >> ARMCP_PKT_CTL_OPCODE_SHIFT);
+ (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
+ >> CPUCP_PKT_CTL_OPCODE_SHIFT);
rc = -EIO;
} else if (result) {
*result = (long) le64_to_cpu(pkt->result);
@@ -145,14 +145,14 @@ out:
int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
long result;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.value = cpu_to_le64(event_type);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
@@ -167,15 +167,15 @@ int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
size_t irq_arr_size)
{
- struct armcp_unmask_irq_arr_packet *pkt;
+ struct cpucp_unmask_irq_arr_packet *pkt;
size_t total_pkt_size;
long result;
int rc;
- total_pkt_size = sizeof(struct armcp_unmask_irq_arr_packet) +
+ total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
irq_arr_size;
- /* data should be aligned to 8 bytes in order to ArmCP to copy it */
+ /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
/* total_pkt_size is casted to u16 later on */
@@ -191,8 +191,8 @@ int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
memcpy(&pkt->irqs, irq_arr, irq_arr_size);
- pkt->armcp_pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
total_pkt_size, 0, &result);
@@ -207,19 +207,19 @@ int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
int hl_fw_test_cpu_queue(struct hl_device *hdev)
{
- struct armcp_packet test_pkt = {};
+ struct cpucp_packet test_pkt = {};
long result;
int rc;
- test_pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEST <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- test_pkt.value = cpu_to_le64(ARMCP_PACKET_FENCE_VAL);
+ test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
+ test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
sizeof(test_pkt), 0, &result);
if (!rc) {
- if (result != ARMCP_PACKET_FENCE_VAL)
+ if (result != CPUCP_PACKET_FENCE_VAL)
dev_err(hdev->dev,
"CPU queue test failed (0x%08lX)\n", result);
} else {
@@ -251,61 +251,61 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
int hl_fw_send_heartbeat(struct hl_device *hdev)
{
- struct armcp_packet hb_pkt = {};
+ struct cpucp_packet hb_pkt = {};
long result;
int rc;
- hb_pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEST <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- hb_pkt.value = cpu_to_le64(ARMCP_PACKET_FENCE_VAL);
+ hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
+ hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
sizeof(hb_pkt), 0, &result);
- if ((rc) || (result != ARMCP_PACKET_FENCE_VAL))
+ if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
rc = -EIO;
return rc;
}
-int hl_fw_armcp_info_get(struct hl_device *hdev)
+int hl_fw_cpucp_info_get(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct armcp_packet pkt = {};
- void *armcp_info_cpu_addr;
- dma_addr_t armcp_info_dma_addr;
+ struct cpucp_packet pkt = {};
+ void *cpucp_info_cpu_addr;
+ dma_addr_t cpucp_info_dma_addr;
long result;
int rc;
- armcp_info_cpu_addr =
+ cpucp_info_cpu_addr =
hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
- sizeof(struct armcp_info),
- &armcp_info_dma_addr);
- if (!armcp_info_cpu_addr) {
+ sizeof(struct cpucp_info),
+ &cpucp_info_dma_addr);
+ if (!cpucp_info_cpu_addr) {
dev_err(hdev->dev,
- "Failed to allocate DMA memory for ArmCP info packet\n");
+ "Failed to allocate DMA memory for CPU-CP info packet\n");
return -ENOMEM;
}
- memset(armcp_info_cpu_addr, 0, sizeof(struct armcp_info));
+ memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_INFO_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- pkt.addr = cpu_to_le64(armcp_info_dma_addr);
- pkt.data_max_size = cpu_to_le32(sizeof(struct armcp_info));
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
+ pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
- HL_ARMCP_INFO_TIMEOUT_USEC, &result);
+ HL_CPUCP_INFO_TIMEOUT_USEC, &result);
if (rc) {
dev_err(hdev->dev,
- "Failed to handle ArmCP info pkt, error %d\n", rc);
+ "Failed to handle CPU-CP info pkt, error %d\n", rc);
goto out;
}
- memcpy(&prop->armcp_info, armcp_info_cpu_addr,
- sizeof(prop->armcp_info));
+ memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
+ sizeof(prop->cpucp_info));
- rc = hl_build_hwmon_channel_info(hdev, prop->armcp_info.sensors);
+ rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
if (rc) {
dev_err(hdev->dev,
"Failed to build hwmon channel info, error %d\n", rc);
@@ -315,14 +315,14 @@ int hl_fw_armcp_info_get(struct hl_device *hdev)
out:
hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
- sizeof(struct armcp_info), armcp_info_cpu_addr);
+ sizeof(struct cpucp_info), cpucp_info_cpu_addr);
return rc;
}
int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
{
- struct armcp_packet pkt = {};
+ struct cpucp_packet pkt = {};
void *eeprom_info_cpu_addr;
dma_addr_t eeprom_info_dma_addr;
long result;
@@ -333,23 +333,24 @@ int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
max_size, &eeprom_info_dma_addr);
if (!eeprom_info_cpu_addr) {
dev_err(hdev->dev,
- "Failed to allocate DMA memory for ArmCP EEPROM packet\n");
+ "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
return -ENOMEM;
}
memset(eeprom_info_cpu_addr, 0, max_size);
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_EEPROM_DATA_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
pkt.data_max_size = cpu_to_le32(max_size);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
- HL_ARMCP_EEPROM_TIMEOUT_USEC, &result);
+ HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
if (rc) {
dev_err(hdev->dev,
- "Failed to handle ArmCP EEPROM packet, error %d\n", rc);
+ "Failed to handle CPU-CP EEPROM packet, error %d\n",
+ rc);
goto out;
}
@@ -363,6 +364,77 @@ out:
return rc;
}
+int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
+ struct hl_info_pci_counters *counters)
+{
+ struct cpucp_packet pkt = {};
+ long result;
+ int rc;
+
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
+
+ /* Fetch PCI rx counter */
+ pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_CPUCP_INFO_TIMEOUT_USEC, &result);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
+ return rc;
+ }
+ counters->rx_throughput = result;
+
+ /* Fetch PCI tx counter */
+ pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_CPUCP_INFO_TIMEOUT_USEC, &result);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
+ return rc;
+ }
+ counters->tx_throughput = result;
+
+ /* Fetch PCI replay counter */
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_CPUCP_INFO_TIMEOUT_USEC, &result);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
+ return rc;
+ }
+ counters->replay_cnt = (u32) result;
+
+ return rc;
+}
+
+int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
+{
+ struct cpucp_packet pkt = {};
+ long result;
+ int rc;
+
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_CPUCP_INFO_TIMEOUT_USEC, &result);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to handle CpuCP total energy pkt, error %d\n",
+ rc);
+ return rc;
+ }
+
+ *total_energy = result;
+
+ return rc;
+}
+
static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg)
{
u32 err_val;
@@ -402,8 +474,11 @@ static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg)
"Device boot error - NIC F/W initialization failed\n");
}
-static void hl_detect_cpu_boot_status(struct hl_device *hdev, u32 status)
+static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
{
+ /* Some of the status codes below are deprecated in newer f/w
+ * versions but we keep them here for backward compatibility
+ */
switch (status) {
case CPU_BOOT_STATUS_NA:
dev_err(hdev->dev,
@@ -449,6 +524,48 @@ static void hl_detect_cpu_boot_status(struct hl_device *hdev, u32 status)
}
}
+int hl_fw_read_preboot_ver(struct hl_device *hdev, u32 cpu_boot_status_reg,
+ u32 boot_err0_reg, u32 timeout)
+{
+ u32 status;
+ int rc;
+
+ if (!hdev->cpu_enable)
+ return 0;
+
+ /* Need to check two possible scenarios:
+ *
+ * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
+ * the preboot is waiting for the boot fit
+ *
+ * All other status values - for older firmwares where the uboot was
+ * loaded from the FLASH
+ */
+ rc = hl_poll_timeout(
+ hdev,
+ cpu_boot_status_reg,
+ status,
+ (status == CPU_BOOT_STATUS_IN_UBOOT) ||
+ (status == CPU_BOOT_STATUS_DRAM_RDY) ||
+ (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
+ (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
+ (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
+ (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
+ 10000,
+ timeout);
+
+ if (rc) {
+ dev_err(hdev->dev, "Failed to read preboot version\n");
+ detect_cpu_boot_status(hdev, status);
+ fw_read_errors(hdev, boot_err0_reg);
+ return -EIO;
+ }
+
+ hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
+
+ return 0;
+}
+
int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
u32 boot_err0_reg, bool skip_bmc,
@@ -514,15 +631,11 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
10000,
cpu_timeout);
- /* Read U-Boot, preboot versions now in case we will later fail */
+ /* Read U-Boot version now in case we will later fail */
hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT);
- hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
- /* Some of the status codes below are deprecated in newer f/w
- * versions but we keep them here for backward compatibility
- */
if (rc) {
- hl_detect_cpu_boot_status(hdev, status);
+ detect_cpu_boot_status(hdev, status);
rc = -EIO;
goto out;
}
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index edbd627b29d2..80d4d7385ffe 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -8,21 +8,33 @@
#ifndef HABANALABSP_H_
#define HABANALABSP_H_
-#include "../include/common/armcp_if.h"
+#include "../include/common/cpucp_if.h"
#include "../include/common/qman_if.h"
#include <uapi/misc/habanalabs.h>
#include <linux/cdev.h>
#include <linux/iopoll.h>
#include <linux/irqreturn.h>
-#include <linux/dma-fence.h>
#include <linux/dma-direction.h>
#include <linux/scatterlist.h>
#include <linux/hashtable.h>
+#include <linux/bitfield.h>
#define HL_NAME "habanalabs"
-#define HL_MMAP_CB_MASK (0x8000000000000000ull >> PAGE_SHIFT)
+/* Use upper bits of mmap offset to store habana driver specific information.
+ * bits[63:62] - Encode mmap type
+ * bits[45:0] - mmap offset value
+ *
+ * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
+ * defines are w.r.t to PAGE_SIZE
+ */
+#define HL_MMAP_TYPE_SHIFT (62 - PAGE_SHIFT)
+#define HL_MMAP_TYPE_MASK (0x3ull << HL_MMAP_TYPE_SHIFT)
+#define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT)
+
+#define HL_MMAP_OFFSET_VALUE_MASK (0x3FFFFFFFFFFFull >> PAGE_SHIFT)
+#define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK)
#define HL_PENDING_RESET_PER_SEC 30
@@ -34,8 +46,8 @@
#define HL_PLL_LOW_JOB_FREQ_USEC 5000000 /* 5 s */
-#define HL_ARMCP_INFO_TIMEOUT_USEC 10000000 /* 10s */
-#define HL_ARMCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */
+#define HL_CPUCP_INFO_TIMEOUT_USEC 10000000 /* 10s */
+#define HL_CPUCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */
#define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */
@@ -66,6 +78,8 @@
#define HL_PCI_NUM_BARS 6
+#define HL_MAX_DCORES 4
+
/**
* struct pgt_info - MMU hop page info.
* @node: hash linked-list node for the pgts shadow hash of pgts.
@@ -222,12 +236,15 @@ enum hl_device_hw_state {
* @hop2_shift: shift of hop 2 mask.
* @hop3_shift: shift of hop 3 mask.
* @hop4_shift: shift of hop 4 mask.
+ * @hop5_shift: shift of hop 5 mask.
* @hop0_mask: mask to get the PTE address in hop 0.
* @hop1_mask: mask to get the PTE address in hop 1.
* @hop2_mask: mask to get the PTE address in hop 2.
* @hop3_mask: mask to get the PTE address in hop 3.
* @hop4_mask: mask to get the PTE address in hop 4.
+ * @hop5_mask: mask to get the PTE address in hop 5.
* @page_size: default page size used to allocate memory.
+ * @num_hops: The amount of hops supported by the translation table.
*/
struct hl_mmu_properties {
u64 start_addr;
@@ -237,18 +254,21 @@ struct hl_mmu_properties {
u64 hop2_shift;
u64 hop3_shift;
u64 hop4_shift;
+ u64 hop5_shift;
u64 hop0_mask;
u64 hop1_mask;
u64 hop2_mask;
u64 hop3_mask;
u64 hop4_mask;
+ u64 hop5_mask;
u32 page_size;
+ u32 num_hops;
};
/**
* struct asic_fixed_properties - ASIC specific immutable properties.
* @hw_queues_props: H/W queues properties.
- * @armcp_info: received various information from ArmCP regarding the H/W, e.g.
+ * @cpucp_info: received various information from CPU-CP regarding the H/W, e.g.
* available sensors.
* @uboot_ver: F/W U-boot version.
* @preboot_ver: F/W Preboot version.
@@ -271,6 +291,10 @@ struct hl_mmu_properties {
* @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register.
* @mmu_pgt_addr: base physical address in DRAM of MMU page tables.
* @mmu_dram_default_page_addr: DRAM default page physical address.
+ * @cb_va_start_addr: virtual start address of command buffers which are mapped
+ * to the device's MMU.
+ * @cb_va_end_addr: virtual end address of command buffers which are mapped to
+ * the device's MMU.
* @mmu_pgt_size: MMU page tables total size.
* @mmu_pte_size: PTE size in MMU page tables.
* @mmu_hop_table_size: MMU hop table size.
@@ -292,12 +316,16 @@ struct hl_mmu_properties {
* @max_queues: maximum amount of queues in the system
* @sync_stream_first_sob: first sync object available for sync stream use
* @sync_stream_first_mon: first monitor available for sync stream use
+ * @first_available_user_sob: first sob available for the user
+ * @first_available_user_mon: first monitor available for the user
* @tpc_enabled_mask: which TPCs are enabled.
* @completion_queues_count: number of completion queues.
+ * @fw_security_disabled: true if security measures are disabled in firmware,
+ * false otherwise
*/
struct asic_fixed_properties {
struct hw_queue_properties *hw_queues_props;
- struct armcp_info armcp_info;
+ struct cpucp_info cpucp_info;
char uboot_ver[VERSION_MAX_LEN];
char preboot_ver[VERSION_MAX_LEN];
struct hl_mmu_properties dmmu;
@@ -317,6 +345,8 @@ struct asic_fixed_properties {
u64 pcie_aux_dbi_reg_addr;
u64 mmu_pgt_addr;
u64 mmu_dram_default_page_addr;
+ u64 cb_va_start_addr;
+ u64 cb_va_end_addr;
u32 mmu_pgt_size;
u32 mmu_pte_size;
u32 mmu_hop_table_size;
@@ -338,13 +368,29 @@ struct asic_fixed_properties {
u32 max_queues;
u16 sync_stream_first_sob;
u16 sync_stream_first_mon;
+ u16 first_available_user_sob[HL_MAX_DCORES];
+ u16 first_available_user_mon[HL_MAX_DCORES];
u8 tpc_enabled_mask;
u8 completion_queues_count;
+ u8 fw_security_disabled;
+};
+
+/**
+ * struct hl_fence - software synchronization primitive
+ * @completion: fence is implemented using completion
+ * @refcount: refcount for this fence
+ * @error: mark this fence with error
+ *
+ */
+struct hl_fence {
+ struct completion completion;
+ struct kref refcount;
+ int error;
};
/**
* struct hl_cs_compl - command submission completion object.
- * @base_fence: kernel fence object.
+ * @base_fence: hl fence object.
* @lock: spinlock to protect fence.
* @hdev: habanalabs device structure.
* @hw_sob: the H/W SOB used in this signal/wait CS.
@@ -353,7 +399,7 @@ struct asic_fixed_properties {
* @sob_val: the SOB value that is used in this signal/wait CS.
*/
struct hl_cs_compl {
- struct dma_fence base_fence;
+ struct hl_fence base_fence;
spinlock_t lock;
struct hl_device *hdev;
struct hl_hw_sob *hw_sob;
@@ -380,36 +426,41 @@ struct hl_cb_mgr {
* struct hl_cb - describes a Command Buffer.
* @refcount: reference counter for usage of the CB.
* @hdev: pointer to device this CB belongs to.
+ * @ctx: pointer to the CB owner's context.
* @lock: spinlock to protect mmap/cs flows.
* @debugfs_list: node in debugfs list of command buffers.
* @pool_list: node in pool list of command buffers.
+ * @va_block_list: list of virtual addresses blocks of the CB if it is mapped to
+ * the device's MMU.
+ * @id: the CB's ID.
* @kernel_address: Holds the CB's kernel virtual address.
* @bus_address: Holds the CB's DMA address.
* @mmap_size: Holds the CB's size that was mmaped.
* @size: holds the CB's size.
- * @id: the CB's ID.
* @cs_cnt: holds number of CS that this CB participates in.
- * @ctx_id: holds the ID of the owner's context.
* @mmap: true if the CB is currently mmaped to user.
* @is_pool: true if CB was acquired from the pool, false otherwise.
* @is_internal: internaly allocated
+ * @is_mmu_mapped: true if the CB is mapped to the device's MMU.
*/
struct hl_cb {
struct kref refcount;
struct hl_device *hdev;
+ struct hl_ctx *ctx;
spinlock_t lock;
struct list_head debugfs_list;
struct list_head pool_list;
+ struct list_head va_block_list;
+ u64 id;
u64 kernel_address;
dma_addr_t bus_address;
u32 mmap_size;
u32 size;
- u32 id;
u32 cs_cnt;
- u32 ctx_id;
u8 mmap;
u8 is_pool;
u8 is_internal;
+ u8 is_mmu_mapped;
};
@@ -435,7 +486,7 @@ struct hl_cs_job;
#define HL_EQ_LENGTH 64
#define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
-/* Host <-> ArmCP shared memory size */
+/* Host <-> CPU-CP shared memory size */
#define HL_CPU_ACCESSIBLE_MEM_SIZE SZ_2M
/**
@@ -617,7 +668,7 @@ enum div_select_defs {
* @debugfs_read32: debug interface for reading u32 from DRAM/SRAM.
* @debugfs_write32: debug interface for writing u32 to DRAM/SRAM.
* @add_device_attr: add ASIC specific device attributes.
- * @handle_eqe: handle event queue entry (IRQ) from ArmCP.
+ * @handle_eqe: handle event queue entry (IRQ) from CPU-CP.
* @set_pll_profile: change PLL profile (manual/automatic).
* @get_events_stat: retrieve event queue entries histogram.
* @read_pte: read MMU page table entry from DRAM.
@@ -626,7 +677,7 @@ enum div_select_defs {
* (L1 only) or hard (L0 & L1) flush.
* @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with
* ASID-VA-size mask.
- * @send_heartbeat: send is-alive packet to ArmCP and verify response.
+ * @send_heartbeat: send is-alive packet to CPU-CP and verify response.
* @set_clock_gating: enable/disable clock gating per engine according to
* clock gating mask in hdev
* @disable_clock_gating: disable clock gating completely
@@ -644,8 +695,6 @@ enum div_select_defs {
* ASIC
* @get_hw_state: retrieve the H/W state
* @pci_bars_map: Map PCI BARs.
- * @set_dram_bar_base: Set DRAM BAR to map specific device address. Returns
- * old address the bar pointed to or U64_MAX for failure
* @init_iatu: Initialize the iATU unit inside the PCI controller.
* @rreg: Read a register. Needed for simulator support.
* @wreg: Write a register. Needed for simulator support.
@@ -679,7 +728,7 @@ struct hl_asic_funcs {
int (*suspend)(struct hl_device *hdev);
int (*resume)(struct hl_device *hdev);
int (*cb_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
- u64 kaddress, phys_addr_t paddress, u32 size);
+ void *cpu_addr, dma_addr_t dma_addr, size_t size);
void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
struct hl_bd *bd);
@@ -736,7 +785,7 @@ struct hl_asic_funcs {
void (*set_clock_gating)(struct hl_device *hdev);
void (*disable_clock_gating)(struct hl_device *hdev);
int (*debug_coresight)(struct hl_device *hdev, void *data);
- bool (*is_device_idle)(struct hl_device *hdev, u32 *mask,
+ bool (*is_device_idle)(struct hl_device *hdev, u64 *mask,
struct seq_file *s);
int (*soft_reset_late_init)(struct hl_device *hdev);
void (*hw_queues_lock)(struct hl_device *hdev);
@@ -748,7 +797,6 @@ struct hl_asic_funcs {
u16 len, u32 timeout, long *result);
enum hl_device_hw_state (*get_hw_state)(struct hl_device *hdev);
int (*pci_bars_map)(struct hl_device *hdev);
- u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
int (*init_iatu)(struct hl_device *hdev);
u32 (*rreg)(struct hl_device *hdev, u32 reg);
void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
@@ -800,7 +848,7 @@ struct hl_va_range {
* @hdev: pointer to the device structure.
* @refcount: reference counter for the context. Context is released only when
* this hits 0l. It is incremented on CS and CS_WAIT.
- * @cs_pending: array of DMA fence objects representing pending CS.
+ * @cs_pending: array of hl fence objects representing pending CS.
* @host_va_range: holds available virtual addresses for host mappings.
* @host_huge_va_range: holds available virtual addresses for host mappings
* with huge pages.
@@ -809,6 +857,8 @@ struct hl_va_range {
* @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
* MMU hash or walking the PGT requires talking this lock.
* @debugfs_list: node in debugfs list of contexts.
+ * @cb_va_pool: device VA pool for command buffers which are mapped to the
+ * device's MMU.
* @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
* to user so user could inquire about CS. It is used as
* index to cs_pending array.
@@ -832,7 +882,7 @@ struct hl_ctx {
struct hl_fpriv *hpriv;
struct hl_device *hdev;
struct kref refcount;
- struct dma_fence **cs_pending;
+ struct hl_fence **cs_pending;
struct hl_va_range *host_va_range;
struct hl_va_range *host_huge_va_range;
struct hl_va_range *dram_va_range;
@@ -840,6 +890,7 @@ struct hl_ctx {
struct mutex mmu_lock;
struct list_head debugfs_list;
struct hl_cs_counters cs_counters;
+ struct gen_pool *cb_va_pool;
u64 cs_sequence;
u64 *dram_default_hops;
spinlock_t cs_lock;
@@ -919,8 +970,8 @@ struct hl_cs {
struct list_head job_list;
spinlock_t job_lock;
struct kref refcount;
- struct dma_fence *fence;
- struct dma_fence *signal_fence;
+ struct hl_fence *fence;
+ struct hl_fence *signal_fence;
struct work_struct finish_work;
struct delayed_work work_tdr;
struct list_head mirror_node;
@@ -1395,6 +1446,44 @@ struct hl_device_idle_busy_ts {
ktime_t busy_to_idle_ts;
};
+
+/**
+ * struct hl_mmu_priv - used for holding per-device mmu internal information.
+ * @mmu_pgt_pool: pool of page tables used by MMU for allocating hops.
+ * @mmu_shadow_hop0: shadow array of hop0 tables.
+ */
+struct hl_mmu_priv {
+ struct gen_pool *mmu_pgt_pool;
+ void *mmu_shadow_hop0;
+};
+
+/**
+ * struct hl_mmu_funcs - Device related MMU functions.
+ * @init: initialize the MMU module.
+ * @fini: release the MMU module.
+ * @ctx_init: Initialize a context for using the MMU module.
+ * @ctx_fini: disable a ctx from using the mmu module.
+ * @map: maps a virtual address to physical address for a context.
+ * @unmap: unmap a virtual address of a context.
+ * @flush: flush all writes from all cores to reach device MMU.
+ * @swap_out: marks all mapping of the given context as swapped out.
+ * @swap_in: marks all mapping of the given context as swapped in.
+ */
+struct hl_mmu_funcs {
+ int (*init)(struct hl_device *hdev);
+ void (*fini)(struct hl_device *hdev);
+ int (*ctx_init)(struct hl_ctx *ctx);
+ void (*ctx_fini)(struct hl_ctx *ctx);
+ int (*map)(struct hl_ctx *ctx,
+ u64 virt_addr, u64 phys_addr, u32 page_size,
+ bool is_dram_addr);
+ int (*unmap)(struct hl_ctx *ctx,
+ u64 virt_addr, bool is_dram_addr);
+ void (*flush)(struct hl_ctx *ctx);
+ void (*swap_out)(struct hl_ctx *ctx);
+ void (*swap_in)(struct hl_ctx *ctx);
+};
+
/**
* struct hl_device - habanalabs device structure.
* @pdev: pointer to PCI device, can be NULL in case of simulator device.
@@ -1407,8 +1496,8 @@ struct hl_device_idle_busy_ts {
* @dev: related kernel basic device structure.
* @dev_ctrl: related kernel device structure for the control device
* @work_freq: delayed work to lower device frequency if possible.
- * @work_heartbeat: delayed work for ArmCP is-alive check.
- * @asic_name: ASIC specific nmae.
+ * @work_heartbeat: delayed work for CPU-CP is-alive check.
+ * @asic_name: ASIC specific name.
* @asic_type: ASIC specific type.
* @completion_queue: array of hl_cq.
* @cq_wq: work queues of completion queues for executing work in process
@@ -1419,22 +1508,20 @@ struct hl_device_idle_busy_ts {
* @hw_queues_mirror_list: CS mirror list for TDR.
* @hw_queues_mirror_lock: protects hw_queues_mirror_list.
* @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
- * @event_queue: event queue for IRQ from ArmCP.
+ * @event_queue: event queue for IRQ from CPU-CP.
* @dma_pool: DMA pool for small allocations.
- * @cpu_accessible_dma_mem: Host <-> ArmCP shared memory CPU address.
- * @cpu_accessible_dma_address: Host <-> ArmCP shared memory DMA address.
- * @cpu_accessible_dma_pool: Host <-> ArmCP shared memory pool.
+ * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address.
+ * @cpu_accessible_dma_address: Host <-> CPU-CP shared memory DMA address.
+ * @cpu_accessible_dma_pool: Host <-> CPU-CP shared memory pool.
* @asid_bitmap: holds used/available ASIDs.
* @asid_mutex: protects asid_bitmap.
- * @send_cpu_message_lock: enforces only one message in Host <-> ArmCP queue.
+ * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue.
* @debug_lock: protects critical section of setting debug mode for device
* @asic_prop: ASIC specific immutable properties.
* @asic_funcs: ASIC specific functions.
* @asic_specific: ASIC specific information to use only from ASIC files.
- * @mmu_pgt_pool: pool of available MMU hops.
* @vm: virtual memory manager for MMU.
* @mmu_cache_lock: protects MMU cache invalidation as it can serve one context.
- * @mmu_shadow_hop0: shadow mapping of the MMU hop 0 zone.
* @hwmon_dev: H/W monitor device.
* @pm_mng_profile: current power management profile.
* @hl_chip_info: ASIC's sensors information.
@@ -1452,6 +1539,8 @@ struct hl_device_idle_busy_ts {
* @idle_busy_ts_arr: array to hold time stamps of transitions from idle to busy
* and vice-versa
* @aggregated_cs_counters: aggregated cs counters among all contexts
+ * @mmu_priv: device-specific MMU data.
+ * @mmu_func: device-related MMU functions.
* @dram_used_mem: current DRAM memory consumption.
* @timeout_jiffies: device CS timeout value.
* @max_power: the max power of the device, as configured by the sysadmin. This
@@ -1471,6 +1560,7 @@ struct hl_device_idle_busy_ts {
* @soft_reset_cnt: number of soft reset since the driver was loaded.
* @hard_reset_cnt: number of hard reset since the driver was loaded.
* @idle_busy_ts_idx: index of current entry in idle_busy_ts_arr
+ * @clk_throttling_reason: bitmask represents the current clk throttling reasons
* @id: device minor.
* @id_control: minor of the control device
* @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
@@ -1479,7 +1569,7 @@ struct hl_device_idle_busy_ts {
* @late_init_done: is late init stage was done during initialization.
* @hwmon_initialized: is H/W monitor sensors was initialized.
* @hard_reset_pending: is there a hard reset work pending.
- * @heartbeat: is heartbeat sanity check towards ArmCP enabled.
+ * @heartbeat: is heartbeat sanity check towards CPU-CP enabled.
* @reset_on_lockup: true if a reset should be done in case of stuck CS, false
* otherwise.
* @dram_supports_virtual_memory: is MMU enabled towards DRAM.
@@ -1501,6 +1591,7 @@ struct hl_device_idle_busy_ts {
* @sync_stream_queue_idx: helper index for sync stream queues initialization.
* @supports_coresight: is CoreSight supported.
* @supports_soft_reset: is soft reset supported.
+ * @supports_cb_mapping: is mapping a CB to the device's MMU supported.
*/
struct hl_device {
struct pci_dev *pdev;
@@ -1513,7 +1604,7 @@ struct hl_device {
struct device *dev_ctrl;
struct delayed_work work_freq;
struct delayed_work work_heartbeat;
- char asic_name[16];
+ char asic_name[32];
enum hl_asic_type asic_type;
struct hl_cq *completion_queue;
struct workqueue_struct **cq_wq;
@@ -1535,10 +1626,8 @@ struct hl_device {
struct asic_fixed_properties asic_prop;
const struct hl_asic_funcs *asic_funcs;
void *asic_specific;
- struct gen_pool *mmu_pgt_pool;
struct hl_vm vm;
struct mutex mmu_cache_lock;
- void *mmu_shadow_hop0;
struct device *hwmon_dev;
enum hl_pm_mng_profile pm_mng_profile;
struct hwmon_chip_info *hl_chip_info;
@@ -1562,19 +1651,23 @@ struct hl_device {
struct hl_cs_counters aggregated_cs_counters;
+ struct hl_mmu_priv mmu_priv;
+ struct hl_mmu_funcs mmu_func;
+
atomic64_t dram_used_mem;
u64 timeout_jiffies;
u64 max_power;
u64 clock_gating_mask;
atomic_t in_reset;
enum hl_pll_frequency curr_pll_profile;
- enum armcp_card_types card_type;
+ enum cpucp_card_types card_type;
int cs_active_cnt;
u32 major;
u32 high_pll;
u32 soft_reset_cnt;
u32 hard_reset_cnt;
u32 idle_busy_ts_idx;
+ u32 clk_throttling_reason;
u16 id;
u16 id_control;
u16 cpu_pci_msb_addr;
@@ -1598,6 +1691,7 @@ struct hl_device {
u8 sync_stream_queue_idx;
u8 supports_coresight;
u8 supports_soft_reset;
+ u8 supports_cb_mapping;
/* Parameters for bring-up */
u8 mmu_enable;
@@ -1739,7 +1833,7 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
void hl_ctx_do_release(struct kref *ref);
void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx);
int hl_ctx_put(struct hl_ctx *ctx);
-struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
+struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
@@ -1755,7 +1849,7 @@ int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq);
uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms);
int hl_build_hwmon_channel_info(struct hl_device *hdev,
- struct armcp_sensor *sensors_arr);
+ struct cpucp_sensor *sensors_arr);
int hl_sysfs_init(struct hl_device *hdev);
void hl_sysfs_fini(struct hl_device *hdev);
@@ -1763,8 +1857,9 @@ void hl_sysfs_fini(struct hl_device *hdev);
int hl_hwmon_init(struct hl_device *hdev);
void hl_hwmon_fini(struct hl_device *hdev);
-int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 cb_size,
- u64 *handle, int ctx_id, bool internal_cb);
+int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
+ struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
+ bool map_cb, u64 *handle);
int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
@@ -1776,11 +1871,15 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
bool internal_cb);
int hl_cb_pool_init(struct hl_device *hdev);
int hl_cb_pool_fini(struct hl_device *hdev);
+int hl_cb_va_pool_init(struct hl_ctx *ctx);
+void hl_cb_va_pool_fini(struct hl_ctx *ctx);
void hl_cs_rollback_all(struct hl_device *hdev);
struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
void hl_sob_reset_error(struct kref *ref);
+void hl_fence_put(struct hl_fence *fence);
+void hl_fence_get(struct hl_fence *fence);
void goya_set_asic_funcs(struct hl_device *hdev);
void gaudi_set_asic_funcs(struct hl_device *hdev);
@@ -1810,6 +1909,8 @@ int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
bool flush_pte);
void hl_mmu_swap_out(struct hl_ctx *ctx);
void hl_mmu_swap_in(struct hl_ctx *ctx);
+int hl_mmu_if_set_funcs(struct hl_device *hdev);
+void hl_mmu_v1_set_funcs(struct hl_device *hdev);
int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
void __iomem *dst);
@@ -1825,23 +1926,28 @@ void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
void *vaddr);
int hl_fw_send_heartbeat(struct hl_device *hdev);
-int hl_fw_armcp_info_get(struct hl_device *hdev);
+int hl_fw_cpucp_info_get(struct hl_device *hdev);
int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
+int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
+ struct hl_info_pci_counters *counters);
+int hl_fw_cpucp_total_energy_get(struct hl_device *hdev,
+ u64 *total_energy);
int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
u32 boot_err0_reg, bool skip_bmc,
u32 cpu_timeout, u32 boot_fit_timeout);
+int hl_fw_read_preboot_ver(struct hl_device *hdev, u32 cpu_boot_status_reg,
+ u32 boot_err0_reg, u32 timeout);
int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
bool is_wc[3]);
int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
-int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
- u64 addr);
int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
struct hl_inbound_pci_region *pci_region);
int hl_pci_set_outbound_region(struct hl_device *hdev,
struct hl_outbound_pci_region *pci_region);
-int hl_pci_init(struct hl_device *hdev);
+int hl_pci_init(struct hl_device *hdev, u32 cpu_boot_status_reg,
+ u32 boot_err0_reg, u32 preboot_ver_timeout);
void hl_pci_fini(struct hl_device *hdev);
long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c
index c6b31e93fb5e..f9067d3ef437 100644
--- a/drivers/misc/habanalabs/common/habanalabs_drv.c
+++ b/drivers/misc/habanalabs/common/habanalabs_drv.c
@@ -11,6 +11,7 @@
#include "habanalabs.h"
#include <linux/pci.h>
+#include <linux/aer.h>
#include <linux/module.h>
#define HL_DRIVER_AUTHOR "HabanaLabs Kernel Driver Team"
@@ -408,6 +409,8 @@ static int hl_pci_probe(struct pci_dev *pdev,
pci_set_drvdata(pdev, hdev);
+ pci_enable_pcie_error_reporting(pdev);
+
rc = hl_device_init(hdev, hl_class);
if (rc) {
dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
@@ -440,22 +443,93 @@ static void hl_pci_remove(struct pci_dev *pdev)
return;
hl_device_fini(hdev);
+ pci_disable_pcie_error_reporting(pdev);
pci_set_drvdata(pdev, NULL);
-
destroy_hdev(hdev);
}
+/**
+ * hl_pci_err_detected - a PCI bus error detected on this device
+ *
+ * @pdev: pointer to pci device
+ * @state: PCI error type
+ *
+ * Called by the PCI subsystem whenever a non-correctable
+ * PCI bus error is detected
+ */
+static pci_ers_result_t
+hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state)
+{
+ struct hl_device *hdev = pci_get_drvdata(pdev);
+ enum pci_ers_result result;
+
+ switch (state) {
+ case pci_channel_io_normal:
+ return PCI_ERS_RESULT_CAN_RECOVER;
+
+ case pci_channel_io_frozen:
+ dev_warn(hdev->dev, "frozen state error detected\n");
+ result = PCI_ERS_RESULT_NEED_RESET;
+ break;
+
+ case pci_channel_io_perm_failure:
+ dev_warn(hdev->dev, "failure state error detected\n");
+ result = PCI_ERS_RESULT_DISCONNECT;
+ break;
+
+ default:
+ result = PCI_ERS_RESULT_NONE;
+ }
+
+ hdev->asic_funcs->halt_engines(hdev, true);
+
+ return result;
+}
+
+/**
+ * hl_pci_err_resume - resume after a PCI slot reset
+ *
+ * @pdev: pointer to pci device
+ *
+ */
+static void hl_pci_err_resume(struct pci_dev *pdev)
+{
+ struct hl_device *hdev = pci_get_drvdata(pdev);
+
+ dev_warn(hdev->dev, "Resuming device after PCI slot reset\n");
+ hl_device_resume(hdev);
+}
+
+/**
+ * hl_pci_err_slot_reset - a PCI slot reset has just happened
+ *
+ * @pdev: pointer to pci device
+ *
+ * Determine if the driver can recover from the PCI slot reset
+ */
+static pci_ers_result_t hl_pci_err_slot_reset(struct pci_dev *pdev)
+{
+ return PCI_ERS_RESULT_RECOVERED;
+}
+
static const struct dev_pm_ops hl_pm_ops = {
.suspend = hl_pmops_suspend,
.resume = hl_pmops_resume,
};
+static const struct pci_error_handlers hl_pci_err_handler = {
+ .error_detected = hl_pci_err_detected,
+ .slot_reset = hl_pci_err_slot_reset,
+ .resume = hl_pci_err_resume,
+};
+
static struct pci_driver hl_pci_driver = {
.name = HL_NAME,
.id_table = ids,
.probe = hl_pci_probe,
.remove = hl_pci_remove,
.driver.pm = &hl_pm_ops,
+ .err_handler = &hl_pci_err_handler,
};
/*
diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c
index 5af1c03da473..07317ea49129 100644
--- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c
+++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c
@@ -8,6 +8,7 @@
#include <uapi/misc/habanalabs.h>
#include "habanalabs.h"
+#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
@@ -64,14 +65,14 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
hw_ip.dram_enabled = 1;
hw_ip.num_of_events = prop->num_of_events;
- memcpy(hw_ip.armcp_version, prop->armcp_info.armcp_version,
+ memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version,
min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
- memcpy(hw_ip.card_name, prop->armcp_info.card_name,
+ memcpy(hw_ip.card_name, prop->cpucp_info.card_name,
min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
- hw_ip.armcp_cpld_version = le32_to_cpu(prop->armcp_info.cpld_version);
- hw_ip.module_id = le32_to_cpu(prop->armcp_info.card_location);
+ hw_ip.cpld_version = le32_to_cpu(prop->cpucp_info.cpld_version);
+ hw_ip.module_id = le32_to_cpu(prop->cpucp_info.card_location);
hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
@@ -131,7 +132,7 @@ static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
return -EINVAL;
hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
- &hw_idle.busy_engines_mask, NULL);
+ &hw_idle.busy_engines_mask_ext, NULL);
return copy_to_user(out, &hw_idle,
min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
@@ -276,10 +277,45 @@ static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
}
+static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ struct hl_info_pci_counters pci_counters = {0};
+ u32 max_size = args->return_size;
+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+ int rc;
+
+ if ((!max_size) || (!out))
+ return -EINVAL;
+
+ rc = hl_fw_cpucp_pci_counters_get(hdev, &pci_counters);
+ if (rc)
+ return rc;
+
+ return copy_to_user(out, &pci_counters,
+ min((size_t) max_size, sizeof(pci_counters))) ? -EFAULT : 0;
+}
+
+static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ struct hl_info_clk_throttle clk_throttle = {0};
+ u32 max_size = args->return_size;
+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+
+ if ((!max_size) || (!out))
+ return -EINVAL;
+
+ clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason;
+
+ return copy_to_user(out, &clk_throttle,
+ min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0;
+}
+
static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
{
struct hl_device *hdev = hpriv->hdev;
- struct hl_info_cs_counters cs_counters = {0};
+ struct hl_info_cs_counters cs_counters = { {0} };
u32 max_size = args->return_size;
void __user *out = (void __user *) (uintptr_t) args->return_pointer;
@@ -297,6 +333,51 @@ static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0;
}
+static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_info_sync_manager sm_info = {0};
+ u32 max_size = args->return_size;
+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+
+ if ((!max_size) || (!out))
+ return -EINVAL;
+
+ if (args->dcore_id >= HL_MAX_DCORES)
+ return -EINVAL;
+
+ sm_info.first_available_sync_object =
+ prop->first_available_user_sob[args->dcore_id];
+ sm_info.first_available_monitor =
+ prop->first_available_user_mon[args->dcore_id];
+
+
+ return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size,
+ sizeof(sm_info))) ? -EFAULT : 0;
+}
+
+static int total_energy_consumption_info(struct hl_fpriv *hpriv,
+ struct hl_info_args *args)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ struct hl_info_energy total_energy = {0};
+ u32 max_size = args->return_size;
+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+ int rc;
+
+ if ((!max_size) || (!out))
+ return -EINVAL;
+
+ rc = hl_fw_cpucp_total_energy_get(hdev,
+ &total_energy.total_energy_consumption);
+ if (rc)
+ return rc;
+
+ return copy_to_user(out, &total_energy,
+ min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0;
+}
+
static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
struct device *dev)
{
@@ -360,6 +441,18 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
case HL_INFO_CS_COUNTERS:
return cs_counters_info(hpriv, args);
+ case HL_INFO_PCI_COUNTERS:
+ return pci_counters_info(hpriv, args);
+
+ case HL_INFO_CLK_THROTTLE_REASON:
+ return clk_throttle_info(hpriv, args);
+
+ case HL_INFO_SYNC_MANAGER:
+ return sync_manager_info(hpriv, args);
+
+ case HL_INFO_TOTAL_ENERGY:
+ return total_energy_consumption_info(hpriv, args);
+
default:
dev_err(dev, "Invalid request %d\n", args->op);
rc = -ENOTTY;
diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c
index 287681646071..5e66c98fb0d3 100644
--- a/drivers/misc/habanalabs/common/hw_queue.c
+++ b/drivers/misc/habanalabs/common/hw_queue.c
@@ -288,10 +288,10 @@ static void ext_queue_schedule_job(struct hl_cs_job *job)
ptr = cb->bus_address;
cq_pkt.data = cpu_to_le32(
- ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)
- & CQ_ENTRY_SHADOW_INDEX_MASK) |
- (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) |
- (1 << CQ_ENTRY_READY_SHIFT));
+ ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)
+ & CQ_ENTRY_SHADOW_INDEX_MASK) |
+ FIELD_PREP(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 1) |
+ FIELD_PREP(CQ_ENTRY_READY_MASK, 1));
/*
* No need to protect pi_offset because scheduling to the
@@ -474,7 +474,7 @@ static void init_signal_wait_cs(struct hl_cs *cs)
* wait CS was submitted.
*/
mb();
- dma_fence_put(cs->signal_fence);
+ hl_fence_put(cs->signal_fence);
cs->signal_fence = NULL;
}
}
diff --git a/drivers/misc/habanalabs/common/hwmon.c b/drivers/misc/habanalabs/common/hwmon.c
index b997336fa75f..2ac29cb2fe61 100644
--- a/drivers/misc/habanalabs/common/hwmon.c
+++ b/drivers/misc/habanalabs/common/hwmon.c
@@ -13,7 +13,7 @@
#define HWMON_NR_SENSOR_TYPES (hwmon_pwm + 1)
int hl_build_hwmon_channel_info(struct hl_device *hdev,
- struct armcp_sensor *sensors_arr)
+ struct cpucp_sensor *sensors_arr)
{
u32 counts[HWMON_NR_SENSOR_TYPES] = {0};
u32 *sensors_by_type[HWMON_NR_SENSOR_TYPES] = {NULL};
@@ -24,7 +24,7 @@ int hl_build_hwmon_channel_info(struct hl_device *hdev,
enum hwmon_sensor_types type;
int rc, i, j;
- for (i = 0 ; i < ARMCP_MAX_SENSORS ; i++) {
+ for (i = 0 ; i < CPUCP_MAX_SENSORS ; i++) {
type = le32_to_cpu(sensors_arr[i].type);
if ((type == 0) && (sensors_arr[i].flags == 0))
@@ -311,13 +311,13 @@ static const struct hwmon_ops hl_hwmon_ops = {
int hl_get_temperature(struct hl_device *hdev,
int sensor_index, u32 attr, long *value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEMPERATURE_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEMPERATURE_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
@@ -337,13 +337,13 @@ int hl_get_temperature(struct hl_device *hdev,
int hl_set_temperature(struct hl_device *hdev,
int sensor_index, u32 attr, long value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEMPERATURE_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEMPERATURE_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
pkt.value = __cpu_to_le64(value);
@@ -362,13 +362,13 @@ int hl_set_temperature(struct hl_device *hdev,
int hl_get_voltage(struct hl_device *hdev,
int sensor_index, u32 attr, long *value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_VOLTAGE_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_VOLTAGE_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
@@ -388,13 +388,13 @@ int hl_get_voltage(struct hl_device *hdev,
int hl_get_current(struct hl_device *hdev,
int sensor_index, u32 attr, long *value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_CURRENT_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_CURRENT_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
@@ -414,13 +414,13 @@ int hl_get_current(struct hl_device *hdev,
int hl_get_fan_speed(struct hl_device *hdev,
int sensor_index, u32 attr, long *value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_FAN_SPEED_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_FAN_SPEED_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
@@ -440,13 +440,13 @@ int hl_get_fan_speed(struct hl_device *hdev,
int hl_get_pwm_info(struct hl_device *hdev,
int sensor_index, u32 attr, long *value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_PWM_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_PWM_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
@@ -466,13 +466,13 @@ int hl_get_pwm_info(struct hl_device *hdev,
void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
long value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_PWM_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_PWM_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
pkt.value = cpu_to_le64(value);
@@ -489,13 +489,13 @@ void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
int hl_set_voltage(struct hl_device *hdev,
int sensor_index, u32 attr, long value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_VOLTAGE_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_VOLTAGE_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
pkt.value = __cpu_to_le64(value);
@@ -514,13 +514,13 @@ int hl_set_voltage(struct hl_device *hdev,
int hl_set_current(struct hl_device *hdev,
int sensor_index, u32 attr, long value)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_CURRENT_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_CURRENT_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.sensor_index = __cpu_to_le16(sensor_index);
pkt.type = __cpu_to_le16(attr);
pkt.value = __cpu_to_le64(value);
@@ -549,7 +549,7 @@ int hl_hwmon_init(struct hl_device *hdev)
hdev->hl_chip_info->ops = &hl_hwmon_ops;
hdev->hwmon_dev = hwmon_device_register_with_info(dev,
- prop->armcp_info.card_name, hdev,
+ prop->cpucp_info.card_name, hdev,
hdev->hl_chip_info, NULL);
if (IS_ERR(hdev->hwmon_dev)) {
rc = PTR_ERR(hdev->hwmon_dev);
diff --git a/drivers/misc/habanalabs/common/irq.c b/drivers/misc/habanalabs/common/irq.c
index c8db717023f5..d20e40a53d70 100644
--- a/drivers/misc/habanalabs/common/irq.c
+++ b/drivers/misc/habanalabs/common/irq.c
@@ -11,7 +11,7 @@
/**
* struct hl_eqe_work - This structure is used to schedule work of EQ
- * entry and armcp_reset event
+ * entry and cpucp_reset event
*
* @eq_work: workqueue object to run when EQ entry is received
* @hdev: pointer to device structure
diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c
index 5ff4688683fd..84227819e4d1 100644
--- a/drivers/misc/habanalabs/common/memory.c
+++ b/drivers/misc/habanalabs/common/memory.c
@@ -77,8 +77,8 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size);
if (!paddr) {
dev_err(hdev->dev,
- "failed to allocate %llu huge contiguous pages\n",
- num_pgs);
+ "failed to allocate %llu contiguous pages with total size of %llu\n",
+ num_pgs, total_size);
return -ENOMEM;
}
}
@@ -505,41 +505,32 @@ static inline int add_va_block(struct hl_device *hdev,
}
/*
- * get_va_block - get a virtual block with the requested size
- *
- * @hdev : pointer to the habanalabs device structure
- * @va_range : pointer to the virtual addresses range
- * @size : requested block size
- * @hint_addr : hint for request address by the user
- * @is_userptr : is host or DRAM memory
+ * get_va_block() - get a virtual block for the given size and alignment.
+ * @hdev: pointer to the habanalabs device structure.
+ * @va_range: pointer to the virtual addresses range.
+ * @size: requested block size.
+ * @hint_addr: hint for requested address by the user.
+ * @va_block_align: required alignment of the virtual block start address.
*
* This function does the following:
* - Iterate on the virtual block list to find a suitable virtual block for the
- * requested size
- * - Reserve the requested block and update the list
- * - Return the start address of the virtual block
+ * given size and alignment.
+ * - Reserve the requested block and update the list.
+ * - Return the start address of the virtual block.
*/
-static u64 get_va_block(struct hl_device *hdev,
- struct hl_va_range *va_range, u64 size, u64 hint_addr,
- bool is_userptr)
+static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range,
+ u64 size, u64 hint_addr, u32 va_block_align)
{
struct hl_vm_va_block *va_block, *new_va_block = NULL;
- u64 valid_start, valid_size, prev_start, prev_end, page_mask,
+ u64 valid_start, valid_size, prev_start, prev_end, align_mask,
res_valid_start = 0, res_valid_size = 0;
- u32 page_size;
bool add_prev = false;
- if (is_userptr)
- /*
- * We cannot know if the user allocated memory with huge pages
- * or not, hence we continue with the biggest possible
- * granularity.
- */
- page_size = hdev->asic_prop.pmmu_huge.page_size;
- else
- page_size = hdev->asic_prop.dmmu.page_size;
+ align_mask = ~((u64)va_block_align - 1);
- page_mask = ~((u64)page_size - 1);
+ /* check if hint_addr is aligned */
+ if (hint_addr & (va_block_align - 1))
+ hint_addr = 0;
mutex_lock(&va_range->lock);
@@ -549,9 +540,9 @@ static u64 get_va_block(struct hl_device *hdev,
/* calc the first possible aligned addr */
valid_start = va_block->start;
- if (valid_start & (page_size - 1)) {
- valid_start &= page_mask;
- valid_start += page_size;
+ if (valid_start & (va_block_align - 1)) {
+ valid_start &= align_mask;
+ valid_start += va_block_align;
if (valid_start > va_block->end)
continue;
}
@@ -863,7 +854,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
struct hl_va_range *va_range;
enum vm_type_t *vm_type;
u64 ret_vaddr, hint_addr;
- u32 handle = 0;
+ u32 handle = 0, va_block_align;
int rc;
bool is_userptr = args->flags & HL_MEM_USERPTR;
@@ -873,6 +864,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
if (is_userptr) {
u64 addr = args->map_host.host_virt_addr,
size = args->map_host.mem_size;
+ u32 page_size = hdev->asic_prop.pmmu.page_size,
+ huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
rc = dma_map_host_va(hdev, addr, size, &userptr);
if (rc) {
@@ -892,6 +885,27 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
vm_type = (enum vm_type_t *) userptr;
hint_addr = args->map_host.hint_addr;
handle = phys_pg_pack->handle;
+
+ /* get required alignment */
+ if (phys_pg_pack->page_size == page_size) {
+ va_range = ctx->host_va_range;
+
+ /*
+ * huge page alignment may be needed in case of regular
+ * page mapping, depending on the host VA alignment
+ */
+ if (addr & (huge_page_size - 1))
+ va_block_align = page_size;
+ else
+ va_block_align = huge_page_size;
+ } else {
+ /*
+ * huge page alignment is needed in case of huge page
+ * mapping
+ */
+ va_range = ctx->host_huge_va_range;
+ va_block_align = huge_page_size;
+ }
} else {
handle = lower_32_bits(args->map_device.handle);
@@ -912,6 +926,10 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
vm_type = (enum vm_type_t *) phys_pg_pack;
hint_addr = args->map_device.hint_addr;
+
+ /* DRAM VA alignment is the same as the DRAM page size */
+ va_range = ctx->dram_va_range;
+ va_block_align = hdev->asic_prop.dmmu.page_size;
}
/*
@@ -933,16 +951,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
goto hnode_err;
}
- if (is_userptr)
- if (phys_pg_pack->page_size == hdev->asic_prop.pmmu.page_size)
- va_range = ctx->host_va_range;
- else
- va_range = ctx->host_huge_va_range;
- else
- va_range = ctx->dram_va_range;
-
ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
- hint_addr, is_userptr);
+ hint_addr, va_block_align);
if (!ret_vaddr) {
dev_err(hdev->dev, "no available va block for handle %u\n",
handle);
diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu.c
index 3fc0f497fab3..b5058798aeb9 100644
--- a/drivers/misc/habanalabs/common/mmu.c
+++ b/drivers/misc/habanalabs/common/mmu.c
@@ -1,258 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
/*
- * Copyright 2016-2019 HabanaLabs, Ltd.
+ * Copyright 2016-2020 HabanaLabs, Ltd.
* All Rights Reserved.
*/
-#include "habanalabs.h"
-#include "../include/hw_ip/mmu/mmu_general.h"
-
-#include <linux/genalloc.h>
#include <linux/slab.h>
-static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr);
-
-static struct pgt_info *get_pgt_info(struct hl_ctx *ctx, u64 hop_addr)
-{
- struct pgt_info *pgt_info = NULL;
-
- hash_for_each_possible(ctx->mmu_shadow_hash, pgt_info, node,
- (unsigned long) hop_addr)
- if (hop_addr == pgt_info->shadow_addr)
- break;
-
- return pgt_info;
-}
-
-static void _free_hop(struct hl_ctx *ctx, struct pgt_info *pgt_info)
-{
- struct hl_device *hdev = ctx->hdev;
-
- gen_pool_free(hdev->mmu_pgt_pool, pgt_info->phys_addr,
- hdev->asic_prop.mmu_hop_table_size);
- hash_del(&pgt_info->node);
- kfree((u64 *) (uintptr_t) pgt_info->shadow_addr);
- kfree(pgt_info);
-}
-
-static void free_hop(struct hl_ctx *ctx, u64 hop_addr)
-{
- struct pgt_info *pgt_info = get_pgt_info(ctx, hop_addr);
-
- _free_hop(ctx, pgt_info);
-}
-
-static u64 alloc_hop(struct hl_ctx *ctx)
-{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct pgt_info *pgt_info;
- u64 phys_addr, shadow_addr;
-
- pgt_info = kmalloc(sizeof(*pgt_info), GFP_KERNEL);
- if (!pgt_info)
- return ULLONG_MAX;
-
- phys_addr = (u64) gen_pool_alloc(hdev->mmu_pgt_pool,
- prop->mmu_hop_table_size);
- if (!phys_addr) {
- dev_err(hdev->dev, "failed to allocate page\n");
- goto pool_add_err;
- }
-
- shadow_addr = (u64) (uintptr_t) kzalloc(prop->mmu_hop_table_size,
- GFP_KERNEL);
- if (!shadow_addr)
- goto shadow_err;
-
- pgt_info->phys_addr = phys_addr;
- pgt_info->shadow_addr = shadow_addr;
- pgt_info->ctx = ctx;
- pgt_info->num_of_ptes = 0;
- hash_add(ctx->mmu_shadow_hash, &pgt_info->node, shadow_addr);
-
- return shadow_addr;
-
-shadow_err:
- gen_pool_free(hdev->mmu_pgt_pool, phys_addr, prop->mmu_hop_table_size);
-pool_add_err:
- kfree(pgt_info);
-
- return ULLONG_MAX;
-}
-
-static inline u64 get_phys_hop0_addr(struct hl_ctx *ctx)
-{
- return ctx->hdev->asic_prop.mmu_pgt_addr +
- (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
-}
-
-static inline u64 get_hop0_addr(struct hl_ctx *ctx)
-{
- return (u64) (uintptr_t) ctx->hdev->mmu_shadow_hop0 +
- (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
-}
-
-static inline void flush(struct hl_ctx *ctx)
-{
- /* flush all writes from all cores to reach PCI */
- mb();
- ctx->hdev->asic_funcs->read_pte(ctx->hdev, get_phys_hop0_addr(ctx));
-}
-
-/* transform the value to physical address when writing to H/W */
-static inline void write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val)
-{
- /*
- * The value to write is actually the address of the next shadow hop +
- * flags at the 12 LSBs.
- * Hence in order to get the value to write to the physical PTE, we
- * clear the 12 LSBs and translate the shadow hop to its associated
- * physical hop, and add back the original 12 LSBs.
- */
- u64 phys_val = get_phys_addr(ctx, val & HOP_PHYS_ADDR_MASK) |
- (val & FLAGS_MASK);
-
- ctx->hdev->asic_funcs->write_pte(ctx->hdev,
- get_phys_addr(ctx, shadow_pte_addr),
- phys_val);
-
- *(u64 *) (uintptr_t) shadow_pte_addr = val;
-}
-
-/* do not transform the value to physical address when writing to H/W */
-static inline void write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr,
- u64 val)
-{
- ctx->hdev->asic_funcs->write_pte(ctx->hdev,
- get_phys_addr(ctx, shadow_pte_addr),
- val);
- *(u64 *) (uintptr_t) shadow_pte_addr = val;
-}
-
-/* clear the last and present bits */
-static inline void clear_pte(struct hl_ctx *ctx, u64 pte_addr)
-{
- /* no need to transform the value to physical address */
- write_final_pte(ctx, pte_addr, 0);
-}
-
-static inline void get_pte(struct hl_ctx *ctx, u64 hop_addr)
-{
- get_pgt_info(ctx, hop_addr)->num_of_ptes++;
-}
-
-/*
- * put_pte - decrement the num of ptes and free the hop if possible
- *
- * @ctx: pointer to the context structure
- * @hop_addr: addr of the hop
- *
- * This function returns the number of ptes left on this hop. If the number is
- * 0, it means the pte was freed.
- */
-static inline int put_pte(struct hl_ctx *ctx, u64 hop_addr)
-{
- struct pgt_info *pgt_info = get_pgt_info(ctx, hop_addr);
- int num_of_ptes_left;
-
- pgt_info->num_of_ptes--;
-
- /*
- * Need to save the number of ptes left because free_hop might free
- * the pgt_info
- */
- num_of_ptes_left = pgt_info->num_of_ptes;
- if (!num_of_ptes_left)
- _free_hop(ctx, pgt_info);
-
- return num_of_ptes_left;
-}
-
-static inline u64 get_hopN_pte_addr(struct hl_ctx *ctx, u64 hop_addr,
- u64 virt_addr, u64 mask, u64 shift)
-{
- return hop_addr + ctx->hdev->asic_prop.mmu_pte_size *
- ((virt_addr & mask) >> shift);
-}
-
-static inline u64 get_hop0_pte_addr(struct hl_ctx *ctx,
- struct hl_mmu_properties *mmu_prop,
- u64 hop_addr, u64 vaddr)
-{
- return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop0_mask,
- mmu_prop->hop0_shift);
-}
-
-static inline u64 get_hop1_pte_addr(struct hl_ctx *ctx,
- struct hl_mmu_properties *mmu_prop,
- u64 hop_addr, u64 vaddr)
-{
- return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop1_mask,
- mmu_prop->hop1_shift);
-}
-
-static inline u64 get_hop2_pte_addr(struct hl_ctx *ctx,
- struct hl_mmu_properties *mmu_prop,
- u64 hop_addr, u64 vaddr)
-{
- return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop2_mask,
- mmu_prop->hop2_shift);
-}
-
-static inline u64 get_hop3_pte_addr(struct hl_ctx *ctx,
- struct hl_mmu_properties *mmu_prop,
- u64 hop_addr, u64 vaddr)
-{
- return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop3_mask,
- mmu_prop->hop3_shift);
-}
-
-static inline u64 get_hop4_pte_addr(struct hl_ctx *ctx,
- struct hl_mmu_properties *mmu_prop,
- u64 hop_addr, u64 vaddr)
-{
- return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop4_mask,
- mmu_prop->hop4_shift);
-}
-
-static inline u64 get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte)
-{
- if (curr_pte & PAGE_PRESENT_MASK)
- return curr_pte & HOP_PHYS_ADDR_MASK;
- else
- return ULLONG_MAX;
-}
-
-static inline u64 get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte,
- bool *is_new_hop)
-{
- u64 hop_addr = get_next_hop_addr(ctx, curr_pte);
-
- if (hop_addr == ULLONG_MAX) {
- hop_addr = alloc_hop(ctx);
- *is_new_hop = (hop_addr != ULLONG_MAX);
- }
-
- return hop_addr;
-}
-
-/* translates shadow address inside hop to a physical address */
-static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr)
-{
- u64 page_mask = (ctx->hdev->asic_prop.mmu_hop_table_size - 1);
- u64 shadow_hop_addr = shadow_addr & ~page_mask;
- u64 pte_offset = shadow_addr & page_mask;
- u64 phys_hop_addr;
-
- if (shadow_hop_addr != get_hop0_addr(ctx))
- phys_hop_addr = get_pgt_info(ctx, shadow_hop_addr)->phys_addr;
- else
- phys_hop_addr = get_phys_hop0_addr(ctx);
-
- return phys_hop_addr + pte_offset;
-}
+#include "habanalabs.h"
static bool is_dram_va(struct hl_device *hdev, u64 virt_addr)
{
@@ -263,155 +18,6 @@ static bool is_dram_va(struct hl_device *hdev, u64 virt_addr)
prop->dmmu.end_addr);
}
-static int dram_default_mapping_init(struct hl_ctx *ctx)
-{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,
- hop2_pte_addr, hop3_pte_addr, pte_val;
- int rc, i, j, hop3_allocated = 0;
-
- if ((!hdev->dram_supports_virtual_memory) ||
- (!hdev->dram_default_page_mapping) ||
- (ctx->asid == HL_KERNEL_ASID_ID))
- return 0;
-
- num_of_hop3 = prop->dram_size_for_default_page_mapping;
- do_div(num_of_hop3, prop->dram_page_size);
- do_div(num_of_hop3, PTE_ENTRIES_IN_HOP);
-
- /* add hop1 and hop2 */
- total_hops = num_of_hop3 + 2;
-
- ctx->dram_default_hops = kzalloc(HL_PTE_SIZE * total_hops, GFP_KERNEL);
- if (!ctx->dram_default_hops)
- return -ENOMEM;
-
- hop0_addr = get_hop0_addr(ctx);
-
- hop1_addr = alloc_hop(ctx);
- if (hop1_addr == ULLONG_MAX) {
- dev_err(hdev->dev, "failed to alloc hop 1\n");
- rc = -ENOMEM;
- goto hop1_err;
- }
-
- ctx->dram_default_hops[total_hops - 1] = hop1_addr;
-
- hop2_addr = alloc_hop(ctx);
- if (hop2_addr == ULLONG_MAX) {
- dev_err(hdev->dev, "failed to alloc hop 2\n");
- rc = -ENOMEM;
- goto hop2_err;
- }
-
- ctx->dram_default_hops[total_hops - 2] = hop2_addr;
-
- for (i = 0 ; i < num_of_hop3 ; i++) {
- ctx->dram_default_hops[i] = alloc_hop(ctx);
- if (ctx->dram_default_hops[i] == ULLONG_MAX) {
- dev_err(hdev->dev, "failed to alloc hop 3, i: %d\n", i);
- rc = -ENOMEM;
- goto hop3_err;
- }
- hop3_allocated++;
- }
-
- /* need only pte 0 in hops 0 and 1 */
- pte_val = (hop1_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- write_pte(ctx, hop0_addr, pte_val);
-
- pte_val = (hop2_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- write_pte(ctx, hop1_addr, pte_val);
- get_pte(ctx, hop1_addr);
-
- hop2_pte_addr = hop2_addr;
- for (i = 0 ; i < num_of_hop3 ; i++) {
- pte_val = (ctx->dram_default_hops[i] & HOP_PHYS_ADDR_MASK) |
- PAGE_PRESENT_MASK;
- write_pte(ctx, hop2_pte_addr, pte_val);
- get_pte(ctx, hop2_addr);
- hop2_pte_addr += HL_PTE_SIZE;
- }
-
- pte_val = (prop->mmu_dram_default_page_addr & HOP_PHYS_ADDR_MASK) |
- LAST_MASK | PAGE_PRESENT_MASK;
-
- for (i = 0 ; i < num_of_hop3 ; i++) {
- hop3_pte_addr = ctx->dram_default_hops[i];
- for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
- write_final_pte(ctx, hop3_pte_addr, pte_val);
- get_pte(ctx, ctx->dram_default_hops[i]);
- hop3_pte_addr += HL_PTE_SIZE;
- }
- }
-
- flush(ctx);
-
- return 0;
-
-hop3_err:
- for (i = 0 ; i < hop3_allocated ; i++)
- free_hop(ctx, ctx->dram_default_hops[i]);
-
- free_hop(ctx, hop2_addr);
-hop2_err:
- free_hop(ctx, hop1_addr);
-hop1_err:
- kfree(ctx->dram_default_hops);
-
- return rc;
-}
-
-static void dram_default_mapping_fini(struct hl_ctx *ctx)
-{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,
- hop2_pte_addr, hop3_pte_addr;
- int i, j;
-
- if ((!hdev->dram_supports_virtual_memory) ||
- (!hdev->dram_default_page_mapping) ||
- (ctx->asid == HL_KERNEL_ASID_ID))
- return;
-
- num_of_hop3 = prop->dram_size_for_default_page_mapping;
- do_div(num_of_hop3, prop->dram_page_size);
- do_div(num_of_hop3, PTE_ENTRIES_IN_HOP);
-
- hop0_addr = get_hop0_addr(ctx);
- /* add hop1 and hop2 */
- total_hops = num_of_hop3 + 2;
- hop1_addr = ctx->dram_default_hops[total_hops - 1];
- hop2_addr = ctx->dram_default_hops[total_hops - 2];
-
- for (i = 0 ; i < num_of_hop3 ; i++) {
- hop3_pte_addr = ctx->dram_default_hops[i];
- for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
- clear_pte(ctx, hop3_pte_addr);
- put_pte(ctx, ctx->dram_default_hops[i]);
- hop3_pte_addr += HL_PTE_SIZE;
- }
- }
-
- hop2_pte_addr = hop2_addr;
- hop2_pte_addr = hop2_addr;
- for (i = 0 ; i < num_of_hop3 ; i++) {
- clear_pte(ctx, hop2_pte_addr);
- put_pte(ctx, hop2_addr);
- hop2_pte_addr += HL_PTE_SIZE;
- }
-
- clear_pte(ctx, hop1_addr);
- put_pte(ctx, hop1_addr);
- clear_pte(ctx, hop0_addr);
-
- kfree(ctx->dram_default_hops);
-
- flush(ctx);
-}
-
/**
* hl_mmu_init() - initialize the MMU module.
* @hdev: habanalabs device structure.
@@ -424,45 +30,10 @@ static void dram_default_mapping_fini(struct hl_ctx *ctx)
*/
int hl_mmu_init(struct hl_device *hdev)
{
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- int rc;
-
- if (!hdev->mmu_enable)
- return 0;
-
- hdev->mmu_pgt_pool =
- gen_pool_create(__ffs(prop->mmu_hop_table_size), -1);
-
- if (!hdev->mmu_pgt_pool) {
- dev_err(hdev->dev, "Failed to create page gen pool\n");
- return -ENOMEM;
- }
-
- rc = gen_pool_add(hdev->mmu_pgt_pool, prop->mmu_pgt_addr +
- prop->mmu_hop0_tables_total_size,
- prop->mmu_pgt_size - prop->mmu_hop0_tables_total_size,
- -1);
- if (rc) {
- dev_err(hdev->dev, "Failed to add memory to page gen pool\n");
- goto err_pool_add;
- }
-
- hdev->mmu_shadow_hop0 = kvmalloc_array(prop->max_asid,
- prop->mmu_hop_table_size,
- GFP_KERNEL | __GFP_ZERO);
- if (ZERO_OR_NULL_PTR(hdev->mmu_shadow_hop0)) {
- rc = -ENOMEM;
- goto err_pool_add;
- }
-
- /* MMU H/W init will be done in device hw_init() */
+ if (hdev->mmu_enable)
+ return hdev->mmu_func.init(hdev);
return 0;
-
-err_pool_add:
- gen_pool_destroy(hdev->mmu_pgt_pool);
-
- return rc;
}
/**
@@ -477,13 +48,8 @@ err_pool_add:
*/
void hl_mmu_fini(struct hl_device *hdev)
{
- if (!hdev->mmu_enable)
- return;
-
- /* MMU H/W fini was already done in device hw_fini() */
-
- kvfree(hdev->mmu_shadow_hop0);
- gen_pool_destroy(hdev->mmu_pgt_pool);
+ if (hdev->mmu_enable)
+ hdev->mmu_func.fini(hdev);
}
/**
@@ -498,13 +64,10 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
- if (!hdev->mmu_enable)
- return 0;
+ if (hdev->mmu_enable)
+ return hdev->mmu_func.ctx_init(ctx);
- mutex_init(&ctx->mmu_lock);
- hash_init(ctx->mmu_shadow_hash);
-
- return dram_default_mapping_init(ctx);
+ return 0;
}
/*
@@ -520,160 +83,9 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
void hl_mmu_ctx_fini(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
- struct pgt_info *pgt_info;
- struct hlist_node *tmp;
- int i;
-
- if (!hdev->mmu_enable)
- return;
-
- dram_default_mapping_fini(ctx);
-
- if (!hash_empty(ctx->mmu_shadow_hash))
- dev_err(hdev->dev, "ctx %d is freed while it has pgts in use\n",
- ctx->asid);
-
- hash_for_each_safe(ctx->mmu_shadow_hash, i, tmp, pgt_info, node) {
- dev_err_ratelimited(hdev->dev,
- "pgt_info of addr 0x%llx of asid %d was not destroyed, num_ptes: %d\n",
- pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes);
- _free_hop(ctx, pgt_info);
- }
-
- mutex_destroy(&ctx->mmu_lock);
-}
-
-static int _hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr)
-{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct hl_mmu_properties *mmu_prop;
- u64 hop0_addr = 0, hop0_pte_addr = 0,
- hop1_addr = 0, hop1_pte_addr = 0,
- hop2_addr = 0, hop2_pte_addr = 0,
- hop3_addr = 0, hop3_pte_addr = 0,
- hop4_addr = 0, hop4_pte_addr = 0,
- curr_pte;
- bool is_huge, clear_hop3 = true;
-
- /* shifts and masks are the same in PMMU and HPMMU, use one of them */
- mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;
-
- hop0_addr = get_hop0_addr(ctx);
- hop0_pte_addr = get_hop0_pte_addr(ctx, mmu_prop, hop0_addr, virt_addr);
-
- curr_pte = *(u64 *) (uintptr_t) hop0_pte_addr;
-
- hop1_addr = get_next_hop_addr(ctx, curr_pte);
-
- if (hop1_addr == ULLONG_MAX)
- goto not_mapped;
-
- hop1_pte_addr = get_hop1_pte_addr(ctx, mmu_prop, hop1_addr, virt_addr);
-
- curr_pte = *(u64 *) (uintptr_t) hop1_pte_addr;
-
- hop2_addr = get_next_hop_addr(ctx, curr_pte);
-
- if (hop2_addr == ULLONG_MAX)
- goto not_mapped;
-
- hop2_pte_addr = get_hop2_pte_addr(ctx, mmu_prop, hop2_addr, virt_addr);
-
- curr_pte = *(u64 *) (uintptr_t) hop2_pte_addr;
-
- hop3_addr = get_next_hop_addr(ctx, curr_pte);
-
- if (hop3_addr == ULLONG_MAX)
- goto not_mapped;
-
- hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);
-
- curr_pte = *(u64 *) (uintptr_t) hop3_pte_addr;
-
- is_huge = curr_pte & LAST_MASK;
-
- if (is_dram_addr && !is_huge) {
- dev_err(hdev->dev,
- "DRAM unmapping should use huge pages only\n");
- return -EFAULT;
- }
-
- if (!is_huge) {
- hop4_addr = get_next_hop_addr(ctx, curr_pte);
-
- if (hop4_addr == ULLONG_MAX)
- goto not_mapped;
-
- hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop4_addr,
- virt_addr);
-
- curr_pte = *(u64 *) (uintptr_t) hop4_pte_addr;
-
- clear_hop3 = false;
- }
-
- if (hdev->dram_default_page_mapping && is_dram_addr) {
- u64 default_pte = (prop->mmu_dram_default_page_addr &
- HOP_PHYS_ADDR_MASK) | LAST_MASK |
- PAGE_PRESENT_MASK;
- if (curr_pte == default_pte) {
- dev_err(hdev->dev,
- "DRAM: hop3 PTE points to zero page, can't unmap, va: 0x%llx\n",
- virt_addr);
- goto not_mapped;
- }
-
- if (!(curr_pte & PAGE_PRESENT_MASK)) {
- dev_err(hdev->dev,
- "DRAM: hop3 PTE is cleared! can't unmap, va: 0x%llx\n",
- virt_addr);
- goto not_mapped;
- }
-
- write_final_pte(ctx, hop3_pte_addr, default_pte);
- put_pte(ctx, hop3_addr);
- } else {
- if (!(curr_pte & PAGE_PRESENT_MASK))
- goto not_mapped;
-
- if (hop4_addr)
- clear_pte(ctx, hop4_pte_addr);
- else
- clear_pte(ctx, hop3_pte_addr);
-
- if (hop4_addr && !put_pte(ctx, hop4_addr))
- clear_hop3 = true;
-
- if (!clear_hop3)
- goto mapped;
-
- clear_pte(ctx, hop3_pte_addr);
- if (put_pte(ctx, hop3_addr))
- goto mapped;
-
- clear_pte(ctx, hop2_pte_addr);
-
- if (put_pte(ctx, hop2_addr))
- goto mapped;
-
- clear_pte(ctx, hop1_pte_addr);
-
- if (put_pte(ctx, hop1_addr))
- goto mapped;
-
- clear_pte(ctx, hop0_pte_addr);
- }
-
-mapped:
- return 0;
-
-not_mapped:
- dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",
- virt_addr);
-
- return -EINVAL;
+ if (hdev->mmu_enable)
+ hdev->mmu_func.ctx_fini(ctx);
}
/*
@@ -738,7 +150,7 @@ int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
real_virt_addr = virt_addr;
for (i = 0 ; i < npages ; i++) {
- rc = _hl_mmu_unmap(ctx, real_virt_addr, is_dram_addr);
+ rc = hdev->mmu_func.unmap(ctx, real_virt_addr, is_dram_addr);
if (rc)
break;
@@ -746,172 +158,7 @@ int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
}
if (flush_pte)
- flush(ctx);
-
- return rc;
-}
-
-static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
- u32 page_size, bool is_dram_addr)
-{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct hl_mmu_properties *mmu_prop;
- u64 hop0_addr = 0, hop0_pte_addr = 0,
- hop1_addr = 0, hop1_pte_addr = 0,
- hop2_addr = 0, hop2_pte_addr = 0,
- hop3_addr = 0, hop3_pte_addr = 0,
- hop4_addr = 0, hop4_pte_addr = 0,
- curr_pte = 0;
- bool hop1_new = false, hop2_new = false, hop3_new = false,
- hop4_new = false, is_huge;
- int rc = -ENOMEM;
-
- /*
- * This mapping function can map a page or a huge page. For huge page
- * there are only 3 hops rather than 4. Currently the DRAM allocation
- * uses huge pages only but user memory could have been allocated with
- * one of the two page sizes. Since this is a common code for all the
- * three cases, we need this hugs page check.
- */
- if (is_dram_addr) {
- mmu_prop = &prop->dmmu;
- is_huge = true;
- } else if (page_size == prop->pmmu_huge.page_size) {
- mmu_prop = &prop->pmmu_huge;
- is_huge = true;
- } else {
- mmu_prop = &prop->pmmu;
- is_huge = false;
- }
-
- hop0_addr = get_hop0_addr(ctx);
- hop0_pte_addr = get_hop0_pte_addr(ctx, mmu_prop, hop0_addr, virt_addr);
- curr_pte = *(u64 *) (uintptr_t) hop0_pte_addr;
-
- hop1_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop1_new);
- if (hop1_addr == ULLONG_MAX)
- goto err;
-
- hop1_pte_addr = get_hop1_pte_addr(ctx, mmu_prop, hop1_addr, virt_addr);
- curr_pte = *(u64 *) (uintptr_t) hop1_pte_addr;
-
- hop2_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop2_new);
- if (hop2_addr == ULLONG_MAX)
- goto err;
-
- hop2_pte_addr = get_hop2_pte_addr(ctx, mmu_prop, hop2_addr, virt_addr);
- curr_pte = *(u64 *) (uintptr_t) hop2_pte_addr;
-
- hop3_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop3_new);
- if (hop3_addr == ULLONG_MAX)
- goto err;
-
- hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);
- curr_pte = *(u64 *) (uintptr_t) hop3_pte_addr;
-
- if (!is_huge) {
- hop4_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop4_new);
- if (hop4_addr == ULLONG_MAX)
- goto err;
-
- hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop4_addr,
- virt_addr);
- curr_pte = *(u64 *) (uintptr_t) hop4_pte_addr;
- }
-
- if (hdev->dram_default_page_mapping && is_dram_addr) {
- u64 default_pte = (prop->mmu_dram_default_page_addr &
- HOP_PHYS_ADDR_MASK) | LAST_MASK |
- PAGE_PRESENT_MASK;
-
- if (curr_pte != default_pte) {
- dev_err(hdev->dev,
- "DRAM: mapping already exists for virt_addr 0x%llx\n",
- virt_addr);
- rc = -EINVAL;
- goto err;
- }
-
- if (hop1_new || hop2_new || hop3_new || hop4_new) {
- dev_err(hdev->dev,
- "DRAM mapping should not allocate more hops\n");
- rc = -EFAULT;
- goto err;
- }
- } else if (curr_pte & PAGE_PRESENT_MASK) {
- dev_err(hdev->dev,
- "mapping already exists for virt_addr 0x%llx\n",
- virt_addr);
-
- dev_dbg(hdev->dev, "hop0 pte: 0x%llx (0x%llx)\n",
- *(u64 *) (uintptr_t) hop0_pte_addr, hop0_pte_addr);
- dev_dbg(hdev->dev, "hop1 pte: 0x%llx (0x%llx)\n",
- *(u64 *) (uintptr_t) hop1_pte_addr, hop1_pte_addr);
- dev_dbg(hdev->dev, "hop2 pte: 0x%llx (0x%llx)\n",
- *(u64 *) (uintptr_t) hop2_pte_addr, hop2_pte_addr);
- dev_dbg(hdev->dev, "hop3 pte: 0x%llx (0x%llx)\n",
- *(u64 *) (uintptr_t) hop3_pte_addr, hop3_pte_addr);
-
- if (!is_huge)
- dev_dbg(hdev->dev, "hop4 pte: 0x%llx (0x%llx)\n",
- *(u64 *) (uintptr_t) hop4_pte_addr,
- hop4_pte_addr);
-
- rc = -EINVAL;
- goto err;
- }
-
- curr_pte = (phys_addr & HOP_PHYS_ADDR_MASK) | LAST_MASK
- | PAGE_PRESENT_MASK;
-
- if (is_huge)
- write_final_pte(ctx, hop3_pte_addr, curr_pte);
- else
- write_final_pte(ctx, hop4_pte_addr, curr_pte);
-
- if (hop1_new) {
- curr_pte =
- (hop1_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- write_pte(ctx, hop0_pte_addr, curr_pte);
- }
- if (hop2_new) {
- curr_pte =
- (hop2_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- write_pte(ctx, hop1_pte_addr, curr_pte);
- get_pte(ctx, hop1_addr);
- }
- if (hop3_new) {
- curr_pte =
- (hop3_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- write_pte(ctx, hop2_pte_addr, curr_pte);
- get_pte(ctx, hop2_addr);
- }
-
- if (!is_huge) {
- if (hop4_new) {
- curr_pte = (hop4_addr & HOP_PHYS_ADDR_MASK) |
- PAGE_PRESENT_MASK;
- write_pte(ctx, hop3_pte_addr, curr_pte);
- get_pte(ctx, hop3_addr);
- }
-
- get_pte(ctx, hop4_addr);
- } else {
- get_pte(ctx, hop3_addr);
- }
-
- return 0;
-
-err:
- if (hop4_new)
- free_hop(ctx, hop4_addr);
- if (hop3_new)
- free_hop(ctx, hop3_addr);
- if (hop2_new)
- free_hop(ctx, hop2_addr);
- if (hop1_new)
- free_hop(ctx, hop1_addr);
+ hdev->mmu_func.flush(ctx);
return rc;
}
@@ -984,7 +231,7 @@ int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
real_phys_addr = phys_addr;
for (i = 0 ; i < npages ; i++) {
- rc = _hl_mmu_map(ctx, real_virt_addr, real_phys_addr,
+ rc = hdev->mmu_func.map(ctx, real_virt_addr, real_phys_addr,
real_page_size, is_dram_addr);
if (rc)
goto err;
@@ -995,21 +242,21 @@ int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
}
if (flush_pte)
- flush(ctx);
+ hdev->mmu_func.flush(ctx);
return 0;
err:
real_virt_addr = virt_addr;
for (i = 0 ; i < mapped_cnt ; i++) {
- if (_hl_mmu_unmap(ctx, real_virt_addr, is_dram_addr))
+ if (hdev->mmu_func.unmap(ctx, real_virt_addr, is_dram_addr))
dev_warn_ratelimited(hdev->dev,
"failed to unmap va: 0x%llx\n", real_virt_addr);
real_virt_addr += real_page_size;
}
- flush(ctx);
+ hdev->mmu_func.flush(ctx);
return rc;
}
@@ -1022,7 +269,10 @@ err:
*/
void hl_mmu_swap_out(struct hl_ctx *ctx)
{
+ struct hl_device *hdev = ctx->hdev;
+ if (hdev->mmu_enable)
+ hdev->mmu_func.swap_out(ctx);
}
/*
@@ -1033,5 +283,27 @@ void hl_mmu_swap_out(struct hl_ctx *ctx)
*/
void hl_mmu_swap_in(struct hl_ctx *ctx)
{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (hdev->mmu_enable)
+ hdev->mmu_func.swap_in(ctx);
+}
+
+int hl_mmu_if_set_funcs(struct hl_device *hdev)
+{
+ if (!hdev->mmu_enable)
+ return 0;
+
+ switch (hdev->asic_type) {
+ case ASIC_GOYA:
+ case ASIC_GAUDI:
+ hl_mmu_v1_set_funcs(hdev);
+ break;
+ default:
+ dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
+ hdev->asic_type);
+ return -EOPNOTSUPP;
+ }
+ return 0;
}
diff --git a/drivers/misc/habanalabs/common/mmu_v1.c b/drivers/misc/habanalabs/common/mmu_v1.c
new file mode 100644
index 000000000000..8d1eb5265419
--- /dev/null
+++ b/drivers/misc/habanalabs/common/mmu_v1.c
@@ -0,0 +1,863 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2019 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "habanalabs.h"
+#include "../include/hw_ip/mmu/mmu_general.h"
+
+#include <linux/genalloc.h>
+#include <linux/slab.h>
+
+static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr);
+
+static struct pgt_info *get_pgt_info(struct hl_ctx *ctx, u64 hop_addr)
+{
+ struct pgt_info *pgt_info = NULL;
+
+ hash_for_each_possible(ctx->mmu_shadow_hash, pgt_info, node,
+ (unsigned long) hop_addr)
+ if (hop_addr == pgt_info->shadow_addr)
+ break;
+
+ return pgt_info;
+}
+
+static void _free_hop(struct hl_ctx *ctx, struct pgt_info *pgt_info)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ gen_pool_free(hdev->mmu_priv.mmu_pgt_pool, pgt_info->phys_addr,
+ hdev->asic_prop.mmu_hop_table_size);
+ hash_del(&pgt_info->node);
+ kfree((u64 *) (uintptr_t) pgt_info->shadow_addr);
+ kfree(pgt_info);
+}
+
+static void free_hop(struct hl_ctx *ctx, u64 hop_addr)
+{
+ struct pgt_info *pgt_info = get_pgt_info(ctx, hop_addr);
+
+ _free_hop(ctx, pgt_info);
+}
+
+static u64 alloc_hop(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct pgt_info *pgt_info;
+ u64 phys_addr, shadow_addr;
+
+ pgt_info = kmalloc(sizeof(*pgt_info), GFP_KERNEL);
+ if (!pgt_info)
+ return ULLONG_MAX;
+
+ phys_addr = (u64) gen_pool_alloc(hdev->mmu_priv.mmu_pgt_pool,
+ prop->mmu_hop_table_size);
+ if (!phys_addr) {
+ dev_err(hdev->dev, "failed to allocate page\n");
+ goto pool_add_err;
+ }
+
+ shadow_addr = (u64) (uintptr_t) kzalloc(prop->mmu_hop_table_size,
+ GFP_KERNEL);
+ if (!shadow_addr)
+ goto shadow_err;
+
+ pgt_info->phys_addr = phys_addr;
+ pgt_info->shadow_addr = shadow_addr;
+ pgt_info->ctx = ctx;
+ pgt_info->num_of_ptes = 0;
+ hash_add(ctx->mmu_shadow_hash, &pgt_info->node, shadow_addr);
+
+ return shadow_addr;
+
+shadow_err:
+ gen_pool_free(hdev->mmu_priv.mmu_pgt_pool, phys_addr,
+ prop->mmu_hop_table_size);
+pool_add_err:
+ kfree(pgt_info);
+
+ return ULLONG_MAX;
+}
+
+static inline u64 get_phys_hop0_addr(struct hl_ctx *ctx)
+{
+ return ctx->hdev->asic_prop.mmu_pgt_addr +
+ (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
+}
+
+static inline u64 get_hop0_addr(struct hl_ctx *ctx)
+{
+ return (u64) (uintptr_t) ctx->hdev->mmu_priv.mmu_shadow_hop0 +
+ (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
+}
+
+static void flush(struct hl_ctx *ctx)
+{
+ /* flush all writes from all cores to reach PCI */
+ mb();
+ ctx->hdev->asic_funcs->read_pte(ctx->hdev, get_phys_hop0_addr(ctx));
+}
+
+/* transform the value to physical address when writing to H/W */
+static inline void write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val)
+{
+ /*
+ * The value to write is actually the address of the next shadow hop +
+ * flags at the 12 LSBs.
+ * Hence in order to get the value to write to the physical PTE, we
+ * clear the 12 LSBs and translate the shadow hop to its associated
+ * physical hop, and add back the original 12 LSBs.
+ */
+ u64 phys_val = get_phys_addr(ctx, val & HOP_PHYS_ADDR_MASK) |
+ (val & FLAGS_MASK);
+
+ ctx->hdev->asic_funcs->write_pte(ctx->hdev,
+ get_phys_addr(ctx, shadow_pte_addr),
+ phys_val);
+
+ *(u64 *) (uintptr_t) shadow_pte_addr = val;
+}
+
+/* do not transform the value to physical address when writing to H/W */
+static inline void write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr,
+ u64 val)
+{
+ ctx->hdev->asic_funcs->write_pte(ctx->hdev,
+ get_phys_addr(ctx, shadow_pte_addr),
+ val);
+ *(u64 *) (uintptr_t) shadow_pte_addr = val;
+}
+
+/* clear the last and present bits */
+static inline void clear_pte(struct hl_ctx *ctx, u64 pte_addr)
+{
+ /* no need to transform the value to physical address */
+ write_final_pte(ctx, pte_addr, 0);
+}
+
+static inline void get_pte(struct hl_ctx *ctx, u64 hop_addr)
+{
+ get_pgt_info(ctx, hop_addr)->num_of_ptes++;
+}
+
+/*
+ * put_pte - decrement the num of ptes and free the hop if possible
+ *
+ * @ctx: pointer to the context structure
+ * @hop_addr: addr of the hop
+ *
+ * This function returns the number of ptes left on this hop. If the number is
+ * 0, it means the pte was freed.
+ */
+static inline int put_pte(struct hl_ctx *ctx, u64 hop_addr)
+{
+ struct pgt_info *pgt_info = get_pgt_info(ctx, hop_addr);
+ int num_of_ptes_left;
+
+ pgt_info->num_of_ptes--;
+
+ /*
+ * Need to save the number of ptes left because free_hop might free
+ * the pgt_info
+ */
+ num_of_ptes_left = pgt_info->num_of_ptes;
+ if (!num_of_ptes_left)
+ _free_hop(ctx, pgt_info);
+
+ return num_of_ptes_left;
+}
+
+static inline u64 get_hopN_pte_addr(struct hl_ctx *ctx, u64 hop_addr,
+ u64 virt_addr, u64 mask, u64 shift)
+{
+ return hop_addr + ctx->hdev->asic_prop.mmu_pte_size *
+ ((virt_addr & mask) >> shift);
+}
+
+static inline u64 get_hop0_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_prop,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop0_mask,
+ mmu_prop->hop0_shift);
+}
+
+static inline u64 get_hop1_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_prop,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop1_mask,
+ mmu_prop->hop1_shift);
+}
+
+static inline u64 get_hop2_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_prop,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop2_mask,
+ mmu_prop->hop2_shift);
+}
+
+static inline u64 get_hop3_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_prop,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop3_mask,
+ mmu_prop->hop3_shift);
+}
+
+static inline u64 get_hop4_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_prop,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_prop->hop4_mask,
+ mmu_prop->hop4_shift);
+}
+
+static inline u64 get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte)
+{
+ if (curr_pte & PAGE_PRESENT_MASK)
+ return curr_pte & HOP_PHYS_ADDR_MASK;
+ else
+ return ULLONG_MAX;
+}
+
+static inline u64 get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte,
+ bool *is_new_hop)
+{
+ u64 hop_addr = get_next_hop_addr(ctx, curr_pte);
+
+ if (hop_addr == ULLONG_MAX) {
+ hop_addr = alloc_hop(ctx);
+ *is_new_hop = (hop_addr != ULLONG_MAX);
+ }
+
+ return hop_addr;
+}
+
+/* translates shadow address inside hop to a physical address */
+static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr)
+{
+ u64 page_mask = (ctx->hdev->asic_prop.mmu_hop_table_size - 1);
+ u64 shadow_hop_addr = shadow_addr & ~page_mask;
+ u64 pte_offset = shadow_addr & page_mask;
+ u64 phys_hop_addr;
+
+ if (shadow_hop_addr != get_hop0_addr(ctx))
+ phys_hop_addr = get_pgt_info(ctx, shadow_hop_addr)->phys_addr;
+ else
+ phys_hop_addr = get_phys_hop0_addr(ctx);
+
+ return phys_hop_addr + pte_offset;
+}
+
+static int dram_default_mapping_init(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,
+ hop2_pte_addr, hop3_pte_addr, pte_val;
+ int rc, i, j, hop3_allocated = 0;
+
+ if ((!hdev->dram_supports_virtual_memory) ||
+ (!hdev->dram_default_page_mapping) ||
+ (ctx->asid == HL_KERNEL_ASID_ID))
+ return 0;
+
+ num_of_hop3 = prop->dram_size_for_default_page_mapping;
+ do_div(num_of_hop3, prop->dram_page_size);
+ do_div(num_of_hop3, PTE_ENTRIES_IN_HOP);
+
+ /* add hop1 and hop2 */
+ total_hops = num_of_hop3 + 2;
+
+ ctx->dram_default_hops = kzalloc(HL_PTE_SIZE * total_hops, GFP_KERNEL);
+ if (!ctx->dram_default_hops)
+ return -ENOMEM;
+
+ hop0_addr = get_hop0_addr(ctx);
+
+ hop1_addr = alloc_hop(ctx);
+ if (hop1_addr == ULLONG_MAX) {
+ dev_err(hdev->dev, "failed to alloc hop 1\n");
+ rc = -ENOMEM;
+ goto hop1_err;
+ }
+
+ ctx->dram_default_hops[total_hops - 1] = hop1_addr;
+
+ hop2_addr = alloc_hop(ctx);
+ if (hop2_addr == ULLONG_MAX) {
+ dev_err(hdev->dev, "failed to alloc hop 2\n");
+ rc = -ENOMEM;
+ goto hop2_err;
+ }
+
+ ctx->dram_default_hops[total_hops - 2] = hop2_addr;
+
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ ctx->dram_default_hops[i] = alloc_hop(ctx);
+ if (ctx->dram_default_hops[i] == ULLONG_MAX) {
+ dev_err(hdev->dev, "failed to alloc hop 3, i: %d\n", i);
+ rc = -ENOMEM;
+ goto hop3_err;
+ }
+ hop3_allocated++;
+ }
+
+ /* need only pte 0 in hops 0 and 1 */
+ pte_val = (hop1_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop0_addr, pte_val);
+
+ pte_val = (hop2_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop1_addr, pte_val);
+ get_pte(ctx, hop1_addr);
+
+ hop2_pte_addr = hop2_addr;
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ pte_val = (ctx->dram_default_hops[i] & HOP_PHYS_ADDR_MASK) |
+ PAGE_PRESENT_MASK;
+ write_pte(ctx, hop2_pte_addr, pte_val);
+ get_pte(ctx, hop2_addr);
+ hop2_pte_addr += HL_PTE_SIZE;
+ }
+
+ pte_val = (prop->mmu_dram_default_page_addr & HOP_PHYS_ADDR_MASK) |
+ LAST_MASK | PAGE_PRESENT_MASK;
+
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ hop3_pte_addr = ctx->dram_default_hops[i];
+ for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
+ write_final_pte(ctx, hop3_pte_addr, pte_val);
+ get_pte(ctx, ctx->dram_default_hops[i]);
+ hop3_pte_addr += HL_PTE_SIZE;
+ }
+ }
+
+ flush(ctx);
+
+ return 0;
+
+hop3_err:
+ for (i = 0 ; i < hop3_allocated ; i++)
+ free_hop(ctx, ctx->dram_default_hops[i]);
+
+ free_hop(ctx, hop2_addr);
+hop2_err:
+ free_hop(ctx, hop1_addr);
+hop1_err:
+ kfree(ctx->dram_default_hops);
+
+ return rc;
+}
+
+static void dram_default_mapping_fini(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,
+ hop2_pte_addr, hop3_pte_addr;
+ int i, j;
+
+ if ((!hdev->dram_supports_virtual_memory) ||
+ (!hdev->dram_default_page_mapping) ||
+ (ctx->asid == HL_KERNEL_ASID_ID))
+ return;
+
+ num_of_hop3 = prop->dram_size_for_default_page_mapping;
+ do_div(num_of_hop3, prop->dram_page_size);
+ do_div(num_of_hop3, PTE_ENTRIES_IN_HOP);
+
+ hop0_addr = get_hop0_addr(ctx);
+ /* add hop1 and hop2 */
+ total_hops = num_of_hop3 + 2;
+ hop1_addr = ctx->dram_default_hops[total_hops - 1];
+ hop2_addr = ctx->dram_default_hops[total_hops - 2];
+
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ hop3_pte_addr = ctx->dram_default_hops[i];
+ for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
+ clear_pte(ctx, hop3_pte_addr);
+ put_pte(ctx, ctx->dram_default_hops[i]);
+ hop3_pte_addr += HL_PTE_SIZE;
+ }
+ }
+
+ hop2_pte_addr = hop2_addr;
+ hop2_pte_addr = hop2_addr;
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ clear_pte(ctx, hop2_pte_addr);
+ put_pte(ctx, hop2_addr);
+ hop2_pte_addr += HL_PTE_SIZE;
+ }
+
+ clear_pte(ctx, hop1_addr);
+ put_pte(ctx, hop1_addr);
+ clear_pte(ctx, hop0_addr);
+
+ kfree(ctx->dram_default_hops);
+
+ flush(ctx);
+}
+
+/**
+ * hl_mmu_v1_init() - initialize the MMU module.
+ * @hdev: habanalabs device structure.
+ *
+ * This function does the following:
+ * - Create a pool of pages for pgt_infos.
+ * - Create a shadow table for pgt
+ *
+ * Return: 0 for success, non-zero for failure.
+ */
+static int hl_mmu_v1_init(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ int rc;
+
+ hdev->mmu_priv.mmu_pgt_pool =
+ gen_pool_create(__ffs(prop->mmu_hop_table_size), -1);
+
+ if (!hdev->mmu_priv.mmu_pgt_pool) {
+ dev_err(hdev->dev, "Failed to create page gen pool\n");
+ return -ENOMEM;
+ }
+
+ rc = gen_pool_add(hdev->mmu_priv.mmu_pgt_pool, prop->mmu_pgt_addr +
+ prop->mmu_hop0_tables_total_size,
+ prop->mmu_pgt_size - prop->mmu_hop0_tables_total_size,
+ -1);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to add memory to page gen pool\n");
+ goto err_pool_add;
+ }
+
+ hdev->mmu_priv.mmu_shadow_hop0 = kvmalloc_array(prop->max_asid,
+ prop->mmu_hop_table_size,
+ GFP_KERNEL | __GFP_ZERO);
+ if (ZERO_OR_NULL_PTR(hdev->mmu_priv.mmu_shadow_hop0)) {
+ rc = -ENOMEM;
+ goto err_pool_add;
+ }
+
+ /* MMU H/W init will be done in device hw_init() */
+
+ return 0;
+
+err_pool_add:
+ gen_pool_destroy(hdev->mmu_priv.mmu_pgt_pool);
+
+ return rc;
+}
+
+/**
+ * hl_mmu_fini() - release the MMU module.
+ * @hdev: habanalabs device structure.
+ *
+ * This function does the following:
+ * - Disable MMU in H/W.
+ * - Free the pgt_infos pool.
+ *
+ * All contexts should be freed before calling this function.
+ */
+static void hl_mmu_v1_fini(struct hl_device *hdev)
+{
+ /* MMU H/W fini was already done in device hw_fini() */
+
+ kvfree(hdev->mmu_priv.mmu_shadow_hop0);
+ gen_pool_destroy(hdev->mmu_priv.mmu_pgt_pool);
+}
+
+/**
+ * hl_mmu_ctx_init() - initialize a context for using the MMU module.
+ * @ctx: pointer to the context structure to initialize.
+ *
+ * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all
+ * page tables hops related to this context.
+ * Return: 0 on success, non-zero otherwise.
+ */
+static int hl_mmu_v1_ctx_init(struct hl_ctx *ctx)
+{
+ mutex_init(&ctx->mmu_lock);
+ hash_init(ctx->mmu_shadow_hash);
+
+ return dram_default_mapping_init(ctx);
+}
+
+/*
+ * hl_mmu_ctx_fini - disable a ctx from using the mmu module
+ *
+ * @ctx: pointer to the context structure
+ *
+ * This function does the following:
+ * - Free any pgts which were not freed yet
+ * - Free the mutex
+ * - Free DRAM default page mapping hops
+ */
+static void hl_mmu_v1_ctx_fini(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct pgt_info *pgt_info;
+ struct hlist_node *tmp;
+ int i;
+
+ dram_default_mapping_fini(ctx);
+
+ if (!hash_empty(ctx->mmu_shadow_hash))
+ dev_err(hdev->dev, "ctx %d is freed while it has pgts in use\n",
+ ctx->asid);
+
+ hash_for_each_safe(ctx->mmu_shadow_hash, i, tmp, pgt_info, node) {
+ dev_err_ratelimited(hdev->dev,
+ "pgt_info of addr 0x%llx of asid %d was not destroyed, num_ptes: %d\n",
+ pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes);
+ _free_hop(ctx, pgt_info);
+ }
+
+ mutex_destroy(&ctx->mmu_lock);
+}
+
+static int _hl_mmu_v1_unmap(struct hl_ctx *ctx,
+ u64 virt_addr, bool is_dram_addr)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_mmu_properties *mmu_prop;
+ u64 hop0_addr = 0, hop0_pte_addr = 0,
+ hop1_addr = 0, hop1_pte_addr = 0,
+ hop2_addr = 0, hop2_pte_addr = 0,
+ hop3_addr = 0, hop3_pte_addr = 0,
+ hop4_addr = 0, hop4_pte_addr = 0,
+ curr_pte;
+ bool is_huge, clear_hop3 = true;
+
+ /* shifts and masks are the same in PMMU and HPMMU, use one of them */
+ mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;
+
+ hop0_addr = get_hop0_addr(ctx);
+ hop0_pte_addr = get_hop0_pte_addr(ctx, mmu_prop, hop0_addr, virt_addr);
+
+ curr_pte = *(u64 *) (uintptr_t) hop0_pte_addr;
+
+ hop1_addr = get_next_hop_addr(ctx, curr_pte);
+
+ if (hop1_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop1_pte_addr = get_hop1_pte_addr(ctx, mmu_prop, hop1_addr, virt_addr);
+
+ curr_pte = *(u64 *) (uintptr_t) hop1_pte_addr;
+
+ hop2_addr = get_next_hop_addr(ctx, curr_pte);
+
+ if (hop2_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop2_pte_addr = get_hop2_pte_addr(ctx, mmu_prop, hop2_addr, virt_addr);
+
+ curr_pte = *(u64 *) (uintptr_t) hop2_pte_addr;
+
+ hop3_addr = get_next_hop_addr(ctx, curr_pte);
+
+ if (hop3_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);
+
+ curr_pte = *(u64 *) (uintptr_t) hop3_pte_addr;
+
+ is_huge = curr_pte & LAST_MASK;
+
+ if (is_dram_addr && !is_huge) {
+ dev_err(hdev->dev,
+ "DRAM unmapping should use huge pages only\n");
+ return -EFAULT;
+ }
+
+ if (!is_huge) {
+ hop4_addr = get_next_hop_addr(ctx, curr_pte);
+
+ if (hop4_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop4_addr,
+ virt_addr);
+
+ curr_pte = *(u64 *) (uintptr_t) hop4_pte_addr;
+
+ clear_hop3 = false;
+ }
+
+ if (hdev->dram_default_page_mapping && is_dram_addr) {
+ u64 default_pte = (prop->mmu_dram_default_page_addr &
+ HOP_PHYS_ADDR_MASK) | LAST_MASK |
+ PAGE_PRESENT_MASK;
+ if (curr_pte == default_pte) {
+ dev_err(hdev->dev,
+ "DRAM: hop3 PTE points to zero page, can't unmap, va: 0x%llx\n",
+ virt_addr);
+ goto not_mapped;
+ }
+
+ if (!(curr_pte & PAGE_PRESENT_MASK)) {
+ dev_err(hdev->dev,
+ "DRAM: hop3 PTE is cleared! can't unmap, va: 0x%llx\n",
+ virt_addr);
+ goto not_mapped;
+ }
+
+ write_final_pte(ctx, hop3_pte_addr, default_pte);
+ put_pte(ctx, hop3_addr);
+ } else {
+ if (!(curr_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+
+ if (hop4_addr)
+ clear_pte(ctx, hop4_pte_addr);
+ else
+ clear_pte(ctx, hop3_pte_addr);
+
+ if (hop4_addr && !put_pte(ctx, hop4_addr))
+ clear_hop3 = true;
+
+ if (!clear_hop3)
+ goto mapped;
+
+ clear_pte(ctx, hop3_pte_addr);
+
+ if (put_pte(ctx, hop3_addr))
+ goto mapped;
+
+ clear_pte(ctx, hop2_pte_addr);
+
+ if (put_pte(ctx, hop2_addr))
+ goto mapped;
+
+ clear_pte(ctx, hop1_pte_addr);
+
+ if (put_pte(ctx, hop1_addr))
+ goto mapped;
+
+ clear_pte(ctx, hop0_pte_addr);
+ }
+
+mapped:
+ return 0;
+
+not_mapped:
+ dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",
+ virt_addr);
+
+ return -EINVAL;
+}
+
+static int _hl_mmu_v1_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
+ u32 page_size, bool is_dram_addr)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_mmu_properties *mmu_prop;
+ u64 hop0_addr = 0, hop0_pte_addr = 0,
+ hop1_addr = 0, hop1_pte_addr = 0,
+ hop2_addr = 0, hop2_pte_addr = 0,
+ hop3_addr = 0, hop3_pte_addr = 0,
+ hop4_addr = 0, hop4_pte_addr = 0,
+ curr_pte = 0;
+ bool hop1_new = false, hop2_new = false, hop3_new = false,
+ hop4_new = false, is_huge;
+ int rc = -ENOMEM;
+
+ /*
+ * This mapping function can map a page or a huge page. For huge page
+ * there are only 3 hops rather than 4. Currently the DRAM allocation
+ * uses huge pages only but user memory could have been allocated with
+ * one of the two page sizes. Since this is a common code for all the
+ * three cases, we need this hugs page check.
+ */
+ if (is_dram_addr) {
+ mmu_prop = &prop->dmmu;
+ is_huge = true;
+ } else if (page_size == prop->pmmu_huge.page_size) {
+ mmu_prop = &prop->pmmu_huge;
+ is_huge = true;
+ } else {
+ mmu_prop = &prop->pmmu;
+ is_huge = false;
+ }
+
+ hop0_addr = get_hop0_addr(ctx);
+ hop0_pte_addr = get_hop0_pte_addr(ctx, mmu_prop, hop0_addr, virt_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop0_pte_addr;
+
+ hop1_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop1_new);
+ if (hop1_addr == ULLONG_MAX)
+ goto err;
+
+ hop1_pte_addr = get_hop1_pte_addr(ctx, mmu_prop, hop1_addr, virt_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop1_pte_addr;
+
+ hop2_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop2_new);
+ if (hop2_addr == ULLONG_MAX)
+ goto err;
+
+ hop2_pte_addr = get_hop2_pte_addr(ctx, mmu_prop, hop2_addr, virt_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop2_pte_addr;
+
+ hop3_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop3_new);
+ if (hop3_addr == ULLONG_MAX)
+ goto err;
+
+ hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop3_pte_addr;
+
+ if (!is_huge) {
+ hop4_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop4_new);
+ if (hop4_addr == ULLONG_MAX)
+ goto err;
+
+ hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop4_addr,
+ virt_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop4_pte_addr;
+ }
+
+ if (hdev->dram_default_page_mapping && is_dram_addr) {
+ u64 default_pte = (prop->mmu_dram_default_page_addr &
+ HOP_PHYS_ADDR_MASK) | LAST_MASK |
+ PAGE_PRESENT_MASK;
+
+ if (curr_pte != default_pte) {
+ dev_err(hdev->dev,
+ "DRAM: mapping already exists for virt_addr 0x%llx\n",
+ virt_addr);
+ rc = -EINVAL;
+ goto err;
+ }
+
+ if (hop1_new || hop2_new || hop3_new || hop4_new) {
+ dev_err(hdev->dev,
+ "DRAM mapping should not allocate more hops\n");
+ rc = -EFAULT;
+ goto err;
+ }
+ } else if (curr_pte & PAGE_PRESENT_MASK) {
+ dev_err(hdev->dev,
+ "mapping already exists for virt_addr 0x%llx\n",
+ virt_addr);
+
+ dev_dbg(hdev->dev, "hop0 pte: 0x%llx (0x%llx)\n",
+ *(u64 *) (uintptr_t) hop0_pte_addr, hop0_pte_addr);
+ dev_dbg(hdev->dev, "hop1 pte: 0x%llx (0x%llx)\n",
+ *(u64 *) (uintptr_t) hop1_pte_addr, hop1_pte_addr);
+ dev_dbg(hdev->dev, "hop2 pte: 0x%llx (0x%llx)\n",
+ *(u64 *) (uintptr_t) hop2_pte_addr, hop2_pte_addr);
+ dev_dbg(hdev->dev, "hop3 pte: 0x%llx (0x%llx)\n",
+ *(u64 *) (uintptr_t) hop3_pte_addr, hop3_pte_addr);
+
+ if (!is_huge)
+ dev_dbg(hdev->dev, "hop4 pte: 0x%llx (0x%llx)\n",
+ *(u64 *) (uintptr_t) hop4_pte_addr,
+ hop4_pte_addr);
+
+ rc = -EINVAL;
+ goto err;
+ }
+
+ curr_pte = (phys_addr & HOP_PHYS_ADDR_MASK) | LAST_MASK
+ | PAGE_PRESENT_MASK;
+
+ if (is_huge)
+ write_final_pte(ctx, hop3_pte_addr, curr_pte);
+ else
+ write_final_pte(ctx, hop4_pte_addr, curr_pte);
+
+ if (hop1_new) {
+ curr_pte =
+ (hop1_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop0_pte_addr, curr_pte);
+ }
+ if (hop2_new) {
+ curr_pte =
+ (hop2_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop1_pte_addr, curr_pte);
+ get_pte(ctx, hop1_addr);
+ }
+ if (hop3_new) {
+ curr_pte =
+ (hop3_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop2_pte_addr, curr_pte);
+ get_pte(ctx, hop2_addr);
+ }
+
+ if (!is_huge) {
+ if (hop4_new) {
+ curr_pte = (hop4_addr & HOP_PHYS_ADDR_MASK) |
+ PAGE_PRESENT_MASK;
+ write_pte(ctx, hop3_pte_addr, curr_pte);
+ get_pte(ctx, hop3_addr);
+ }
+
+ get_pte(ctx, hop4_addr);
+ } else {
+ get_pte(ctx, hop3_addr);
+ }
+
+ return 0;
+
+err:
+ if (hop4_new)
+ free_hop(ctx, hop4_addr);
+ if (hop3_new)
+ free_hop(ctx, hop3_addr);
+ if (hop2_new)
+ free_hop(ctx, hop2_addr);
+ if (hop1_new)
+ free_hop(ctx, hop1_addr);
+
+ return rc;
+}
+
+/*
+ * hl_mmu_v1_swap_out - marks all mapping of the given ctx as swapped out
+ *
+ * @ctx: pointer to the context structure
+ *
+ */
+static void hl_mmu_v1_swap_out(struct hl_ctx *ctx)
+{
+
+}
+
+/*
+ * hl_mmu_v1_swap_in - marks all mapping of the given ctx as swapped in
+ *
+ * @ctx: pointer to the context structure
+ *
+ */
+static void hl_mmu_v1_swap_in(struct hl_ctx *ctx)
+{
+
+}
+
+/*
+ * hl_mmu_v1_prepare - prepare mmu for working with mmu v1
+ *
+ * @hdev: pointer to the device structure
+ */
+void hl_mmu_v1_set_funcs(struct hl_device *hdev)
+{
+ struct hl_mmu_funcs *mmu = &hdev->mmu_func;
+
+ mmu->init = hl_mmu_v1_init;
+ mmu->fini = hl_mmu_v1_fini;
+ mmu->ctx_init = hl_mmu_v1_ctx_init;
+ mmu->ctx_fini = hl_mmu_v1_ctx_fini;
+ mmu->map = _hl_mmu_v1_map;
+ mmu->unmap = _hl_mmu_v1_unmap;
+ mmu->flush = flush;
+ mmu->swap_out = hl_mmu_v1_swap_out;
+ mmu->swap_in = hl_mmu_v1_swap_in;
+}
diff --git a/drivers/misc/habanalabs/common/pci.c b/drivers/misc/habanalabs/common/pci.c
index 2770f03b6cbb..4327e5704ebb 100644
--- a/drivers/misc/habanalabs/common/pci.c
+++ b/drivers/misc/habanalabs/common/pci.c
@@ -9,7 +9,6 @@
#include "../include/hw_ip/pci/pci_general.h"
#include <linux/pci.h>
-#include <linux/bitfield.h>
#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10)
@@ -339,12 +338,17 @@ static int hl_pci_set_dma_mask(struct hl_device *hdev)
/**
* hl_pci_init() - PCI initialization code.
* @hdev: Pointer to hl_device structure.
+ * @cpu_boot_status_reg: status register of the device's CPU
+ * @boot_err0_reg: boot error register of the device's CPU
+ * @preboot_ver_timeout: how much to wait before bailing out on reading
+ * the preboot version
*
* Set DMA masks, initialize the PCI controller and map the PCI BARs.
*
* Return: 0 on success, non-zero for failure.
*/
-int hl_pci_init(struct hl_device *hdev)
+int hl_pci_init(struct hl_device *hdev, u32 cpu_boot_status_reg,
+ u32 boot_err0_reg, u32 preboot_ver_timeout)
{
struct pci_dev *pdev = hdev->pdev;
int rc;
@@ -376,6 +380,15 @@ int hl_pci_init(struct hl_device *hdev)
if (rc)
goto unmap_pci_bars;
+ /* Before continuing in the initialization, we need to read the preboot
+ * version to determine whether we run with a security-enabled firmware
+ * The check will be done in each ASIC's specific code
+ */
+ rc = hl_fw_read_preboot_ver(hdev, cpu_boot_status_reg, boot_err0_reg,
+ preboot_ver_timeout);
+ if (rc)
+ goto unmap_pci_bars;
+
return 0;
unmap_pci_bars:
diff --git a/drivers/misc/habanalabs/common/sysfs.c b/drivers/misc/habanalabs/common/sysfs.c
index 5ae484cc84cd..3ceae87016b1 100644
--- a/drivers/misc/habanalabs/common/sysfs.c
+++ b/drivers/misc/habanalabs/common/sysfs.c
@@ -11,18 +11,18 @@
long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
long result;
int rc;
memset(&pkt, 0, sizeof(pkt));
if (curr)
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_FREQUENCY_CURR_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
else
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_FREQUENCY_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.pll_index = cpu_to_le32(pll_index);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
@@ -40,13 +40,13 @@ long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_FREQUENCY_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.pll_index = cpu_to_le32(pll_index);
pkt.value = cpu_to_le64(freq);
@@ -61,14 +61,14 @@ void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
u64 hl_get_max_power(struct hl_device *hdev)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
long result;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_MAX_POWER_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
0, &result);
@@ -83,13 +83,13 @@ u64 hl_get_max_power(struct hl_device *hdev)
void hl_set_max_power(struct hl_device *hdev)
{
- struct armcp_packet pkt;
+ struct cpucp_packet pkt;
int rc;
memset(&pkt, 0, sizeof(pkt));
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_MAX_POWER_SET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET <<
+ CPUCP_PKT_CTL_OPCODE_SHIFT);
pkt.value = cpu_to_le64(hdev->max_power);
rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
@@ -112,7 +112,7 @@ static ssize_t armcp_kernel_ver_show(struct device *dev,
{
struct hl_device *hdev = dev_get_drvdata(dev);
- return sprintf(buf, "%s", hdev->asic_prop.armcp_info.kernel_version);
+ return sprintf(buf, "%s", hdev->asic_prop.cpucp_info.kernel_version);
}
static ssize_t armcp_ver_show(struct device *dev, struct device_attribute *attr,
@@ -120,7 +120,7 @@ static ssize_t armcp_ver_show(struct device *dev, struct device_attribute *attr,
{
struct hl_device *hdev = dev_get_drvdata(dev);
- return sprintf(buf, "%s\n", hdev->asic_prop.armcp_info.armcp_version);
+ return sprintf(buf, "%s\n", hdev->asic_prop.cpucp_info.cpucp_version);
}
static ssize_t cpld_ver_show(struct device *dev, struct device_attribute *attr,
@@ -129,7 +129,23 @@ static ssize_t cpld_ver_show(struct device *dev, struct device_attribute *attr,
struct hl_device *hdev = dev_get_drvdata(dev);
return sprintf(buf, "0x%08x\n",
- hdev->asic_prop.armcp_info.cpld_version);
+ hdev->asic_prop.cpucp_info.cpld_version);
+}
+
+static ssize_t cpucp_kernel_ver_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct hl_device *hdev = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%s", hdev->asic_prop.cpucp_info.kernel_version);
+}
+
+static ssize_t cpucp_ver_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct hl_device *hdev = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%s\n", hdev->asic_prop.cpucp_info.cpucp_version);
}
static ssize_t infineon_ver_show(struct device *dev,
@@ -138,7 +154,7 @@ static ssize_t infineon_ver_show(struct device *dev,
struct hl_device *hdev = dev_get_drvdata(dev);
return sprintf(buf, "0x%04x\n",
- hdev->asic_prop.armcp_info.infineon_version);
+ hdev->asic_prop.cpucp_info.infineon_version);
}
static ssize_t fuse_ver_show(struct device *dev, struct device_attribute *attr,
@@ -146,7 +162,7 @@ static ssize_t fuse_ver_show(struct device *dev, struct device_attribute *attr,
{
struct hl_device *hdev = dev_get_drvdata(dev);
- return sprintf(buf, "%s\n", hdev->asic_prop.armcp_info.fuse_version);
+ return sprintf(buf, "%s\n", hdev->asic_prop.cpucp_info.fuse_version);
}
static ssize_t thermal_ver_show(struct device *dev,
@@ -154,7 +170,7 @@ static ssize_t thermal_ver_show(struct device *dev,
{
struct hl_device *hdev = dev_get_drvdata(dev);
- return sprintf(buf, "%s", hdev->asic_prop.armcp_info.thermal_version);
+ return sprintf(buf, "%s", hdev->asic_prop.cpucp_info.thermal_version);
}
static ssize_t preboot_btl_ver_show(struct device *dev,
@@ -356,6 +372,8 @@ out:
static DEVICE_ATTR_RO(armcp_kernel_ver);
static DEVICE_ATTR_RO(armcp_ver);
static DEVICE_ATTR_RO(cpld_ver);
+static DEVICE_ATTR_RO(cpucp_kernel_ver);
+static DEVICE_ATTR_RO(cpucp_ver);
static DEVICE_ATTR_RO(device_type);
static DEVICE_ATTR_RO(fuse_ver);
static DEVICE_ATTR_WO(hard_reset);
@@ -380,6 +398,8 @@ static struct attribute *hl_dev_attrs[] = {
&dev_attr_armcp_kernel_ver.attr,
&dev_attr_armcp_ver.attr,
&dev_attr_cpld_ver.attr,
+ &dev_attr_cpucp_kernel_ver.attr,
+ &dev_attr_cpucp_ver.attr,
&dev_attr_device_type.attr,
&dev_attr_fuse_ver.attr,
&dev_attr_hard_reset.attr,