summaryrefslogtreecommitdiff
path: root/drivers/accel/ivpu/ivpu_mmu.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/accel/ivpu/ivpu_mmu.c')
-rw-r--r--drivers/accel/ivpu/ivpu_mmu.c1029
1 files changed, 1029 insertions, 0 deletions
diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c
new file mode 100644
index 000000000000..e1baf6b64935
--- /dev/null
+++ b/drivers/accel/ivpu/ivpu_mmu.c
@@ -0,0 +1,1029 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020-2024 Intel Corporation
+ */
+
+#include <linux/circ_buf.h>
+#include <linux/highmem.h>
+
+#include "ivpu_drv.h"
+#include "ivpu_hw.h"
+#include "ivpu_hw_reg_io.h"
+#include "ivpu_mmu.h"
+#include "ivpu_mmu_context.h"
+#include "ivpu_pm.h"
+
+#define IVPU_MMU_REG_IDR0 0x00200000u
+#define IVPU_MMU_REG_IDR1 0x00200004u
+#define IVPU_MMU_REG_IDR3 0x0020000cu
+#define IVPU_MMU_REG_IDR5 0x00200014u
+#define IVPU_MMU_REG_CR0 0x00200020u
+#define IVPU_MMU_REG_CR0ACK 0x00200024u
+#define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0)
+#define IVPU_MMU_REG_CR0_ATSCHK_MASK BIT(4)
+#define IVPU_MMU_REG_CR0_CMDQEN_MASK BIT(3)
+#define IVPU_MMU_REG_CR0_EVTQEN_MASK BIT(2)
+#define IVPU_MMU_REG_CR0_PRIQEN_MASK BIT(1)
+#define IVPU_MMU_REG_CR0_SMMUEN_MASK BIT(0)
+
+#define IVPU_MMU_REG_CR1 0x00200028u
+#define IVPU_MMU_REG_CR2 0x0020002cu
+#define IVPU_MMU_REG_IRQ_CTRL 0x00200050u
+#define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u
+#define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0)
+
+#define IVPU_MMU_REG_GERROR 0x00200060u
+#define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0)
+#define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2)
+#define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3)
+#define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4)
+#define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5)
+#define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6)
+#define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7)
+
+#define IVPU_MMU_REG_GERRORN 0x00200064u
+
+#define IVPU_MMU_REG_STRTAB_BASE 0x00200080u
+#define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u
+#define IVPU_MMU_REG_CMDQ_BASE 0x00200090u
+#define IVPU_MMU_REG_CMDQ_PROD 0x00200098u
+#define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu
+#define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0)
+#define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24)
+#define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u
+#define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u
+#define IVPU_MMU_REG_EVTQ_CONS 0x002000acu
+#define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K)
+#define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K)
+
+#define IVPU_MMU_IDR0_REF 0x080f3e0f
+#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f
+#define IVPU_MMU_IDR1_REF 0x0e739d18
+#define IVPU_MMU_IDR3_REF 0x0000003c
+#define IVPU_MMU_IDR5_REF 0x00040070
+#define IVPU_MMU_IDR5_REF_SIMICS 0x00000075
+#define IVPU_MMU_IDR5_REF_FPGA 0x00800075
+
+#define IVPU_MMU_CDTAB_ENT_SIZE 64
+#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */
+#define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2)
+
+#define IVPU_MMU_STREAM_ID0 0
+#define IVPU_MMU_STREAM_ID3 3
+
+#define IVPU_MMU_STRTAB_ENT_SIZE 64
+#define IVPU_MMU_STRTAB_ENT_COUNT 4
+#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2
+#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE
+
+#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */
+#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
+#define IVPU_MMU_Q_WRAP_MASK GENMASK(IVPU_MMU_Q_COUNT_LOG2, 0)
+#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)
+#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)
+#define IVPU_MMU_Q_WRP(val) ((val) & IVPU_MMU_Q_COUNT)
+
+#define IVPU_MMU_CMDQ_CMD_SIZE 16
+#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
+
+#define IVPU_MMU_EVTQ_CMD_SIZE 32
+#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
+
+#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)
+
+#define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12)
+#define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22)
+#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
+#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
+#define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32)
+
+#define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10)
+#define IVPU_MMU_CMD_CFGI_0_SSV BIT(11)
+#define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12)
+#define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32)
+#define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0)
+
+#define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48)
+#define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32)
+
+#define CMD_PREFETCH_CFG 0x1
+#define CMD_CFGI_STE 0x3
+#define CMD_CFGI_ALL 0x4
+#define CMD_CFGI_CD 0x5
+#define CMD_CFGI_CD_ALL 0x6
+#define CMD_TLBI_NH_ASID 0x11
+#define CMD_TLBI_EL2_ALL 0x20
+#define CMD_TLBI_NSNH_ALL 0x30
+#define CMD_SYNC 0x46
+
+#define IVPU_MMU_EVT_F_UUT 0x01
+#define IVPU_MMU_EVT_C_BAD_STREAMID 0x02
+#define IVPU_MMU_EVT_F_STE_FETCH 0x03
+#define IVPU_MMU_EVT_C_BAD_STE 0x04
+#define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05
+#define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06
+#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07
+#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08
+#define IVPU_MMU_EVT_F_CD_FETCH 0x09
+#define IVPU_MMU_EVT_C_BAD_CD 0x0a
+#define IVPU_MMU_EVT_F_WALK_EABT 0x0b
+#define IVPU_MMU_EVT_F_TRANSLATION 0x10
+#define IVPU_MMU_EVT_F_ADDR_SIZE 0x11
+#define IVPU_MMU_EVT_F_ACCESS 0x12
+#define IVPU_MMU_EVT_F_PERMISSION 0x13
+#define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20
+#define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21
+#define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24
+#define IVPU_MMU_EVT_F_VMS_FETCH 0x25
+
+#define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0)
+#define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12)
+
+#define IVPU_MMU_Q_BASE_RWA BIT(62)
+#define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
+#define IVPU_MMU_STRTAB_BASE_RA BIT(62)
+#define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
+
+#define IVPU_MMU_IRQ_EVTQ_EN BIT(2)
+#define IVPU_MMU_IRQ_GERROR_EN BIT(0)
+
+#define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10)
+#define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8)
+#define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6)
+#define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4)
+#define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2)
+#define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0)
+#define IVPU_MMU_CACHE_NC 0
+#define IVPU_MMU_CACHE_WB 1
+#define IVPU_MMU_CACHE_WT 2
+#define IVPU_MMU_SH_NSH 0
+#define IVPU_MMU_SH_OSH 2
+#define IVPU_MMU_SH_ISH 3
+
+#define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0)
+
+#define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
+#define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
+#define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
+#define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
+#define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
+#define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14)
+#define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30)
+#define IVPU_MMU_CD_0_ENDI BIT(15)
+#define IVPU_MMU_CD_0_V BIT(31)
+#define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32)
+#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)
+#define IVPU_MMU_CD_0_AA64 BIT(41)
+#define IVPU_MMU_CD_0_S BIT(44)
+#define IVPU_MMU_CD_0_R BIT(45)
+#define IVPU_MMU_CD_0_A BIT(46)
+#define IVPU_MMU_CD_0_ASET BIT(47)
+#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)
+
+#define IVPU_MMU_T0SZ_48BIT 16
+#define IVPU_MMU_T0SZ_38BIT 26
+
+#define IVPU_MMU_IPS_48BIT 5
+#define IVPU_MMU_IPS_44BIT 4
+#define IVPU_MMU_IPS_42BIT 3
+#define IVPU_MMU_IPS_40BIT 2
+#define IVPU_MMU_IPS_36BIT 1
+#define IVPU_MMU_IPS_32BIT 0
+
+#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
+
+#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)
+#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)
+#define IVPU_MMU_STE_0_S1FMT_LINEAR 0
+#define IVPU_MMU_STE_DWORDS 8
+#define IVPU_MMU_STE_0_CFG_S1_TRANS 5
+#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)
+#define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
+#define IVPU_MMU_STE_0_V BIT(0)
+
+#define IVPU_MMU_STE_1_STRW_NSEL1 0ul
+#define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13)
+#define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30)
+#define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48)
+#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul
+#define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50)
+#define IVPU_MMU_STE_1_INSTCFG_DATA 2ul
+#define IVPU_MMU_STE_1_MEV BIT(19)
+#define IVPU_MMU_STE_1_S1STALLD BIT(27)
+#define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul
+#define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul
+#define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul
+#define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul
+#define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2)
+#define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4)
+#define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6)
+#define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0)
+#define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0
+
+#define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC)
+#define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC)
+
+#define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \
+ (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \
+ (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \
+ (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \
+ (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \
+ (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \
+ (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT)))
+
+#define IVPU_MMU_CERROR_NONE 0x0
+#define IVPU_MMU_CERROR_ILL 0x1
+#define IVPU_MMU_CERROR_ABT 0x2
+#define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3
+
+static const char *ivpu_mmu_event_to_str(u32 cmd)
+{
+ switch (cmd) {
+ case IVPU_MMU_EVT_F_UUT:
+ return "Unsupported Upstream Transaction";
+ case IVPU_MMU_EVT_C_BAD_STREAMID:
+ return "Transaction StreamID out of range";
+ case IVPU_MMU_EVT_F_STE_FETCH:
+ return "Fetch of STE caused external abort";
+ case IVPU_MMU_EVT_C_BAD_STE:
+ return "Used STE invalid";
+ case IVPU_MMU_EVT_F_BAD_ATS_TREQ:
+ return "Address Request disallowed for a StreamID";
+ case IVPU_MMU_EVT_F_STREAM_DISABLED:
+ return "Transaction marks non-substream disabled";
+ case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN:
+ return "MMU bypass is disallowed for this StreamID";
+ case IVPU_MMU_EVT_C_BAD_SUBSTREAMID:
+ return "Invalid StreamID";
+ case IVPU_MMU_EVT_F_CD_FETCH:
+ return "Fetch of CD caused external abort";
+ case IVPU_MMU_EVT_C_BAD_CD:
+ return "Fetched CD invalid";
+ case IVPU_MMU_EVT_F_WALK_EABT:
+ return " An external abort occurred fetching a TLB";
+ case IVPU_MMU_EVT_F_TRANSLATION:
+ return "Translation fault";
+ case IVPU_MMU_EVT_F_ADDR_SIZE:
+ return " Output address caused address size fault";
+ case IVPU_MMU_EVT_F_ACCESS:
+ return "Access flag fault";
+ case IVPU_MMU_EVT_F_PERMISSION:
+ return "Permission fault occurred on page access";
+ case IVPU_MMU_EVT_F_TLB_CONFLICT:
+ return "A TLB conflict";
+ case IVPU_MMU_EVT_F_CFG_CONFLICT:
+ return "A configuration cache conflict";
+ case IVPU_MMU_EVT_E_PAGE_REQUEST:
+ return "Page request hint from a client device";
+ case IVPU_MMU_EVT_F_VMS_FETCH:
+ return "Fetch of VMS caused external abort";
+ default:
+ return "Unknown event";
+ }
+}
+
+static const char *ivpu_mmu_cmdq_err_to_str(u32 err)
+{
+ switch (err) {
+ case IVPU_MMU_CERROR_NONE:
+ return "No error";
+ case IVPU_MMU_CERROR_ILL:
+ return "Illegal command";
+ case IVPU_MMU_CERROR_ABT:
+ return "External abort on command queue read";
+ case IVPU_MMU_CERROR_ATC_INV_SYNC:
+ return "Sync failed to complete ATS invalidation";
+ default:
+ return "Unknown error";
+ }
+}
+
+static void ivpu_mmu_config_check(struct ivpu_device *vdev)
+{
+ u32 val_ref;
+ u32 val;
+
+ if (ivpu_is_simics(vdev))
+ val_ref = IVPU_MMU_IDR0_REF_SIMICS;
+ else
+ val_ref = IVPU_MMU_IDR0_REF;
+
+ val = REGV_RD32(IVPU_MMU_REG_IDR0);
+ if (val != val_ref)
+ ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);
+
+ val = REGV_RD32(IVPU_MMU_REG_IDR1);
+ if (val != IVPU_MMU_IDR1_REF)
+ ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);
+
+ val = REGV_RD32(IVPU_MMU_REG_IDR3);
+ if (val != IVPU_MMU_IDR3_REF)
+ ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);
+
+ if (ivpu_is_simics(vdev))
+ val_ref = IVPU_MMU_IDR5_REF_SIMICS;
+ else if (ivpu_is_fpga(vdev))
+ val_ref = IVPU_MMU_IDR5_REF_FPGA;
+ else
+ val_ref = IVPU_MMU_IDR5_REF;
+
+ val = REGV_RD32(IVPU_MMU_REG_IDR5);
+ if (val != val_ref)
+ ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);
+}
+
+static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
+ size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE;
+
+ cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);
+ if (!cdtab->base)
+ return -ENOMEM;
+
+ ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);
+
+ return 0;
+}
+
+static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_strtab *strtab = &mmu->strtab;
+ size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE;
+
+ strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);
+ if (!strtab->base)
+ return -ENOMEM;
+
+ strtab->base_cfg = IVPU_MMU_STRTAB_CFG;
+ strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA;
+ strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK;
+
+ ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",
+ &strtab->dma, &strtab->dma_q, size);
+
+ return 0;
+}
+
+static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_queue *q = &mmu->cmdq;
+
+ q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);
+ if (!q->base)
+ return -ENOMEM;
+
+ q->dma_q = IVPU_MMU_Q_BASE_RWA;
+ q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
+ q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
+
+ ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",
+ &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE);
+
+ return 0;
+}
+
+static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_queue *q = &mmu->evtq;
+
+ q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);
+ if (!q->base)
+ return -ENOMEM;
+
+ q->dma_q = IVPU_MMU_Q_BASE_RWA;
+ q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
+ q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
+
+ ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",
+ &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE);
+
+ return 0;
+}
+
+static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)
+{
+ int ret;
+
+ ret = ivpu_mmu_cdtab_alloc(vdev);
+ if (ret) {
+ ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);
+ return ret;
+ }
+
+ ret = ivpu_mmu_strtab_alloc(vdev);
+ if (ret) {
+ ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);
+ return ret;
+ }
+
+ ret = ivpu_mmu_cmdq_alloc(vdev);
+ if (ret) {
+ ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);
+ return ret;
+ }
+
+ ret = ivpu_mmu_evtq_alloc(vdev);
+ if (ret)
+ ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);
+
+ return ret;
+}
+
+static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val)
+{
+ REGV_WR32(IVPU_MMU_REG_CR0, val);
+
+ return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
+}
+
+static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val)
+{
+ REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val);
+
+ return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
+}
+
+static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
+{
+ u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN;
+ int ret;
+
+ ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0);
+ if (ret)
+ return ret;
+
+ return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl);
+}
+
+static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
+ int ret;
+
+ ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod,
+ IVPU_MMU_QUEUE_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ cmdq->cons = cmdq->prod;
+
+ return 0;
+}
+
+static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q)
+{
+ return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&
+ (IVPU_MMU_Q_WRP(q->prod) != IVPU_MMU_Q_WRP(q->cons)));
+}
+
+static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q)
+{
+ return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&
+ (IVPU_MMU_Q_WRP(q->prod) == IVPU_MMU_Q_WRP(q->cons)));
+}
+
+static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
+{
+ struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
+ u64 *queue_buffer = cmdq->base;
+ int idx = IVPU_MMU_Q_IDX(cmdq->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
+
+ if (ivpu_mmu_queue_is_full(cmdq)) {
+ ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
+ return -EBUSY;
+ }
+
+ queue_buffer[idx] = data0;
+ queue_buffer[idx + 1] = data1;
+ cmdq->prod = (cmdq->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
+
+ ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
+
+ return 0;
+}
+
+static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
+ u64 val;
+ int ret;
+
+ val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC);
+
+ ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
+ if (ret)
+ return ret;
+
+ if (!ivpu_is_force_snoop_enabled(vdev))
+ clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE);
+ REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod);
+
+ ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
+ if (ret) {
+ u32 err;
+
+ val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS);
+ err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val);
+
+ ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,
+ ivpu_mmu_cmdq_err_to_str(err));
+ ivpu_hw_diagnose_failure(vdev);
+ }
+
+ return ret;
+}
+
+static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)
+{
+ u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL);
+ u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f);
+
+ return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);
+}
+
+static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)
+{
+ u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) |
+ FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid);
+
+ return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);
+}
+
+static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)
+{
+ u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL);
+
+ return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);
+}
+
+static int ivpu_mmu_reset(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ u32 val;
+ int ret;
+
+ memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);
+ if (!ivpu_is_force_snoop_enabled(vdev))
+ clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);
+ mmu->cmdq.prod = 0;
+ mmu->cmdq.cons = 0;
+
+ memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);
+ mmu->evtq.prod = 0;
+ mmu->evtq.cons = 0;
+
+ ret = ivpu_mmu_reg_write_cr0(vdev, 0);
+ if (ret)
+ return ret;
+
+ val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) |
+ FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) |
+ FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) |
+ FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) |
+ FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) |
+ FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB);
+ REGV_WR32(IVPU_MMU_REG_CR1, val);
+
+ REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q);
+ REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg);
+
+ REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q);
+ REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0);
+ REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0);
+
+ val = REG_SET_FLD(IVPU_MMU_REG_CR0, CMDQEN, 0);
+ ret = ivpu_mmu_reg_write_cr0(vdev, val);
+ if (ret)
+ return ret;
+
+ ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
+ if (ret)
+ return ret;
+
+ ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
+ if (ret)
+ return ret;
+
+ ret = ivpu_mmu_cmdq_sync(vdev);
+ if (ret)
+ return ret;
+
+ REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q);
+ REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0);
+ REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0);
+
+ val = REG_SET_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
+ ret = ivpu_mmu_reg_write_cr0(vdev, val);
+ if (ret)
+ return ret;
+
+ val = REG_SET_FLD(IVPU_MMU_REG_CR0, ATSCHK, val);
+ ret = ivpu_mmu_reg_write_cr0(vdev, val);
+ if (ret)
+ return ret;
+
+ ret = ivpu_mmu_irqs_setup(vdev);
+ if (ret)
+ return ret;
+
+ val = REG_SET_FLD(IVPU_MMU_REG_CR0, SMMUEN, val);
+ return ivpu_mmu_reg_write_cr0(vdev, val);
+}
+
+static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_strtab *strtab = &mmu->strtab;
+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
+ u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE);
+ u64 str[2];
+
+ str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) |
+ FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) |
+ FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) |
+ IVPU_MMU_STE_0_V |
+ (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK);
+
+ str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) |
+ FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
+ FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
+ FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) |
+ FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) |
+ FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) |
+ FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) |
+ FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) |
+ IVPU_MMU_STE_1_MEV |
+ IVPU_MMU_STE_1_S1STALLD;
+
+ WRITE_ONCE(entry[1], str[1]);
+ WRITE_ONCE(entry[0], str[0]);
+
+ if (!ivpu_is_force_snoop_enabled(vdev))
+ clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE);
+
+ ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);
+}
+
+static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)
+{
+ ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);
+ ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
+
+ return 0;
+}
+
+int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ int ret = 0;
+
+ mutex_lock(&mmu->lock);
+ if (!mmu->on)
+ goto unlock;
+
+ ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
+ if (ret)
+ goto unlock;
+
+ ret = ivpu_mmu_cmdq_sync(vdev);
+unlock:
+ mutex_unlock(&mmu->lock);
+ return ret;
+}
+
+static int ivpu_mmu_cdtab_entry_set(struct ivpu_device *vdev, u32 ssid, u64 cd_dma, bool valid)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
+ u64 *entry;
+ u64 cd[4];
+ int ret = 0;
+
+ if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
+ return -EINVAL;
+
+ entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
+ drm_WARN_ON(&vdev->drm, (entry[0] & IVPU_MMU_CD_0_V) == valid);
+
+ cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) |
+ FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |
+ FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |
+ FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |
+ FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |
+ FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) |
+ FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |
+ IVPU_MMU_CD_0_TCR_EPD1 |
+ IVPU_MMU_CD_0_AA64 |
+ IVPU_MMU_CD_0_R |
+ IVPU_MMU_CD_0_ASET;
+ cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
+ cd[2] = 0;
+ cd[3] = 0x0000000000007444;
+
+ /* For global and reserved contexts generate memory fault on VPU */
+ if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID || ssid == IVPU_RESERVED_CONTEXT_MMU_SSID)
+ cd[0] |= IVPU_MMU_CD_0_A;
+
+ if (valid)
+ cd[0] |= IVPU_MMU_CD_0_V;
+
+ WRITE_ONCE(entry[1], cd[1]);
+ WRITE_ONCE(entry[2], cd[2]);
+ WRITE_ONCE(entry[3], cd[3]);
+ WRITE_ONCE(entry[0], cd[0]);
+
+ if (!ivpu_is_force_snoop_enabled(vdev))
+ clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
+
+ ivpu_dbg(vdev, MMU, "CDTAB set %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
+ valid ? "valid" : "invalid", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
+
+ mutex_lock(&mmu->lock);
+ if (!mmu->on)
+ goto unlock;
+
+ ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
+ if (ret)
+ goto err_invalidate;
+
+ ret = ivpu_mmu_cmdq_sync(vdev);
+ if (ret)
+ goto err_invalidate;
+unlock:
+ mutex_unlock(&mmu->lock);
+ return 0;
+
+err_invalidate:
+ WRITE_ONCE(entry[0], 0);
+ mutex_unlock(&mmu->lock);
+ return ret;
+}
+
+int ivpu_mmu_init(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ int ret;
+
+ ivpu_dbg(vdev, MMU, "Init..\n");
+
+ ivpu_mmu_config_check(vdev);
+
+ ret = drmm_mutex_init(&vdev->drm, &mmu->lock);
+ if (ret)
+ return ret;
+
+ ret = ivpu_mmu_structs_alloc(vdev);
+ if (ret)
+ return ret;
+
+ ret = ivpu_mmu_strtab_init(vdev);
+ if (ret) {
+ ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
+ return ret;
+ }
+
+ ret = ivpu_mmu_enable(vdev);
+ if (ret) {
+ ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
+ return ret;
+ }
+
+ ivpu_dbg(vdev, MMU, "Init done\n");
+
+ return 0;
+}
+
+int ivpu_mmu_enable(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ int ret;
+
+ mutex_lock(&mmu->lock);
+
+ mmu->on = true;
+
+ ret = ivpu_mmu_reset(vdev);
+ if (ret) {
+ ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);
+ goto err;
+ }
+
+ ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
+ if (ret)
+ goto err;
+
+ ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
+ if (ret)
+ goto err;
+
+ ret = ivpu_mmu_cmdq_sync(vdev);
+ if (ret)
+ goto err;
+
+ mutex_unlock(&mmu->lock);
+
+ return 0;
+err:
+ mmu->on = false;
+ mutex_unlock(&mmu->lock);
+ return ret;
+}
+
+void ivpu_mmu_disable(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+
+ mutex_lock(&mmu->lock);
+ mmu->on = false;
+ mutex_unlock(&mmu->lock);
+}
+
+static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)
+{
+ u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
+ u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]);
+ u64 fetch_addr = ((u64)event[7]) << 32 | event[6];
+ u64 in_addr = ((u64)event[5]) << 32 | event[4];
+ u32 sid = event[1];
+
+ ivpu_err_ratelimited(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n",
+ op, ivpu_mmu_event_to_str(op), ssid, sid,
+ event[2], event[3], in_addr, fetch_addr);
+}
+
+static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
+ u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
+ u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);
+
+ evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
+ if (ivpu_mmu_queue_is_empty(evtq))
+ return NULL;
+
+ evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
+ return evt;
+}
+
+static int ivpu_mmu_evtq_set(struct ivpu_device *vdev, bool enable)
+{
+ u32 val = REGV_RD32(IVPU_MMU_REG_CR0);
+
+ if (enable)
+ val = REG_SET_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
+ else
+ val = REG_CLR_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
+ REGV_WR32(IVPU_MMU_REG_CR0, val);
+
+ return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
+}
+
+static int ivpu_mmu_evtq_enable(struct ivpu_device *vdev)
+{
+ return ivpu_mmu_evtq_set(vdev, true);
+}
+
+static int ivpu_mmu_evtq_disable(struct ivpu_device *vdev)
+{
+ return ivpu_mmu_evtq_set(vdev, false);
+}
+
+void ivpu_mmu_discard_events(struct ivpu_device *vdev)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+
+ mutex_lock(&mmu->lock);
+ /*
+ * Disable event queue (stop MMU from updating the producer)
+ * to allow synchronization of consumer and producer indexes
+ */
+ ivpu_mmu_evtq_disable(vdev);
+
+ vdev->mmu->evtq.cons = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
+ REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons);
+ vdev->mmu->evtq.prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
+
+ ivpu_mmu_evtq_enable(vdev);
+
+ drm_WARN_ON_ONCE(&vdev->drm, vdev->mmu->evtq.cons != vdev->mmu->evtq.prod);
+
+ mutex_unlock(&mmu->lock);
+}
+
+int ivpu_mmu_disable_ssid_events(struct ivpu_device *vdev, u32 ssid)
+{
+ struct ivpu_mmu_info *mmu = vdev->mmu;
+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
+ u64 *entry;
+ u64 val;
+
+ if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
+ return -EINVAL;
+
+ mutex_lock(&mmu->lock);
+
+ entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
+
+ val = READ_ONCE(entry[0]);
+ val &= ~IVPU_MMU_CD_0_R;
+ WRITE_ONCE(entry[0], val);
+
+ if (!ivpu_is_force_snoop_enabled(vdev))
+ clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
+
+ ivpu_mmu_cmdq_write_cfgi_all(vdev);
+ ivpu_mmu_cmdq_sync(vdev);
+
+ mutex_unlock(&mmu->lock);
+
+ return 0;
+}
+
+void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
+{
+ struct ivpu_file_priv *file_priv;
+ u32 *event;
+ u32 ssid;
+
+ ivpu_dbg(vdev, IRQ, "MMU event queue\n");
+
+ while ((event = ivpu_mmu_get_event(vdev))) {
+ ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, *event);
+ if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID ||
+ ssid == IVPU_RESERVED_CONTEXT_MMU_SSID) {
+ ivpu_mmu_dump_event(vdev, event);
+ ivpu_pm_trigger_recovery(vdev, "MMU event");
+ return;
+ }
+
+ file_priv = xa_load(&vdev->context_xa, ssid);
+ if (file_priv) {
+ if (!READ_ONCE(file_priv->has_mmu_faults)) {
+ ivpu_mmu_dump_event(vdev, event);
+ WRITE_ONCE(file_priv->has_mmu_faults, true);
+ }
+ }
+ }
+
+ queue_work(system_percpu_wq, &vdev->context_abort_work);
+}
+
+void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)
+{
+ u32 *event;
+
+ while ((event = ivpu_mmu_get_event(vdev)) != NULL)
+ ivpu_mmu_dump_event(vdev, event);
+}
+
+void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)
+{
+ u32 gerror_val, gerrorn_val, active;
+
+ ivpu_dbg(vdev, IRQ, "MMU error\n");
+
+ gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR);
+ gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN);
+
+ active = gerror_val ^ gerrorn_val;
+ if (!(active & IVPU_MMU_GERROR_ERR_MASK))
+ return;
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active))
+ ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active))
+ ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active))
+ ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active))
+ ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active))
+ ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active))
+ ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");
+
+ if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active))
+ ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");
+
+ REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val);
+}
+
+int ivpu_mmu_cd_set(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)
+{
+ return ivpu_mmu_cdtab_entry_set(vdev, ssid, pgtable->pgd_dma, true);
+}
+
+void ivpu_mmu_cd_clear(struct ivpu_device *vdev, int ssid)
+{
+ ivpu_mmu_cdtab_entry_set(vdev, ssid, 0, false);
+}
generated by cgit (git 2.34.1) at 2025-12-25 17:48:08 +0000