diff options
Diffstat (limited to 'drivers/iommu/mtk_iommu.c')
| -rw-r--r-- | drivers/iommu/mtk_iommu.c | 435 |
1 files changed, 336 insertions, 99 deletions
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index e93906d6e112..60fcd3d3b5eb 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -3,6 +3,7 @@ * Copyright (c) 2015-2016 MediaTek Inc. * Author: Yong Wu <yong.wu@mediatek.com> */ +#include <linux/arm-smccc.h> #include <linux/bitfield.h> #include <linux/bug.h> #include <linux/clk.h> @@ -27,6 +28,8 @@ #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/soc/mediatek/infracfg.h> +#include <linux/soc/mediatek/mtk_sip_svc.h> +#include <linux/string_choices.h> #include <asm/barrier.h> #include <soc/mediatek/smi.h> @@ -136,6 +139,7 @@ /* 2 bits: iommu type */ #define MTK_IOMMU_TYPE_MM (0x0 << 13) #define MTK_IOMMU_TYPE_INFRA (0x1 << 13) +#define MTK_IOMMU_TYPE_APU (0x2 << 13) #define MTK_IOMMU_TYPE_MASK (0x3 << 13) /* PM and clock always on. e.g. infra iommu */ #define PM_CLK_AO BIT(15) @@ -143,6 +147,8 @@ #define PGTABLE_PA_35_EN BIT(17) #define TF_PORT_TO_ADDR_MT8173 BIT(18) #define INT_ID_PORT_WIDTH_6 BIT(19) +#define CFG_IFA_MASTER_IN_ATF BIT(20) +#define DL_WITH_MULTI_LARB BIT(21) #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ ((((pdata)->flags) & (mask)) == (_x)) @@ -167,6 +173,8 @@ enum mtk_iommu_plat { M4U_MT8173, M4U_MT8183, M4U_MT8186, + M4U_MT8188, + M4U_MT8189, M4U_MT8192, M4U_MT8195, M4U_MT8365, @@ -258,6 +266,8 @@ struct mtk_iommu_data { struct device *smicomm_dev; struct mtk_iommu_bank_data *bank; + struct mtk_iommu_domain *share_dom; + struct regmap *pericfg; struct mutex mutex; /* Protect m4u_group/m4u_dom above */ @@ -328,6 +338,8 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int ban */ #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL +static LIST_HEAD(apulist); /* List the apu iommu HWs */ +static LIST_HEAD(infralist); /* List the iommu_infra HW */ static LIST_HEAD(m4ulist); /* List all the M4U HWs */ #define for_each_m4u(data, head) list_for_each_entry(data, head, list) @@ -343,6 +355,15 @@ static const struct mtk_iommu_iova_region single_domain[] = { #define MT8192_MULTI_REGION_NR (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) ? \ MT8192_MULTI_REGION_NR_MAX : 1) +static const struct mtk_iommu_iova_region mt8189_multi_dom_apu[] = { + { .iova_base = 0x200000ULL, .size = SZ_512M}, /* APU SECURE */ +#if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) + { .iova_base = SZ_1G, .size = 0xc0000000}, /* APU CODE */ + { .iova_base = 0x70000000ULL, .size = 0x12600000}, /* APU VLM */ + { .iova_base = SZ_4G, .size = SZ_4G * 3}, /* APU VPU */ +#endif +}; + static const struct mtk_iommu_iova_region mt8192_multi_dom[MT8192_MULTI_REGION_NR] = { { .iova_base = 0x0, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 0 ~ 4G, */ #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) @@ -504,7 +525,7 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) bank->parent_dev, "fault type=0x%x iova=0x%llx pa=0x%llx master=0x%x(larb=%d port=%d) layer=%d %s\n", int_state, fault_iova, fault_pa, regval, fault_larb, fault_port, - layer, write ? "write" : "read"); + layer, str_write_read(write)); } /* Interrupt clear */ @@ -577,41 +598,55 @@ static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, unsigned int larbid, portid; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); const struct mtk_iommu_iova_region *region; - u32 peri_mmuen, peri_mmuen_msk; + unsigned long portid_msk = 0; + struct arm_smccc_res res; int i, ret = 0; for (i = 0; i < fwspec->num_ids; ++i) { - larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); portid = MTK_M4U_TO_PORT(fwspec->ids[i]); + portid_msk |= BIT(portid); + } - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { - larb_mmu = &data->larb_imu[larbid]; + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + /* All ports should be in the same larb. just use 0 here */ + larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); + larb_mmu = &data->larb_imu[larbid]; + region = data->plat_data->iova_region + regionid; - region = data->plat_data->iova_region + regionid; + for_each_set_bit(portid, &portid_msk, 32) larb_mmu->bank[portid] = upper_32_bits(region->iova_base); - dev_dbg(dev, "%s iommu for larb(%s) port %d region %d rgn-bank %d.\n", - enable ? "enable" : "disable", dev_name(larb_mmu->dev), - portid, regionid, larb_mmu->bank[portid]); + dev_dbg(dev, "%s iommu for larb(%s) port 0x%lx region %d rgn-bank %d.\n", + str_enable_disable(enable), dev_name(larb_mmu->dev), + portid_msk, regionid, upper_32_bits(region->iova_base)); - if (enable) - larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); - else - larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); - } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { - peri_mmuen_msk = BIT(portid); + if (enable) + larb_mmu->mmu |= portid_msk; + else + larb_mmu->mmu &= ~portid_msk; + } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { + if (MTK_IOMMU_HAS_FLAG(data->plat_data, CFG_IFA_MASTER_IN_ATF)) { + arm_smccc_smc(MTK_SIP_KERNEL_IOMMU_CONTROL, + IOMMU_ATF_CMD_CONFIG_INFRA_IOMMU, + portid_msk, enable, 0, 0, 0, 0, &res); + ret = res.a0; + } else { /* PCI dev has only one output id, enable the next writing bit for PCIe */ - if (dev_is_pci(dev)) - peri_mmuen_msk |= BIT(portid + 1); + if (dev_is_pci(dev)) { + if (fwspec->num_ids != 1) { + dev_err(dev, "PCI dev can only have one port.\n"); + return -ENODEV; + } + portid_msk |= BIT(portid + 1); + } - peri_mmuen = enable ? peri_mmuen_msk : 0; ret = regmap_update_bits(data->pericfg, PERICFG_IOMMU_1, - peri_mmuen_msk, peri_mmuen); - if (ret) - dev_err(dev, "%s iommu(%s) inframaster 0x%x fail(%d).\n", - enable ? "enable" : "disable", - dev_name(data->dev), peri_mmuen_msk, ret); + (u32)portid_msk, enable ? (u32)portid_msk : 0); } + if (ret) + dev_err(dev, "%s iommu(%s) inframaster 0x%lx fail(%d).\n", + str_enable_disable(enable), dev_name(data->dev), + portid_msk, ret); } return ret; } @@ -620,15 +655,14 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, struct mtk_iommu_data *data, unsigned int region_id) { + struct mtk_iommu_domain *share_dom = data->share_dom; const struct mtk_iommu_iova_region *region; - struct mtk_iommu_domain *m4u_dom; - - /* Always use bank0 in sharing pgtable case */ - m4u_dom = data->bank[0].m4u_dom; - if (m4u_dom) { - dom->iop = m4u_dom->iop; - dom->cfg = m4u_dom->cfg; - dom->domain.pgsize_bitmap = m4u_dom->cfg.pgsize_bitmap; + + /* Share pgtable when 2 MM IOMMU share the pgtable or one IOMMU use multiple iova ranges */ + if (share_dom) { + dom->iop = share_dom->iop; + dom->cfg = share_dom->cfg; + dom->domain.pgsize_bitmap = share_dom->domain.pgsize_bitmap; goto update_iova_region; } @@ -636,7 +670,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, .quirks = IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | IO_PGTABLE_QUIRK_ARM_MTK_EXT, - .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, + .pgsize_bitmap = dom->domain.pgsize_bitmap, .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, .iommu_dev = data->dev, }; @@ -655,8 +689,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, return -ENOMEM; } - /* Update our support page sizes bitmap */ - dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; + data->share_dom = dom; update_iova_region: /* Update the iova region for this domain */ @@ -667,17 +700,15 @@ update_iova_region: return 0; } -static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) +static struct iommu_domain *mtk_iommu_domain_alloc_paging(struct device *dev) { struct mtk_iommu_domain *dom; - if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED) - return NULL; - dom = kzalloc(sizeof(*dom), GFP_KERNEL); if (!dom) return NULL; mutex_init(&dom->mutex); + dom->domain.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M; return &dom->domain; } @@ -688,7 +719,7 @@ static void mtk_iommu_domain_free(struct iommu_domain *domain) } static int mtk_iommu_attach_device(struct iommu_domain *domain, - struct device *dev) + struct device *dev, struct iommu_domain *old) { struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; struct mtk_iommu_domain *dom = to_mtk_domain(domain); @@ -708,7 +739,9 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, /* Data is in the frstdata in sharing pgtable case. */ frstdata = mtk_iommu_get_frst_data(hw_list); + mutex_lock(&frstdata->mutex); ret = mtk_iommu_domain_finalise(dom, frstdata, region_id); + mutex_unlock(&frstdata->mutex); if (ret) { mutex_unlock(&dom->mutex); return ret; @@ -753,6 +786,28 @@ err_unlock: return ret; } +static int mtk_iommu_identity_attach(struct iommu_domain *identity_domain, + struct device *dev, + struct iommu_domain *old) +{ + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); + + if (old == identity_domain || !old) + return 0; + + mtk_iommu_config(data, dev, false, 0); + return 0; +} + +static struct iommu_domain_ops mtk_iommu_identity_ops = { + .attach_dev = mtk_iommu_identity_attach, +}; + +static struct iommu_domain mtk_iommu_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &mtk_iommu_identity_ops, +}; + static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) @@ -794,12 +849,13 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->bank); } -static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, - size_t size) +static int mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, + size_t size) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); mtk_iommu_tlb_flush_range_sync(iova, size, dom->bank); + return 0; } static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, @@ -820,45 +876,61 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, static struct iommu_device *mtk_iommu_probe_device(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct mtk_iommu_data *data; + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct device_link *link; struct device *larbdev; + unsigned long larbid_msk = 0; unsigned int larbid, larbidx, i; - if (!fwspec || fwspec->ops != &mtk_iommu_ops) - return ERR_PTR(-ENODEV); /* Not a iommu client device */ - - data = dev_iommu_priv_get(dev); - if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) return &data->iommu; /* * Link the consumer device with the smi-larb device(supplier). - * The device that connects with each a larb is a independent HW. - * All the ports in each a device should be in the same larbs. + * w/DL_WITH_MULTI_LARB: the master may connect with multi larbs, + * we should create device link with each larb. + * w/o DL_WITH_MULTI_LARB: the master must connect with one larb, + * otherwise fail. */ larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); if (larbid >= MTK_LARB_NR_MAX) return ERR_PTR(-EINVAL); + larbid_msk |= BIT(larbid); + for (i = 1; i < fwspec->num_ids; i++) { larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); - if (larbid != larbidx) { + if (MTK_IOMMU_HAS_FLAG(data->plat_data, DL_WITH_MULTI_LARB)) { + larbid_msk |= BIT(larbidx); + } else if (larbid != larbidx) { dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n", larbid, larbidx); return ERR_PTR(-EINVAL); } } - larbdev = data->larb_imu[larbid].dev; - if (!larbdev) - return ERR_PTR(-EINVAL); - link = device_link_add(dev, larbdev, - DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); - if (!link) - dev_err(dev, "Unable to link %s\n", dev_name(larbdev)); + for_each_set_bit(larbid, &larbid_msk, 32) { + larbdev = data->larb_imu[larbid].dev; + if (!larbdev) + return ERR_PTR(-EINVAL); + + link = device_link_add(dev, larbdev, + DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); + if (!link) { + dev_err(dev, "Unable to link %s\n", dev_name(larbdev)); + goto link_remove; + } + } + return &data->iommu; + +link_remove: + for_each_set_bit(i, &larbid_msk, larbid) { + larbdev = data->larb_imu[i].dev; + device_link_remove(dev, larbdev); + } + + return ERR_PTR(-ENODEV); } static void mtk_iommu_release_device(struct device *dev) @@ -866,11 +938,19 @@ static void mtk_iommu_release_device(struct device *dev) struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_data *data; struct device *larbdev; - unsigned int larbid; + unsigned int larbid, i; + unsigned long larbid_msk = 0; data = dev_iommu_priv_get(dev); - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { - larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); + if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) + return; + + for (i = 0; i < fwspec->num_ids; i++) { + larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); + larbid_msk |= BIT(larbid); + } + + for_each_set_bit(larbid, &larbid_msk, 32) { larbdev = data->larb_imu[larbid].dev; device_link_remove(dev, larbdev); } @@ -919,7 +999,8 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) return group; } -static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *m4updev; @@ -936,6 +1017,8 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) return -EINVAL; dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); + + put_device(&m4updev->dev); } return iommu_fwspec_add_ids(dev, args->args, 1); @@ -972,13 +1055,13 @@ static void mtk_iommu_get_resv_regions(struct device *dev, } static const struct iommu_ops mtk_iommu_ops = { - .domain_alloc = mtk_iommu_domain_alloc, + .identity_domain = &mtk_iommu_identity_domain, + .domain_alloc_paging = mtk_iommu_domain_alloc_paging, .probe_device = mtk_iommu_probe_device, .release_device = mtk_iommu_release_device, .device_group = mtk_iommu_device_group, .of_xlate = mtk_iommu_of_xlate, .get_resv_regions = mtk_iommu_get_resv_regions, - .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = mtk_iommu_attach_device, @@ -1173,16 +1256,19 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m } component_match_add(dev, match, component_compare_dev, &plarbdev->dev); - platform_device_put(plarbdev); } - if (!frst_avail_smicomm_node) - return -EINVAL; + if (!frst_avail_smicomm_node) { + ret = -EINVAL; + goto err_larbdev_put; + } pcommdev = of_find_device_by_node(frst_avail_smicomm_node); of_node_put(frst_avail_smicomm_node); - if (!pcommdev) - return -ENODEV; + if (!pcommdev) { + ret = -ENODEV; + goto err_larbdev_put; + } data->smicomm_dev = &pcommdev->dev; link = device_link_add(data->smicomm_dev, dev, @@ -1190,16 +1276,16 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m platform_device_put(pcommdev); if (!link) { dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); - return -EINVAL; + ret = -EINVAL; + goto err_larbdev_put; } return 0; err_larbdev_put: - for (i = MTK_LARB_NR_MAX - 1; i >= 0; i--) { - if (!data->larb_imu[i].dev) - continue; + /* id mapping may not be linear, loop the whole array */ + for (i = 0; i < MTK_LARB_NR_MAX; i++) put_device(data->larb_imu[i].dev); - } + return ret; } @@ -1225,7 +1311,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->plat_data = of_device_get_match_data(dev); /* Protect memory. HW will access here while translation fault.*/ - protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); + protect = devm_kcalloc(dev, 2, MTK_PROTECT_PA_ALIGN, GFP_KERNEL); if (!protect) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); @@ -1318,7 +1404,8 @@ static int mtk_iommu_probe(struct platform_device *pdev) dev_err_probe(dev, ret, "mm dts parse fail\n"); goto out_runtime_disable; } - } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { + } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && + !MTK_IOMMU_HAS_FLAG(data->plat_data, CFG_IFA_MASTER_IN_ATF)) { p = data->plat_data->pericfg_comp_str; data->pericfg = syscon_regmap_lookup_by_compatible(p); if (IS_ERR(data->pericfg)) { @@ -1330,15 +1417,6 @@ static int mtk_iommu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); mutex_init(&data->mutex); - ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, - "mtk-iommu.%pa", &ioaddr); - if (ret) - goto out_link_remove; - - ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); - if (ret) - goto out_sysfs_remove; - if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { list_add_tail(&data->list, data->plat_data->hw_list); data->hw_list = data->plat_data->hw_list; @@ -1348,21 +1426,34 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->hw_list = &data->hw_list_head; } + ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, + "mtk-iommu.%pa", &ioaddr); + if (ret) + goto out_list_del; + + ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); + if (ret) + goto out_sysfs_remove; + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); if (ret) - goto out_list_del; + goto out_device_unregister; } return ret; -out_list_del: - list_del(&data->list); +out_device_unregister: iommu_device_unregister(&data->iommu); out_sysfs_remove: iommu_device_sysfs_remove(&data->iommu); -out_link_remove: - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) +out_list_del: + list_del(&data->list); + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, dev); + + for (i = 0; i < MTK_LARB_NR_MAX; i++) + put_device(data->larb_imu[i].dev); + } out_runtime_disable: pm_runtime_disable(dev); return ret; @@ -1382,6 +1473,9 @@ static void mtk_iommu_remove(struct platform_device *pdev) if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, &pdev->dev); component_master_del(&pdev->dev, &mtk_iommu_com_ops); + + for (i = 0; i < MTK_LARB_NR_MAX; i++) + put_device(data->larb_imu[i].dev); } pm_runtime_disable(&pdev->dev); for (i = 0; i < data->plat_data->banks_num; i++) { @@ -1508,6 +1602,31 @@ static const struct mtk_iommu_plat_data mt6795_data = { .larbid_remap = {{0}, {1}, {2}, {3}, {4}}, /* Linear mapping. */ }; +static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { + [0] = {~0, ~0}, /* Region0: larb0/1 */ + [1] = {0, 0, 0, 0, ~0, ~0, 0, ~0}, /* Region1: larb4/5/7 */ + [2] = {0, 0, ~0, 0, 0, 0, 0, 0, /* Region2: larb2/9/11/13/14/16/17/18/19/20 */ + 0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), ~(u32)(BIT(4) | BIT(5)), 0, + ~0, ~0, ~0, ~0, ~0}, + [3] = {0}, + [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ + [5] = {[14] = BIT(4) | BIT(5)}, /* larb14 port4/5 */ +}; + +static const struct mtk_iommu_plat_data mt6893_data = { + .m4u_plat = M4U_MT8192, + .flags = HAS_BCLK | OUT_ORDER_WR_EN | HAS_SUB_COMM_2BITS | + WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8192_larb_region_msk, + .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, + {0, 14, 16}, {0, 13, 18, 17}}, +}; + static const struct mtk_iommu_plat_data mt8167_data = { .m4u_plat = M4U_MT8167, .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, @@ -1558,7 +1677,7 @@ static const unsigned int mt8186_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK static const struct mtk_iommu_plat_data mt8186_data_mm = { .m4u_plat = M4U_MT8186, .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | - WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, + WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM | PGTABLE_PA_35_EN, .larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20}, {MTK_INVALID_LARBID, 14, 16}, {MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}}, @@ -1570,15 +1689,125 @@ static const struct mtk_iommu_plat_data mt8186_data_mm = { .iova_region_larb_msk = mt8186_larb_region_msk, }; -static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { - [0] = {~0, ~0}, /* Region0: larb0/1 */ - [1] = {0, 0, 0, 0, ~0, ~0, 0, ~0}, /* Region1: larb4/5/7 */ - [2] = {0, 0, ~0, 0, 0, 0, 0, 0, /* Region2: larb2/9/11/13/14/16/17/18/19/20 */ - 0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), ~(u32)(BIT(4) | BIT(5)), 0, - ~0, ~0, ~0, ~0, ~0}, +static const struct mtk_iommu_plat_data mt8188_data_infra = { + .m4u_plat = M4U_MT8188, + .flags = WR_THROT_EN | DCM_DISABLE | STD_AXI_MODE | PM_CLK_AO | + MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT | + PGTABLE_PA_35_EN | CFG_IFA_MASTER_IN_ATF, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), +}; + +static const u32 mt8188_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { + [0] = {~0, ~0, ~0, ~0}, /* Region0: all ports for larb0/1/2/3 */ + [1] = {0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, ~0, ~0, ~0}, /* Region1: larb19(21)/21(22)/23 */ + [2] = {0, 0, 0, 0, ~0, ~0, ~0, ~0, /* Region2: the other larbs. */ + ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, + ~0, ~0, ~0, ~0, ~0, 0, 0, 0, + 0, ~0}, [3] = {0}, - [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ - [5] = {[14] = BIT(4) | BIT(5)}, /* larb14 port4/5 */ + [4] = {[24] = BIT(0) | BIT(1)}, /* Only larb27(24) port0/1 */ + [5] = {[24] = BIT(2) | BIT(3)}, /* Only larb27(24) port2/3 */ +}; + +static const struct mtk_iommu_plat_data mt8188_data_vdo = { + .m4u_plat = M4U_MT8188, + .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | + PGTABLE_PA_35_EN | MTK_IOMMU_TYPE_MM, + .hw_list = &m4ulist, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8188_larb_region_msk, + .larbid_remap = {{2}, {0}, {21}, {0}, {19}, {9, 10, + 11 /* 11a */, 25 /* 11c */}, + {13, 0, 29 /* 16b */, 30 /* 17b */, 0}, {5}}, +}; + +static const struct mtk_iommu_plat_data mt8188_data_vpp = { + .m4u_plat = M4U_MT8188, + .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | + PGTABLE_PA_35_EN | MTK_IOMMU_TYPE_MM, + .hw_list = &m4ulist, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8188_larb_region_msk, + .larbid_remap = {{1}, {3}, {23}, {7}, {MTK_INVALID_LARBID}, + {12, 15, 24 /* 11b */}, {14, MTK_INVALID_LARBID, + 16 /* 16a */, 17 /* 17a */, MTK_INVALID_LARBID, + 27, 28 /* ccu0 */, MTK_INVALID_LARBID}, {4, 6}}, +}; + +static const unsigned int mt8189_apu_region_msk[][MTK_LARB_NR_MAX] = { + [0] = {[0] = BIT(2)}, /* Region0: fake larb 0 APU_SECURE */ + [1] = {[0] = BIT(1)}, /* Region1: fake larb 0 APU_CODE */ + [2] = {[0] = BIT(3)}, /* Region2: fake larb 0 APU_VLM */ + [3] = {[0] = BIT(0)}, /* Region3: fake larb 0 APU_DATA */ +}; + +static const struct mtk_iommu_plat_data mt8189_data_apu = { + .m4u_plat = M4U_MT8189, + .flags = IOVA_34_EN | DCM_DISABLE | + MTK_IOMMU_TYPE_APU | PGTABLE_PA_35_EN, + .hw_list = &apulist, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = mt8189_multi_dom_apu, + .iova_region_nr = ARRAY_SIZE(mt8189_multi_dom_apu), + .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, + .iova_region_larb_msk = mt8189_apu_region_msk, +}; + +static const struct mtk_iommu_plat_data mt8189_data_infra = { + .m4u_plat = M4U_MT8189, + .flags = WR_THROT_EN | DCM_DISABLE | MTK_IOMMU_TYPE_INFRA | + CFG_IFA_MASTER_IN_ATF | SHARE_PGTABLE | PGTABLE_PA_35_EN, + .hw_list = &infralist, + .banks_num = 1, + .banks_enable = {true}, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), +}; + +static const u32 mt8189_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { + [0] = {~0, ~0, ~0, [22] = BIT(0)}, /* Region0: all ports for larb0/1/2 */ + [1] = {[3] = ~0, [4] = ~0}, /* Region1: all ports for larb4(3)/7(4) */ + [2] = {[5] = ~0, [6] = ~0, /* Region2: all ports for larb9(5)/11(6) */ + [7] = ~0, [8] = ~0, /* Region2: all ports for larb13(7)/14(8) */ + [9] = ~0, [10] = ~0, /* Region2: all ports for larb16(9)/17(10) */ + [11] = ~0, [12] = ~0, /* Region2: all ports for larb19(11)/20(12) */ + [21] = ~0}, /* Region2: larb21 fake GCE larb */ +}; + +static const struct mtk_iommu_plat_data mt8189_data_mm = { + .m4u_plat = M4U_MT8189, + .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM | + PGTABLE_PA_35_EN | DL_WITH_MULTI_LARB, + .hw_list = &m4ulist, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 5, + .banks_enable = {true, false, false, false, false}, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8189_larb_region_msk, + .larbid_remap = {{0}, {1}, {21/* GCE_D */, 21/* GCE_M */, 2}, + {19, 20, 9, 11}, {7}, {4}, + {13, 17}, {14, 16}}, }; static const struct mtk_iommu_plat_data mt8192_data = { @@ -1674,10 +1903,17 @@ static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, { .compatible = "mediatek,mt6795-m4u", .data = &mt6795_data}, + { .compatible = "mediatek,mt6893-iommu-mm", .data = &mt6893_data}, { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, { .compatible = "mediatek,mt8186-iommu-mm", .data = &mt8186_data_mm}, /* mm: m4u */ + { .compatible = "mediatek,mt8188-iommu-infra", .data = &mt8188_data_infra}, + { .compatible = "mediatek,mt8188-iommu-vdo", .data = &mt8188_data_vdo}, + { .compatible = "mediatek,mt8188-iommu-vpp", .data = &mt8188_data_vpp}, + { .compatible = "mediatek,mt8189-iommu-apu", .data = &mt8189_data_apu}, + { .compatible = "mediatek,mt8189-iommu-infra", .data = &mt8189_data_infra}, + { .compatible = "mediatek,mt8189-iommu-mm", .data = &mt8189_data_mm}, { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, { .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra}, { .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo}, @@ -1685,10 +1921,11 @@ static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt8365-m4u", .data = &mt8365_data}, {} }; +MODULE_DEVICE_TABLE(of, mtk_iommu_of_ids); static struct platform_driver mtk_iommu_driver = { .probe = mtk_iommu_probe, - .remove_new = mtk_iommu_remove, + .remove = mtk_iommu_remove, .driver = { .name = "mtk-iommu", .of_match_table = mtk_iommu_of_ids, |