From 810871c57011eb3e89e6768932757f169d666cd2 Mon Sep 17 00:00:00 2001 From: Nate Watterson Date: Tue, 20 Dec 2016 23:11:48 -0500 Subject: iommu/arm-smmu-v3: Clear prior settings when updating STEs To prevent corruption of the stage-1 context pointer field when updating STEs, rebuild the entire containing dword instead of clearing individual fields. Signed-off-by: Nate Watterson Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4d6ec444a9d6..94f305db8739 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1042,13 +1042,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, } } - /* Nuke the existing Config, as we're going to rewrite it */ - val &= ~(STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT); - - if (ste->valid) - val |= STRTAB_STE_0_V; - else - val &= ~STRTAB_STE_0_V; + /* Nuke the existing STE_0 value, as we're going to rewrite it */ + val = ste->valid ? STRTAB_STE_0_V : 0; if (ste->bypass) { val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT @@ -1083,7 +1078,6 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK << STRTAB_STE_0_S1CTXPTR_SHIFT) | STRTAB_STE_0_CFG_S1_TRANS; - } if (ste->s2_cfg) { -- cgit From 692c4e422dae023a5247c0c6fc3b7b89305ca974 Mon Sep 17 00:00:00 2001 From: Nate Watterson Date: Tue, 10 Jan 2017 14:47:13 -0500 Subject: iommu/arm-smmu-v3: limit use of 2-level stream tables In the current arm-smmu-v3 driver, all smmus that support 2-level stream tables are being forced to use them. This is suboptimal for smmus that support fewer stream id bits than would fill in a single second level table. This patch limits the use of 2-level tables to smmus that both support the feature and whose first level table can possibly contain more than a single entry. Signed-off-by: Nate Watterson Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 94f305db8739..947807ca9296 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1977,17 +1977,9 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) u32 size, l1size; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - /* - * If we can resolve everything with a single L2 table, then we - * just need a single L1 descriptor. Otherwise, calculate the L1 - * size, capped to the SIDSIZE. - */ - if (smmu->sid_bits < STRTAB_SPLIT) { - size = 0; - } else { - size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); - size = min(size, smmu->sid_bits - STRTAB_SPLIT); - } + /* Calculate the L1 size, capped to the SIDSIZE. */ + size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); + size = min(size, smmu->sid_bits - STRTAB_SPLIT); cfg->num_l1_ents = 1 << size; size += STRTAB_SPLIT; @@ -2498,6 +2490,13 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK; smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK; + /* + * If the SMMU supports fewer bits than would fill a single L2 stream + * table, use a linear table instead. + */ + if (smmu->sid_bits <= STRTAB_SPLIT) + smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; + /* IDR5 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); -- cgit From 65e251a4634c5644efca6f7e15803f0962d8943d Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Tue, 3 Jan 2017 17:34:56 +0000 Subject: iommu: Drop the of_iommu_{set/get}_ops() interface With the introduction of the new iommu_{register/get}_instance() interface in commit e4f10ffe4c9b ("iommu: Make of_iommu_set/get_ops() DT agnostic") (based on struct fwnode_handle as look-up token, so firmware agnostic) to register IOMMU instances with the core IOMMU layer there is no reason to keep the old OF based interface around any longer. Convert all the IOMMU drivers (and OF IOMMU core code) that rely on the of_iommu_{set/get}_ops() to the new kernel interface to register/retrieve IOMMU instances and remove the of_iommu_{set/get}_ops() remaining glue code in order to complete the interface rework. Cc: Matthias Brugger Cc: Will Deacon Cc: Joerg Roedel Cc: Marek Szyprowski Reviewed-by: Robin Murphy Tested-by: Sricharan R Tested-by: Yong Wu Signed-off-by: Lorenzo Pieralisi Signed-off-by: Will Deacon --- drivers/iommu/exynos-iommu.c | 2 +- drivers/iommu/msm_iommu.c | 2 +- drivers/iommu/mtk_iommu.c | 2 +- drivers/iommu/of_iommu.c | 4 ++-- include/linux/of_iommu.h | 11 ----------- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 57ba0d3091ea..b79e4c452b8b 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -628,7 +628,7 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) pm_runtime_enable(dev); - of_iommu_set_ops(dev->of_node, &exynos_iommu_ops); + iommu_register_instance(dev->fwnode, &exynos_iommu_ops); return 0; } diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index b09692bb5b0a..9cd3cee8db4d 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -737,7 +737,7 @@ static int msm_iommu_probe(struct platform_device *pdev) } list_add(&iommu->dev_node, &qcom_iommu_devices); - of_iommu_set_ops(pdev->dev.of_node, &msm_iommu_ops); + iommu_register_instance(pdev->dev.fwnode, &msm_iommu_ops); pr_info("device mapped at %p, irq %d with %d ctx banks\n", iommu->base, iommu->irq, iommu->ncb); diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 1479c76ece9e..0596ab224ff0 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -655,7 +655,7 @@ static int mtk_iommu_init_fn(struct device_node *np) return ret; } - of_iommu_set_ops(np, &mtk_iommu_ops); + iommu_register_instance(&np->fwnode, &mtk_iommu_ops); return 0; } diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 0f57ddc4ecc2..d7f480a52736 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -127,7 +127,7 @@ static const struct iommu_ops "iommu-map-mask", &iommu_spec.np, iommu_spec.args)) return NULL; - ops = of_iommu_get_ops(iommu_spec.np); + ops = iommu_get_instance(&iommu_spec.np->fwnode); if (!ops || !ops->of_xlate || iommu_fwspec_init(&pdev->dev, &iommu_spec.np->fwnode, ops) || ops->of_xlate(&pdev->dev, &iommu_spec)) @@ -157,7 +157,7 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, "#iommu-cells", idx, &iommu_spec)) { np = iommu_spec.np; - ops = of_iommu_get_ops(np); + ops = iommu_get_instance(&np->fwnode); if (!ops || !ops->of_xlate || iommu_fwspec_init(dev, &np->fwnode, ops) || diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h index 6a7fc5051099..13394ac83c66 100644 --- a/include/linux/of_iommu.h +++ b/include/linux/of_iommu.h @@ -31,17 +31,6 @@ static inline const struct iommu_ops *of_iommu_configure(struct device *dev, #endif /* CONFIG_OF_IOMMU */ -static inline void of_iommu_set_ops(struct device_node *np, - const struct iommu_ops *ops) -{ - iommu_register_instance(&np->fwnode, ops); -} - -static inline const struct iommu_ops *of_iommu_get_ops(struct device_node *np) -{ - return iommu_get_instance(&np->fwnode); -} - extern struct of_device_id __iommu_of_table; typedef int (*of_iommu_init_fn)(struct device_node *); -- cgit From dc0eaa4e19a7db1cb046b3979945536fdc914517 Mon Sep 17 00:00:00 2001 From: Aleksey Makarov Date: Thu, 19 Jan 2017 17:36:36 +0300 Subject: iommu/arm-smmu: Support for Extended Stream ID (16 bit) It is the time we have the real 16-bit Stream ID user, which is the ThunderX. Its IO topology uses 1:1 map for Requester ID to Stream ID translation for each root complex which allows to get full 16-bit Stream ID. Firmware assigns bus IDs that are greater than 128 (0x80) to some buses under PEM (external PCIe interface). Eventually SMMU drops devices on that buses because their Stream ID is out of range: pci 0006:90:00.0: stream ID 0x9000 out of range for SMMU (0x7fff) To fix above issue enable the Extended Stream ID optional feature when available. Reviewed-by: Tomasz Nowicki Signed-off-by: Aleksey Makarov Tested-by: Tomasz Nowicki Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 69 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index a60cded8a6ed..93b9177d4dfd 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -24,6 +24,7 @@ * - v7/v8 long-descriptor format * - Non-secure access to the SMMU * - Context fault reporting + * - Extended Stream ID (16 bit) */ #define pr_fmt(fmt) "arm-smmu: " fmt @@ -87,6 +88,7 @@ #define sCR0_CLIENTPD (1 << 0) #define sCR0_GFRE (1 << 1) #define sCR0_GFIE (1 << 2) +#define sCR0_EXIDENABLE (1 << 3) #define sCR0_GCFGFRE (1 << 4) #define sCR0_GCFGFIE (1 << 5) #define sCR0_USFCFG (1 << 10) @@ -126,6 +128,7 @@ #define ID0_NUMIRPT_MASK 0xff #define ID0_NUMSIDB_SHIFT 9 #define ID0_NUMSIDB_MASK 0xf +#define ID0_EXIDS (1 << 8) #define ID0_NUMSMRG_SHIFT 0 #define ID0_NUMSMRG_MASK 0xff @@ -169,6 +172,7 @@ #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) #define S2CR_CBNDX_SHIFT 0 #define S2CR_CBNDX_MASK 0xff +#define S2CR_EXIDVALID (1 << 10) #define S2CR_TYPE_SHIFT 16 #define S2CR_TYPE_MASK 0x3 enum arm_smmu_s2cr_type { @@ -351,6 +355,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) +#define ARM_SMMU_FEAT_EXIDS (1 << 12) u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) @@ -1048,7 +1053,7 @@ static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) struct arm_smmu_smr *smr = smmu->smrs + idx; u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT; - if (smr->valid) + if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) reg |= SMR_VALID; writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx)); } @@ -1060,6 +1065,9 @@ static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT | (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT; + if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && + smmu->smrs[idx].valid) + reg |= S2CR_EXIDVALID; writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx)); } @@ -1070,6 +1078,34 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) arm_smmu_write_smr(smmu, idx); } +/* + * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function + * should be called after sCR0 is written. + */ +static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) +{ + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + u32 smr; + + if (!smmu->smrs) + return; + + /* + * SMR.ID bits may not be preserved if the corresponding MASK + * bits are set, so check each one separately. We can reject + * masters later if they try to claim IDs outside these masks. + */ + smr = smmu->streamid_mask << SMR_ID_SHIFT; + writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + smmu->streamid_mask = smr >> SMR_ID_SHIFT; + + smr = smmu->streamid_mask << SMR_MASK_SHIFT; + writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; +} + static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) { struct arm_smmu_smr *smrs = smmu->smrs; @@ -1648,6 +1684,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_VMID16) reg |= sCR0_VMID16EN; + if (smmu->features & ARM_SMMU_FEAT_EXIDS) + reg |= sCR0_EXIDENABLE; + /* Push the button */ __arm_smmu_tlb_sync(smmu); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); @@ -1735,11 +1774,14 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) "\t(IDR0.CTTW overridden by FW configuration)\n"); /* Max. number of entries we have for stream matching/indexing */ - size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); + if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) { + smmu->features |= ARM_SMMU_FEAT_EXIDS; + size = 1 << 16; + } else { + size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); + } smmu->streamid_mask = size - 1; if (id & ID0_SMS) { - u32 smr; - smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK; if (size == 0) { @@ -1748,21 +1790,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return -ENODEV; } - /* - * SMR.ID bits may not be preserved if the corresponding MASK - * bits are set, so check each one separately. We can reject - * masters later if they try to claim IDs outside these masks. - */ - smr = smmu->streamid_mask << SMR_ID_SHIFT; - writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); - smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); - smmu->streamid_mask = smr >> SMR_ID_SHIFT; - - smr = smmu->streamid_mask << SMR_MASK_SHIFT; - writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); - smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); - smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; - /* Zero-initialised to mark as invalid */ smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), GFP_KERNEL); @@ -1770,8 +1797,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return -ENOMEM; dev_notice(smmu->dev, - "\tstream matching with %lu register groups, mask 0x%x", - size, smmu->smr_mask_mask); + "\tstream matching with %lu register groups", size); } /* s2cr->type == 0 means translation, so initialise explicitly */ smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), @@ -2094,6 +2120,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) iommu_register_instance(dev->fwnode, &arm_smmu_ops); platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); + arm_smmu_test_smr_masks(smmu); /* Oh, for a proper bus abstraction */ if (!iommu_present(&platform_bus_type)) -- cgit From 3677a649a751c8f061ba379a98c369473ccac980 Mon Sep 17 00:00:00 2001 From: Tomasz Nowicki Date: Mon, 16 Jan 2017 08:16:07 +0100 Subject: iommu/arm-smmu: Fix for ThunderX erratum #27704 The goal of erratum #27704 workaround was to make sure that ASIDs and VMIDs are unique across all SMMU instances on affected Cavium systems. Currently, the workaround code partitions ASIDs and VMIDs by increasing global cavium_smmu_context_count which in turn becomes the base ASID and VMID value for the given SMMU instance upon the context bank initialization. For systems with multiple SMMU instances this approach implies the risk of crossing 8-bit ASID, like for 1-socket CN88xx capable of 4 SMMUv2, 128 context banks each: SMMU_0 (0-127 ASID RANGE) SMMU_1 (127-255 ASID RANGE) SMMU_2 (256-383 ASID RANGE) <--- crossing 8-bit ASID SMMU_3 (384-511 ASID RANGE) <--- crossing 8-bit ASID Since now we use 8-bit ASID (SMMU_CBn_TCR2.AS = 0) we effectively misconfigure ASID[15:8] bits of SMMU_CBn_TTBRm register for SMMU_2/3. Moreover, we still assume non-zero ASID[15:8] bits upon context invalidation. In the end, except SMMU_0/1 devices all other devices under other SMMUs will fail on guest power off/on. Since we try to invalidate TLB with 16-bit ASID but we actually have 8-bit zero padded 16-bit entry. This patch adds 16-bit ASID support for stage-1 AArch64 contexts so that we use ASIDs consistently for all SMMU instances. Signed-off-by: Tomasz Nowicki Reviewed-by: Robin Murphy Reviewed-by: Tirumalesh Chalamarla Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 93b9177d4dfd..d01802e7cbfd 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -264,6 +264,7 @@ enum arm_smmu_s2cr_privcfg { #define TTBCR2_SEP_SHIFT 15 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT) +#define TTBCR2_AS (1 << 4) #define TTBRn_ASID_SHIFT 48 @@ -783,6 +784,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr; reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; reg2 |= TTBCR2_SEP_UPSTREAM; + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) + reg2 |= TTBCR2_AS; } if (smmu->version > ARM_SMMU_V1) writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2); -- cgit