From 0d3642883b092ccfc0b044c6581ee2c1f32ab165 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Tue, 5 Sep 2017 17:56:17 -0500 Subject: iommu/omap: Change the attach detection logic The OMAP IOMMU driver allows only a single device (eg: a rproc device) to be attached per domain. The current attach detection logic relies on a check for an attached iommu for the respective client device. Change this logic to use the client device pointer instead in preparation for supporting multiple iommu devices to be bound to a single iommu domain, and thereby to a client device. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index bd67e1b2c64e..81ef729994ce 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -805,7 +805,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data) struct iommu_domain *domain = obj->domain; struct omap_iommu_domain *omap_domain = to_omap_domain(domain); - if (!omap_domain->iommu_dev) + if (!omap_domain->dev) return IRQ_NONE; errs = iommu_report_fault(obj, &da); @@ -1118,8 +1118,8 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) spin_lock(&omap_domain->lock); - /* only a single device is supported per domain for now */ - if (omap_domain->iommu_dev) { + /* only a single client device can be attached to a domain */ + if (omap_domain->dev) { dev_err(dev, "iommu domain is already attached\n"); ret = -EBUSY; goto out; @@ -1148,9 +1148,14 @@ static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, { struct omap_iommu *oiommu = dev_to_omap_iommu(dev); + if (!omap_domain->dev) { + dev_err(dev, "domain has no attached device\n"); + return; + } + /* only a single device is supported per domain for now */ - if (omap_domain->iommu_dev != oiommu) { - dev_err(dev, "invalid iommu device\n"); + if (omap_domain->dev != dev) { + dev_err(dev, "invalid attached device\n"); return; } @@ -1219,7 +1224,7 @@ static void omap_iommu_domain_free(struct iommu_domain *domain) * An iommu device is still attached * (currently, only one device can be attached) ? */ - if (omap_domain->iommu_dev) + if (omap_domain->dev) _omap_iommu_detach_dev(omap_domain, omap_domain->dev); kfree(omap_domain->pgtable); -- cgit From 9d5018deec86673ef8418546a3ac43e47dbff3b9 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Tue, 5 Sep 2017 17:56:18 -0500 Subject: iommu/omap: Add support to program multiple iommus A client user instantiates and attaches to an iommu_domain to program the OMAP IOMMU associated with the domain. The iommus programmed by a client user are bound with the iommu_domain through the user's device archdata. The OMAP IOMMU driver currently supports only one IOMMU per IOMMU domain per user. The OMAP IOMMU driver has been enhanced to support allowing multiple IOMMUs to be programmed by a single client user. This support is being added mainly to handle the DSP subsystems on the DRA7xx SoCs, which have two MMUs within the same subsystem. These MMUs provide translations for a processor core port and an internal EDMA port. This support allows both the MMUs to be programmed together, but with each one retaining it's own internal state objects. The internal EDMA block is managed by the software running on the DSPs, and this design provides on-par functionality with previous generation OMAP DSPs where the EDMA and the DSP core shared the same MMU. The multiple iommus are expected to be provided through a sentinel terminated array of omap_iommu_arch_data objects through the client user's device archdata. The OMAP driver core is enhanced to loop through the array of attached iommus and program them for all common operations. The sentinel-terminated logic is used so as to not change the omap_iommu_arch_data structure. NOTE: 1. The IOMMU group and IOMMU core registration is done only for the DSP processor core MMU even though both MMUs are represented by their own platform device and are probed individually. The IOMMU device linking uses this registered MMU device. The struct iommu_device for the second MMU is not used even though memory for it is allocated. 2. The OMAP IOMMU debugfs code still continues to operate on individual IOMMU objects. Signed-off-by: Suman Anna [t-kristo@ti.com: ported support to 4.13 based kernel] Signed-off-by: Tero Kristo Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 358 ++++++++++++++++++++++++++++++++++----------- drivers/iommu/omap-iommu.h | 30 ++-- 2 files changed, 285 insertions(+), 103 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 81ef729994ce..e135ab830ebf 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -2,6 +2,7 @@ * omap iommu: tlb and pagetable primitives * * Copyright (C) 2008-2010 Nokia Corporation + * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/ * * Written by Hiroshi DOYU , * Paul Mundt and Toshihiro Kobayashi @@ -71,13 +72,23 @@ static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom) **/ void omap_iommu_save_ctx(struct device *dev) { - struct omap_iommu *obj = dev_to_omap_iommu(dev); - u32 *p = obj->ctx; + struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu *obj; + u32 *p; int i; - for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { - p[i] = iommu_read_reg(obj, i * sizeof(u32)); - dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]); + if (!arch_data) + return; + + while (arch_data->iommu_dev) { + obj = arch_data->iommu_dev; + p = obj->ctx; + for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { + p[i] = iommu_read_reg(obj, i * sizeof(u32)); + dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, + p[i]); + } + arch_data++; } } EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); @@ -88,13 +99,23 @@ EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); **/ void omap_iommu_restore_ctx(struct device *dev) { - struct omap_iommu *obj = dev_to_omap_iommu(dev); - u32 *p = obj->ctx; + struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu *obj; + u32 *p; int i; - for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { - iommu_write_reg(obj, p[i], i * sizeof(u32)); - dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]); + if (!arch_data) + return; + + while (arch_data->iommu_dev) { + obj = arch_data->iommu_dev; + p = obj->ctx; + for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { + iommu_write_reg(obj, p[i], i * sizeof(u32)); + dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, + p[i]); + } + arch_data++; } } EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); @@ -893,6 +914,24 @@ static void omap_iommu_detach(struct omap_iommu *obj) dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); } +static bool omap_iommu_can_register(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) + return true; + + /* + * restrict IOMMU core registration only for processor-port MDMA MMUs + * on DRA7 DSPs + */ + if ((!strcmp(dev_name(&pdev->dev), "40d01000.mmu")) || + (!strcmp(dev_name(&pdev->dev), "41501000.mmu"))) + return true; + + return false; +} + static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev, struct omap_iommu *obj) { @@ -984,19 +1023,22 @@ static int omap_iommu_probe(struct platform_device *pdev) return err; platform_set_drvdata(pdev, obj); - obj->group = iommu_group_alloc(); - if (IS_ERR(obj->group)) - return PTR_ERR(obj->group); + if (omap_iommu_can_register(pdev)) { + obj->group = iommu_group_alloc(); + if (IS_ERR(obj->group)) + return PTR_ERR(obj->group); - err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL, obj->name); - if (err) - goto out_group; + err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL, + obj->name); + if (err) + goto out_group; - iommu_device_set_ops(&obj->iommu, &omap_iommu_ops); + iommu_device_set_ops(&obj->iommu, &omap_iommu_ops); - err = iommu_device_register(&obj->iommu); - if (err) - goto out_sysfs; + err = iommu_device_register(&obj->iommu); + if (err) + goto out_sysfs; + } pm_runtime_irq_safe(obj->dev); pm_runtime_enable(obj->dev); @@ -1018,11 +1060,13 @@ static int omap_iommu_remove(struct platform_device *pdev) { struct omap_iommu *obj = platform_get_drvdata(pdev); - iommu_group_put(obj->group); - obj->group = NULL; + if (obj->group) { + iommu_group_put(obj->group); + obj->group = NULL; - iommu_device_sysfs_remove(&obj->iommu); - iommu_device_unregister(&obj->iommu); + iommu_device_sysfs_remove(&obj->iommu); + iommu_device_unregister(&obj->iommu); + } omap_iommu_debugfs_remove(obj); @@ -1068,11 +1112,13 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, phys_addr_t pa, size_t bytes, int prot) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); - struct omap_iommu *oiommu = omap_domain->iommu_dev; - struct device *dev = oiommu->dev; + struct device *dev = omap_domain->dev; + struct omap_iommu_device *iommu; + struct omap_iommu *oiommu; struct iotlb_entry e; int omap_pgsz; - u32 ret; + u32 ret = -EINVAL; + int i; omap_pgsz = bytes_to_iopgsz(bytes); if (omap_pgsz < 0) { @@ -1084,9 +1130,24 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, iotlb_init_entry(&e, da, pa, omap_pgsz); - ret = omap_iopgtable_store_entry(oiommu, &e); - if (ret) - dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret); + iommu = omap_domain->iommus; + for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { + oiommu = iommu->iommu_dev; + ret = omap_iopgtable_store_entry(oiommu, &e); + if (ret) { + dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", + ret); + break; + } + } + + if (ret) { + while (i--) { + iommu--; + oiommu = iommu->iommu_dev; + iopgtable_clear_entry(oiommu, da); + } + } return ret; } @@ -1095,12 +1156,90 @@ static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, size_t size) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); - struct omap_iommu *oiommu = omap_domain->iommu_dev; - struct device *dev = oiommu->dev; + struct device *dev = omap_domain->dev; + struct omap_iommu_device *iommu; + struct omap_iommu *oiommu; + bool error = false; + size_t bytes = 0; + int i; dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size); - return iopgtable_clear_entry(oiommu, da); + iommu = omap_domain->iommus; + for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { + oiommu = iommu->iommu_dev; + bytes = iopgtable_clear_entry(oiommu, da); + if (!bytes) + error = true; + } + + /* + * simplify return - we are only checking if any of the iommus + * reported an error, but not if all of them are unmapping the + * same number of entries. This should not occur due to the + * mirror programming. + */ + return error ? 0 : bytes; +} + +static int omap_iommu_count(struct device *dev) +{ + struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + int count = 0; + + while (arch_data->iommu_dev) { + count++; + arch_data++; + } + + return count; +} + +/* caller should call cleanup if this function fails */ +static int omap_iommu_attach_init(struct device *dev, + struct omap_iommu_domain *odomain) +{ + struct omap_iommu_device *iommu; + int i; + + odomain->num_iommus = omap_iommu_count(dev); + if (!odomain->num_iommus) + return -EINVAL; + + odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu), + GFP_ATOMIC); + if (!odomain->iommus) + return -ENOMEM; + + iommu = odomain->iommus; + for (i = 0; i < odomain->num_iommus; i++, iommu++) { + iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC); + if (!iommu->pgtable) + return -ENOMEM; + + /* + * should never fail, but please keep this around to ensure + * we keep the hardware happy + */ + if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable, + IOPGD_TABLE_SIZE))) + return -EINVAL; + } + + return 0; +} + +static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain) +{ + int i; + struct omap_iommu_device *iommu = odomain->iommus; + + for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++) + kfree(iommu->pgtable); + + kfree(odomain->iommus); + odomain->num_iommus = 0; + odomain->iommus = NULL; } static int @@ -1108,8 +1247,10 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_device *iommu; struct omap_iommu *oiommu; int ret = 0; + int i; if (!arch_data || !arch_data->iommu_dev) { dev_err(dev, "device doesn't have an associated iommu\n"); @@ -1125,19 +1266,42 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) goto out; } - oiommu = arch_data->iommu_dev; - - /* get a handle to and enable the omap iommu */ - ret = omap_iommu_attach(oiommu, omap_domain->pgtable); + ret = omap_iommu_attach_init(dev, omap_domain); if (ret) { - dev_err(dev, "can't get omap iommu: %d\n", ret); - goto out; + dev_err(dev, "failed to allocate required iommu data %d\n", + ret); + goto init_fail; + } + + iommu = omap_domain->iommus; + for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) { + /* configure and enable the omap iommu */ + oiommu = arch_data->iommu_dev; + ret = omap_iommu_attach(oiommu, iommu->pgtable); + if (ret) { + dev_err(dev, "can't get omap iommu: %d\n", ret); + goto attach_fail; + } + + oiommu->domain = domain; + iommu->iommu_dev = oiommu; } - omap_domain->iommu_dev = oiommu; omap_domain->dev = dev; - oiommu->domain = domain; + goto out; + +attach_fail: + while (i--) { + iommu--; + arch_data--; + oiommu = iommu->iommu_dev; + omap_iommu_detach(oiommu); + iommu->iommu_dev = NULL; + oiommu->domain = NULL; + } +init_fail: + omap_iommu_detach_fini(omap_domain); out: spin_unlock(&omap_domain->lock); return ret; @@ -1146,7 +1310,10 @@ out: static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, struct device *dev) { - struct omap_iommu *oiommu = dev_to_omap_iommu(dev); + struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_device *iommu = omap_domain->iommus; + struct omap_iommu *oiommu; + int i; if (!omap_domain->dev) { dev_err(dev, "domain has no attached device\n"); @@ -1159,13 +1326,24 @@ static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, return; } - iopgtable_clear_entry_all(oiommu); + /* + * cleanup in the reverse order of attachment - this addresses + * any h/w dependencies between multiple instances, if any + */ + iommu += (omap_domain->num_iommus - 1); + arch_data += (omap_domain->num_iommus - 1); + for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) { + oiommu = iommu->iommu_dev; + iopgtable_clear_entry_all(oiommu); + + omap_iommu_detach(oiommu); + iommu->iommu_dev = NULL; + oiommu->domain = NULL; + } - omap_iommu_detach(oiommu); + omap_iommu_detach_fini(omap_domain); - omap_domain->iommu_dev = NULL; omap_domain->dev = NULL; - oiommu->domain = NULL; } static void omap_iommu_detach_dev(struct iommu_domain *domain, @@ -1187,18 +1365,7 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); if (!omap_domain) - goto out; - - omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); - if (!omap_domain->pgtable) - goto fail_nomem; - - /* - * should never fail, but please keep this around to ensure - * we keep the hardware happy - */ - if (WARN_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE))) - goto fail_align; + return NULL; spin_lock_init(&omap_domain->lock); @@ -1207,13 +1374,6 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) omap_domain->domain.geometry.force_aperture = true; return &omap_domain->domain; - -fail_align: - kfree(omap_domain->pgtable); -fail_nomem: - kfree(omap_domain); -out: - return NULL; } static void omap_iommu_domain_free(struct iommu_domain *domain) @@ -1227,7 +1387,6 @@ static void omap_iommu_domain_free(struct iommu_domain *domain) if (omap_domain->dev) _omap_iommu_detach_dev(omap_domain, omap_domain->dev); - kfree(omap_domain->pgtable); kfree(omap_domain); } @@ -1235,11 +1394,16 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t da) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); - struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct omap_iommu_device *iommu = omap_domain->iommus; + struct omap_iommu *oiommu = iommu->iommu_dev; struct device *dev = oiommu->dev; u32 *pgd, *pte; phys_addr_t ret = 0; + /* + * all the iommus within the domain will have identical programming, + * so perform the lookup using just the first iommu + */ iopgtable_lookup_entry(oiommu, da, &pgd, &pte); if (pte) { @@ -1265,11 +1429,12 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, static int omap_iommu_add_device(struct device *dev) { - struct omap_iommu_arch_data *arch_data; + struct omap_iommu_arch_data *arch_data, *tmp; struct omap_iommu *oiommu; struct iommu_group *group; struct device_node *np; struct platform_device *pdev; + int num_iommus, i; int ret; /* @@ -1281,36 +1446,57 @@ static int omap_iommu_add_device(struct device *dev) if (!dev->of_node) return 0; - np = of_parse_phandle(dev->of_node, "iommus", 0); - if (!np) + /* + * retrieve the count of IOMMU nodes using phandle size as element size + * since #iommu-cells = 0 for OMAP + */ + num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus", + sizeof(phandle)); + if (num_iommus < 0) return 0; - pdev = of_find_device_by_node(np); - if (WARN_ON(!pdev)) { - of_node_put(np); - return -EINVAL; - } + arch_data = kzalloc((num_iommus + 1) * sizeof(*arch_data), GFP_KERNEL); + if (!arch_data) + return -ENOMEM; - oiommu = platform_get_drvdata(pdev); - if (!oiommu) { - of_node_put(np); - return -EINVAL; - } + for (i = 0, tmp = arch_data; i < num_iommus; i++, tmp++) { + np = of_parse_phandle(dev->of_node, "iommus", i); + if (!np) { + kfree(arch_data); + return -EINVAL; + } + + pdev = of_find_device_by_node(np); + if (WARN_ON(!pdev)) { + of_node_put(np); + kfree(arch_data); + return -EINVAL; + } + + oiommu = platform_get_drvdata(pdev); + if (!oiommu) { + of_node_put(np); + kfree(arch_data); + return -EINVAL; + } + + tmp->iommu_dev = oiommu; - arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL); - if (!arch_data) { of_node_put(np); - return -ENOMEM; } + /* + * use the first IOMMU alone for the sysfs device linking. + * TODO: Evaluate if a single iommu_group needs to be + * maintained for both IOMMUs + */ + oiommu = arch_data->iommu_dev; ret = iommu_device_link(&oiommu->iommu, dev); if (ret) { kfree(arch_data); - of_node_put(np); return ret; } - arch_data->iommu_dev = oiommu; dev->archdata.iommu = arch_data; /* @@ -1326,8 +1512,6 @@ static int omap_iommu_add_device(struct device *dev) } iommu_group_put(group); - of_node_put(np); - return 0; } diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h index a675af29a6ec..1703159ef5af 100644 --- a/drivers/iommu/omap-iommu.h +++ b/drivers/iommu/omap-iommu.h @@ -28,18 +28,27 @@ struct iotlb_entry { u32 endian, elsz, mixed; }; +/** + * struct omap_iommu_device - omap iommu device data + * @pgtable: page table used by an omap iommu attached to a domain + * @iommu_dev: pointer to store an omap iommu instance attached to a domain + */ +struct omap_iommu_device { + u32 *pgtable; + struct omap_iommu *iommu_dev; +}; + /** * struct omap_iommu_domain - omap iommu domain - * @pgtable: the page table - * @iommu_dev: an omap iommu device attached to this domain. only a single - * iommu device can be attached for now. + * @num_iommus: number of iommus in this domain + * @iommus: omap iommu device data for all iommus in this domain * @dev: Device using this domain. * @lock: domain lock, should be taken when attaching/detaching * @domain: generic domain handle used by iommu core code */ struct omap_iommu_domain { - u32 *pgtable; - struct omap_iommu *iommu_dev; + u32 num_iommus; + struct omap_iommu_device *iommus; struct device *dev; spinlock_t lock; struct iommu_domain domain; @@ -97,17 +106,6 @@ struct iotlb_lock { short vict; }; -/** - * dev_to_omap_iommu() - retrieves an omap iommu object from a user device - * @dev: iommu client device - */ -static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev) -{ - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; - - return arch_data->iommu_dev; -} - /* * MMU Register offsets */ -- cgit From a4aaeccc7a91e24cc17f72a7eb2f586f5c3d811d Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Sun, 10 Sep 2017 13:43:37 -0700 Subject: iommu: Add missing dependencies parisc:allmodconfig, xtensa:allmodconfig, and possibly others generate the following Kconfig warning. warning: (IPMMU_VMSA && ARM_SMMU && ARM_SMMU_V3 && QCOM_IOMMU) selects IOMMU_IO_PGTABLE_LPAE which has unmet direct dependencies (IOMMU_SUPPORT && HAS_DMA && (ARM || ARM64 || COMPILE_TEST && !GENERIC_ATOMIC64)) IOMMU_IO_PGTABLE_LPAE depends on (COMPILE_TEST && !GENERIC_ATOMIC64), so any configuration option selecting it needs to have the same dependencies. Signed-off-by: Guenter Roeck Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 49bd2ab8c507..8530ef78925d 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -278,7 +278,7 @@ config EXYNOS_IOMMU_DEBUG config IPMMU_VMSA bool "Renesas VMSA-compatible IPMMU" depends on ARM || IOMMU_DMA - depends on ARCH_RENESAS || COMPILE_TEST + depends on ARCH_RENESAS || (COMPILE_TEST && !GENERIC_ATOMIC64) select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU @@ -373,7 +373,7 @@ config MTK_IOMMU_V1 config QCOM_IOMMU # Note: iommu drivers cannot (yet?) be built as modules bool "Qualcomm IOMMU Support" - depends on ARCH_QCOM || COMPILE_TEST + depends on ARCH_QCOM || (COMPILE_TEST && !GENERIC_ATOMIC64) select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU -- cgit From 3bd71e18c5803c23014df962bdd74af1b34697fe Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 12 Sep 2017 22:10:21 +0200 Subject: iommu/vt-d: Fix harmless section mismatch warning Building with gcc-4.6 results in this warning due to dmar_table_print_dmar_entry being inlined as in newer compiler versions: WARNING: vmlinux.o(.text+0x5c8bee): Section mismatch in reference from the function dmar_walk_remapping_entries() to the function .init.text:dmar_table_print_dmar_entry() The function dmar_walk_remapping_entries() references the function __init dmar_table_print_dmar_entry(). This is often because dmar_walk_remapping_entries lacks a __init annotation or the annotation of dmar_table_print_dmar_entry is wrong. This removes the __init annotation to avoid the warning. On compilers that don't show the warning today, this should have no impact since the function gets inlined anyway. Signed-off-by: Arnd Bergmann Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index ca5ebaeafd6a..57c920c1372d 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -497,7 +497,7 @@ static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg) #define dmar_parse_one_rhsa dmar_res_noop #endif -static void __init +static void dmar_table_print_dmar_entry(struct acpi_dmar_header *header) { struct acpi_dmar_hardware_unit *drhd; -- cgit From 7a974b29fe5d3704eafec707ba6390c3288c80fe Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Fri, 15 Sep 2017 13:05:08 +0200 Subject: iommu/exynos: Rework runtime PM links management add_device is a bit more suitable for establishing runtime PM links than the xlate callback. This change also makes it possible to implement proper cleanup - in remove_device callback. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index f596fcc32898..91c548d49b92 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -263,6 +263,7 @@ struct exynos_iommu_domain { struct sysmmu_drvdata { struct device *sysmmu; /* SYSMMU controller device */ struct device *master; /* master device (owner) */ + struct device_link *link; /* runtime PM link to master */ void __iomem *sfrbase; /* our registers */ struct clk *clk; /* SYSMMU's clock */ struct clk *aclk; /* SYSMMU's aclk clock */ @@ -1250,6 +1251,8 @@ static struct iommu_group *get_device_iommu_group(struct device *dev) static int exynos_iommu_add_device(struct device *dev) { + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct sysmmu_drvdata *data; struct iommu_group *group; if (!has_sysmmu(dev)) @@ -1260,6 +1263,15 @@ static int exynos_iommu_add_device(struct device *dev) if (IS_ERR(group)) return PTR_ERR(group); + list_for_each_entry(data, &owner->controllers, owner_node) { + /* + * SYSMMU will be runtime activated via device link + * (dependency) to its master device, so there are no + * direct calls to pm_runtime_get/put in this driver. + */ + data->link = device_link_add(dev, data->sysmmu, + DL_FLAG_PM_RUNTIME); + } iommu_group_put(group); return 0; @@ -1268,6 +1280,7 @@ static int exynos_iommu_add_device(struct device *dev) static void exynos_iommu_remove_device(struct device *dev) { struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct sysmmu_drvdata *data; if (!has_sysmmu(dev)) return; @@ -1283,6 +1296,9 @@ static void exynos_iommu_remove_device(struct device *dev) } } iommu_group_remove_device(dev); + + list_for_each_entry(data, &owner->controllers, owner_node) + device_link_del(data->link); } static int exynos_iommu_of_xlate(struct device *dev, @@ -1316,13 +1332,6 @@ static int exynos_iommu_of_xlate(struct device *dev, list_add_tail(&data->owner_node, &owner->controllers); data->master = dev; - /* - * SYSMMU will be runtime activated via device link (dependency) to its - * master device, so there are no direct calls to pm_runtime_get/put - * in this driver. - */ - device_link_add(dev, data->sysmmu, DL_FLAG_PM_RUNTIME); - return 0; } -- cgit From 986a5f70173626eea9863fee6d6e029f0f2bc361 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 11 Sep 2017 14:34:34 +0200 Subject: iommu/qcom: Depend on HAS_DMA to fix compile error If NO_DMA=y: warning: (IPMMU_VMSA && ARM_SMMU && ARM_SMMU_V3 && QCOM_IOMMU) selects IOMMU_IO_PGTABLE_LPAE which has unmet direct dependencies (IOMMU_SUPPORT && HAS_DMA && (ARM || ARM64 || COMPILE_TEST && !GENERIC_ATOMIC64)) and drivers/iommu/io-pgtable-arm.o: In function `__arm_lpae_sync_pte': io-pgtable-arm.c:(.text+0x206): undefined reference to `bad_dma_ops' drivers/iommu/io-pgtable-arm.o: In function `__arm_lpae_free_pages': io-pgtable-arm.c:(.text+0x6a6): undefined reference to `bad_dma_ops' drivers/iommu/io-pgtable-arm.o: In function `__arm_lpae_alloc_pages': io-pgtable-arm.c:(.text+0x812): undefined reference to `bad_dma_ops' io-pgtable-arm.c:(.text+0x81c): undefined reference to `bad_dma_ops' io-pgtable-arm.c:(.text+0x862): undefined reference to `bad_dma_ops' drivers/iommu/io-pgtable-arm.o: In function `arm_lpae_run_tests': io-pgtable-arm.c:(.init.text+0x86): undefined reference to `alloc_io_pgtable_ops' io-pgtable-arm.c:(.init.text+0x47c): undefined reference to `free_io_pgtable_ops' drivers/iommu/qcom_iommu.o: In function `qcom_iommu_init_domain': qcom_iommu.c:(.text+0x1ce): undefined reference to `alloc_io_pgtable_ops' drivers/iommu/qcom_iommu.o: In function `qcom_iommu_domain_free': qcom_iommu.c:(.text+0x754): undefined reference to `free_io_pgtable_ops' QCOM_IOMMU selects IOMMU_IO_PGTABLE_LPAE, which bypasses its dependency on HAS_DMA. Make QCOM_IOMMU depend on HAS_DMA to fix this. Fixes: 0ae349a0f33fb040 ("iommu/qcom: Add qcom_iommu") Signed-off-by: Geert Uytterhoeven Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 8530ef78925d..f3a21343e636 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -374,6 +374,7 @@ config QCOM_IOMMU # Note: iommu drivers cannot (yet?) be built as modules bool "Qualcomm IOMMU Support" depends on ARCH_QCOM || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on HAS_DMA select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU -- cgit From c0d05cde2a685cd6486390c62be684ce456d84d6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Sep 2017 11:20:58 +0100 Subject: iommu/of: Remove PCI host bridge node check of_pci_iommu_init() tries to be clever and stop its alias walk at the device represented by master_np, in case of weird PCI topologies where the bridge to the IOMMU and the rest of the system is not at the root. It turns out this is a bit short-sighted, since there are plenty of other callers of pci_for_each_dma_alias() which would also need the same behaviour in that situation, and the only platform so far with such a topology (Cavium ThunderX2) already solves it more generally via a PCI quirk. As this check is effectively redundant, and returning a boolean value as an int is a bit broken anyway, let's just get rid of it. Reported-by: Jean-Philippe Brucker Fixes: d87beb749281 ("iommu/of: Handle PCI aliases properly") Signed-off-by: Robin Murphy Tested-by: Jean-Philippe Brucker Signed-off-by: Joerg Roedel --- drivers/iommu/of_iommu.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index e60e3dba85a0..50947ebb6d17 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -157,10 +157,7 @@ static int of_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data) err = of_iommu_xlate(info->dev, &iommu_spec); of_node_put(iommu_spec.np); - if (err) - return err; - - return info->np == pdev->bus->dev.of_node; + return err; } const struct iommu_ops *of_iommu_configure(struct device *dev, -- cgit From 5c62c1c67903621cfa715d6f690548ee53301620 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 25 Sep 2017 17:28:47 +0800 Subject: iommu/io-pgtable-arm-v7s: Need dma-sync while there is no QUIRK_NO_DMA Fix the commit 81b3c2521844 ("iommu/io-pgtable: Introduce explicit coherency"). If there is no IO_PGTABLE_QUIRK_NO_DMA, we should call dma_sync_single_for_device for cache synchronization. Signed-off-by: Yong Wu Fixes: 81b3c2521844 ('iommu/io-pgtable: Introduce explicit coherency') Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index d665d0dc16e8..6961fc393f0b 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -245,7 +245,7 @@ static void __arm_v7s_free_table(void *table, int lvl, static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, struct io_pgtable_cfg *cfg) { - if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) + if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) return; dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), -- cgit From 1ff9b17cedb39bc78f9e3f82485765f9b467177d Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 25 Sep 2017 18:15:26 +0800 Subject: iommu/mediatek: Limit the physical address in 32bit for v7s The ARM short descriptor has already limited the physical address to 32bit after the commit <76557391433c> ("iommu/io-pgtable: Sanitise map/unmap addresses"). But in MediaTek 4GB mode, the physical address is from 0x1_0000_0000 to 0x1_ffff_ffff. this will cause: WARNING: CPU: 4 PID: 3900 at xxx/drivers/iommu/io-pgtable-arm-v7s.c:482 arm_v7s_map+0x40/0xf8 Modules linked in: CPU: 4 PID: 3900 Comm: weston Tainted: G S W 4.9.44 #1 Hardware name: MediaTek MT2712m1v1 board (DT) task: ffffffc0eaa5b280 task.stack: ffffffc0e9858000 PC is at arm_v7s_map+0x40/0xf8 LR is at mtk_iommu_map+0x64/0x90 pc : [] lr : [] pstate: 000001c5 sp : ffffffc0e985b920 x29: ffffffc0e985b920 x28: 0000000127d00000 x27: 0000000000100000 x26: ffffff8008f9e000 x25: 0000000000000003 x24: 0000000000100000 x23: 0000000127d00000 x22: 00000000ff800000 x21: ffffffc0f7ec8ce0 x20: 0000000000000003 x19: 0000000000000003 x18: 0000000000000002 x17: 0000007f7e5d72c0 x16: ffffff80082b0f08 x15: 0000000000000001 x14: 000000000000003f x13: 0000000000000000 x12: 0000000000000028 x11: 0088000000000000 x10: 0000000000000000 x9 : ffffff80092fa000 x8 : ffffffc0e9858000 x7 : ffffff80085b29d8 x6 : 0000000000000000 x5 : ffffff80085b09a8 x4 : 0000000000000003 x3 : 0000000000100000 x2 : 0000000127d00000 x1 : 00000000ff800000 x0 : 0000000000000001 ... Call trace: [] arm_v7s_map+0x40/0xf8 [] mtk_iommu_map+0x64/0x90 [] iommu_map+0x100/0x3a0 [] default_iommu_map_sg+0x104/0x168 [] iommu_dma_alloc+0x238/0x3f8 [] __iommu_alloc_attrs+0xa8/0x260 [] mtk_drm_gem_create+0xac/0x180 [] mtk_drm_gem_dumb_create+0x54/0xc8 [] drm_mode_create_dumb_ioctl+0xa4/0xd8 [] drm_ioctl+0x1c0/0x490 In order to satify this, Limit the physical address to 32bit. Signed-off-by: Yong Wu Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index bd515be5b380..16d33ac19db0 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -371,7 +371,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, int ret; spin_lock_irqsave(&dom->pgtlock, flags); - ret = dom->iop->map(dom->iop, iova, paddr, size, prot); + ret = dom->iop->map(dom->iop, iova, paddr & DMA_BIT_MASK(32), + size, prot); spin_unlock_irqrestore(&dom->pgtlock, flags); return ret; -- cgit From 3c6bae62136ba5b24f0b113e68121b783457ca4b Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Tue, 26 Sep 2017 13:07:46 +0530 Subject: iommu/amd: pr_err() strings should end with newlines pr_err() messages should end with a new-line to avoid other messages being concatenated. So replace '/n' with '\n'. Signed-off-by: Arvind Yadav Fixes: 45a01c42933b ('iommu/amd: Add function copy_dev_tables()') Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 382de42b8359..6fe2d0346073 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -874,7 +874,7 @@ static bool copy_device_table(void) hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); entry = (((u64) hi) << 32) + lo; if (last_entry && last_entry != entry) { - pr_err("IOMMU:%d should use the same dev table as others!/n", + pr_err("IOMMU:%d should use the same dev table as others!\n", iommu->index); return false; } @@ -882,7 +882,7 @@ static bool copy_device_table(void) old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; if (old_devtb_size != dev_table_size) { - pr_err("The device table size of IOMMU:%d is not expected!/n", + pr_err("The device table size of IOMMU:%d is not expected!\n", iommu->index); return false; } @@ -890,7 +890,7 @@ static bool copy_device_table(void) old_devtb_phys = entry & PAGE_MASK; if (old_devtb_phys >= 0x100000000ULL) { - pr_err("The address of old device table is above 4G, not trustworthy!/n"); + pr_err("The address of old device table is above 4G, not trustworthy!\n"); return false; } old_devtb = memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB); @@ -901,7 +901,7 @@ static bool copy_device_table(void) old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, get_order(dev_table_size)); if (old_dev_tbl_cpy == NULL) { - pr_err("Failed to allocate memory for copying old device table!/n"); + pr_err("Failed to allocate memory for copying old device table!\n"); return false; } -- cgit From 2070f940a6d5148cf2df0d0087ff0a64d9f15237 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Thu, 21 Sep 2017 16:52:42 +0100 Subject: iommu/iova: Optimise rbtree searching Checking the IOVA bounds separately before deciding which direction to continue the search (if necessary) results in redundantly comparing both pfns twice each. GCC can already determine that the final comparison op is redundant and optimise it down to 3 in total, but we can go one further with a little tweak of the ordering (which makes the intent of the code that much cleaner as a bonus). Signed-off-by: Zhen Lei Tested-by: Ard Biesheuvel Tested-by: Zhen Lei Tested-by: Nate Watterson [rm: rewrote commit message to clarify] Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 33edfa794ae9..f129ff4f5c89 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -342,15 +342,12 @@ private_find_iova(struct iova_domain *iovad, unsigned long pfn) while (node) { struct iova *iova = rb_entry(node, struct iova, node); - /* If pfn falls within iova's range, return iova */ - if ((pfn >= iova->pfn_lo) && (pfn <= iova->pfn_hi)) { - return iova; - } - if (pfn < iova->pfn_lo) node = node->rb_left; - else if (pfn > iova->pfn_lo) + else if (pfn > iova->pfn_hi) node = node->rb_right; + else + return iova; /* pfn falls within iova's range */ } return NULL; -- cgit From 086c83acb70fc6da044c9ca45c1c9780c64545b0 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Thu, 21 Sep 2017 16:52:43 +0100 Subject: iommu/iova: Optimise the padding calculation The mask for calculating the padding size doesn't change, so there's no need to recalculate it every loop iteration. Furthermore, Once we've done that, it becomes clear that we don't actually need to calculate a padding size at all - by flipping the arithmetic around, we can just combine the upper limit, size, and mask directly to check against the lower limit. For an arm64 build, this alone knocks 20% off the object code size of the entire alloc_iova() function! Signed-off-by: Zhen Lei Tested-by: Ard Biesheuvel Tested-by: Zhen Lei Tested-by: Nate Watterson [rm: simplified more of the arithmetic, rewrote commit message] Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index f129ff4f5c89..20be9a8b3188 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -182,24 +182,17 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova, rb_insert_color(&iova->node, root); } -/* - * Computes the padding size required, to make the start address - * naturally aligned on the power-of-two order of its size - */ -static unsigned int -iova_get_pad_size(unsigned int size, unsigned int limit_pfn) -{ - return (limit_pfn - size) & (__roundup_pow_of_two(size) - 1); -} - static int __alloc_and_insert_iova_range(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, struct iova *new, bool size_aligned) { struct rb_node *prev, *curr = NULL; unsigned long flags; - unsigned long saved_pfn; - unsigned int pad_size = 0; + unsigned long saved_pfn, new_pfn; + unsigned long align_mask = ~0UL; + + if (size_aligned) + align_mask <<= fls_long(size - 1); /* Walk the tree backwards */ spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); @@ -209,31 +202,26 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, while (curr) { struct iova *curr_iova = rb_entry(curr, struct iova, node); - if (limit_pfn <= curr_iova->pfn_lo) { + if (limit_pfn <= curr_iova->pfn_lo) goto move_left; - } else if (limit_pfn > curr_iova->pfn_hi) { - if (size_aligned) - pad_size = iova_get_pad_size(size, limit_pfn); - if ((curr_iova->pfn_hi + size + pad_size) < limit_pfn) - break; /* found a free slot */ - } + + if (((limit_pfn - size) & align_mask) > curr_iova->pfn_hi) + break; /* found a free slot */ + limit_pfn = curr_iova->pfn_lo; move_left: prev = curr; curr = rb_prev(curr); } - if (!curr) { - if (size_aligned) - pad_size = iova_get_pad_size(size, limit_pfn); - if ((iovad->start_pfn + size + pad_size) > limit_pfn) { - spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); - return -ENOMEM; - } + new_pfn = (limit_pfn - size) & align_mask; + if (limit_pfn < size || new_pfn < iovad->start_pfn) { + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + return -ENOMEM; } /* pfn_lo will point to size aligned address if size_aligned is set */ - new->pfn_lo = limit_pfn - (size + pad_size); + new->pfn_lo = new_pfn; new->pfn_hi = new->pfn_lo + size - 1; /* If we have 'prev', it's a valid place to start the insertion. */ -- cgit From e60aa7b53845a261dd419652f12ab9f89e668843 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Sep 2017 16:52:44 +0100 Subject: iommu/iova: Extend rbtree node caching The cached node mechanism provides a significant performance benefit for allocations using a 32-bit DMA mask, but in the case of non-PCI devices or where the 32-bit space is full, the loss of this benefit can be significant - on large systems there can be many thousands of entries in the tree, such that walking all the way down to find free space every time becomes increasingly awful. Maintain a similar cached node for the whole IOVA space as a superset of the 32-bit space so that performance can remain much more consistent. Inspired by work by Zhen Lei . Tested-by: Ard Biesheuvel Tested-by: Zhen Lei Tested-by: Nate Watterson Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 60 ++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 20be9a8b3188..c6f5a22f8d20 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -48,6 +48,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, spin_lock_init(&iovad->iova_rbtree_lock); iovad->rbroot = RB_ROOT; + iovad->cached_node = NULL; iovad->cached32_node = NULL; iovad->granule = granule; iovad->start_pfn = start_pfn; @@ -110,48 +111,44 @@ EXPORT_SYMBOL_GPL(init_iova_flush_queue); static struct rb_node * __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn) { - if ((*limit_pfn > iovad->dma_32bit_pfn) || - (iovad->cached32_node == NULL)) + struct rb_node *cached_node = NULL; + struct iova *curr_iova; + + if (*limit_pfn <= iovad->dma_32bit_pfn) + cached_node = iovad->cached32_node; + if (!cached_node) + cached_node = iovad->cached_node; + if (!cached_node) return rb_last(&iovad->rbroot); - else { - struct rb_node *prev_node = rb_prev(iovad->cached32_node); - struct iova *curr_iova = - rb_entry(iovad->cached32_node, struct iova, node); - *limit_pfn = curr_iova->pfn_lo; - return prev_node; - } + + curr_iova = rb_entry(cached_node, struct iova, node); + *limit_pfn = min(*limit_pfn, curr_iova->pfn_lo); + + return rb_prev(cached_node); } static void -__cached_rbnode_insert_update(struct iova_domain *iovad, - unsigned long limit_pfn, struct iova *new) +__cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) { - if (limit_pfn != iovad->dma_32bit_pfn) - return; - iovad->cached32_node = &new->node; + if (new->pfn_hi < iovad->dma_32bit_pfn) + iovad->cached32_node = &new->node; + else + iovad->cached_node = &new->node; } static void __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) { struct iova *cached_iova; - struct rb_node *curr; - if (!iovad->cached32_node) - return; - curr = iovad->cached32_node; - cached_iova = rb_entry(curr, struct iova, node); + cached_iova = rb_entry(iovad->cached32_node, struct iova, node); + if (free->pfn_hi < iovad->dma_32bit_pfn && + iovad->cached32_node && free->pfn_lo >= cached_iova->pfn_lo) + iovad->cached32_node = rb_next(&free->node); - if (free->pfn_lo >= cached_iova->pfn_lo) { - struct rb_node *node = rb_next(&free->node); - struct iova *iova = rb_entry(node, struct iova, node); - - /* only cache if it's below 32bit pfn */ - if (node && iova->pfn_lo < iovad->dma_32bit_pfn) - iovad->cached32_node = node; - else - iovad->cached32_node = NULL; - } + cached_iova = rb_entry(iovad->cached_node, struct iova, node); + if (iovad->cached_node && free->pfn_lo >= cached_iova->pfn_lo) + iovad->cached_node = rb_next(&free->node); } /* Insert the iova into domain rbtree by holding writer lock */ @@ -188,7 +185,7 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, { struct rb_node *prev, *curr = NULL; unsigned long flags; - unsigned long saved_pfn, new_pfn; + unsigned long new_pfn; unsigned long align_mask = ~0UL; if (size_aligned) @@ -196,7 +193,6 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, /* Walk the tree backwards */ spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); - saved_pfn = limit_pfn; curr = __get_cached_rbnode(iovad, &limit_pfn); prev = curr; while (curr) { @@ -226,7 +222,7 @@ move_left: /* If we have 'prev', it's a valid place to start the insertion. */ iova_insert_rbtree(&iovad->rbroot, new, prev); - __cached_rbnode_insert_update(iovad, saved_pfn, new); + __cached_rbnode_insert_update(iovad, new); spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); -- cgit From aa3ac9469c1850ed00741955b975c3a19029763a Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Thu, 21 Sep 2017 16:52:45 +0100 Subject: iommu/iova: Make dma_32bit_pfn implicit Now that the cached node optimisation can apply to all allocations, the couple of users which were playing tricks with dma_32bit_pfn in order to benefit from it can stop doing so. Conversely, there is also no need for all the other users to explicitly calculate a 'real' 32-bit PFN, when init_iova_domain() can happily do that itself from the page granularity. CC: Thierry Reding CC: Jonathan Hunter CC: David Airlie CC: Sudeep Dutt CC: Ashutosh Dixit Signed-off-by: Zhen Lei Tested-by: Ard Biesheuvel Tested-by: Zhen Lei Tested-by: Nate Watterson [rm: use iova_shift(), rewrote commit message] Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 7 ++----- drivers/iommu/dma-iommu.c | 18 +----------------- drivers/iommu/intel-iommu.c | 11 +++-------- drivers/iommu/iova.c | 4 ++-- 4 files changed, 8 insertions(+), 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 51f8215877f5..647ab7691aee 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -63,7 +63,6 @@ /* IO virtual address start page frame number */ #define IOVA_START_PFN (1) #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) -#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) /* Reserved IOVA ranges */ #define MSI_RANGE_START (0xfee00000) @@ -1788,8 +1787,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (!dma_dom->domain.pt_root) goto free_dma_dom; - init_iova_domain(&dma_dom->iovad, PAGE_SIZE, - IOVA_START_PFN, DMA_32BIT_PFN); + init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_START_PFN); if (init_iova_flush_queue(&dma_dom->iovad, iova_domain_flush_tlb, NULL)) goto free_dma_dom; @@ -2696,8 +2694,7 @@ static int init_reserved_iova_ranges(void) struct pci_dev *pdev = NULL; struct iova *val; - init_iova_domain(&reserved_iova_ranges, PAGE_SIZE, - IOVA_START_PFN, DMA_32BIT_PFN); + init_iova_domain(&reserved_iova_ranges, PAGE_SIZE, IOVA_START_PFN); lockdep_set_class(&reserved_iova_ranges.iova_rbtree_lock, &reserved_rbtree_key); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 9d1cebe7f6cb..191be9c80a8a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -292,18 +292,7 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, /* ...then finally give it a kicking to make sure it fits */ base_pfn = max_t(unsigned long, base_pfn, domain->geometry.aperture_start >> order); - end_pfn = min_t(unsigned long, end_pfn, - domain->geometry.aperture_end >> order); } - /* - * PCI devices may have larger DMA masks, but still prefer allocating - * within a 32-bit mask to avoid DAC addressing. Such limitations don't - * apply to the typical platform device, so for those we may as well - * leave the cache limit at the top of their range to save an rb_last() - * traversal on every allocation. - */ - if (dev && dev_is_pci(dev)) - end_pfn &= DMA_BIT_MASK(32) >> order; /* start_pfn is always nonzero for an already-initialised domain */ if (iovad->start_pfn) { @@ -312,16 +301,11 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, pr_warn("Incompatible range for DMA domain\n"); return -EFAULT; } - /* - * If we have devices with different DMA masks, move the free - * area cache limit down for the benefit of the smaller one. - */ - iovad->dma_32bit_pfn = min(end_pfn + 1, iovad->dma_32bit_pfn); return 0; } - init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn); + init_iova_domain(iovad, 1UL << order, base_pfn); if (!dev) return 0; diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 6784a05dd6b2..ebb48353dd39 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -82,8 +82,6 @@ #define IOVA_START_PFN (1) #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) -#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) -#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64)) /* page table handling */ #define LEVEL_STRIDE (9) @@ -1878,8 +1876,7 @@ static int dmar_init_reserved_ranges(void) struct iova *iova; int i; - init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN, - DMA_32BIT_PFN); + init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN); lockdep_set_class(&reserved_iova_list.iova_rbtree_lock, &reserved_rbtree_key); @@ -1938,8 +1935,7 @@ static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu, unsigned long sagaw; int err; - init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN, - DMA_32BIT_PFN); + init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN); err = init_iova_flush_queue(&domain->iovad, iommu_flush_iova, iova_entry_free); @@ -4897,8 +4893,7 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width) { int adjust_width; - init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN, - DMA_32BIT_PFN); + init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN); domain_reserve_special_ranges(domain); /* calculate AGAW */ diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index c6f5a22f8d20..65032e60a5d1 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -37,7 +37,7 @@ static void fq_flush_timeout(unsigned long data); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, - unsigned long start_pfn, unsigned long pfn_32bit) + unsigned long start_pfn) { /* * IOVA granularity will normally be equal to the smallest @@ -52,7 +52,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->cached32_node = NULL; iovad->granule = granule; iovad->start_pfn = start_pfn; - iovad->dma_32bit_pfn = pfn_32bit + 1; + iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); iovad->flush_cb = NULL; iovad->fq = NULL; init_iova_rcaches(iovad); -- cgit From bb68b2fbfbd643d4407541f9c7a16a2c9b3a57c7 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Sep 2017 16:52:46 +0100 Subject: iommu/iova: Add rbtree anchor node Add a permanent dummy IOVA reservation to the rbtree, such that we can always access the top of the address space instantly. The immediate benefit is that we remove the overhead of the rb_last() traversal when not using the cached node, but it also paves the way for further simplifications. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 65032e60a5d1..9e04c1f3e740 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -24,6 +24,9 @@ #include #include +/* The anchor node sits above the top of the usable address space */ +#define IOVA_ANCHOR ~0UL + static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, unsigned long size); @@ -55,6 +58,9 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); iovad->flush_cb = NULL; iovad->fq = NULL; + iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; + rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); + rb_insert_color(&iovad->anchor.node, &iovad->rbroot); init_iova_rcaches(iovad); } EXPORT_SYMBOL_GPL(init_iova_domain); @@ -119,7 +125,7 @@ __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn) if (!cached_node) cached_node = iovad->cached_node; if (!cached_node) - return rb_last(&iovad->rbroot); + return rb_prev(&iovad->anchor.node); curr_iova = rb_entry(cached_node, struct iova, node); *limit_pfn = min(*limit_pfn, curr_iova->pfn_lo); @@ -242,7 +248,8 @@ EXPORT_SYMBOL(alloc_iova_mem); void free_iova_mem(struct iova *iova) { - kmem_cache_free(iova_cache, iova); + if (iova->pfn_lo != IOVA_ANCHOR) + kmem_cache_free(iova_cache, iova); } EXPORT_SYMBOL(free_iova_mem); @@ -676,6 +683,10 @@ reserve_iova(struct iova_domain *iovad, struct iova *iova; unsigned int overlap = 0; + /* Don't allow nonsensical pfns */ + if (WARN_ON((pfn_hi | pfn_lo) > (ULLONG_MAX >> iova_shift(iovad)))) + return NULL; + spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) { if (__is_range_overlap(node, pfn_lo, pfn_hi)) { -- cgit From 973f5fbedb0721ab964386a5fe5120998e71580c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Sep 2017 16:52:47 +0100 Subject: iommu/iova: Simplify cached node logic The logic of __get_cached_rbnode() is a little obtuse, but then __get_prev_node_of_cached_rbnode_or_last_node_and_update_limit_pfn() wouldn't exactly roll off the tongue... Now that we have the invariant that there is always a valid node to start searching downwards from, everything gets a bit easier to follow if we simplify that function to do what it says on the tin and return the cached node (or anchor node as appropriate) directly. In turn, we can then deduplicate the rb_prev() and limit_pfn logic into the main loop itself, further reduce the amount of code under the lock, and generally make the inner workings a bit less subtle. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 51 +++++++++++++++++---------------------------------- 1 file changed, 17 insertions(+), 34 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 9e04c1f3e740..7b7363518733 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -51,8 +51,8 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, spin_lock_init(&iovad->iova_rbtree_lock); iovad->rbroot = RB_ROOT; - iovad->cached_node = NULL; - iovad->cached32_node = NULL; + iovad->cached_node = &iovad->anchor.node; + iovad->cached32_node = &iovad->anchor.node; iovad->granule = granule; iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); @@ -115,22 +115,12 @@ int init_iova_flush_queue(struct iova_domain *iovad, EXPORT_SYMBOL_GPL(init_iova_flush_queue); static struct rb_node * -__get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn) +__get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) { - struct rb_node *cached_node = NULL; - struct iova *curr_iova; - - if (*limit_pfn <= iovad->dma_32bit_pfn) - cached_node = iovad->cached32_node; - if (!cached_node) - cached_node = iovad->cached_node; - if (!cached_node) - return rb_prev(&iovad->anchor.node); + if (limit_pfn <= iovad->dma_32bit_pfn) + return iovad->cached32_node; - curr_iova = rb_entry(cached_node, struct iova, node); - *limit_pfn = min(*limit_pfn, curr_iova->pfn_lo); - - return rb_prev(cached_node); + return iovad->cached_node; } static void @@ -149,11 +139,11 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) cached_iova = rb_entry(iovad->cached32_node, struct iova, node); if (free->pfn_hi < iovad->dma_32bit_pfn && - iovad->cached32_node && free->pfn_lo >= cached_iova->pfn_lo) + free->pfn_lo >= cached_iova->pfn_lo) iovad->cached32_node = rb_next(&free->node); cached_iova = rb_entry(iovad->cached_node, struct iova, node); - if (iovad->cached_node && free->pfn_lo >= cached_iova->pfn_lo) + if (free->pfn_lo >= cached_iova->pfn_lo) iovad->cached_node = rb_next(&free->node); } @@ -189,7 +179,8 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, struct iova *new, bool size_aligned) { - struct rb_node *prev, *curr = NULL; + struct rb_node *curr, *prev; + struct iova *curr_iova; unsigned long flags; unsigned long new_pfn; unsigned long align_mask = ~0UL; @@ -199,24 +190,16 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, /* Walk the tree backwards */ spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); - curr = __get_cached_rbnode(iovad, &limit_pfn); - prev = curr; - while (curr) { - struct iova *curr_iova = rb_entry(curr, struct iova, node); - - if (limit_pfn <= curr_iova->pfn_lo) - goto move_left; - - if (((limit_pfn - size) & align_mask) > curr_iova->pfn_hi) - break; /* found a free slot */ - - limit_pfn = curr_iova->pfn_lo; -move_left: + curr = __get_cached_rbnode(iovad, limit_pfn); + curr_iova = rb_entry(curr, struct iova, node); + do { + limit_pfn = min(limit_pfn, curr_iova->pfn_lo); + new_pfn = (limit_pfn - size) & align_mask; prev = curr; curr = rb_prev(curr); - } + curr_iova = rb_entry(curr, struct iova, node); + } while (curr && new_pfn <= curr_iova->pfn_hi); - new_pfn = (limit_pfn - size) & align_mask; if (limit_pfn < size || new_pfn < iovad->start_pfn) { spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); return -ENOMEM; -- cgit From 7595dc588a39c37091ddf65f6c0a3cd40f128e7a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 19 Sep 2017 14:48:39 +0100 Subject: iommu/iova: Simplify domain destruction All put_iova_domain() should have to worry about is freeing memory - by that point the domain must no longer be live, so the act of cleaning up doesn't need to be concurrency-safe or maintain the rbtree in a self-consistent state. There's no need to waste time with locking or emptying the rcache magazines, and we can just use the postorder traversal helper to clear out the remaining rbtree entries in-place. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 48 +++++++++--------------------------------------- 1 file changed, 9 insertions(+), 39 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 7b7363518733..ca21196c1f2d 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -583,21 +583,12 @@ EXPORT_SYMBOL_GPL(queue_iova); */ void put_iova_domain(struct iova_domain *iovad) { - struct rb_node *node; - unsigned long flags; + struct iova *iova, *tmp; free_iova_flush_queue(iovad); free_iova_rcaches(iovad); - spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); - node = rb_first(&iovad->rbroot); - while (node) { - struct iova *iova = rb_entry(node, struct iova, node); - - rb_erase(node, &iovad->rbroot); + rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node) free_iova_mem(iova); - node = rb_first(&iovad->rbroot); - } - spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); } EXPORT_SYMBOL_GPL(put_iova_domain); @@ -989,47 +980,26 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn); } -/* - * Free a cpu's rcache. - */ -static void free_cpu_iova_rcache(unsigned int cpu, struct iova_domain *iovad, - struct iova_rcache *rcache) -{ - struct iova_cpu_rcache *cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu); - unsigned long flags; - - spin_lock_irqsave(&cpu_rcache->lock, flags); - - iova_magazine_free_pfns(cpu_rcache->loaded, iovad); - iova_magazine_free(cpu_rcache->loaded); - - iova_magazine_free_pfns(cpu_rcache->prev, iovad); - iova_magazine_free(cpu_rcache->prev); - - spin_unlock_irqrestore(&cpu_rcache->lock, flags); -} - /* * free rcache data structures. */ static void free_iova_rcaches(struct iova_domain *iovad) { struct iova_rcache *rcache; - unsigned long flags; + struct iova_cpu_rcache *cpu_rcache; unsigned int cpu; int i, j; for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { rcache = &iovad->rcaches[i]; - for_each_possible_cpu(cpu) - free_cpu_iova_rcache(cpu, iovad, rcache); - spin_lock_irqsave(&rcache->lock, flags); + for_each_possible_cpu(cpu) { + cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu); + iova_magazine_free(cpu_rcache->loaded); + iova_magazine_free(cpu_rcache->prev); + } free_percpu(rcache->cpu_rcaches); - for (j = 0; j < rcache->depot_size; ++j) { - iova_magazine_free_pfns(rcache->depot[j], iovad); + for (j = 0; j < rcache->depot_size; ++j) iova_magazine_free(rcache->depot[j]); - } - spin_unlock_irqrestore(&rcache->lock, flags); } } -- cgit From b826ee9a4f1cbf83cadc5a307de8eea27637699a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 19 Sep 2017 14:48:40 +0100 Subject: iommu/iova: Make rcache limit_pfn handling more robust When popping a pfn from an rcache, we are currently checking it directly against limit_pfn for viability. Since this represents iova->pfn_lo, it is technically possible for the corresponding iova->pfn_hi to be greater than limit_pfn. Although we generally get away with it in practice since limit_pfn is typically a power-of-two boundary and the IOVAs are size-aligned, it's pretty trivial to make the iova_rcache_get() path take the allocation size into account for complete safety. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ca21196c1f2d..15ff3033bbd7 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -406,7 +406,7 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long iova_pfn; struct iova *new_iova; - iova_pfn = iova_rcache_get(iovad, size, limit_pfn); + iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1); if (iova_pfn) return iova_pfn; @@ -823,7 +823,7 @@ static unsigned long iova_magazine_pop(struct iova_magazine *mag, { BUG_ON(iova_magazine_empty(mag)); - if (mag->pfns[mag->size - 1] >= limit_pfn) + if (mag->pfns[mag->size - 1] > limit_pfn) return 0; return mag->pfns[--mag->size]; @@ -977,7 +977,7 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) return 0; - return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn); + return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); } /* -- cgit From e8b198402745ed413ed8229b2eb45d34016eb5d8 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 28 Sep 2017 11:31:23 +0100 Subject: iommu/iova: Try harder to allocate from rcache magazine When devices with different DMA masks are using the same domain, or for PCI devices where we usually try a speculative 32-bit allocation first, there is a fair possibility that the top PFN of the rcache stack at any given time may be unsuitable for the lower limit, prompting a fallback to allocating anew from the rbtree. Consequently, we may end up artifically increasing pressure on the 32-bit IOVA space as unused IOVAs accumulate lower down in the rcache stacks, while callers with 32-bit masks also impose unnecessary rbtree overhead. In such cases, let's try a bit harder to satisfy the allocation locally first - scanning the whole stack should still be relatively inexpensive. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 15ff3033bbd7..b0ca23682008 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -821,12 +821,21 @@ static bool iova_magazine_empty(struct iova_magazine *mag) static unsigned long iova_magazine_pop(struct iova_magazine *mag, unsigned long limit_pfn) { + int i; + unsigned long pfn; + BUG_ON(iova_magazine_empty(mag)); - if (mag->pfns[mag->size - 1] > limit_pfn) - return 0; + /* Only fall back to the rbtree if we have no suitable pfns at all */ + for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--) + if (i == 0) + return 0; + + /* Swap it to pop it */ + pfn = mag->pfns[i]; + mag->pfns[i] = mag->pfns[--mag->size]; - return mag->pfns[--mag->size]; + return pfn; } static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) -- cgit From abbb8a09384f69f7bb05936879e51933c146afba Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 2 Oct 2017 11:53:31 +0100 Subject: iommu/iova: Don't try to copy anchor nodes Anchor nodes are not reserved IOVAs in the way that copy_reserved_iova() cares about - while the failure from reserve_iova() is benign since the target domain will already have its own anchor, we still don't want to be triggering spurious warnings. Reported-by: kernel test robot Signed-off-by: Robin Murphy Fixes: bb68b2fbfbd6 ('iommu/iova: Add rbtree anchor node') Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index b0ca23682008..3aee64b99df1 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -704,6 +704,9 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to) struct iova *iova = rb_entry(node, struct iova, node); struct iova *new_iova; + if (iova->pfn_lo == IOVA_ANCHOR) + continue; + new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi); if (!new_iova) printk(KERN_ERR "Reserve iova range %lx@%lx failed\n", -- cgit From 32b124492bdf974f68eaef1bde80dc8058aef002 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 28 Sep 2017 15:55:01 +0100 Subject: iommu/io-pgtable-arm: Convert to IOMMU API TLB sync Now that the core API issues its own post-unmap TLB sync call, push that operation out from the io-pgtable-arm internals into the users. For now, we leave the invalidation implicit in the unmap operation, since none of the current users would benefit much from any change to that. CC: Magnus Damm CC: Laurent Pinchart Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 10 ++++++++++ drivers/iommu/arm-smmu.c | 20 +++++++++++++++----- drivers/iommu/io-pgtable-arm.c | 7 +------ drivers/iommu/ipmmu-vmsa.c | 10 ++++++++++ 4 files changed, 36 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index e67ba6c40faf..ee0c7b73cff7 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1743,6 +1743,14 @@ arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) return ops->unmap(ops, iova, size); } +static void arm_smmu_iotlb_sync(struct iommu_domain *domain) +{ + struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; + + if (smmu) + __arm_smmu_tlb_sync(smmu); +} + static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -1963,6 +1971,8 @@ static struct iommu_ops arm_smmu_ops = { .map = arm_smmu_map, .unmap = arm_smmu_unmap, .map_sg = default_iommu_map_sg, + .flush_iotlb_all = arm_smmu_iotlb_sync, + .iotlb_sync = arm_smmu_iotlb_sync, .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 3bdb799d3b4b..e4a82d70d446 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -250,6 +250,7 @@ enum arm_smmu_domain_stage { struct arm_smmu_domain { struct arm_smmu_device *smmu; struct io_pgtable_ops *pgtbl_ops; + const struct iommu_gather_ops *tlb_ops; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; struct mutex init_mutex; /* Protects smmu pointer */ @@ -735,7 +736,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - const struct iommu_gather_ops *tlb_ops; mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) @@ -813,7 +813,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, ias = min(ias, 32UL); oas = min(oas, 32UL); } - tlb_ops = &arm_smmu_s1_tlb_ops; + smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops; break; case ARM_SMMU_DOMAIN_NESTED: /* @@ -833,9 +833,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, oas = min(oas, 40UL); } if (smmu->version == ARM_SMMU_V2) - tlb_ops = &arm_smmu_s2_tlb_ops_v2; + smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2; else - tlb_ops = &arm_smmu_s2_tlb_ops_v1; + smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1; break; default: ret = -EINVAL; @@ -863,7 +863,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, - .tlb = tlb_ops, + .tlb = smmu_domain->tlb_ops, .iommu_dev = smmu->dev, }; @@ -1259,6 +1259,14 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, return ops->unmap(ops, iova, size); } +static void arm_smmu_iotlb_sync(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + if (smmu_domain->tlb_ops) + smmu_domain->tlb_ops->tlb_sync(smmu_domain); +} + static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { @@ -1562,6 +1570,8 @@ static struct iommu_ops arm_smmu_ops = { .map = arm_smmu_map, .unmap = arm_smmu_unmap, .map_sg = default_iommu_map_sg, + .flush_iotlb_all = arm_smmu_iotlb_sync, + .iotlb_sync = arm_smmu_iotlb_sync, .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index e8018a308868..51e5c43caed1 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -609,7 +609,6 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t size) { - size_t unmapped; struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); @@ -617,11 +616,7 @@ static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) return 0; - unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); - if (unmapped) - io_pgtable_tlb_sync(&data->iop); - - return unmapped; + return __arm_lpae_unmap(data, iova, size, lvl, ptep); } static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 195d6e93ac71..af8140054273 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -619,6 +619,14 @@ static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, return domain->iop->unmap(domain->iop, iova, size); } +static void ipmmu_iotlb_sync(struct iommu_domain *io_domain) +{ + struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); + + if (domain->mmu) + ipmmu_tlb_flush_all(domain); +} + static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, dma_addr_t iova) { @@ -876,6 +884,8 @@ static const struct iommu_ops ipmmu_ops = { .detach_dev = ipmmu_detach_device, .map = ipmmu_map, .unmap = ipmmu_unmap, + .flush_iotlb_all = ipmmu_iotlb_sync, + .iotlb_sync = ipmmu_iotlb_sync, .map_sg = default_iommu_map_sg, .iova_to_phys = ipmmu_iova_to_phys, .add_device = ipmmu_add_device_dma, -- cgit From 4d689b619445894f6b6fcbc496f6d302bd9e44a5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 28 Sep 2017 15:55:02 +0100 Subject: iommu/io-pgtable-arm-v7s: Convert to IOMMU API TLB sync Now that the core API issues its own post-unmap TLB sync call, push that operation out from the io-pgtable-arm-v7s internals into the users. For now, we leave the invalidation implicit in the unmap operation, since none of the current users would benefit much from any change to that. Note that the conversion of msm_iommu is implicit, since that apparently has no specific TLB sync operation anyway. CC: Yong Wu CC: Rob Clark Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 7 +------ drivers/iommu/mtk_iommu.c | 7 +++++++ drivers/iommu/qcom_iommu.c | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index d665d0dc16e8..397531da8d9c 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -660,16 +660,11 @@ static int arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t size) { struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); - size_t unmapped; if (WARN_ON(upper_32_bits(iova))) return 0; - unmapped = __arm_v7s_unmap(data, iova, size, 1, data->pgd); - if (unmapped) - io_pgtable_tlb_sync(&data->iop); - - return unmapped; + return __arm_v7s_unmap(data, iova, size, 1, data->pgd); } static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index bd515be5b380..d0c8dfbbd74d 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -391,6 +391,11 @@ static size_t mtk_iommu_unmap(struct iommu_domain *domain, return unmapsz; } +static void mtk_iommu_iotlb_sync(struct iommu_domain *domain) +{ + mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data()); +} + static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -490,6 +495,8 @@ static struct iommu_ops mtk_iommu_ops = { .map = mtk_iommu_map, .unmap = mtk_iommu_unmap, .map_sg = default_iommu_map_sg, + .flush_iotlb_all = mtk_iommu_iotlb_sync, + .iotlb_sync = mtk_iommu_iotlb_sync, .iova_to_phys = mtk_iommu_iova_to_phys, .add_device = mtk_iommu_add_device, .remove_device = mtk_iommu_remove_device, diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index c8a587d034b0..4a2c4378b3db 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -443,6 +443,19 @@ static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova, return ret; } +static void qcom_iommu_iotlb_sync(struct iommu_domain *domain) +{ + struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); + struct io_pgtable *pgtable = container_of(qcom_domain->pgtbl_ops, + struct io_pgtable, ops); + if (!qcom_domain->pgtbl_ops) + return; + + pm_runtime_get_sync(qcom_domain->iommu->dev); + qcom_iommu_tlb_sync(pgtable->cookie); + pm_runtime_put_sync(qcom_domain->iommu->dev); +} + static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -570,6 +583,8 @@ static const struct iommu_ops qcom_iommu_ops = { .map = qcom_iommu_map, .unmap = qcom_iommu_unmap, .map_sg = default_iommu_map_sg, + .flush_iotlb_all = qcom_iommu_iotlb_sync, + .iotlb_sync = qcom_iommu_iotlb_sync, .iova_to_phys = qcom_iommu_iova_to_phys, .add_device = qcom_iommu_add_device, .remove_device = qcom_iommu_remove_device, -- cgit From ec154bf56b276a0bb36079a5d22a267b5f417801 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 6 Oct 2017 15:00:53 +0200 Subject: iommu/vt-d: Don't register bus-notifier under dmar_global_lock The notifier function will take the dmar_global_lock too, so lockdep complains about inverse locking order when the notifier is registered under the dmar_global_lock. Reported-by: Jan Kiszka Fixes: 59ce0515cdaf ('iommu/vt-d: Update DRHD/RMRR/ATSR device scope caches when PCI hotplug happens') Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 7 +++++-- drivers/iommu/intel-iommu.c | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 57c920c1372d..1ea7cd537873 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -801,13 +801,16 @@ int __init dmar_dev_scope_init(void) dmar_free_pci_notify_info(info); } } - - bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb); } return dmar_dev_scope_status; } +void dmar_register_bus_notifier(void) +{ + bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb); +} + int __init dmar_table_init(void) { diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 6784a05dd6b2..934cef924461 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4752,6 +4752,16 @@ int __init intel_iommu_init(void) goto out_free_dmar; } + up_write(&dmar_global_lock); + + /* + * The bus notifier takes the dmar_global_lock, so lockdep will + * complain later when we register it under the lock. + */ + dmar_register_bus_notifier(); + + down_write(&dmar_global_lock); + if (no_iommu || dmar_disabled) { /* * We exit the function here to ensure IOMMU's remapping and -- cgit From b117e0380513c186065f247a9af09dc0cd3e703d Mon Sep 17 00:00:00 2001 From: Christos Gkekas Date: Sun, 8 Oct 2017 23:33:31 +0100 Subject: iommu/vt-d: Delete unnecessary check in domain_context_mapping_one() Variable did_old is unsigned so checking whether it is greater or equal to zero is not necessary. Signed-off-by: Christos Gkekas Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 934cef924461..1dab9f73a20b 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2058,7 +2058,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, if (context_copied(context)) { u16 did_old = context_domain_id(context); - if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) { + if (did_old < cap_ndoms(iommu->cap)) { iommu->flush.flush_context(iommu, did_old, (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, -- cgit From 37946d95fc1a41ed79efb613b0818c2cdecbb2fa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 6 Oct 2017 12:16:39 +0200 Subject: iommu/amd: Add align parameter to alloc_irq_index() For multi-MSI IRQ ranges the IRQ index needs to be aligned to the power-of-two of the requested IRQ count. Extend the alloc_irq_index() function to allow such an allocation. Reported-by: Thomas Gleixner Fixes: 2b324506341cb ('iommu/amd: Add routines to manage irq remapping tables') Reviewed-by: Thomas Gleixner Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 51f8215877f5..2d4ee2555a0d 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3660,11 +3660,11 @@ out_unlock: return table; } -static int alloc_irq_index(u16 devid, int count) +static int alloc_irq_index(u16 devid, int count, bool align) { struct irq_remap_table *table; + int index, c, alignment = 1; unsigned long flags; - int index, c; struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; if (!iommu) @@ -3674,16 +3674,22 @@ static int alloc_irq_index(u16 devid, int count) if (!table) return -ENODEV; + if (align) + alignment = roundup_pow_of_two(count); + spin_lock_irqsave(&table->lock, flags); /* Scan table for free entries */ - for (c = 0, index = table->min_index; + for (index = ALIGN(table->min_index, alignment), c = 0; index < MAX_IRQS_PER_TABLE; - ++index) { - if (!iommu->irte_ops->is_allocated(table, index)) + index++) { + if (!iommu->irte_ops->is_allocated(table, index)) { c += 1; - else - c = 0; + } else { + c = 0; + index = ALIGN(index, alignment); + continue; + } if (c == count) { for (; c != 0; --c) @@ -4096,7 +4102,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, else ret = -ENOMEM; } else { - index = alloc_irq_index(devid, nr_irqs); + index = alloc_irq_index(devid, nr_irqs, false); } if (index < 0) { pr_warn("Failed to allocate IRTE\n"); -- cgit From 53b9ec3fbb7da97d13951debbd42e3a0c4a7c9f7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 6 Oct 2017 12:22:06 +0200 Subject: iommu/amd: Enforce alignment for MSI IRQs Make use of the new alignment capability of alloc_irq_index() to enforce IRQ index alignment for MSI. Reported-by: Thomas Gleixner Fixes: 2b324506341cb ('iommu/amd: Add routines to manage irq remapping tables') Reviewed-by: Thomas Gleixner Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 2d4ee2555a0d..cb7c531542da 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -4102,7 +4102,9 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, else ret = -ENOMEM; } else { - index = alloc_irq_index(devid, nr_irqs, false); + bool align = (info->type == X86_IRQ_ALLOC_TYPE_MSI); + + index = alloc_irq_index(devid, nr_irqs, align); } if (index < 0) { pr_warn("Failed to allocate IRTE\n"); -- cgit From aba2d9a6385a5cc4f7a7e8eb5788e1ddbc213fc0 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Fri, 6 Oct 2017 16:35:40 -0500 Subject: iommu/amd: Do not disable SWIOTLB if SME is active When SME memory encryption is active it will rely on SWIOTLB to handle DMA for devices that cannot support the addressing requirements of having the encryption mask set in the physical address. The IOMMU currently disables SWIOTLB if it is not running in passthrough mode. This is not desired as non-PCI devices attempting DMA may fail. Update the code to check if SME is active and not disable SWIOTLB. Fixes: 2543a786aa25 ("iommu/amd: Allow the AMD IOMMU to work with memory encryption") Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 51f8215877f5..822679ac90a1 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2773,14 +2773,16 @@ int __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { - swiotlb = iommu_pass_through ? 1 : 0; + swiotlb = (iommu_pass_through || sme_me_mask) ? 1 : 0; iommu_detected = 1; /* * In case we don't initialize SWIOTLB (actually the common case - * when AMD IOMMU is enabled), make sure there are global - * dma_ops set as a fall-back for devices not handled by this - * driver (for example non-PCI devices). + * when AMD IOMMU is enabled and SME is not active), make sure there + * are global dma_ops set as a fall-back for devices not handled by + * this driver (for example non-PCI devices). When SME is active, + * make sure that swiotlb variable remains set so the global dma_ops + * continue to be SWIOTLB. */ if (!swiotlb) dma_ops = &nommu_dma_ops; -- cgit From 9d25e3cc83d731ae4eeb017fd07562fde3f80bef Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 9 Oct 2017 13:40:23 +0200 Subject: iommu/exynos: Remove initconst attribute to avoid potential kernel oops Exynos SYSMMU registers standard platform device with sysmmu_of_match table, what means that this table is accessed every time a new platform device is registered in a system. This might happen also after the boot, so the table must not be attributed as initconst to avoid potential kernel oops caused by access to freed memory. Fixes: 6b21a5db3642 ("iommu/exynos: Support for device tree") Signed-off-by: Marek Szyprowski Reviewed-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index f596fcc32898..25c2c75f5332 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -709,7 +709,7 @@ static const struct dev_pm_ops sysmmu_pm_ops = { pm_runtime_force_resume) }; -static const struct of_device_id sysmmu_of_match[] __initconst = { +static const struct of_device_id sysmmu_of_match[] = { { .compatible = "samsung,exynos-sysmmu", }, { }, }; -- cgit From 538d5b333216c3daa7a5821307164f10af73ec8c Mon Sep 17 00:00:00 2001 From: Tomasz Nowicki Date: Wed, 20 Sep 2017 10:52:02 +0200 Subject: iommu/iova: Make rcache flush optional on IOVA allocation failure Since IOVA allocation failure is not unusual case we need to flush CPUs' rcache in hope we will succeed in next round. However, it is useful to decide whether we need rcache flush step because of two reasons: - Not scalability. On large system with ~100 CPUs iterating and flushing rcache for each CPU becomes serious bottleneck so we may want to defer it. - free_cpu_cached_iovas() does not care about max PFN we are interested in. Thus we may flush our rcaches and still get no new IOVA like in the commonly used scenario: if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) iova = alloc_iova_fast(iovad, iova_len, DMA_BIT_MASK(32) >> shift); if (!iova) iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift); 1. First alloc_iova_fast() call is limited to DMA_BIT_MASK(32) to get PCI devices a SAC address 2. alloc_iova() fails due to full 32-bit space 3. rcaches contain PFNs out of 32-bit space so free_cpu_cached_iovas() throws entries away for nothing and alloc_iova() fails again 4. Next alloc_iova_fast() call cannot take advantage of rcache since we have just defeated caches. In this case we pick the slowest option to proceed. This patch reworks flushed_rcache local flag to be additional function argument instead and control rcache flush step. Also, it updates all users to do the flush as the last chance. Signed-off-by: Tomasz Nowicki Reviewed-by: Robin Murphy Tested-by: Nate Watterson Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 5 +++-- drivers/iommu/dma-iommu.c | 6 ++++-- drivers/iommu/intel-iommu.c | 5 +++-- drivers/iommu/iova.c | 11 ++++++----- 4 files changed, 16 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 647ab7691aee..3d64c844d8b1 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1546,10 +1546,11 @@ static unsigned long dma_ops_alloc_iova(struct device *dev, if (dma_mask > DMA_BIT_MASK(32)) pfn = alloc_iova_fast(&dma_dom->iovad, pages, - IOVA_PFN(DMA_BIT_MASK(32))); + IOVA_PFN(DMA_BIT_MASK(32)), false); if (!pfn) - pfn = alloc_iova_fast(&dma_dom->iovad, pages, IOVA_PFN(dma_mask)); + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(dma_mask), true); return (pfn << PAGE_SHIFT); } diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 191be9c80a8a..25914d36c5ac 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -370,10 +370,12 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, /* Try to get PCI devices a SAC address */ if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) - iova = alloc_iova_fast(iovad, iova_len, DMA_BIT_MASK(32) >> shift); + iova = alloc_iova_fast(iovad, iova_len, + DMA_BIT_MASK(32) >> shift, false); if (!iova) - iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift); + iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, + true); return (dma_addr_t)iova << shift; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ebb48353dd39..b3914fce8254 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3469,11 +3469,12 @@ static unsigned long intel_alloc_iova(struct device *dev, * from higher range */ iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, - IOVA_PFN(DMA_BIT_MASK(32))); + IOVA_PFN(DMA_BIT_MASK(32)), false); if (iova_pfn) return iova_pfn; } - iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask)); + iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, + IOVA_PFN(dma_mask), true); if (unlikely(!iova_pfn)) { pr_err("Allocating %ld-page iova for %s failed", nrpages, dev_name(dev)); diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 3aee64b99df1..84bda3a4dafc 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -395,14 +395,15 @@ EXPORT_SYMBOL_GPL(free_iova); * @iovad: - iova domain in question * @size: - size of page frames to allocate * @limit_pfn: - max limit address + * @flush_rcache: - set to flush rcache on regular allocation failure * This function tries to satisfy an iova allocation from the rcache, - * and falls back to regular allocation on failure. + * and falls back to regular allocation on failure. If regular allocation + * fails too and the flush_rcache flag is set then the rcache will be flushed. */ unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, - unsigned long limit_pfn) + unsigned long limit_pfn, bool flush_rcache) { - bool flushed_rcache = false; unsigned long iova_pfn; struct iova *new_iova; @@ -415,11 +416,11 @@ retry: if (!new_iova) { unsigned int cpu; - if (flushed_rcache) + if (!flush_rcache) return 0; /* Try replenishing IOVAs by flushing rcache. */ - flushed_rcache = true; + flush_rcache = false; for_each_online_cpu(cpu) free_cpu_cached_iovas(cpu, iovad); goto retry; -- cgit From ce76353f169a6471542d999baf3d29b121dce9c0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 13 Oct 2017 14:32:37 +0200 Subject: iommu/amd: Finish TLB flush in amd_iommu_unmap() The function only sends the flush command to the IOMMU(s), but does not wait for its completion when it returns. Fix that. Fixes: 601367d76bd1 ('x86/amd-iommu: Remove iommu_flush_domain function') Cc: stable@vger.kernel.org # >= 2.6.33 Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 822679ac90a1..8e8874d23717 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3048,6 +3048,7 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, mutex_unlock(&domain->api_lock); domain_flush_tlb_pde(domain); + domain_flush_complete(domain); return unmap_size; } -- cgit From 07d1c91b6c649705fdd9acf58001071845ecf068 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Fri, 3 Nov 2017 10:50:31 -0600 Subject: iommu/amd: Fix alloc_irq_index() increment On an is_allocated() interrupt index, we ALIGN() the current index and then increment it via the for loop, guaranteeing that it is no longer aligned for alignments >1. We instead need to align the next index, to guarantee forward progress, moving the increment-only to the case where the index was found to be unallocated. Fixes: 37946d95fc1a ('iommu/amd: Add align parameter to alloc_irq_index()') Signed-off-by: Alex Williamson --- drivers/iommu/amd_iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 9dc7facfd2e5..3c1a29104f0e 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3682,13 +3682,12 @@ static int alloc_irq_index(u16 devid, int count, bool align) /* Scan table for free entries */ for (index = ALIGN(table->min_index, alignment), c = 0; - index < MAX_IRQS_PER_TABLE; - index++) { + index < MAX_IRQS_PER_TABLE;) { if (!iommu->irte_ops->is_allocated(table, index)) { c += 1; } else { c = 0; - index = ALIGN(index, alignment); + index = ALIGN(index + 1, alignment); continue; } @@ -3699,6 +3698,8 @@ static int alloc_irq_index(u16 devid, int count, bool align) index -= count - 1; goto out; } + + index++; } index = -ENOSPC; -- cgit From 2c40367cbff6f9ed1efda238685837fb5f0d9e3c Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 3 Nov 2017 10:50:32 -0600 Subject: iommu/amd: remove unused variable flush_addr Variable flush_addr is being assigned but is never read; it is redundant and can be removed. Cleans up the clang warning: drivers/iommu/amd_iommu.c:2388:2: warning: Value stored to 'flush_addr' is never read Signed-off-by: Colin Ian King Signed-off-by: Alex Williamson --- drivers/iommu/amd_iommu.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 3c1a29104f0e..797e6454afd5 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2382,11 +2382,9 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, size_t size, int dir) { - dma_addr_t flush_addr; dma_addr_t i, start; unsigned int pages; - flush_addr = dma_addr; pages = iommu_num_pages(dma_addr, size, PAGE_SIZE); dma_addr &= PAGE_MASK; start = dma_addr; -- cgit From 049541e178d5b1d003584aa0ad2a96101f0694d2 Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Fri, 3 Nov 2017 10:50:33 -0600 Subject: iommu: qcom: wire up fault handler This is quite useful for debugging. Currently, always TERMINATE the translation when the fault handler returns (since this is all we need for debugging drivers). But I expect the SVM work should eventually let us do something more clever. Signed-off-by: Rob Clark Signed-off-by: Alex Williamson --- drivers/iommu/qcom_iommu.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 4a2c4378b3db..e07f02d00c68 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -66,6 +66,7 @@ struct qcom_iommu_ctx { void __iomem *base; bool secure_init; u8 asid; /* asid and ctx bank # are 1:1 */ + struct iommu_domain *domain; }; struct qcom_iommu_domain { @@ -194,12 +195,15 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0); iova = iommu_readq(ctx, ARM_SMMU_CB_FAR); - dev_err_ratelimited(ctx->dev, - "Unhandled context fault: fsr=0x%x, " - "iova=0x%016llx, fsynr=0x%x, cb=%d\n", - fsr, iova, fsynr, ctx->asid); + if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { + dev_err_ratelimited(ctx->dev, + "Unhandled context fault: fsr=0x%x, " + "iova=0x%016llx, fsynr=0x%x, cb=%d\n", + fsr, iova, fsynr, ctx->asid); + } iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr); + iommu_writel(ctx, ARM_SMMU_CB_RESUME, RESUME_TERMINATE); return IRQ_HANDLED; } @@ -274,12 +278,14 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, /* SCTLR */ reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | - SCTLR_M | SCTLR_S1_ASIDPNE; + SCTLR_M | SCTLR_S1_ASIDPNE | SCTLR_CFCFG; if (IS_ENABLED(CONFIG_BIG_ENDIAN)) reg |= SCTLR_E; iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg); + + ctx->domain = domain; } mutex_unlock(&qcom_domain->init_mutex); @@ -395,6 +401,8 @@ static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *de /* Disable the context bank: */ iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0); + + ctx->domain = NULL; } pm_runtime_put_sync(qcom_iommu->dev); -- cgit From b92b4fb5c14257c0e7eae291ecc1f7b1962e1699 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Fri, 3 Nov 2017 10:50:34 -0600 Subject: iommu/amd: Limit the IOVA page range to the specified addresses The extent of pages specified when applying a reserved region should include up to the last page of the range, but not the page following the range. Signed-off-by: Gary R Hook Fixes: 8d54d6c8b8f3 ('iommu/amd: Implement apply_dm_region call-back') Signed-off-by: Alex Williamson --- drivers/iommu/amd_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 797e6454afd5..a8c111e96cc3 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3151,7 +3151,7 @@ static void amd_iommu_apply_resv_region(struct device *dev, unsigned long start, end; start = IOVA_PFN(region->start); - end = IOVA_PFN(region->start + region->length); + end = IOVA_PFN(region->start + region->length - 1); WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL); } -- cgit From 2e2e35d512798193b4165ed884a5d1d9b181e231 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 3 Nov 2017 10:51:32 -0600 Subject: iommu/vt-d: Missing checks for pasid tables if allocation fails intel_svm_alloc_pasid_tables() might return an error but never be checked by the callers. Later when intel_svm_bind_mm() is called, there are no checks for valid pasid tables before enabling them. Signed-off-by: Ashok Raj Signed-off-by: Lu Baolu Reviewed-by: Liu, Yi L Signed-off-by: Alex Williamson --- drivers/iommu/intel-svm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index f6697e55c2d4..43280ca282df 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -292,7 +292,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ int pasid_max; int ret; - if (WARN_ON(!iommu)) + if (WARN_ON(!iommu || !iommu->pasid_table)) return -EINVAL; if (dev_is_pci(dev)) { -- cgit From 973b546451fdf11e518cc96d1b137af893a38db5 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 3 Nov 2017 10:51:33 -0600 Subject: iommu/vt-d: Clear Page Request Overflow fault bit Currently Page Request Overflow bit in IOMMU Fault Status register is not cleared. Not clearing this bit would mean that any future page-request is going to be automatically dropped by IOMMU. Suggested-by: Ashok Raj Signed-off-by: Lu Baolu Signed-off-by: Alex Williamson --- drivers/iommu/dmar.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 1ea7cd537873..9a7ffd13c7f0 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1679,7 +1679,8 @@ irqreturn_t dmar_fault(int irq, void *dev_id) raw_spin_lock_irqsave(&iommu->register_lock, flag); } - writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG); + writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO, + iommu->reg + DMAR_FSTS_REG); unlock_exit: raw_spin_unlock_irqrestore(&iommu->register_lock, flag); -- cgit From 4fa064b26c2eb9e1c38460ba327765dff16a4b14 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 3 Nov 2017 10:51:34 -0600 Subject: iommu/vt-d: Clear pasid table entry when memory unbound In intel_svm_unbind_mm(), pasid table entry must be cleared during svm free. Otherwise, hardware may be set up with a wild pointer. Suggested-by: Ashok Raj Signed-off-by: Lu Baolu Signed-off-by: Alex Williamson --- drivers/iommu/intel-svm.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 43280ca282df..ed1cf7c5a43b 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -458,6 +458,8 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) kfree_rcu(sdev, rcu); if (list_empty(&svm->devs)) { + svm->iommu->pasid_table[svm->pasid].val = 0; + wmb(); idr_remove(&svm->iommu->pasid_idr, svm->pasid); if (svm->mm) -- cgit From 105a004e2187609a74f75d55fd0f9a054b49d60a Mon Sep 17 00:00:00 2001 From: "weiyongjun (A)" Date: Tue, 17 Oct 2017 12:11:22 +0000 Subject: iommu/ipmmu-vmsa: Fix return value check in ipmmu_find_group_dma() In case of error, the function iommu_group_get() returns NULL pointer not ERR_PTR(). The IS_ERR() test in the return value check should be replaced with NULL test. Fixes: 3ae47292024f ("iommu/ipmmu-vmsa: Add new IOMMU_DOMAIN_DMA ops") Signed-off-by: Wei Yongjun Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index af8140054273..00e88a88ee3a 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -871,7 +871,7 @@ static struct iommu_group *ipmmu_find_group_dma(struct device *dev) sibling = ipmmu_find_sibling_device(dev); if (sibling) group = iommu_group_get(sibling); - if (!sibling || IS_ERR(group)) + if (!sibling || !group) group = generic_device_group(dev); return group; -- cgit From 1c7e7c0278df968221a5edb1a293423e13b13814 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 13 Oct 2017 19:23:39 +0100 Subject: iommu/ipmmu-vmsa: Unify domain alloc/free We have two implementations for ipmmu_ops->alloc depending on CONFIG_IOMMU_DMA, the difference being whether they accept the IOMMU_DOMAIN_DMA type or not. However, iommu_dma_get_cookie() is guaranteed to return an error when !CONFIG_IOMMU_DMA, so if ipmmu_domain_alloc_dma() was actually checking and handling the return value correctly, it would behave the same as ipmmu_domain_alloc() anyway. Similarly for freeing; iommu_put_dma_cookie() is robust by design. Signed-off-by: Robin Murphy Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 65 +++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 41 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 00e88a88ee3a..37154075c00a 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -528,6 +528,27 @@ static struct iommu_domain *__ipmmu_domain_alloc(unsigned type) return &domain->io_domain; } +static struct iommu_domain *ipmmu_domain_alloc(unsigned type) +{ + struct iommu_domain *io_domain = NULL; + + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + io_domain = __ipmmu_domain_alloc(type); + break; + + case IOMMU_DOMAIN_DMA: + io_domain = __ipmmu_domain_alloc(type); + if (io_domain && iommu_get_dma_cookie(io_domain)) { + kfree(io_domain); + io_domain = NULL; + } + break; + } + + return io_domain; +} + static void ipmmu_domain_free(struct iommu_domain *io_domain) { struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); @@ -536,6 +557,7 @@ static void ipmmu_domain_free(struct iommu_domain *io_domain) * Free the domain resources. We assume that all devices have already * been detached. */ + iommu_put_dma_cookie(io_domain); ipmmu_domain_destroy_context(domain); free_io_pgtable_ops(domain->iop); kfree(domain); @@ -671,14 +693,6 @@ static int ipmmu_of_xlate(struct device *dev, #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) -static struct iommu_domain *ipmmu_domain_alloc(unsigned type) -{ - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; - - return __ipmmu_domain_alloc(type); -} - static int ipmmu_add_device(struct device *dev) { struct ipmmu_vmsa_device *mmu = NULL; @@ -779,37 +793,6 @@ static const struct iommu_ops ipmmu_ops = { static DEFINE_SPINLOCK(ipmmu_slave_devices_lock); static LIST_HEAD(ipmmu_slave_devices); -static struct iommu_domain *ipmmu_domain_alloc_dma(unsigned type) -{ - struct iommu_domain *io_domain = NULL; - - switch (type) { - case IOMMU_DOMAIN_UNMANAGED: - io_domain = __ipmmu_domain_alloc(type); - break; - - case IOMMU_DOMAIN_DMA: - io_domain = __ipmmu_domain_alloc(type); - if (io_domain) - iommu_get_dma_cookie(io_domain); - break; - } - - return io_domain; -} - -static void ipmmu_domain_free_dma(struct iommu_domain *io_domain) -{ - switch (io_domain->type) { - case IOMMU_DOMAIN_DMA: - iommu_put_dma_cookie(io_domain); - /* fall-through */ - default: - ipmmu_domain_free(io_domain); - break; - } -} - static int ipmmu_add_device_dma(struct device *dev) { struct iommu_group *group; @@ -878,8 +861,8 @@ static struct iommu_group *ipmmu_find_group_dma(struct device *dev) } static const struct iommu_ops ipmmu_ops = { - .domain_alloc = ipmmu_domain_alloc_dma, - .domain_free = ipmmu_domain_free_dma, + .domain_alloc = ipmmu_domain_alloc, + .domain_free = ipmmu_domain_free, .attach_dev = ipmmu_attach_device, .detach_dev = ipmmu_detach_device, .map = ipmmu_map, -- cgit From b354c73edc7eb8d6ee643866e9e4de7842213b06 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 13 Oct 2017 19:23:40 +0100 Subject: iommu/ipmmu-vmsa: Simplify group allocation We go through quite the merry dance in order to find masters behind the same IPMMU instance, so that we can ensure they are grouped together. None of which is really necessary, since the master's private data already points to the particular IPMMU it is associated with, and that IPMMU instance data is the perfect place to keep track of a per-instance group directly. Signed-off-by: Robin Murphy Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 53 ++++++++-------------------------------------- 1 file changed, 9 insertions(+), 44 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 37154075c00a..6e6a86f3c375 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -43,6 +43,7 @@ struct ipmmu_vmsa_device { DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX]; + struct iommu_group *group; struct dma_iommu_mapping *mapping; }; @@ -59,8 +60,6 @@ struct ipmmu_vmsa_domain { struct ipmmu_vmsa_iommu_priv { struct ipmmu_vmsa_device *mmu; - struct device *dev; - struct list_head list; }; static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) @@ -674,7 +673,6 @@ static int ipmmu_init_platform_device(struct device *dev, return -ENOMEM; priv->mmu = platform_get_drvdata(ipmmu_pdev); - priv->dev = dev; dev->iommu_fwspec->iommu_priv = priv; return 0; } @@ -790,9 +788,6 @@ static const struct iommu_ops ipmmu_ops = { #ifdef CONFIG_IOMMU_DMA -static DEFINE_SPINLOCK(ipmmu_slave_devices_lock); -static LIST_HEAD(ipmmu_slave_devices); - static int ipmmu_add_device_dma(struct device *dev) { struct iommu_group *group; @@ -807,55 +802,25 @@ static int ipmmu_add_device_dma(struct device *dev) if (IS_ERR(group)) return PTR_ERR(group); - spin_lock(&ipmmu_slave_devices_lock); - list_add(&to_priv(dev)->list, &ipmmu_slave_devices); - spin_unlock(&ipmmu_slave_devices_lock); return 0; } static void ipmmu_remove_device_dma(struct device *dev) { - struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev); - - spin_lock(&ipmmu_slave_devices_lock); - list_del(&priv->list); - spin_unlock(&ipmmu_slave_devices_lock); - iommu_group_remove_device(dev); } -static struct device *ipmmu_find_sibling_device(struct device *dev) +static struct iommu_group *ipmmu_find_group(struct device *dev) { struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev); - struct ipmmu_vmsa_iommu_priv *sibling_priv = NULL; - bool found = false; - - spin_lock(&ipmmu_slave_devices_lock); - - list_for_each_entry(sibling_priv, &ipmmu_slave_devices, list) { - if (priv == sibling_priv) - continue; - if (sibling_priv->mmu == priv->mmu) { - found = true; - break; - } - } - - spin_unlock(&ipmmu_slave_devices_lock); - - return found ? sibling_priv->dev : NULL; -} - -static struct iommu_group *ipmmu_find_group_dma(struct device *dev) -{ struct iommu_group *group; - struct device *sibling; - sibling = ipmmu_find_sibling_device(dev); - if (sibling) - group = iommu_group_get(sibling); - if (!sibling || !group) - group = generic_device_group(dev); + if (priv->mmu->group) + return iommu_group_ref_get(priv->mmu->group); + + group = iommu_group_alloc(); + if (!IS_ERR(group)) + priv->mmu->group = group; return group; } @@ -873,7 +838,7 @@ static const struct iommu_ops ipmmu_ops = { .iova_to_phys = ipmmu_iova_to_phys, .add_device = ipmmu_add_device_dma, .remove_device = ipmmu_remove_device_dma, - .device_group = ipmmu_find_group_dma, + .device_group = ipmmu_find_group, .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, .of_xlate = ipmmu_of_xlate, }; -- cgit From e4efe4a9a2ace658a36b5a4f515c11d4d36400a8 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 13 Oct 2017 19:23:41 +0100 Subject: iommu/ipmmu-vmsa: Clean up struct ipmmu_vmsa_iommu_priv Now that the IPMMU instance pointer is the only thing remaining in the private data structure, we no longer need the extra level of indirection and can simply stash that directlty in the fwspec. Signed-off-by: Robin Murphy Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 6e6a86f3c375..fd05a5f5a47e 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -58,16 +58,12 @@ struct ipmmu_vmsa_domain { spinlock_t lock; /* Protects mappings */ }; -struct ipmmu_vmsa_iommu_priv { - struct ipmmu_vmsa_device *mmu; -}; - static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) { return container_of(dom, struct ipmmu_vmsa_domain, io_domain); } -static struct ipmmu_vmsa_iommu_priv *to_priv(struct device *dev) +static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) { return dev->iommu_fwspec ? dev->iommu_fwspec->iommu_priv : NULL; } @@ -565,15 +561,14 @@ static void ipmmu_domain_free(struct iommu_domain *io_domain) static int ipmmu_attach_device(struct iommu_domain *io_domain, struct device *dev) { - struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev); struct iommu_fwspec *fwspec = dev->iommu_fwspec; - struct ipmmu_vmsa_device *mmu = priv->mmu; + struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); unsigned long flags; unsigned int i; int ret = 0; - if (!priv || !priv->mmu) { + if (!mmu) { dev_err(dev, "Cannot attach to IPMMU\n"); return -ENXIO; } @@ -662,18 +657,12 @@ static int ipmmu_init_platform_device(struct device *dev, struct of_phandle_args *args) { struct platform_device *ipmmu_pdev; - struct ipmmu_vmsa_iommu_priv *priv; ipmmu_pdev = of_find_device_by_node(args->np); if (!ipmmu_pdev) return -ENODEV; - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->mmu = platform_get_drvdata(ipmmu_pdev); - dev->iommu_fwspec->iommu_priv = priv; + dev->iommu_fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); return 0; } @@ -683,7 +672,7 @@ static int ipmmu_of_xlate(struct device *dev, iommu_fwspec_add_ids(dev, spec->args, 1); /* Initialize once - xlate() will call multiple times */ - if (to_priv(dev)) + if (to_ipmmu(dev)) return 0; return ipmmu_init_platform_device(dev, spec); @@ -693,14 +682,14 @@ static int ipmmu_of_xlate(struct device *dev, static int ipmmu_add_device(struct device *dev) { - struct ipmmu_vmsa_device *mmu = NULL; + struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); struct iommu_group *group; int ret; /* * Only let through devices that have been verified in xlate() */ - if (!to_priv(dev)) + if (!mmu) return -ENODEV; /* Create a device group and add the device to it. */ @@ -729,7 +718,6 @@ static int ipmmu_add_device(struct device *dev) * - Make the mapping size configurable ? We currently use a 2GB mapping * at a 1GB offset to ensure that NULL VAs will fault. */ - mmu = to_priv(dev)->mmu; if (!mmu->mapping) { struct dma_iommu_mapping *mapping; @@ -795,7 +783,7 @@ static int ipmmu_add_device_dma(struct device *dev) /* * Only let through devices that have been verified in xlate() */ - if (!to_priv(dev)) + if (!to_ipmmu(dev)) return -ENODEV; group = iommu_group_get_for_dev(dev); @@ -812,15 +800,15 @@ static void ipmmu_remove_device_dma(struct device *dev) static struct iommu_group *ipmmu_find_group(struct device *dev) { - struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev); + struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); struct iommu_group *group; - if (priv->mmu->group) - return iommu_group_ref_get(priv->mmu->group); + if (mmu->group) + return iommu_group_ref_get(mmu->group); group = iommu_group_alloc(); if (!IS_ERR(group)) - priv->mmu->group = group; + mmu->group = group; return group; } -- cgit From 49c875f030523d676a508e53f7dc3e592e9439d7 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 13 Oct 2017 19:23:42 +0100 Subject: iommu/ipmmu-vmsa: Unify ipmmu_ops The remaining difference between the ARM-specific and iommu-dma ops is in the {add,remove}_device implementations, but even those have some overlap and duplication. By stubbing out the few arm_iommu_*() calls, we can get rid of the rest of the inline #ifdeffery to both simplify the code and improve build coverage. Signed-off-by: Robin Murphy Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 69 +++++++++++++--------------------------------- 1 file changed, 19 insertions(+), 50 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index fd05a5f5a47e..f6d2e8e650be 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -27,6 +27,11 @@ #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) #include #include +#else +#define arm_iommu_create_mapping(...) NULL +#define arm_iommu_attach_device(...) -ENODEV +#define arm_iommu_release_mapping(...) do {} while (0) +#define arm_iommu_detach_device(...) do {} while (0) #endif #include "io-pgtable.h" @@ -678,26 +683,17 @@ static int ipmmu_of_xlate(struct device *dev, return ipmmu_init_platform_device(dev, spec); } -#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) - -static int ipmmu_add_device(struct device *dev) +static int ipmmu_init_arm_mapping(struct device *dev) { struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); struct iommu_group *group; int ret; - /* - * Only let through devices that have been verified in xlate() - */ - if (!mmu) - return -ENODEV; - /* Create a device group and add the device to it. */ group = iommu_group_alloc(); if (IS_ERR(group)) { dev_err(dev, "Failed to allocate IOMMU group\n"); - ret = PTR_ERR(group); - goto error; + return PTR_ERR(group); } ret = iommu_group_add_device(group, dev); @@ -705,8 +701,7 @@ static int ipmmu_add_device(struct device *dev) if (ret < 0) { dev_err(dev, "Failed to add device to IPMMU group\n"); - group = NULL; - goto error; + return ret; } /* @@ -742,41 +737,14 @@ static int ipmmu_add_device(struct device *dev) return 0; error: - if (mmu) + iommu_group_remove_device(dev); + if (mmu->mapping) arm_iommu_release_mapping(mmu->mapping); - if (!IS_ERR_OR_NULL(group)) - iommu_group_remove_device(dev); - return ret; } -static void ipmmu_remove_device(struct device *dev) -{ - arm_iommu_detach_device(dev); - iommu_group_remove_device(dev); -} - -static const struct iommu_ops ipmmu_ops = { - .domain_alloc = ipmmu_domain_alloc, - .domain_free = ipmmu_domain_free, - .attach_dev = ipmmu_attach_device, - .detach_dev = ipmmu_detach_device, - .map = ipmmu_map, - .unmap = ipmmu_unmap, - .map_sg = default_iommu_map_sg, - .iova_to_phys = ipmmu_iova_to_phys, - .add_device = ipmmu_add_device, - .remove_device = ipmmu_remove_device, - .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, - .of_xlate = ipmmu_of_xlate, -}; - -#endif /* !CONFIG_ARM && CONFIG_IOMMU_DMA */ - -#ifdef CONFIG_IOMMU_DMA - -static int ipmmu_add_device_dma(struct device *dev) +static int ipmmu_add_device(struct device *dev) { struct iommu_group *group; @@ -786,15 +754,20 @@ static int ipmmu_add_device_dma(struct device *dev) if (!to_ipmmu(dev)) return -ENODEV; + if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) + return ipmmu_init_arm_mapping(dev); + group = iommu_group_get_for_dev(dev); if (IS_ERR(group)) return PTR_ERR(group); + iommu_group_put(group); return 0; } -static void ipmmu_remove_device_dma(struct device *dev) +static void ipmmu_remove_device(struct device *dev) { + arm_iommu_detach_device(dev); iommu_group_remove_device(dev); } @@ -824,15 +797,13 @@ static const struct iommu_ops ipmmu_ops = { .iotlb_sync = ipmmu_iotlb_sync, .map_sg = default_iommu_map_sg, .iova_to_phys = ipmmu_iova_to_phys, - .add_device = ipmmu_add_device_dma, - .remove_device = ipmmu_remove_device_dma, + .add_device = ipmmu_add_device, + .remove_device = ipmmu_remove_device, .device_group = ipmmu_find_group, .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, .of_xlate = ipmmu_of_xlate, }; -#endif /* CONFIG_IOMMU_DMA */ - /* ----------------------------------------------------------------------------- * Probe/remove and init */ @@ -929,9 +900,7 @@ static int ipmmu_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&mmu->iommu); iommu_device_unregister(&mmu->iommu); -#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) arm_iommu_release_mapping(mmu->mapping); -#endif ipmmu_device_reset(mmu); -- cgit From 33f3ac9b511612153bae1d328b0c84c0367cd08d Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:29:25 +0900 Subject: iommu/ipmmu-vmsa: Introduce features, break out alias Introduce struct ipmmu_features to track various hardware and software implementation changes inside the driver for different kinds of IPMMU hardware. Add use_ns_alias_offset as a first example of a feature to control if the secure register bank offset should be used or not. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index f6d2e8e650be..5ce7879cb58d 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -38,11 +39,15 @@ #define IPMMU_CTX_MAX 1 +struct ipmmu_features { + bool use_ns_alias_offset; +}; + struct ipmmu_vmsa_device { struct device *dev; void __iomem *base; struct iommu_device iommu; - + const struct ipmmu_features *features; unsigned int num_utlbs; spinlock_t lock; /* Protects ctx and domains[] */ DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); @@ -817,6 +822,21 @@ static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0); } +static const struct ipmmu_features ipmmu_features_default = { + .use_ns_alias_offset = true, +}; + +static const struct of_device_id ipmmu_of_ids[] = { + { + .compatible = "renesas,ipmmu-vmsa", + .data = &ipmmu_features_default, + }, { + /* Terminator */ + }, +}; + +MODULE_DEVICE_TABLE(of, ipmmu_of_ids); + static int ipmmu_probe(struct platform_device *pdev) { struct ipmmu_vmsa_device *mmu; @@ -834,6 +854,7 @@ static int ipmmu_probe(struct platform_device *pdev) mmu->num_utlbs = 32; spin_lock_init(&mmu->lock); bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); + mmu->features = of_device_get_match_data(&pdev->dev); /* Map I/O memory and request IRQ. */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -853,7 +874,8 @@ static int ipmmu_probe(struct platform_device *pdev) * Offset the registers base unconditionally to point to the non-secure * alias space for now. */ - mmu->base += IM_NS_ALIAS_OFFSET; + if (mmu->features->use_ns_alias_offset) + mmu->base += IM_NS_ALIAS_OFFSET; irq = platform_get_irq(pdev, 0); if (irq < 0) { @@ -907,11 +929,6 @@ static int ipmmu_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id ipmmu_of_ids[] = { - { .compatible = "renesas,ipmmu-vmsa", }, - { } -}; - static struct platform_driver ipmmu_driver = { .driver = { .name = "ipmmu-vmsa", -- cgit From fd5140e29a59e04a6c3e8cc56536bda3e60bbf49 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:29:36 +0900 Subject: iommu/ipmmu-vmsa: Add optional root device feature Add root device handling to the IPMMU driver by allowing certain DT compat strings to enable has_cache_leaf_nodes that in turn will support both root devices with interrupts and leaf devices that face the actual IPMMU consumer devices. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 90 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 17 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 5ce7879cb58d..5db853b92d3b 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -41,12 +41,14 @@ struct ipmmu_features { bool use_ns_alias_offset; + bool has_cache_leaf_nodes; }; struct ipmmu_vmsa_device { struct device *dev; void __iomem *base; struct iommu_device iommu; + struct ipmmu_vmsa_device *root; const struct ipmmu_features *features; unsigned int num_utlbs; spinlock_t lock; /* Protects ctx and domains[] */ @@ -198,6 +200,36 @@ static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) #define IMUASID_ASID0_MASK (0xff << 0) #define IMUASID_ASID0_SHIFT 0 +/* ----------------------------------------------------------------------------- + * Root device handling + */ + +static struct platform_driver ipmmu_driver; + +static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu) +{ + return mmu->root == mmu; +} + +static int __ipmmu_check_device(struct device *dev, void *data) +{ + struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); + struct ipmmu_vmsa_device **rootp = data; + + if (ipmmu_is_root(mmu)) + *rootp = mmu; + + return 0; +} + +static struct ipmmu_vmsa_device *ipmmu_find_root(void) +{ + struct ipmmu_vmsa_device *root = NULL; + + return driver_for_each_device(&ipmmu_driver.driver, NULL, &root, + __ipmmu_check_device) == 0 ? root : NULL; +} + /* ----------------------------------------------------------------------------- * Read/Write Access */ @@ -215,13 +247,15 @@ static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg) { - return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg); + return ipmmu_read(domain->mmu->root, + domain->context_id * IM_CTX_SIZE + reg); } static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg, u32 data) { - ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data); + ipmmu_write(domain->mmu->root, + domain->context_id * IM_CTX_SIZE + reg, data); } /* ----------------------------------------------------------------------------- @@ -369,12 +403,12 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) * TODO: Add support for coherent walk through CCI with DVM and remove * cache handling. For now, delegate it to the io-pgtable code. */ - domain->cfg.iommu_dev = domain->mmu->dev; + domain->cfg.iommu_dev = domain->mmu->root->dev; /* * Find an unused context. */ - ret = ipmmu_domain_allocate_context(domain->mmu, domain); + ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); if (ret == IPMMU_CTX_MAX) return -EBUSY; @@ -383,7 +417,8 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, domain); if (!domain->iop) { - ipmmu_domain_free_context(domain->mmu, domain->context_id); + ipmmu_domain_free_context(domain->mmu->root, + domain->context_id); return -EINVAL; } @@ -437,7 +472,7 @@ static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) */ ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH); ipmmu_tlb_sync(domain); - ipmmu_domain_free_context(domain->mmu, domain->context_id); + ipmmu_domain_free_context(domain->mmu->root, domain->context_id); } /* ----------------------------------------------------------------------------- @@ -824,6 +859,7 @@ static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) static const struct ipmmu_features ipmmu_features_default = { .use_ns_alias_offset = true, + .has_cache_leaf_nodes = false, }; static const struct of_device_id ipmmu_of_ids[] = { @@ -878,19 +914,39 @@ static int ipmmu_probe(struct platform_device *pdev) mmu->base += IM_NS_ALIAS_OFFSET; irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "no IRQ found\n"); - return irq; - } - ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, - dev_name(&pdev->dev), mmu); - if (ret < 0) { - dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); - return ret; - } + /* + * Determine if this IPMMU instance is a root device by checking for + * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. + */ + if (!mmu->features->has_cache_leaf_nodes || + !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL)) + mmu->root = mmu; + else + mmu->root = ipmmu_find_root(); - ipmmu_device_reset(mmu); + /* + * Wait until the root device has been registered for sure. + */ + if (!mmu->root) + return -EPROBE_DEFER; + + /* Root devices have mandatory IRQs */ + if (ipmmu_is_root(mmu)) { + if (irq < 0) { + dev_err(&pdev->dev, "no IRQ found\n"); + return irq; + } + + ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, + dev_name(&pdev->dev), mmu); + if (ret < 0) { + dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); + return ret; + } + + ipmmu_device_reset(mmu); + } ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, dev_name(&pdev->dev)); -- cgit From 5fd163416fb7b6592521c39f867d5ae6360e7924 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:29:46 +0900 Subject: iommu/ipmmu-vmsa: Enable multi context support Add support for up to 8 contexts. Each context is mapped to one domain. One domain is assigned one or more slave devices. Contexts are allocated dynamically and slave devices are grouped together based on which IPMMU device they are connected to. This makes slave devices tied to the same IPMMU device share the same IOVA space. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 5db853b92d3b..c70efd80f740 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -37,11 +37,12 @@ #include "io-pgtable.h" -#define IPMMU_CTX_MAX 1 +#define IPMMU_CTX_MAX 8 struct ipmmu_features { bool use_ns_alias_offset; bool has_cache_leaf_nodes; + unsigned int number_of_contexts; }; struct ipmmu_vmsa_device { @@ -51,6 +52,7 @@ struct ipmmu_vmsa_device { struct ipmmu_vmsa_device *root; const struct ipmmu_features *features; unsigned int num_utlbs; + unsigned int num_ctx; spinlock_t lock; /* Protects ctx and domains[] */ DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX]; @@ -352,11 +354,12 @@ static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu, spin_lock_irqsave(&mmu->lock, flags); - ret = find_first_zero_bit(mmu->ctx, IPMMU_CTX_MAX); - if (ret != IPMMU_CTX_MAX) { + ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx); + if (ret != mmu->num_ctx) { mmu->domains[ret] = domain; set_bit(ret, mmu->ctx); - } + } else + ret = -EBUSY; spin_unlock_irqrestore(&mmu->lock, flags); @@ -409,8 +412,8 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) * Find an unused context. */ ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); - if (ret == IPMMU_CTX_MAX) - return -EBUSY; + if (ret < 0) + return ret; domain->context_id = ret; @@ -539,7 +542,7 @@ static irqreturn_t ipmmu_irq(int irq, void *dev) /* * Check interrupts for all active contexts. */ - for (i = 0; i < IPMMU_CTX_MAX; i++) { + for (i = 0; i < mmu->num_ctx; i++) { if (!mmu->domains[i]) continue; if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED) @@ -624,6 +627,13 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, /* The domain hasn't been used yet, initialize it. */ domain->mmu = mmu; ret = ipmmu_domain_init_context(domain); + if (ret < 0) { + dev_err(dev, "Unable to initialize IPMMU context\n"); + domain->mmu = NULL; + } else { + dev_info(dev, "Using IPMMU context %u\n", + domain->context_id); + } } else if (domain->mmu != mmu) { /* * Something is wrong, we can't attach two devices using @@ -853,13 +863,14 @@ static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) unsigned int i; /* Disable all contexts. */ - for (i = 0; i < 4; ++i) + for (i = 0; i < mmu->num_ctx; ++i) ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0); } static const struct ipmmu_features ipmmu_features_default = { .use_ns_alias_offset = true, .has_cache_leaf_nodes = false, + .number_of_contexts = 1, /* software only tested with one context */ }; static const struct of_device_id ipmmu_of_ids[] = { @@ -913,6 +924,9 @@ static int ipmmu_probe(struct platform_device *pdev) if (mmu->features->use_ns_alias_offset) mmu->base += IM_NS_ALIAS_OFFSET; + mmu->num_ctx = min_t(unsigned int, IPMMU_CTX_MAX, + mmu->features->number_of_contexts); + irq = platform_get_irq(pdev, 0); /* -- cgit From cda52fcd999f389c6f24f079910a62e53912d411 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:29:57 +0900 Subject: iommu/ipmmu-vmsa: Make use of IOMMU_OF_DECLARE() Hook up IOMMU_OF_DECLARE() support in case CONFIG_IOMMU_DMA is enabled. The only current supported case for 32-bit ARM is disabled, however for 64-bit ARM usage of OF is required. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 50 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index c70efd80f740..9cde61970bb5 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -962,17 +963,30 @@ static int ipmmu_probe(struct platform_device *pdev) ipmmu_device_reset(mmu); } - ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, - dev_name(&pdev->dev)); - if (ret) - return ret; + /* + * Register the IPMMU to the IOMMU subsystem in the following cases: + * - R-Car Gen2 IPMMU (all devices registered) + * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) + */ + if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { + ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, + dev_name(&pdev->dev)); + if (ret) + return ret; - iommu_device_set_ops(&mmu->iommu, &ipmmu_ops); - iommu_device_set_fwnode(&mmu->iommu, &pdev->dev.of_node->fwnode); + iommu_device_set_ops(&mmu->iommu, &ipmmu_ops); + iommu_device_set_fwnode(&mmu->iommu, + &pdev->dev.of_node->fwnode); - ret = iommu_device_register(&mmu->iommu); - if (ret) - return ret; + ret = iommu_device_register(&mmu->iommu); + if (ret) + return ret; + +#if defined(CONFIG_IOMMU_DMA) + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &ipmmu_ops); +#endif + } /* * We can't create the ARM mapping here as it requires the bus to have @@ -1010,15 +1024,22 @@ static struct platform_driver ipmmu_driver = { static int __init ipmmu_init(void) { + static bool setup_done; int ret; + if (setup_done) + return 0; + ret = platform_driver_register(&ipmmu_driver); if (ret < 0) return ret; +#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) if (!iommu_present(&platform_bus_type)) bus_set_iommu(&platform_bus_type, &ipmmu_ops); +#endif + setup_done = true; return 0; } @@ -1030,6 +1051,17 @@ static void __exit ipmmu_exit(void) subsys_initcall(ipmmu_init); module_exit(ipmmu_exit); +#ifdef CONFIG_IOMMU_DMA +static int __init ipmmu_vmsa_iommu_of_setup(struct device_node *np) +{ + ipmmu_init(); + return 0; +} + +IOMMU_OF_DECLARE(ipmmu_vmsa_iommu_of, "renesas,ipmmu-vmsa", + ipmmu_vmsa_iommu_of_setup); +#endif + MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU"); MODULE_AUTHOR("Laurent Pinchart "); MODULE_LICENSE("GPL v2"); -- cgit From 1c894225bf5b1cdffac0c6ef935b61273203d7d5 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:30:07 +0900 Subject: iommu/ipmmu-vmsa: IPMMU device is 40-bit bus master The r8a7795 IPMMU supports 40-bit bus mastering. Both the coherent DMA mask and the streaming DMA mask are set to unlock the 40-bit address space for coherent allocations and streaming operations. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 9cde61970bb5..6b74ec62f4b4 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -903,6 +903,7 @@ static int ipmmu_probe(struct platform_device *pdev) spin_lock_init(&mmu->lock); bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); mmu->features = of_device_get_match_data(&pdev->dev); + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); /* Map I/O memory and request IRQ. */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); -- cgit From d574893aee991efa67fefa849347c49de5df8108 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:30:18 +0900 Subject: iommu/ipmmu-vmsa: Write IMCTR twice Write IMCTR both in the root device and the leaf node. To allow access of IMCTR introduce the following function: - ipmmu_ctx_write_all() While at it also rename context functions: - ipmmu_ctx_read() -> ipmmu_ctx_read_root() - ipmmu_ctx_write() -> ipmmu_ctx_write_root() Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 56 +++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 6b74ec62f4b4..7587017972b0 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -248,19 +248,31 @@ static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, iowrite32(data, mmu->base + offset); } -static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg) +static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain, + unsigned int reg) { return ipmmu_read(domain->mmu->root, domain->context_id * IM_CTX_SIZE + reg); } -static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg, - u32 data) +static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain, + unsigned int reg, u32 data) { ipmmu_write(domain->mmu->root, domain->context_id * IM_CTX_SIZE + reg, data); } +static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain, + unsigned int reg, u32 data) +{ + if (domain->mmu != domain->mmu->root) + ipmmu_write(domain->mmu, + domain->context_id * IM_CTX_SIZE + reg, data); + + ipmmu_write(domain->mmu->root, + domain->context_id * IM_CTX_SIZE + reg, data); +} + /* ----------------------------------------------------------------------------- * TLB and microTLB Management */ @@ -270,7 +282,7 @@ static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) { unsigned int count = 0; - while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) { + while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) { cpu_relax(); if (++count == TLB_LOOP_TIMEOUT) { dev_err_ratelimited(domain->mmu->dev, @@ -285,9 +297,9 @@ static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) { u32 reg; - reg = ipmmu_ctx_read(domain, IMCTR); + reg = ipmmu_ctx_read_root(domain, IMCTR); reg |= IMCTR_FLUSH; - ipmmu_ctx_write(domain, IMCTR, reg); + ipmmu_ctx_write_all(domain, IMCTR, reg); ipmmu_tlb_sync(domain); } @@ -428,31 +440,32 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) /* TTBR0 */ ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; - ipmmu_ctx_write(domain, IMTTLBR0, ttbr); - ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32); + ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr); + ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32); /* * TTBCR * We use long descriptors with inner-shareable WBWA tables and allocate * the whole 32-bit VA space to TTBR0. */ - ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE | - IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | - IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); + ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | + IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | + IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); /* MAIR0 */ - ipmmu_ctx_write(domain, IMMAIR0, domain->cfg.arm_lpae_s1_cfg.mair[0]); + ipmmu_ctx_write_root(domain, IMMAIR0, + domain->cfg.arm_lpae_s1_cfg.mair[0]); /* IMBUSCR */ - ipmmu_ctx_write(domain, IMBUSCR, - ipmmu_ctx_read(domain, IMBUSCR) & - ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); + ipmmu_ctx_write_root(domain, IMBUSCR, + ipmmu_ctx_read_root(domain, IMBUSCR) & + ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); /* * IMSTR * Clear all interrupt flags. */ - ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR)); + ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR)); /* * IMCTR @@ -461,7 +474,8 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) * software management as we have no use for it. Flush the TLB as * required when modifying the context registers. */ - ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); + ipmmu_ctx_write_all(domain, IMCTR, + IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); return 0; } @@ -474,7 +488,7 @@ static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) * * TODO: Is TLB flush really needed ? */ - ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH); + ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH); ipmmu_tlb_sync(domain); ipmmu_domain_free_context(domain->mmu->root, domain->context_id); } @@ -490,11 +504,11 @@ static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) u32 status; u32 iova; - status = ipmmu_ctx_read(domain, IMSTR); + status = ipmmu_ctx_read_root(domain, IMSTR); if (!(status & err_mask)) return IRQ_NONE; - iova = ipmmu_ctx_read(domain, IMEAR); + iova = ipmmu_ctx_read_root(domain, IMEAR); /* * Clear the error status flags. Unlike traditional interrupt flag @@ -502,7 +516,7 @@ static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) * seems to require 0. The error address register must be read before, * otherwise its value will be 0. */ - ipmmu_ctx_write(domain, IMSTR, 0); + ipmmu_ctx_write_root(domain, IMSTR, 0); /* Log fatal errors. */ if (status & IMSTR_MHIT) -- cgit From f5c858912acd2b17059ebe6f34abac183bdfbf80 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:30:28 +0900 Subject: iommu/ipmmu-vmsa: Make IMBUSCTR setup optional Introduce a feature to allow opt-out of setting up IMBUSCR. The default case is unchanged. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 7587017972b0..49f2c697b108 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -44,6 +44,7 @@ struct ipmmu_features { bool use_ns_alias_offset; bool has_cache_leaf_nodes; unsigned int number_of_contexts; + bool setup_imbuscr; }; struct ipmmu_vmsa_device { @@ -457,9 +458,10 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) domain->cfg.arm_lpae_s1_cfg.mair[0]); /* IMBUSCR */ - ipmmu_ctx_write_root(domain, IMBUSCR, - ipmmu_ctx_read_root(domain, IMBUSCR) & - ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); + if (domain->mmu->features->setup_imbuscr) + ipmmu_ctx_write_root(domain, IMBUSCR, + ipmmu_ctx_read_root(domain, IMBUSCR) & + ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); /* * IMSTR @@ -886,6 +888,7 @@ static const struct ipmmu_features ipmmu_features_default = { .use_ns_alias_offset = true, .has_cache_leaf_nodes = false, .number_of_contexts = 1, /* software only tested with one context */ + .setup_imbuscr = true, }; static const struct of_device_id ipmmu_of_ids[] = { -- cgit From c295f504fb5a38abbb4094e687ee333a75613a0c Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:30:39 +0900 Subject: iommu/ipmmu-vmsa: Allow two bit SL0 Introduce support for two bit SL0 bitfield in IMTTBCR by using a separate feature flag. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 49f2c697b108..65ad6910cb70 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -45,6 +45,7 @@ struct ipmmu_features { bool has_cache_leaf_nodes; unsigned int number_of_contexts; bool setup_imbuscr; + bool twobit_imttbcr_sl0; }; struct ipmmu_vmsa_device { @@ -144,6 +145,10 @@ static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) #define IMTTBCR_TSZ0_MASK (7 << 0) #define IMTTBCR_TSZ0_SHIFT O +#define IMTTBCR_SL0_TWOBIT_LVL_3 (0 << 6) +#define IMTTBCR_SL0_TWOBIT_LVL_2 (1 << 6) +#define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) + #define IMBUSCR 0x000c #define IMBUSCR_DVM (1 << 2) #define IMBUSCR_BUSSEL_SYS (0 << 0) @@ -396,6 +401,7 @@ static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu, static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) { u64 ttbr; + u32 tmp; int ret; /* @@ -449,9 +455,14 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) * We use long descriptors with inner-shareable WBWA tables and allocate * the whole 32-bit VA space to TTBR0. */ + if (domain->mmu->features->twobit_imttbcr_sl0) + tmp = IMTTBCR_SL0_TWOBIT_LVL_1; + else + tmp = IMTTBCR_SL0_LVL_1; + ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | - IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); + IMTTBCR_IRGN0_WB_WA | tmp); /* MAIR0 */ ipmmu_ctx_write_root(domain, IMMAIR0, @@ -889,6 +900,7 @@ static const struct ipmmu_features ipmmu_features_default = { .has_cache_leaf_nodes = false, .number_of_contexts = 1, /* software only tested with one context */ .setup_imbuscr = true, + .twobit_imttbcr_sl0 = false, }; static const struct of_device_id ipmmu_of_ids[] = { -- cgit From 58b8e8bf409236cdea379b8a3ab5d7b85a003d22 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 16 Oct 2017 21:30:50 +0900 Subject: iommu/ipmmu-vmsa: Hook up r8a7795 DT matching code Tie in r8a7795 features and update the IOMMU_OF_DECLARE compat string to include the updated compat string. Signed-off-by: Magnus Damm Signed-off-by: Alex Williamson --- drivers/iommu/ipmmu-vmsa.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 65ad6910cb70..8dce3a9de9d8 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -25,6 +25,7 @@ #include #include #include +#include #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) #include @@ -749,9 +750,24 @@ static int ipmmu_init_platform_device(struct device *dev, return 0; } +static bool ipmmu_slave_whitelist(struct device *dev) +{ + /* By default, do not allow use of IPMMU */ + return false; +} + +static const struct soc_device_attribute soc_r8a7795[] = { + { .soc_id = "r8a7795", }, + { /* sentinel */ } +}; + static int ipmmu_of_xlate(struct device *dev, struct of_phandle_args *spec) { + /* For R-Car Gen3 use a white list to opt-in slave devices */ + if (soc_device_match(soc_r8a7795) && !ipmmu_slave_whitelist(dev)) + return -ENODEV; + iommu_fwspec_add_ids(dev, spec->args, 1); /* Initialize once - xlate() will call multiple times */ @@ -903,10 +919,21 @@ static const struct ipmmu_features ipmmu_features_default = { .twobit_imttbcr_sl0 = false, }; +static const struct ipmmu_features ipmmu_features_r8a7795 = { + .use_ns_alias_offset = false, + .has_cache_leaf_nodes = true, + .number_of_contexts = 8, + .setup_imbuscr = false, + .twobit_imttbcr_sl0 = true, +}; + static const struct of_device_id ipmmu_of_ids[] = { { .compatible = "renesas,ipmmu-vmsa", .data = &ipmmu_features_default, + }, { + .compatible = "renesas,ipmmu-r8a7795", + .data = &ipmmu_features_r8a7795, }, { /* Terminator */ }, @@ -1090,6 +1117,8 @@ static int __init ipmmu_vmsa_iommu_of_setup(struct device_node *np) IOMMU_OF_DECLARE(ipmmu_vmsa_iommu_of, "renesas,ipmmu-vmsa", ipmmu_vmsa_iommu_of_setup); +IOMMU_OF_DECLARE(ipmmu_r8a7795_iommu_of, "renesas,ipmmu-r8a7795", + ipmmu_vmsa_iommu_of_setup); #endif MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU"); -- cgit From 395df08d2e1de238a9c8c33fdcd0e2160efd63a9 Mon Sep 17 00:00:00 2001 From: Matthias Brugger Date: Mon, 30 Oct 2017 12:37:55 +0100 Subject: iommu/mediatek: Fix driver name There exist two Mediatek iommu drivers for the two different generations of the device. But both drivers have the same name "mtk-iommu". This breaks the registration of the second driver: Error: Driver 'mtk-iommu' is already registered, aborting... Fix this by changing the name for first generation to "mtk-iommu-v1". Fixes: b17336c55d89 ("iommu/mediatek: add support for mtk iommu generation one HW") Signed-off-by: Matthias Brugger Signed-off-by: Alex Williamson --- drivers/iommu/mtk_iommu_v1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index bc1efbfb9ddf..542930cd183d 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -708,7 +708,7 @@ static struct platform_driver mtk_iommu_driver = { .probe = mtk_iommu_probe, .remove = mtk_iommu_remove, .driver = { - .name = "mtk-iommu", + .name = "mtk-iommu-v1", .of_match_table = mtk_iommu_of_ids, .pm = &mtk_iommu_pm_ops, } -- cgit From 94e2cc4dba39efbb63e84cd4f7243627c16ceac5 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 21 Sep 2017 17:21:40 +0200 Subject: iommu/iova: Use raw_cpu_ptr() instead of get_cpu_ptr() for ->fq get_cpu_ptr() disabled preemption and returns the ->fq object of the current CPU. raw_cpu_ptr() does the same except that it not disable preemption which means the scheduler can move it to another CPU after it obtained the per-CPU object. In this case this is not bad because the data structure itself is protected with a spin_lock. This change shouldn't matter however on RT it does because the sleeping lock can't be accessed with disabled preemption. Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Reported-by: vinadhy@gmail.com Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Alex Williamson --- drivers/iommu/iova.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 84bda3a4dafc..466aaa8ba841 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -542,7 +542,7 @@ void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, unsigned long data) { - struct iova_fq *fq = get_cpu_ptr(iovad->fq); + struct iova_fq *fq = raw_cpu_ptr(iovad->fq); unsigned long flags; unsigned idx; @@ -572,8 +572,6 @@ void queue_iova(struct iova_domain *iovad, if (atomic_cmpxchg(&iovad->fq_timer_on, 0, 1) == 0) mod_timer(&iovad->fq_timer, jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); - - put_cpu_ptr(iovad->fq); } EXPORT_SYMBOL_GPL(queue_iova); -- cgit