summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig13
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/amd_iommu.c313
-rw-r--r--drivers/iommu/amd_iommu_init.c223
-rw-r--r--drivers/iommu/amd_iommu_proto.h2
-rw-r--r--drivers/iommu/amd_iommu_types.h55
-rw-r--r--drivers/iommu/amd_iommu_v2.c18
-rw-r--r--drivers/iommu/arm-smmu-regs.h220
-rw-r--r--drivers/iommu/arm-smmu-v3.c7
-rw-r--r--drivers/iommu/arm-smmu.c384
-rw-r--r--drivers/iommu/dmar.c2
-rw-r--r--drivers/iommu/exynos-iommu.c44
-rw-r--r--drivers/iommu/fsl_pamu.c27
-rw-r--r--drivers/iommu/fsl_pamu_domain.c28
-rw-r--r--drivers/iommu/intel-iommu.c280
-rw-r--r--drivers/iommu/intel-svm.c14
-rw-r--r--drivers/iommu/iommu.c59
-rw-r--r--drivers/iommu/iova.c183
-rw-r--r--drivers/iommu/ipmmu-vmsa.c242
-rw-r--r--drivers/iommu/msm_iommu.c15
-rw-r--r--drivers/iommu/mtk_iommu.c214
-rw-r--r--drivers/iommu/mtk_iommu.h9
-rw-r--r--drivers/iommu/of_iommu.c144
-rw-r--r--drivers/iommu/omap-iommu.c125
-rw-r--r--drivers/iommu/omap-iommu.h1
-rw-r--r--drivers/iommu/qcom_iommu.c930
-rw-r--r--drivers/iommu/rockchip-iommu.c52
-rw-r--r--drivers/iommu/s390-iommu.c37
-rw-r--r--drivers/iommu/tegra-gart.c45
-rw-r--r--drivers/iommu/tegra-smmu.c39
30 files changed, 2525 insertions, 1201 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index f73ff28f77e2..49bd2ab8c507 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -76,6 +76,8 @@ config IOMMU_DMA
config FSL_PAMU
bool "Freescale IOMMU support"
+ depends on PCI
+ depends on PHYS_64BIT
depends on PPC_E500MC || (COMPILE_TEST && PPC)
select IOMMU_API
select GENERIC_ALLOCATOR
@@ -253,6 +255,7 @@ config TEGRA_IOMMU_SMMU
config EXYNOS_IOMMU
bool "Exynos IOMMU Support"
depends on ARCH_EXYNOS && MMU
+ depends on !CPU_BIG_ENDIAN # revisit driver if we can enable big-endian ptes
select IOMMU_API
select ARM_DMA_USE_IOMMU
help
@@ -367,4 +370,14 @@ config MTK_IOMMU_V1
if unsure, say N here.
+config QCOM_IOMMU
+ # Note: iommu drivers cannot (yet?) be built as modules
+ bool "Qualcomm IOMMU Support"
+ depends on ARCH_QCOM || COMPILE_TEST
+ select IOMMU_API
+ select IOMMU_IO_PGTABLE_LPAE
+ select ARM_DMA_USE_IOMMU
+ help
+ Support for IOMMU on certain Qualcomm SoCs.
+
endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 195f7b997d8e..b910aea813a1 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -27,3 +27,4 @@ obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
+obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 4ad7e5e31943..51f8215877f5 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -103,29 +103,6 @@ int amd_iommu_max_glx_val = -1;
static const struct dma_map_ops amd_iommu_dma_ops;
/*
- * This struct contains device specific data for the IOMMU
- */
-struct iommu_dev_data {
- struct list_head list; /* For domain->dev_list */
- struct list_head dev_data_list; /* For global dev_data_list */
- struct protection_domain *domain; /* Domain the device is bound to */
- u16 devid; /* PCI Device ID */
- u16 alias; /* Alias Device ID */
- bool iommu_v2; /* Device can make use of IOMMUv2 */
- bool passthrough; /* Device is identity mapped */
- struct {
- bool enabled;
- int qdep;
- } ats; /* ATS state */
- bool pri_tlp; /* PASID TLB required for
- PPR completions */
- u32 errata; /* Bitmap for errata to apply */
- bool use_vapic; /* Enable device to use vapic mode */
-
- struct ratelimit_state rs; /* Ratelimit IOPF messages */
-};
-
-/*
* general struct to manage commands send to an IOMMU
*/
struct iommu_cmd {
@@ -137,20 +114,7 @@ struct kmem_cache *amd_iommu_irq_cache;
static void update_domain(struct protection_domain *domain);
static int protection_domain_init(struct protection_domain *domain);
static void detach_device(struct device *dev);
-
-#define FLUSH_QUEUE_SIZE 256
-
-struct flush_queue_entry {
- unsigned long iova_pfn;
- unsigned long pages;
- u64 counter; /* Flush counter when this entry was added to the queue */
-};
-
-struct flush_queue {
- struct flush_queue_entry *entries;
- unsigned head, tail;
- spinlock_t lock;
-};
+static void iova_domain_flush_tlb(struct iova_domain *iovad);
/*
* Data container for a dma_ops specific protection domain
@@ -161,36 +125,6 @@ struct dma_ops_domain {
/* IOVA RB-Tree */
struct iova_domain iovad;
-
- struct flush_queue __percpu *flush_queue;
-
- /*
- * We need two counter here to be race-free wrt. IOTLB flushing and
- * adding entries to the flush queue.
- *
- * The flush_start_cnt is incremented _before_ the IOTLB flush starts.
- * New entries added to the flush ring-buffer get their 'counter' value
- * from here. This way we can make sure that entries added to the queue
- * (or other per-cpu queues of the same domain) while the TLB is about
- * to be flushed are not considered to be flushed already.
- */
- atomic64_t flush_start_cnt;
-
- /*
- * The flush_finish_cnt is incremented when an IOTLB flush is complete.
- * This value is always smaller than flush_start_cnt. The queue_add
- * function frees all IOVAs that have a counter value smaller than
- * flush_finish_cnt. This makes sure that we only free IOVAs that are
- * flushed out of the IOTLB of the domain.
- */
- atomic64_t flush_finish_cnt;
-
- /*
- * Timer to make sure we don't keep IOVAs around unflushed
- * for too long
- */
- struct timer_list flush_timer;
- atomic_t flush_timer_on;
};
static struct iova_domain reserved_iova_ranges;
@@ -371,19 +305,25 @@ static u16 get_alias(struct device *dev)
static struct iommu_dev_data *find_dev_data(u16 devid)
{
struct iommu_dev_data *dev_data;
+ struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
dev_data = search_dev_data(devid);
- if (dev_data == NULL)
+ if (dev_data == NULL) {
dev_data = alloc_dev_data(devid);
+ if (translation_pre_enabled(iommu))
+ dev_data->defer_attach = true;
+ }
+
return dev_data;
}
-static struct iommu_dev_data *get_dev_data(struct device *dev)
+struct iommu_dev_data *get_dev_data(struct device *dev)
{
return dev->archdata.iommu;
}
+EXPORT_SYMBOL(get_dev_data);
/*
* Find or create an IOMMU group for a acpihid device.
@@ -1167,7 +1107,7 @@ static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
return iommu_queue_command(iommu, &cmd);
}
-static void iommu_flush_dte_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)
{
u32 devid;
@@ -1181,7 +1121,7 @@ static void iommu_flush_dte_all(struct amd_iommu *iommu)
* This function uses heavy locking and may disable irqs for some time. But
* this is no issue because it is only called during resume.
*/
-static void iommu_flush_tlb_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
{
u32 dom_id;
@@ -1195,7 +1135,7 @@ static void iommu_flush_tlb_all(struct amd_iommu *iommu)
iommu_completion_wait(iommu);
}
-static void iommu_flush_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_all(struct amd_iommu *iommu)
{
struct iommu_cmd cmd;
@@ -1214,7 +1154,7 @@ static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
iommu_queue_command(iommu, &cmd);
}
-static void iommu_flush_irt_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
{
u32 devid;
@@ -1227,11 +1167,11 @@ static void iommu_flush_irt_all(struct amd_iommu *iommu)
void iommu_flush_all_caches(struct amd_iommu *iommu)
{
if (iommu_feature(iommu, FEATURE_IA)) {
- iommu_flush_all(iommu);
+ amd_iommu_flush_all(iommu);
} else {
- iommu_flush_dte_all(iommu);
- iommu_flush_irt_all(iommu);
- iommu_flush_tlb_all(iommu);
+ amd_iommu_flush_dte_all(iommu);
+ amd_iommu_flush_irt_all(iommu);
+ amd_iommu_flush_tlb_all(iommu);
}
}
@@ -1539,9 +1479,9 @@ static int iommu_map_page(struct protection_domain *dom,
if (count > 1) {
__pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
- __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
+ __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
} else
- __pte = __sme_set(phys_addr) | IOMMU_PTE_P | IOMMU_PTE_FC;
+ __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
if (prot & IOMMU_PROT_IR)
__pte |= IOMMU_PTE_IR;
@@ -1790,178 +1730,19 @@ static void free_gcr3_table(struct protection_domain *domain)
free_page((unsigned long)domain->gcr3_tbl);
}
-static void dma_ops_domain_free_flush_queue(struct dma_ops_domain *dom)
-{
- int cpu;
-
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue;
-
- queue = per_cpu_ptr(dom->flush_queue, cpu);
- kfree(queue->entries);
- }
-
- free_percpu(dom->flush_queue);
-
- dom->flush_queue = NULL;
-}
-
-static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
-{
- int cpu;
-
- atomic64_set(&dom->flush_start_cnt, 0);
- atomic64_set(&dom->flush_finish_cnt, 0);
-
- dom->flush_queue = alloc_percpu(struct flush_queue);
- if (!dom->flush_queue)
- return -ENOMEM;
-
- /* First make sure everything is cleared */
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue;
-
- queue = per_cpu_ptr(dom->flush_queue, cpu);
- queue->head = 0;
- queue->tail = 0;
- queue->entries = NULL;
- }
-
- /* Now start doing the allocation */
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue;
-
- queue = per_cpu_ptr(dom->flush_queue, cpu);
- queue->entries = kzalloc(FLUSH_QUEUE_SIZE * sizeof(*queue->entries),
- GFP_KERNEL);
- if (!queue->entries) {
- dma_ops_domain_free_flush_queue(dom);
- return -ENOMEM;
- }
-
- spin_lock_init(&queue->lock);
- }
-
- return 0;
-}
-
static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
{
- atomic64_inc(&dom->flush_start_cnt);
domain_flush_tlb(&dom->domain);
domain_flush_complete(&dom->domain);
- atomic64_inc(&dom->flush_finish_cnt);
-}
-
-static inline bool queue_ring_full(struct flush_queue *queue)
-{
- assert_spin_locked(&queue->lock);
-
- return (((queue->tail + 1) % FLUSH_QUEUE_SIZE) == queue->head);
-}
-
-#define queue_ring_for_each(i, q) \
- for (i = (q)->head; i != (q)->tail; i = (i + 1) % FLUSH_QUEUE_SIZE)
-
-static inline unsigned queue_ring_add(struct flush_queue *queue)
-{
- unsigned idx = queue->tail;
-
- assert_spin_locked(&queue->lock);
- queue->tail = (idx + 1) % FLUSH_QUEUE_SIZE;
-
- return idx;
-}
-
-static inline void queue_ring_remove_head(struct flush_queue *queue)
-{
- assert_spin_locked(&queue->lock);
- queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
-}
-
-static void queue_ring_free_flushed(struct dma_ops_domain *dom,
- struct flush_queue *queue)
-{
- u64 counter = atomic64_read(&dom->flush_finish_cnt);
- int idx;
-
- queue_ring_for_each(idx, queue) {
- /*
- * This assumes that counter values in the ring-buffer are
- * monotonously rising.
- */
- if (queue->entries[idx].counter >= counter)
- break;
-
- free_iova_fast(&dom->iovad,
- queue->entries[idx].iova_pfn,
- queue->entries[idx].pages);
-
- queue_ring_remove_head(queue);
- }
-}
-
-static void queue_add(struct dma_ops_domain *dom,
- unsigned long address, unsigned long pages)
-{
- struct flush_queue *queue;
- unsigned long flags;
- int idx;
-
- pages = __roundup_pow_of_two(pages);
- address >>= PAGE_SHIFT;
-
- queue = get_cpu_ptr(dom->flush_queue);
- spin_lock_irqsave(&queue->lock, flags);
-
- /*
- * First remove the enries from the ring-buffer that are already
- * flushed to make the below queue_ring_full() check less likely
- */
- queue_ring_free_flushed(dom, queue);
-
- /*
- * When ring-queue is full, flush the entries from the IOTLB so
- * that we can free all entries with queue_ring_free_flushed()
- * below.
- */
- if (queue_ring_full(queue)) {
- dma_ops_domain_flush_tlb(dom);
- queue_ring_free_flushed(dom, queue);
- }
-
- idx = queue_ring_add(queue);
-
- queue->entries[idx].iova_pfn = address;
- queue->entries[idx].pages = pages;
- queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
-
- spin_unlock_irqrestore(&queue->lock, flags);
-
- if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
- mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
-
- put_cpu_ptr(dom->flush_queue);
}
-static void queue_flush_timeout(unsigned long data)
+static void iova_domain_flush_tlb(struct iova_domain *iovad)
{
- struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
- int cpu;
+ struct dma_ops_domain *dom;
- atomic_set(&dom->flush_timer_on, 0);
+ dom = container_of(iovad, struct dma_ops_domain, iovad);
dma_ops_domain_flush_tlb(dom);
-
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue;
- unsigned long flags;
-
- queue = per_cpu_ptr(dom->flush_queue, cpu);
- spin_lock_irqsave(&queue->lock, flags);
- queue_ring_free_flushed(dom, queue);
- spin_unlock_irqrestore(&queue->lock, flags);
- }
}
/*
@@ -1975,11 +1756,6 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
del_domain_from_list(&dom->domain);
- if (timer_pending(&dom->flush_timer))
- del_timer(&dom->flush_timer);
-
- dma_ops_domain_free_flush_queue(dom);
-
put_iova_domain(&dom->iovad);
free_pagetable(&dom->domain);
@@ -2015,16 +1791,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
init_iova_domain(&dma_dom->iovad, PAGE_SIZE,
IOVA_START_PFN, DMA_32BIT_PFN);
- /* Initialize reserved ranges */
- copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
-
- if (dma_ops_domain_alloc_flush_queue(dma_dom))
+ if (init_iova_flush_queue(&dma_dom->iovad, iova_domain_flush_tlb, NULL))
goto free_dma_dom;
- setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
- (unsigned long)dma_dom);
-
- atomic_set(&dma_dom->flush_timer_on, 0);
+ /* Initialize reserved ranges */
+ copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
add_domain_to_list(&dma_dom->domain);
@@ -2055,7 +1826,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;
- pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
+ pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV;
flags = amd_iommu_dev_table[devid].data[1];
@@ -2088,8 +1859,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
flags |= tmp;
}
-
- flags &= ~(DTE_FLAG_SA | 0xffffULL);
+ flags &= ~DEV_DOMID_MASK;
flags |= domain->id;
amd_iommu_dev_table[devid].data[1] = flags;
@@ -2099,7 +1869,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
static void clear_dte_entry(u16 devid)
{
/* remove entry from the device table seen by the hardware */
- amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
+ amd_iommu_dev_table[devid].data[0] = DTE_FLAG_V | DTE_FLAG_TV;
amd_iommu_dev_table[devid].data[1] &= DTE_FLAG_MASK;
amd_iommu_apply_erratum_63(devid);
@@ -2480,11 +2250,21 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev)
static struct protection_domain *get_domain(struct device *dev)
{
struct protection_domain *domain;
+ struct iommu_domain *io_domain;
if (!check_device(dev))
return ERR_PTR(-EINVAL);
domain = get_dev_data(dev)->domain;
+ if (domain == NULL && get_dev_data(dev)->defer_attach) {
+ get_dev_data(dev)->defer_attach = false;
+ io_domain = iommu_get_domain_for_dev(dev);
+ domain = to_pdomain(io_domain);
+ attach_device(dev, domain);
+ }
+ if (domain == NULL)
+ return ERR_PTR(-EBUSY);
+
if (!dma_ops_domain(domain))
return ERR_PTR(-EBUSY);
@@ -2530,6 +2310,7 @@ static int dir2prot(enum dma_data_direction direction)
else
return 0;
}
+
/*
* This function contains common code for mapping of a physically
* contiguous memory region into DMA address space. It is used by all
@@ -2621,7 +2402,8 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
domain_flush_tlb(&dma_dom->domain);
domain_flush_complete(&dma_dom->domain);
} else {
- queue_add(dma_dom, dma_addr, pages);
+ pages = __roundup_pow_of_two(pages);
+ queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0);
}
}
@@ -3375,6 +3157,13 @@ static void amd_iommu_apply_resv_region(struct device *dev,
WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL);
}
+static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
+ struct device *dev)
+{
+ struct iommu_dev_data *dev_data = dev->archdata.iommu;
+ return dev_data->defer_attach;
+}
+
const struct iommu_ops amd_iommu_ops = {
.capable = amd_iommu_capable,
.domain_alloc = amd_iommu_domain_alloc,
@@ -3391,6 +3180,7 @@ const struct iommu_ops amd_iommu_ops = {
.get_resv_regions = amd_iommu_get_resv_regions,
.put_resv_regions = amd_iommu_put_resv_regions,
.apply_resv_region = amd_iommu_apply_resv_region,
+ .is_attach_deferred = amd_iommu_is_attach_deferred,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
};
@@ -3779,11 +3569,6 @@ EXPORT_SYMBOL(amd_iommu_device_info);
static struct irq_chip amd_ir_chip;
-#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
-#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
-#define DTE_IRQ_TABLE_LEN (8ULL << 1)
-#define DTE_IRQ_REMAP_ENABLE 1ULL
-
static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
{
u64 dte;
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 2292a6cece76..382de42b8359 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -29,7 +29,6 @@
#include <linux/export.h>
#include <linux/iommu.h>
#include <linux/kmemleak.h>
-#include <linux/crash_dump.h>
#include <linux/mem_encrypt.h>
#include <asm/pci-direct.h>
#include <asm/iommu.h>
@@ -39,6 +38,7 @@
#include <asm/io_apic.h>
#include <asm/irq_remapping.h>
+#include <linux/crash_dump.h>
#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
#include "irq_remapping.h"
@@ -197,6 +197,11 @@ spinlock_t amd_iommu_pd_lock;
* page table root pointer.
*/
struct dev_table_entry *amd_iommu_dev_table;
+/*
+ * Pointer to a device table which the content of old device table
+ * will be copied to. It's only be used in kdump kernel.
+ */
+static struct dev_table_entry *old_dev_tbl_cpy;
/*
* The alias table is a driver specific data structure which contains the
@@ -210,6 +215,7 @@ u16 *amd_iommu_alias_table;
* for a specific device. It is also indexed by the PCI device id.
*/
struct amd_iommu **amd_iommu_rlookup_table;
+EXPORT_SYMBOL(amd_iommu_rlookup_table);
/*
* This table is used to find the irq remapping table for a given device id
@@ -259,6 +265,28 @@ static int amd_iommu_enable_interrupts(void);
static int __init iommu_go_to_state(enum iommu_init_state state);
static void init_device_table_dma(void);
+static bool amd_iommu_pre_enabled = true;
+
+bool translation_pre_enabled(struct amd_iommu *iommu)
+{
+ return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
+}
+EXPORT_SYMBOL(translation_pre_enabled);
+
+static void clear_translation_pre_enabled(struct amd_iommu *iommu)
+{
+ iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
+}
+
+static void init_translation_status(struct amd_iommu *iommu)
+{
+ u32 ctrl;
+
+ ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
+ if (ctrl & (1<<CONTROL_IOMMU_EN))
+ iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
+}
+
static inline void update_last_devid(u16 devid)
{
if (devid > amd_iommu_last_bdf)
@@ -616,6 +644,14 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)
amd_iommu_reset_cmd_buffer(iommu);
}
+/*
+ * This function disables the command buffer
+ */
+static void iommu_disable_command_buffer(struct amd_iommu *iommu)
+{
+ iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
+}
+
static void __init free_command_buffer(struct amd_iommu *iommu)
{
free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
@@ -648,6 +684,14 @@ static void iommu_enable_event_buffer(struct amd_iommu *iommu)
iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
}
+/*
+ * This function disables the event log buffer
+ */
+static void iommu_disable_event_buffer(struct amd_iommu *iommu)
+{
+ iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
+}
+
static void __init free_event_buffer(struct amd_iommu *iommu)
{
free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
@@ -809,6 +853,96 @@ static int get_dev_entry_bit(u16 devid, u8 bit)
}
+static bool copy_device_table(void)
+{
+ u64 int_ctl, int_tab_len, entry = 0, last_entry = 0;
+ struct dev_table_entry *old_devtb = NULL;
+ u32 lo, hi, devid, old_devtb_size;
+ phys_addr_t old_devtb_phys;
+ struct amd_iommu *iommu;
+ u16 dom_id, dte_v, irq_v;
+ gfp_t gfp_flag;
+ u64 tmp;
+
+ if (!amd_iommu_pre_enabled)
+ return false;
+
+ pr_warn("Translation is already enabled - trying to copy translation structures\n");
+ for_each_iommu(iommu) {
+ /* All IOMMUs should use the same device table with the same size */
+ lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
+ hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
+ entry = (((u64) hi) << 32) + lo;
+ if (last_entry && last_entry != entry) {
+ pr_err("IOMMU:%d should use the same dev table as others!/n",
+ iommu->index);
+ return false;
+ }
+ last_entry = entry;
+
+ old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
+ if (old_devtb_size != dev_table_size) {
+ pr_err("The device table size of IOMMU:%d is not expected!/n",
+ iommu->index);
+ return false;
+ }
+ }
+
+ old_devtb_phys = entry & PAGE_MASK;
+ if (old_devtb_phys >= 0x100000000ULL) {
+ pr_err("The address of old device table is above 4G, not trustworthy!/n");
+ return false;
+ }
+ old_devtb = memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB);
+ if (!old_devtb)
+ return false;
+
+ gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32;
+ old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,
+ get_order(dev_table_size));
+ if (old_dev_tbl_cpy == NULL) {
+ pr_err("Failed to allocate memory for copying old device table!/n");
+ return false;
+ }
+
+ for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
+ old_dev_tbl_cpy[devid] = old_devtb[devid];
+ dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
+ dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
+
+ if (dte_v && dom_id) {
+ old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
+ old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
+ __set_bit(dom_id, amd_iommu_pd_alloc_bitmap);
+ /* If gcr3 table existed, mask it out */
+ if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
+ tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
+ tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
+ old_dev_tbl_cpy[devid].data[1] &= ~tmp;
+ tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
+ tmp |= DTE_FLAG_GV;
+ old_dev_tbl_cpy[devid].data[0] &= ~tmp;
+ }
+ }
+
+ irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
+ int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
+ int_tab_len = old_devtb[devid].data[2] & DTE_IRQ_TABLE_LEN_MASK;
+ if (irq_v && (int_ctl || int_tab_len)) {
+ if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
+ (int_tab_len != DTE_IRQ_TABLE_LEN)) {
+ pr_err("Wrong old irq remapping flag: %#x\n", devid);
+ return false;
+ }
+
+ old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
+ }
+ }
+ memunmap(old_devtb);
+
+ return true;
+}
+
void amd_iommu_apply_erratum_63(u16 devid)
{
int sysmgt;
@@ -1400,6 +1534,16 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
iommu->int_enabled = false;
+ init_translation_status(iommu);
+ if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
+ iommu_disable(iommu);
+ clear_translation_pre_enabled(iommu);
+ pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
+ iommu->index);
+ }
+ if (amd_iommu_pre_enabled)
+ amd_iommu_pre_enabled = translation_pre_enabled(iommu);
+
ret = init_iommu_from_acpi(iommu, h);
if (ret)
return ret;
@@ -1893,8 +2037,7 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
}
/*
- * Init the device table to not allow DMA access for devices and
- * suppress all page faults
+ * Init the device table to not allow DMA access for devices
*/
static void init_device_table_dma(void)
{
@@ -1903,14 +2046,6 @@ static void init_device_table_dma(void)
for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
set_dev_entry_bit(devid, DEV_ENTRY_VALID);
set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
- /*
- * In kdump kernels in-flight DMA from the old kernel might
- * cause IO_PAGE_FAULTs. There are no reports that a kdump
- * actually failed because of that, so just disable fault
- * reporting in the hardware to get rid of the messages
- */
- if (is_kdump_kernel())
- set_dev_entry_bit(devid, DEV_ENTRY_NO_PAGE_FAULT);
}
}
@@ -2023,24 +2158,62 @@ static void iommu_enable_ga(struct amd_iommu *iommu)
#endif
}
+static void early_enable_iommu(struct amd_iommu *iommu)
+{
+ iommu_disable(iommu);
+ iommu_init_flags(iommu);
+ iommu_set_device_table(iommu);
+ iommu_enable_command_buffer(iommu);
+ iommu_enable_event_buffer(iommu);
+ iommu_set_exclusion_range(iommu);
+ iommu_enable_ga(iommu);
+ iommu_enable(iommu);
+ iommu_flush_all_caches(iommu);
+}
+
/*
* This function finally enables all IOMMUs found in the system after
- * they have been initialized
+ * they have been initialized.
+ *
+ * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
+ * the old content of device table entries. Not this case or copy failed,
+ * just continue as normal kernel does.
*/
static void early_enable_iommus(void)
{
struct amd_iommu *iommu;
- for_each_iommu(iommu) {
- iommu_disable(iommu);
- iommu_init_flags(iommu);
- iommu_set_device_table(iommu);
- iommu_enable_command_buffer(iommu);
- iommu_enable_event_buffer(iommu);
- iommu_set_exclusion_range(iommu);
- iommu_enable_ga(iommu);
- iommu_enable(iommu);
- iommu_flush_all_caches(iommu);
+
+ if (!copy_device_table()) {
+ /*
+ * If come here because of failure in copying device table from old
+ * kernel with all IOMMUs enabled, print error message and try to
+ * free allocated old_dev_tbl_cpy.
+ */
+ if (amd_iommu_pre_enabled)
+ pr_err("Failed to copy DEV table from previous kernel.\n");
+ if (old_dev_tbl_cpy != NULL)
+ free_pages((unsigned long)old_dev_tbl_cpy,
+ get_order(dev_table_size));
+
+ for_each_iommu(iommu) {
+ clear_translation_pre_enabled(iommu);
+ early_enable_iommu(iommu);
+ }
+ } else {
+ pr_info("Copied DEV table from previous kernel.\n");
+ free_pages((unsigned long)amd_iommu_dev_table,
+ get_order(dev_table_size));
+ amd_iommu_dev_table = old_dev_tbl_cpy;
+ for_each_iommu(iommu) {
+ iommu_disable_command_buffer(iommu);
+ iommu_disable_event_buffer(iommu);
+ iommu_enable_command_buffer(iommu);
+ iommu_enable_event_buffer(iommu);
+ iommu_enable_ga(iommu);
+ iommu_set_device_table(iommu);
+ iommu_flush_all_caches(iommu);
+ }
}
#ifdef CONFIG_IRQ_REMAP
@@ -2276,7 +2449,8 @@ static int __init early_amd_iommu_init(void)
/* Device table - directly used by all IOMMUs */
ret = -ENOMEM;
- amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+ amd_iommu_dev_table = (void *)__get_free_pages(
+ GFP_KERNEL | __GFP_ZERO | GFP_DMA32,
get_order(dev_table_size));
if (amd_iommu_dev_table == NULL)
goto out;
@@ -2326,7 +2500,8 @@ static int __init early_amd_iommu_init(void)
goto out;
/* Disable any previously enabled IOMMUs */
- disable_iommus();
+ if (!is_kdump_kernel() || amd_iommu_disabled)
+ disable_iommus();
if (amd_iommu_irq_remap)
amd_iommu_irq_remap = check_ioapic_information();
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 3f12fb2338ea..640c286a0ab9 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -97,4 +97,6 @@ static inline void *iommu_phys_to_virt(unsigned long paddr)
return phys_to_virt(__sme_clr(paddr));
}
+extern bool translation_pre_enabled(struct amd_iommu *iommu);
+extern struct iommu_dev_data *get_dev_data(struct device *dev);
#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 8e3a85759242..f6b24c7d8b70 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -250,6 +250,14 @@
#define GA_GUEST_NR 0x1
+/* Bit value definition for dte irq remapping fields*/
+#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
+#define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
+#define DTE_IRQ_TABLE_LEN_MASK (0xfULL << 1)
+#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
+#define DTE_IRQ_TABLE_LEN (8ULL << 1)
+#define DTE_IRQ_REMAP_ENABLE 1ULL
+
#define PAGE_MODE_NONE 0x00
#define PAGE_MODE_1_LEVEL 0x01
#define PAGE_MODE_2_LEVEL 0x02
@@ -265,7 +273,7 @@
#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL)
#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL)
#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \
- IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
+ IOMMU_PTE_PR | IOMMU_PTE_IR | IOMMU_PTE_IW)
#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL)
#define PM_MAP_4k 0
@@ -314,19 +322,29 @@
#define PTE_LEVEL_PAGE_SIZE(level) \
(1ULL << (12 + (9 * (level))))
-#define IOMMU_PTE_P (1ULL << 0)
-#define IOMMU_PTE_TV (1ULL << 1)
+/*
+ * Bit value definition for I/O PTE fields
+ */
+#define IOMMU_PTE_PR (1ULL << 0)
#define IOMMU_PTE_U (1ULL << 59)
#define IOMMU_PTE_FC (1ULL << 60)
#define IOMMU_PTE_IR (1ULL << 61)
#define IOMMU_PTE_IW (1ULL << 62)
+/*
+ * Bit value definition for DTE fields
+ */
+#define DTE_FLAG_V (1ULL << 0)
+#define DTE_FLAG_TV (1ULL << 1)
+#define DTE_FLAG_IR (1ULL << 61)
+#define DTE_FLAG_IW (1ULL << 62)
+
#define DTE_FLAG_IOTLB (1ULL << 32)
-#define DTE_FLAG_SA (1ULL << 34)
#define DTE_FLAG_GV (1ULL << 55)
#define DTE_FLAG_MASK (0x3ffULL << 32)
#define DTE_GLX_SHIFT (56)
#define DTE_GLX_MASK (3)
+#define DEV_DOMID_MASK 0xffffULL
#define DTE_GCR3_VAL_A(x) (((x) >> 12) & 0x00007ULL)
#define DTE_GCR3_VAL_B(x) (((x) >> 15) & 0x0ffffULL)
@@ -343,7 +361,7 @@
#define GCR3_VALID 0x01ULL
#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
-#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
+#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_PR)
#define IOMMU_PTE_PAGE(pte) (iommu_phys_to_virt((pte) & IOMMU_PAGE_MASK))
#define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07)
@@ -435,6 +453,8 @@ struct iommu_domain;
struct irq_domain;
struct amd_irte_ops;
+#define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
+
/*
* This structure contains generic data for IOMMU protection domains
* independent of their use.
@@ -569,6 +589,7 @@ struct amd_iommu {
struct amd_irte_ops *irte_ops;
#endif
+ u32 flags;
volatile u64 __aligned(8) cmd_sem;
};
@@ -599,6 +620,30 @@ struct devid_map {
bool cmd_line;
};
+/*
+ * This struct contains device specific data for the IOMMU
+ */
+struct iommu_dev_data {
+ struct list_head list; /* For domain->dev_list */
+ struct list_head dev_data_list; /* For global dev_data_list */
+ struct protection_domain *domain; /* Domain the device is bound to */
+ u16 devid; /* PCI Device ID */
+ u16 alias; /* Alias Device ID */
+ bool iommu_v2; /* Device can make use of IOMMUv2 */
+ bool passthrough; /* Device is identity mapped */
+ struct {
+ bool enabled;
+ int qdep;
+ } ats; /* ATS state */
+ bool pri_tlp; /* PASID TLB required for
+ PPR completions */
+ u32 errata; /* Bitmap for errata to apply */
+ bool use_vapic; /* Enable device to use vapic mode */
+ bool defer_attach;
+
+ struct ratelimit_state rs; /* Ratelimit IOPF messages */
+};
+
/* Map HPET and IOAPIC ids to the devid used by the IOMMU */
extern struct list_head ioapic_map;
extern struct list_head hpet_map;
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index dccf5b76eff2..7d94e1d39e5e 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -554,14 +554,30 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
unsigned long flags;
struct fault *fault;
bool finish;
- u16 tag;
+ u16 tag, devid;
int ret;
+ struct iommu_dev_data *dev_data;
+ struct pci_dev *pdev = NULL;
iommu_fault = data;
tag = iommu_fault->tag & 0x1ff;
finish = (iommu_fault->tag >> 9) & 1;
+ devid = iommu_fault->device_id;
+ pdev = pci_get_bus_and_slot(PCI_BUS_NUM(devid), devid & 0xff);
+ if (!pdev)
+ return -ENODEV;
+ dev_data = get_dev_data(&pdev->dev);
+
+ /* In kdump kernel pci dev is not initialized yet -> send INVALID */
ret = NOTIFY_DONE;
+ if (translation_pre_enabled(amd_iommu_rlookup_table[devid])
+ && dev_data->defer_attach) {
+ amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
+ PPR_INVALID, tag);
+ goto out;
+ }
+
dev_state = get_device_state(iommu_fault->device_id);
if (dev_state == NULL)
goto out;
diff --git a/drivers/iommu/arm-smmu-regs.h b/drivers/iommu/arm-smmu-regs.h
new file mode 100644
index 000000000000..a1226e4ab5f8
--- /dev/null
+++ b/drivers/iommu/arm-smmu-regs.h
@@ -0,0 +1,220 @@
+/*
+ * IOMMU API for ARM architected SMMU implementations.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) 2013 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+#ifndef _ARM_SMMU_REGS_H
+#define _ARM_SMMU_REGS_H
+
+/* Configuration registers */
+#define ARM_SMMU_GR0_sCR0 0x0
+#define sCR0_CLIENTPD (1 << 0)
+#define sCR0_GFRE (1 << 1)
+#define sCR0_GFIE (1 << 2)
+#define sCR0_EXIDENABLE (1 << 3)
+#define sCR0_GCFGFRE (1 << 4)
+#define sCR0_GCFGFIE (1 << 5)
+#define sCR0_USFCFG (1 << 10)
+#define sCR0_VMIDPNE (1 << 11)
+#define sCR0_PTM (1 << 12)
+#define sCR0_FB (1 << 13)
+#define sCR0_VMID16EN (1 << 31)
+#define sCR0_BSU_SHIFT 14
+#define sCR0_BSU_MASK 0x3
+
+/* Auxiliary Configuration register */
+#define ARM_SMMU_GR0_sACR 0x10
+
+/* Identification registers */
+#define ARM_SMMU_GR0_ID0 0x20
+#define ARM_SMMU_GR0_ID1 0x24
+#define ARM_SMMU_GR0_ID2 0x28
+#define ARM_SMMU_GR0_ID3 0x2c
+#define ARM_SMMU_GR0_ID4 0x30
+#define ARM_SMMU_GR0_ID5 0x34
+#define ARM_SMMU_GR0_ID6 0x38
+#define ARM_SMMU_GR0_ID7 0x3c
+#define ARM_SMMU_GR0_sGFSR 0x48
+#define ARM_SMMU_GR0_sGFSYNR0 0x50
+#define ARM_SMMU_GR0_sGFSYNR1 0x54
+#define ARM_SMMU_GR0_sGFSYNR2 0x58
+
+#define ID0_S1TS (1 << 30)
+#define ID0_S2TS (1 << 29)
+#define ID0_NTS (1 << 28)
+#define ID0_SMS (1 << 27)
+#define ID0_ATOSNS (1 << 26)
+#define ID0_PTFS_NO_AARCH32 (1 << 25)
+#define ID0_PTFS_NO_AARCH32S (1 << 24)
+#define ID0_CTTW (1 << 14)
+#define ID0_NUMIRPT_SHIFT 16
+#define ID0_NUMIRPT_MASK 0xff
+#define ID0_NUMSIDB_SHIFT 9
+#define ID0_NUMSIDB_MASK 0xf
+#define ID0_EXIDS (1 << 8)
+#define ID0_NUMSMRG_SHIFT 0
+#define ID0_NUMSMRG_MASK 0xff
+
+#define ID1_PAGESIZE (1 << 31)
+#define ID1_NUMPAGENDXB_SHIFT 28
+#define ID1_NUMPAGENDXB_MASK 7
+#define ID1_NUMS2CB_SHIFT 16
+#define ID1_NUMS2CB_MASK 0xff
+#define ID1_NUMCB_SHIFT 0
+#define ID1_NUMCB_MASK 0xff
+
+#define ID2_OAS_SHIFT 4
+#define ID2_OAS_MASK 0xf
+#define ID2_IAS_SHIFT 0
+#define ID2_IAS_MASK 0xf
+#define ID2_UBS_SHIFT 8
+#define ID2_UBS_MASK 0xf
+#define ID2_PTFS_4K (1 << 12)
+#define ID2_PTFS_16K (1 << 13)
+#define ID2_PTFS_64K (1 << 14)
+#define ID2_VMID16 (1 << 15)
+
+#define ID7_MAJOR_SHIFT 4
+#define ID7_MAJOR_MASK 0xf
+
+/* Global TLB invalidation */
+#define ARM_SMMU_GR0_TLBIVMID 0x64
+#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
+#define ARM_SMMU_GR0_TLBIALLH 0x6c
+#define ARM_SMMU_GR0_sTLBGSYNC 0x70
+#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
+#define sTLBGSTATUS_GSACTIVE (1 << 0)
+
+/* Stream mapping registers */
+#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
+#define SMR_VALID (1 << 31)
+#define SMR_MASK_SHIFT 16
+#define SMR_ID_SHIFT 0
+
+#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
+#define S2CR_CBNDX_SHIFT 0
+#define S2CR_CBNDX_MASK 0xff
+#define S2CR_EXIDVALID (1 << 10)
+#define S2CR_TYPE_SHIFT 16
+#define S2CR_TYPE_MASK 0x3
+enum arm_smmu_s2cr_type {
+ S2CR_TYPE_TRANS,
+ S2CR_TYPE_BYPASS,
+ S2CR_TYPE_FAULT,
+};
+
+#define S2CR_PRIVCFG_SHIFT 24
+#define S2CR_PRIVCFG_MASK 0x3
+enum arm_smmu_s2cr_privcfg {
+ S2CR_PRIVCFG_DEFAULT,
+ S2CR_PRIVCFG_DIPAN,
+ S2CR_PRIVCFG_UNPRIV,
+ S2CR_PRIVCFG_PRIV,
+};
+
+/* Context bank attribute registers */
+#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
+#define CBAR_VMID_SHIFT 0
+#define CBAR_VMID_MASK 0xff
+#define CBAR_S1_BPSHCFG_SHIFT 8
+#define CBAR_S1_BPSHCFG_MASK 3
+#define CBAR_S1_BPSHCFG_NSH 3
+#define CBAR_S1_MEMATTR_SHIFT 12
+#define CBAR_S1_MEMATTR_MASK 0xf
+#define CBAR_S1_MEMATTR_WB 0xf
+#define CBAR_TYPE_SHIFT 16
+#define CBAR_TYPE_MASK 0x3
+#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
+#define CBAR_IRPTNDX_SHIFT 24
+#define CBAR_IRPTNDX_MASK 0xff
+
+#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
+#define CBA2R_RW64_32BIT (0 << 0)
+#define CBA2R_RW64_64BIT (1 << 0)
+#define CBA2R_VMID_SHIFT 16
+#define CBA2R_VMID_MASK 0xffff
+
+#define ARM_SMMU_CB_SCTLR 0x0
+#define ARM_SMMU_CB_ACTLR 0x4
+#define ARM_SMMU_CB_RESUME 0x8
+#define ARM_SMMU_CB_TTBCR2 0x10
+#define ARM_SMMU_CB_TTBR0 0x20
+#define ARM_SMMU_CB_TTBR1 0x28
+#define ARM_SMMU_CB_TTBCR 0x30
+#define ARM_SMMU_CB_CONTEXTIDR 0x34
+#define ARM_SMMU_CB_S1_MAIR0 0x38
+#define ARM_SMMU_CB_S1_MAIR1 0x3c
+#define ARM_SMMU_CB_PAR 0x50
+#define ARM_SMMU_CB_FSR 0x58
+#define ARM_SMMU_CB_FAR 0x60
+#define ARM_SMMU_CB_FSYNR0 0x68
+#define ARM_SMMU_CB_S1_TLBIVA 0x600
+#define ARM_SMMU_CB_S1_TLBIASID 0x610
+#define ARM_SMMU_CB_S1_TLBIVAL 0x620
+#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
+#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
+#define ARM_SMMU_CB_TLBSYNC 0x7f0
+#define ARM_SMMU_CB_TLBSTATUS 0x7f4
+#define ARM_SMMU_CB_ATS1PR 0x800
+#define ARM_SMMU_CB_ATSR 0x8f0
+
+#define SCTLR_S1_ASIDPNE (1 << 12)
+#define SCTLR_CFCFG (1 << 7)
+#define SCTLR_CFIE (1 << 6)
+#define SCTLR_CFRE (1 << 5)
+#define SCTLR_E (1 << 4)
+#define SCTLR_AFE (1 << 2)
+#define SCTLR_TRE (1 << 1)
+#define SCTLR_M (1 << 0)
+
+#define CB_PAR_F (1 << 0)
+
+#define ATSR_ACTIVE (1 << 0)
+
+#define RESUME_RETRY (0 << 0)
+#define RESUME_TERMINATE (1 << 0)
+
+#define TTBCR2_SEP_SHIFT 15
+#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_AS (1 << 4)
+
+#define TTBRn_ASID_SHIFT 48
+
+#define FSR_MULTI (1 << 31)
+#define FSR_SS (1 << 30)
+#define FSR_UUT (1 << 8)
+#define FSR_ASF (1 << 7)
+#define FSR_TLBLKF (1 << 6)
+#define FSR_TLBMCF (1 << 5)
+#define FSR_EF (1 << 4)
+#define FSR_PF (1 << 3)
+#define FSR_AFF (1 << 2)
+#define FSR_TF (1 << 1)
+
+#define FSR_IGN (FSR_AFF | FSR_ASF | \
+ FSR_TLBMCF | FSR_TLBLKF)
+#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
+ FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
+
+#define FSYNR0_WNR (1 << 4)
+
+#endif /* _ARM_SMMU_REGS_H */
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 568c400eeaed..e67ba6c40faf 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -2852,9 +2852,15 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
arm_smmu_device_disable(smmu);
+
return 0;
}
+static void arm_smmu_device_shutdown(struct platform_device *pdev)
+{
+ arm_smmu_device_remove(pdev);
+}
+
static const struct of_device_id arm_smmu_of_match[] = {
{ .compatible = "arm,smmu-v3", },
{ },
@@ -2868,6 +2874,7 @@ static struct platform_driver arm_smmu_driver = {
},
.probe = arm_smmu_device_probe,
.remove = arm_smmu_device_remove,
+ .shutdown = arm_smmu_device_shutdown,
};
module_platform_driver(arm_smmu_driver);
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 2d80fa8a0634..3bdb799d3b4b 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -54,6 +54,15 @@
#include <linux/amba/bus.h>
#include "io-pgtable.h"
+#include "arm-smmu-regs.h"
+
+#define ARM_MMU500_ACTLR_CPRE (1 << 1)
+
+#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
+#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
+
+#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
+#define TLB_SPIN_COUNT 10
/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS 128
@@ -83,211 +92,9 @@
#define smmu_write_atomic_lq writel_relaxed
#endif
-/* Configuration registers */
-#define ARM_SMMU_GR0_sCR0 0x0
-#define sCR0_CLIENTPD (1 << 0)
-#define sCR0_GFRE (1 << 1)
-#define sCR0_GFIE (1 << 2)
-#define sCR0_EXIDENABLE (1 << 3)
-#define sCR0_GCFGFRE (1 << 4)
-#define sCR0_GCFGFIE (1 << 5)
-#define sCR0_USFCFG (1 << 10)
-#define sCR0_VMIDPNE (1 << 11)
-#define sCR0_PTM (1 << 12)
-#define sCR0_FB (1 << 13)
-#define sCR0_VMID16EN (1 << 31)
-#define sCR0_BSU_SHIFT 14
-#define sCR0_BSU_MASK 0x3
-
-/* Auxiliary Configuration register */
-#define ARM_SMMU_GR0_sACR 0x10
-
-/* Identification registers */
-#define ARM_SMMU_GR0_ID0 0x20
-#define ARM_SMMU_GR0_ID1 0x24
-#define ARM_SMMU_GR0_ID2 0x28
-#define ARM_SMMU_GR0_ID3 0x2c
-#define ARM_SMMU_GR0_ID4 0x30
-#define ARM_SMMU_GR0_ID5 0x34
-#define ARM_SMMU_GR0_ID6 0x38
-#define ARM_SMMU_GR0_ID7 0x3c
-#define ARM_SMMU_GR0_sGFSR 0x48
-#define ARM_SMMU_GR0_sGFSYNR0 0x50
-#define ARM_SMMU_GR0_sGFSYNR1 0x54
-#define ARM_SMMU_GR0_sGFSYNR2 0x58
-
-#define ID0_S1TS (1 << 30)
-#define ID0_S2TS (1 << 29)
-#define ID0_NTS (1 << 28)
-#define ID0_SMS (1 << 27)
-#define ID0_ATOSNS (1 << 26)
-#define ID0_PTFS_NO_AARCH32 (1 << 25)
-#define ID0_PTFS_NO_AARCH32S (1 << 24)
-#define ID0_CTTW (1 << 14)
-#define ID0_NUMIRPT_SHIFT 16
-#define ID0_NUMIRPT_MASK 0xff
-#define ID0_NUMSIDB_SHIFT 9
-#define ID0_NUMSIDB_MASK 0xf
-#define ID0_EXIDS (1 << 8)
-#define ID0_NUMSMRG_SHIFT 0
-#define ID0_NUMSMRG_MASK 0xff
-
-#define ID1_PAGESIZE (1 << 31)
-#define ID1_NUMPAGENDXB_SHIFT 28
-#define ID1_NUMPAGENDXB_MASK 7
-#define ID1_NUMS2CB_SHIFT 16
-#define ID1_NUMS2CB_MASK 0xff
-#define ID1_NUMCB_SHIFT 0
-#define ID1_NUMCB_MASK 0xff
-
-#define ID2_OAS_SHIFT 4
-#define ID2_OAS_MASK 0xf
-#define ID2_IAS_SHIFT 0
-#define ID2_IAS_MASK 0xf
-#define ID2_UBS_SHIFT 8
-#define ID2_UBS_MASK 0xf
-#define ID2_PTFS_4K (1 << 12)
-#define ID2_PTFS_16K (1 << 13)
-#define ID2_PTFS_64K (1 << 14)
-#define ID2_VMID16 (1 << 15)
-
-#define ID7_MAJOR_SHIFT 4
-#define ID7_MAJOR_MASK 0xf
-
-/* Global TLB invalidation */
-#define ARM_SMMU_GR0_TLBIVMID 0x64
-#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
-#define ARM_SMMU_GR0_TLBIALLH 0x6c
-#define ARM_SMMU_GR0_sTLBGSYNC 0x70
-#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
-#define sTLBGSTATUS_GSACTIVE (1 << 0)
-#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
-#define TLB_SPIN_COUNT 10
-
-/* Stream mapping registers */
-#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
-#define SMR_VALID (1 << 31)
-#define SMR_MASK_SHIFT 16
-#define SMR_ID_SHIFT 0
-
-#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
-#define S2CR_CBNDX_SHIFT 0
-#define S2CR_CBNDX_MASK 0xff
-#define S2CR_EXIDVALID (1 << 10)
-#define S2CR_TYPE_SHIFT 16
-#define S2CR_TYPE_MASK 0x3
-enum arm_smmu_s2cr_type {
- S2CR_TYPE_TRANS,
- S2CR_TYPE_BYPASS,
- S2CR_TYPE_FAULT,
-};
-
-#define S2CR_PRIVCFG_SHIFT 24
-#define S2CR_PRIVCFG_MASK 0x3
-enum arm_smmu_s2cr_privcfg {
- S2CR_PRIVCFG_DEFAULT,
- S2CR_PRIVCFG_DIPAN,
- S2CR_PRIVCFG_UNPRIV,
- S2CR_PRIVCFG_PRIV,
-};
-
-/* Context bank attribute registers */
-#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
-#define CBAR_VMID_SHIFT 0
-#define CBAR_VMID_MASK 0xff
-#define CBAR_S1_BPSHCFG_SHIFT 8
-#define CBAR_S1_BPSHCFG_MASK 3
-#define CBAR_S1_BPSHCFG_NSH 3
-#define CBAR_S1_MEMATTR_SHIFT 12
-#define CBAR_S1_MEMATTR_MASK 0xf
-#define CBAR_S1_MEMATTR_WB 0xf
-#define CBAR_TYPE_SHIFT 16
-#define CBAR_TYPE_MASK 0x3
-#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
-#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
-#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
-#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
-#define CBAR_IRPTNDX_SHIFT 24
-#define CBAR_IRPTNDX_MASK 0xff
-
-#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
-#define CBA2R_RW64_32BIT (0 << 0)
-#define CBA2R_RW64_64BIT (1 << 0)
-#define CBA2R_VMID_SHIFT 16
-#define CBA2R_VMID_MASK 0xffff
-
/* Translation context bank */
#define ARM_SMMU_CB(smmu, n) ((smmu)->cb_base + ((n) << (smmu)->pgshift))
-#define ARM_SMMU_CB_SCTLR 0x0
-#define ARM_SMMU_CB_ACTLR 0x4
-#define ARM_SMMU_CB_RESUME 0x8
-#define ARM_SMMU_CB_TTBCR2 0x10
-#define ARM_SMMU_CB_TTBR0 0x20
-#define ARM_SMMU_CB_TTBR1 0x28
-#define ARM_SMMU_CB_TTBCR 0x30
-#define ARM_SMMU_CB_CONTEXTIDR 0x34
-#define ARM_SMMU_CB_S1_MAIR0 0x38
-#define ARM_SMMU_CB_S1_MAIR1 0x3c
-#define ARM_SMMU_CB_PAR 0x50
-#define ARM_SMMU_CB_FSR 0x58
-#define ARM_SMMU_CB_FAR 0x60
-#define ARM_SMMU_CB_FSYNR0 0x68
-#define ARM_SMMU_CB_S1_TLBIVA 0x600
-#define ARM_SMMU_CB_S1_TLBIASID 0x610
-#define ARM_SMMU_CB_S1_TLBIVAL 0x620
-#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
-#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
-#define ARM_SMMU_CB_TLBSYNC 0x7f0
-#define ARM_SMMU_CB_TLBSTATUS 0x7f4
-#define ARM_SMMU_CB_ATS1PR 0x800
-#define ARM_SMMU_CB_ATSR 0x8f0
-
-#define SCTLR_S1_ASIDPNE (1 << 12)
-#define SCTLR_CFCFG (1 << 7)
-#define SCTLR_CFIE (1 << 6)
-#define SCTLR_CFRE (1 << 5)
-#define SCTLR_E (1 << 4)
-#define SCTLR_AFE (1 << 2)
-#define SCTLR_TRE (1 << 1)
-#define SCTLR_M (1 << 0)
-
-#define ARM_MMU500_ACTLR_CPRE (1 << 1)
-
-#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
-#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
-
-#define CB_PAR_F (1 << 0)
-
-#define ATSR_ACTIVE (1 << 0)
-
-#define RESUME_RETRY (0 << 0)
-#define RESUME_TERMINATE (1 << 0)
-
-#define TTBCR2_SEP_SHIFT 15
-#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
-#define TTBCR2_AS (1 << 4)
-
-#define TTBRn_ASID_SHIFT 48
-
-#define FSR_MULTI (1 << 31)
-#define FSR_SS (1 << 30)
-#define FSR_UUT (1 << 8)
-#define FSR_ASF (1 << 7)
-#define FSR_TLBLKF (1 << 6)
-#define FSR_TLBMCF (1 << 5)
-#define FSR_EF (1 << 4)
-#define FSR_PF (1 << 3)
-#define FSR_AFF (1 << 2)
-#define FSR_TF (1 << 1)
-
-#define FSR_IGN (FSR_AFF | FSR_ASF | \
- FSR_TLBMCF | FSR_TLBLKF)
-#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
- FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
-
-#define FSYNR0_WNR (1 << 4)
-
#define MSI_IOVA_BASE 0x8000000
#define MSI_IOVA_LENGTH 0x100000
@@ -338,6 +145,13 @@ struct arm_smmu_smr {
bool valid;
};
+struct arm_smmu_cb {
+ u64 ttbr[2];
+ u32 tcr[2];
+ u32 mair[2];
+ struct arm_smmu_cfg *cfg;
+};
+
struct arm_smmu_master_cfg {
struct arm_smmu_device *smmu;
s16 smendx[];
@@ -380,6 +194,7 @@ struct arm_smmu_device {
u32 num_context_banks;
u32 num_s2_context_banks;
DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
+ struct arm_smmu_cb *cbs;
atomic_t irptndx;
u32 num_mapping_groups;
@@ -776,17 +591,74 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
struct io_pgtable_cfg *pgtbl_cfg)
{
- u32 reg, reg2;
- u64 reg64;
- bool stage1;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
+ struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
+ bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+
+ cb->cfg = cfg;
+
+ /* TTBCR */
+ if (stage1) {
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
+ } else {
+ cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
+ cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
+ cb->tcr[1] |= TTBCR2_SEP_UPSTREAM;
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
+ cb->tcr[1] |= TTBCR2_AS;
+ }
+ } else {
+ cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
+ }
+
+ /* TTBRs */
+ if (stage1) {
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
+ cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
+ } else {
+ cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
+ cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
+ cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
+ cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
+ }
+ } else {
+ cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
+ }
+
+ /* MAIRs (stage-1 only) */
+ if (stage1) {
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
+ cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
+ } else {
+ cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
+ cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
+ }
+ }
+}
+
+static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
+{
+ u32 reg;
+ bool stage1;
+ struct arm_smmu_cb *cb = &smmu->cbs[idx];
+ struct arm_smmu_cfg *cfg = cb->cfg;
void __iomem *cb_base, *gr1_base;
+ cb_base = ARM_SMMU_CB(smmu, idx);
+
+ /* Unassigned context banks only need disabling */
+ if (!cfg) {
+ writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ return;
+ }
+
gr1_base = ARM_SMMU_GR1(smmu);
stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
- cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
+ /* CBA2R */
if (smmu->version > ARM_SMMU_V1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
reg = CBA2R_RW64_64BIT;
@@ -796,7 +668,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
if (smmu->features & ARM_SMMU_FEAT_VMID16)
reg |= cfg->vmid << CBA2R_VMID_SHIFT;
- writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
+ writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
}
/* CBAR */
@@ -815,72 +687,41 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
/* 8-bit VMIDs live in CBAR */
reg |= cfg->vmid << CBAR_VMID_SHIFT;
}
- writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
+ writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
/*
* TTBCR
* We must write this before the TTBRs, since it determines the
* access behaviour of some fields (in particular, ASID[15:8]).
*/
- if (stage1) {
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- reg = pgtbl_cfg->arm_v7s_cfg.tcr;
- reg2 = 0;
- } else {
- reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
- reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
- reg2 |= TTBCR2_SEP_UPSTREAM;
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
- reg2 |= TTBCR2_AS;
- }
- if (smmu->version > ARM_SMMU_V1)
- writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
- } else {
- reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
- }
- writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
+ if (stage1 && smmu->version > ARM_SMMU_V1)
+ writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2);
+ writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR);
/* TTBRs */
- if (stage1) {
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
- writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
- reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
- writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
- writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
- } else {
- reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
- reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
- writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
- reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
- reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
- writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
- }
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
+ writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
+ writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
} else {
- reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
- writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
+ writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
+ if (stage1)
+ writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
}
/* MAIRs (stage-1 only) */
if (stage1) {
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- reg = pgtbl_cfg->arm_v7s_cfg.prrr;
- reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
- } else {
- reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
- reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
- }
- writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
- writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
+ writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
+ writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
}
/* SCTLR */
reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
if (stage1)
reg |= SCTLR_S1_ASIDPNE;
-#ifdef __BIG_ENDIAN
- reg |= SCTLR_E;
-#endif
+ if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+ reg |= SCTLR_E;
+
writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
}
@@ -1043,6 +884,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
/* Initialise the context bank with our page table cfg */
arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
+ arm_smmu_write_context_bank(smmu, cfg->cbndx);
/*
* Request context fault interrupt. Do this last to avoid the
@@ -1075,7 +917,6 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- void __iomem *cb_base;
int irq;
if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
@@ -1085,8 +926,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
* Disable the context bank and free the page tables before freeing
* it.
*/
- cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
- writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ smmu->cbs[cfg->cbndx].cfg = NULL;
+ arm_smmu_write_context_bank(smmu, cfg->cbndx);
if (cfg->irptndx != INVALID_IRPTNDX) {
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
@@ -1736,7 +1577,6 @@ static struct iommu_ops arm_smmu_ops = {
static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
{
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
- void __iomem *cb_base;
int i;
u32 reg, major;
@@ -1772,8 +1612,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Make sure all context banks are disabled and clear CB_FSR */
for (i = 0; i < smmu->num_context_banks; ++i) {
- cb_base = ARM_SMMU_CB(smmu, i);
- writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
+
+ arm_smmu_write_context_bank(smmu, i);
writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
/*
* Disable MMU-500's not-particularly-beneficial next-page
@@ -1979,6 +1820,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
smmu->cavium_id_base -= smmu->num_context_banks;
dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
}
+ smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
+ sizeof(*smmu->cbs), GFP_KERNEL);
+ if (!smmu->cbs)
+ return -ENOMEM;
/* ID2 */
id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
@@ -2336,13 +2181,30 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
return 0;
}
+static void arm_smmu_device_shutdown(struct platform_device *pdev)
+{
+ arm_smmu_device_remove(pdev);
+}
+
+static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
+{
+ struct arm_smmu_device *smmu = dev_get_drvdata(dev);
+
+ arm_smmu_device_reset(smmu);
+ return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume);
+
static struct platform_driver arm_smmu_driver = {
.driver = {
.name = "arm-smmu",
.of_match_table = of_match_ptr(arm_smmu_of_match),
+ .pm = &arm_smmu_pm_ops,
},
.probe = arm_smmu_device_probe,
.remove = arm_smmu_device_remove,
+ .shutdown = arm_smmu_device_shutdown,
};
module_platform_driver(arm_smmu_driver);
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index c8b0329c85d2..ca5ebaeafd6a 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1343,7 +1343,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
if (mask) {
BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
- addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
+ addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
} else
desc.high = QI_DEV_IOTLB_ADDR(addr);
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 2395478dde75..f596fcc32898 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -54,10 +54,6 @@ typedef u32 sysmmu_pte_t;
#define lv2ent_small(pent) ((*(pent) & 2) == 2)
#define lv2ent_large(pent) ((*(pent) & 3) == 1)
-#ifdef CONFIG_BIG_ENDIAN
-#warning "revisit driver if we can enable big-endian ptes"
-#endif
-
/*
* v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
* v5.0 introduced support for 36bit physical address space by shifting
@@ -569,7 +565,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
spin_unlock_irqrestore(&data->lock, flags);
}
-static struct iommu_ops exynos_iommu_ops;
+static const struct iommu_ops exynos_iommu_ops;
static int __init exynos_sysmmu_probe(struct platform_device *pdev)
{
@@ -659,6 +655,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
}
}
+ /*
+ * use the first registered sysmmu device for performing
+ * dma mapping operations on iommu page tables (cpu cache flush)
+ */
+ if (!dma_dev)
+ dma_dev = &pdev->dev;
+
pm_runtime_enable(dev);
return 0;
@@ -1323,7 +1326,7 @@ static int exynos_iommu_of_xlate(struct device *dev,
return 0;
}
-static struct iommu_ops exynos_iommu_ops = {
+static const struct iommu_ops exynos_iommu_ops = {
.domain_alloc = exynos_iommu_domain_alloc,
.domain_free = exynos_iommu_domain_free,
.attach_dev = exynos_iommu_attach_device,
@@ -1339,8 +1342,6 @@ static struct iommu_ops exynos_iommu_ops = {
.of_xlate = exynos_iommu_of_xlate,
};
-static bool init_done;
-
static int __init exynos_iommu_init(void)
{
int ret;
@@ -1373,8 +1374,6 @@ static int __init exynos_iommu_init(void)
goto err_set_iommu;
}
- init_done = true;
-
return 0;
err_set_iommu:
kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
@@ -1384,27 +1383,6 @@ err_reg_driver:
kmem_cache_destroy(lv2table_kmem_cache);
return ret;
}
+core_initcall(exynos_iommu_init);
-static int __init exynos_iommu_of_setup(struct device_node *np)
-{
- struct platform_device *pdev;
-
- if (!init_done)
- exynos_iommu_init();
-
- pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
- if (!pdev)
- return -ENODEV;
-
- /*
- * use the first registered sysmmu device for performing
- * dma mapping operations on iommu page tables (cpu cache flush)
- */
- if (!dma_dev)
- dma_dev = &pdev->dev;
-
- return 0;
-}
-
-IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu",
- exynos_iommu_of_setup);
+IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", NULL);
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index a34355fca37a..8540625796a1 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -42,6 +42,8 @@ struct pamu_isr_data {
static struct paace *ppaact;
static struct paace *spaact;
+static bool probed; /* Has PAMU been probed? */
+
/*
* Table for matching compatible strings, for device tree
* guts node, for QorIQ SOCs.
@@ -530,8 +532,8 @@ u32 get_stash_id(u32 stash_dest_hint, u32 vcpu)
if (node) {
prop = of_get_property(node, "cache-stash-id", NULL);
if (!prop) {
- pr_debug("missing cache-stash-id at %s\n",
- node->full_name);
+ pr_debug("missing cache-stash-id at %pOF\n",
+ node);
of_node_put(node);
return ~(u32)0;
}
@@ -557,8 +559,8 @@ found_cpu_node:
if (stash_dest_hint == cache_level) {
prop = of_get_property(node, "cache-stash-id", NULL);
if (!prop) {
- pr_debug("missing cache-stash-id at %s\n",
- node->full_name);
+ pr_debug("missing cache-stash-id at %pOF\n",
+ node);
of_node_put(node);
return ~(u32)0;
}
@@ -568,8 +570,7 @@ found_cpu_node:
prop = of_get_property(node, "next-level-cache", NULL);
if (!prop) {
- pr_debug("can't find next-level-cache at %s\n",
- node->full_name);
+ pr_debug("can't find next-level-cache at %pOF\n", node);
of_node_put(node);
return ~(u32)0; /* can't traverse any further */
}
@@ -1033,6 +1034,9 @@ static int fsl_pamu_probe(struct platform_device *pdev)
* NOTE : All PAMUs share the same LIODN tables.
*/
+ if (WARN_ON(probed))
+ return -EBUSY;
+
pamu_regs = of_iomap(dev->of_node, 0);
if (!pamu_regs) {
dev_err(dev, "ioremap of PAMU node failed\n");
@@ -1063,8 +1067,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
guts_node = of_find_matching_node(NULL, guts_device_ids);
if (!guts_node) {
- dev_err(dev, "could not find GUTS node %s\n",
- dev->of_node->full_name);
+ dev_err(dev, "could not find GUTS node %pOF\n", dev->of_node);
ret = -ENODEV;
goto error;
}
@@ -1172,6 +1175,8 @@ static int fsl_pamu_probe(struct platform_device *pdev)
setup_liodns();
+ probed = true;
+
return 0;
error_genpool:
@@ -1246,8 +1251,7 @@ static __init int fsl_pamu_init(void)
pdev = platform_device_alloc("fsl-of-pamu", 0);
if (!pdev) {
- pr_err("could not allocate device %s\n",
- np->full_name);
+ pr_err("could not allocate device %pOF\n", np);
ret = -ENOMEM;
goto error_device_alloc;
}
@@ -1259,8 +1263,7 @@ static __init int fsl_pamu_init(void)
ret = platform_device_add(pdev);
if (ret) {
- pr_err("could not add device %s (err=%i)\n",
- np->full_name, ret);
+ pr_err("could not add device %pOF (err=%i)\n", np, ret);
goto error_device_add;
}
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index da0e1e30ef37..f089136e9c3f 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -33,6 +33,8 @@ static struct kmem_cache *fsl_pamu_domain_cache;
static struct kmem_cache *iommu_devinfo_cache;
static DEFINE_SPINLOCK(device_domain_lock);
+struct iommu_device pamu_iommu; /* IOMMU core code handle */
+
static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
{
return container_of(dom, struct fsl_dma_domain, iommu_domain);
@@ -619,8 +621,8 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain,
for (i = 0; i < num; i++) {
/* Ensure that LIODN value is valid */
if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
- pr_debug("Invalid liodn %d, attach device failed for %s\n",
- liodn[i], dev->of_node->full_name);
+ pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
+ liodn[i], dev->of_node);
ret = -EINVAL;
break;
}
@@ -684,8 +686,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
liodn_cnt = len / sizeof(u32);
ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
} else {
- pr_debug("missing fsl,liodn property at %s\n",
- dev->of_node->full_name);
+ pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
ret = -EINVAL;
}
@@ -720,8 +721,7 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain,
if (prop)
detach_device(dev, dma_domain);
else
- pr_debug("missing fsl,liodn property at %s\n",
- dev->of_node->full_name);
+ pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
}
static int configure_domain_geometry(struct iommu_domain *domain, void *data)
@@ -983,11 +983,14 @@ static int fsl_pamu_add_device(struct device *dev)
iommu_group_put(group);
+ iommu_device_link(&pamu_iommu, dev);
+
return 0;
}
static void fsl_pamu_remove_device(struct device *dev)
{
+ iommu_device_unlink(&pamu_iommu, dev);
iommu_group_remove_device(dev);
}
@@ -1073,6 +1076,19 @@ int __init pamu_domain_init(void)
if (ret)
return ret;
+ ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
+ if (ret)
+ return ret;
+
+ iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops);
+
+ ret = iommu_device_register(&pamu_iommu);
+ if (ret) {
+ iommu_device_sysfs_remove(&pamu_iommu);
+ pr_err("Can't register iommu device\n");
+ return ret;
+ }
+
bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index bb05fc50ee2e..6784a05dd6b2 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -458,31 +458,6 @@ static LIST_HEAD(dmar_rmrr_units);
#define for_each_rmrr_units(rmrr) \
list_for_each_entry(rmrr, &dmar_rmrr_units, list)
-static void flush_unmaps_timeout(unsigned long data);
-
-struct deferred_flush_entry {
- unsigned long iova_pfn;
- unsigned long nrpages;
- struct dmar_domain *domain;
- struct page *freelist;
-};
-
-#define HIGH_WATER_MARK 250
-struct deferred_flush_table {
- int next;
- struct deferred_flush_entry entries[HIGH_WATER_MARK];
-};
-
-struct deferred_flush_data {
- spinlock_t lock;
- int timer_on;
- struct timer_list timer;
- long size;
- struct deferred_flush_table *tables;
-};
-
-static DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
-
/* bitmap for indexing intel_iommus */
static int g_num_of_iommus;
@@ -981,20 +956,6 @@ static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
return ret;
}
-static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
-{
- struct context_entry *context;
- unsigned long flags;
-
- spin_lock_irqsave(&iommu->lock, flags);
- context = iommu_context_addr(iommu, bus, devfn, 0);
- if (context) {
- context_clear_entry(context);
- __iommu_flush_cache(iommu, context, sizeof(*context));
- }
- spin_unlock_irqrestore(&iommu->lock, flags);
-}
-
static void free_context_table(struct intel_iommu *iommu)
{
int i;
@@ -1144,8 +1105,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain,
}
static void dma_pte_free_level(struct dmar_domain *domain, int level,
- struct dma_pte *pte, unsigned long pfn,
- unsigned long start_pfn, unsigned long last_pfn)
+ int retain_level, struct dma_pte *pte,
+ unsigned long pfn, unsigned long start_pfn,
+ unsigned long last_pfn)
{
pfn = max(start_pfn, pfn);
pte = &pte[pfn_level_offset(pfn, level)];
@@ -1160,12 +1122,17 @@ static void dma_pte_free_level(struct dmar_domain *domain, int level,
level_pfn = pfn & level_mask(level);
level_pte = phys_to_virt(dma_pte_addr(pte));
- if (level > 2)
- dma_pte_free_level(domain, level - 1, level_pte,
- level_pfn, start_pfn, last_pfn);
+ if (level > 2) {
+ dma_pte_free_level(domain, level - 1, retain_level,
+ level_pte, level_pfn, start_pfn,
+ last_pfn);
+ }
- /* If range covers entire pagetable, free it */
- if (!(start_pfn > level_pfn ||
+ /*
+ * Free the page table if we're below the level we want to
+ * retain and the range covers the entire table.
+ */
+ if (level < retain_level && !(start_pfn > level_pfn ||
last_pfn < level_pfn + level_size(level) - 1)) {
dma_clear_pte(pte);
domain_flush_cache(domain, pte, sizeof(*pte));
@@ -1176,10 +1143,14 @@ next:
} while (!first_pte_in_page(++pte) && pfn <= last_pfn);
}
-/* clear last level (leaf) ptes and free page table pages. */
+/*
+ * clear last level (leaf) ptes and free page table pages below the
+ * level we wish to keep intact.
+ */
static void dma_pte_free_pagetable(struct dmar_domain *domain,
unsigned long start_pfn,
- unsigned long last_pfn)
+ unsigned long last_pfn,
+ int retain_level)
{
BUG_ON(!domain_pfn_supported(domain, start_pfn));
BUG_ON(!domain_pfn_supported(domain, last_pfn));
@@ -1188,7 +1159,7 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
dma_pte_clear_range(domain, start_pfn, last_pfn);
/* We don't need lock here; nobody else touches the iova range */
- dma_pte_free_level(domain, agaw_to_level(domain->agaw),
+ dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
domain->pgd, 0, start_pfn, last_pfn);
/* free pgd */
@@ -1316,6 +1287,13 @@ static void dma_free_pagelist(struct page *freelist)
}
}
+static void iova_entry_free(unsigned long data)
+{
+ struct page *freelist = (struct page *)data;
+
+ dma_free_pagelist(freelist);
+}
+
/* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{
@@ -1629,6 +1607,25 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
addr, mask);
}
+static void iommu_flush_iova(struct iova_domain *iovad)
+{
+ struct dmar_domain *domain;
+ int idx;
+
+ domain = container_of(iovad, struct dmar_domain, iovad);
+
+ for_each_domain_iommu(idx, domain) {
+ struct intel_iommu *iommu = g_iommus[idx];
+ u16 did = domain->iommu_did[iommu->seq_id];
+
+ iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
+
+ if (!cap_caching_mode(iommu->cap))
+ iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
+ 0, MAX_AGAW_PFN_WIDTH);
+ }
+}
+
static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
{
u32 pmen;
@@ -1939,9 +1936,16 @@ static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
{
int adjust_width, agaw;
unsigned long sagaw;
+ int err;
init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
DMA_32BIT_PFN);
+
+ err = init_iova_flush_queue(&domain->iovad,
+ iommu_flush_iova, iova_entry_free);
+ if (err)
+ return err;
+
domain_reserve_special_ranges(domain);
/* calculate AGAW */
@@ -1993,14 +1997,6 @@ static void domain_exit(struct dmar_domain *domain)
if (!domain)
return;
- /* Flush any lazy unmaps that may reference this domain */
- if (!intel_iommu_strict) {
- int cpu;
-
- for_each_possible_cpu(cpu)
- flush_unmaps_timeout(cpu);
- }
-
/* Remove associated devices and clear attached or cached domains */
rcu_read_lock();
domain_remove_dev_info(domain);
@@ -2284,8 +2280,11 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
/*
* Ensure that old small page tables are
* removed to make room for superpage(s).
+ * We're adding new large pages, so make sure
+ * we don't remove their parent tables.
*/
- dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
+ dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
+ largepage_lvl + 1);
} else {
pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
}
@@ -2358,13 +2357,33 @@ static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long i
static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
+ unsigned long flags;
+ struct context_entry *context;
+ u16 did_old;
+
if (!iommu)
return;
- clear_context_table(iommu, bus, devfn);
- iommu->flush.flush_context(iommu, 0, 0, 0,
- DMA_CCMD_GLOBAL_INVL);
- iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
+ spin_lock_irqsave(&iommu->lock, flags);
+ context = iommu_context_addr(iommu, bus, devfn, 0);
+ if (!context) {
+ spin_unlock_irqrestore(&iommu->lock, flags);
+ return;
+ }
+ did_old = context_domain_id(context);
+ context_clear_entry(context);
+ __iommu_flush_cache(iommu, context, sizeof(*context));
+ spin_unlock_irqrestore(&iommu->lock, flags);
+ iommu->flush.flush_context(iommu,
+ did_old,
+ (((u16)bus) << 8) | devfn,
+ DMA_CCMD_MASK_NOBIT,
+ DMA_CCMD_DEVICE_INVL);
+ iommu->flush.flush_iotlb(iommu,
+ did_old,
+ 0,
+ 0,
+ DMA_TLB_DSI_FLUSH);
}
static inline void unlink_domain_info(struct device_domain_info *info)
@@ -3213,7 +3232,7 @@ static int __init init_dmars(void)
bool copied_tables = false;
struct device *dev;
struct intel_iommu *iommu;
- int i, ret, cpu;
+ int i, ret;
/*
* for each drhd
@@ -3246,22 +3265,6 @@ static int __init init_dmars(void)
goto error;
}
- for_each_possible_cpu(cpu) {
- struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
- cpu);
-
- dfd->tables = kzalloc(g_num_of_iommus *
- sizeof(struct deferred_flush_table),
- GFP_KERNEL);
- if (!dfd->tables) {
- ret = -ENOMEM;
- goto free_g_iommus;
- }
-
- spin_lock_init(&dfd->lock);
- setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
- }
-
for_each_active_iommu(iommu, drhd) {
g_iommus[iommu->seq_id] = iommu;
@@ -3444,10 +3447,9 @@ free_iommu:
disable_dmar_iommu(iommu);
free_dmar_iommu(iommu);
}
-free_g_iommus:
- for_each_possible_cpu(cpu)
- kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
+
kfree(g_iommus);
+
error:
return ret;
}
@@ -3652,110 +3654,6 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
dir, *dev->dma_mask);
}
-static void flush_unmaps(struct deferred_flush_data *flush_data)
-{
- int i, j;
-
- flush_data->timer_on = 0;
-
- /* just flush them all */
- for (i = 0; i < g_num_of_iommus; i++) {
- struct intel_iommu *iommu = g_iommus[i];
- struct deferred_flush_table *flush_table =
- &flush_data->tables[i];
- if (!iommu)
- continue;
-
- if (!flush_table->next)
- continue;
-
- /* In caching mode, global flushes turn emulation expensive */
- if (!cap_caching_mode(iommu->cap))
- iommu->flush.flush_iotlb(iommu, 0, 0, 0,
- DMA_TLB_GLOBAL_FLUSH);
- for (j = 0; j < flush_table->next; j++) {
- unsigned long mask;
- struct deferred_flush_entry *entry =
- &flush_table->entries[j];
- unsigned long iova_pfn = entry->iova_pfn;
- unsigned long nrpages = entry->nrpages;
- struct dmar_domain *domain = entry->domain;
- struct page *freelist = entry->freelist;
-
- /* On real hardware multiple invalidations are expensive */
- if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, domain,
- mm_to_dma_pfn(iova_pfn),
- nrpages, !freelist, 0);
- else {
- mask = ilog2(nrpages);
- iommu_flush_dev_iotlb(domain,
- (uint64_t)iova_pfn << PAGE_SHIFT, mask);
- }
- free_iova_fast(&domain->iovad, iova_pfn, nrpages);
- if (freelist)
- dma_free_pagelist(freelist);
- }
- flush_table->next = 0;
- }
-
- flush_data->size = 0;
-}
-
-static void flush_unmaps_timeout(unsigned long cpuid)
-{
- struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
- unsigned long flags;
-
- spin_lock_irqsave(&flush_data->lock, flags);
- flush_unmaps(flush_data);
- spin_unlock_irqrestore(&flush_data->lock, flags);
-}
-
-static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
- unsigned long nrpages, struct page *freelist)
-{
- unsigned long flags;
- int entry_id, iommu_id;
- struct intel_iommu *iommu;
- struct deferred_flush_entry *entry;
- struct deferred_flush_data *flush_data;
-
- flush_data = raw_cpu_ptr(&deferred_flush);
-
- /* Flush all CPUs' entries to avoid deferring too much. If
- * this becomes a bottleneck, can just flush us, and rely on
- * flush timer for the rest.
- */
- if (flush_data->size == HIGH_WATER_MARK) {
- int cpu;
-
- for_each_online_cpu(cpu)
- flush_unmaps_timeout(cpu);
- }
-
- spin_lock_irqsave(&flush_data->lock, flags);
-
- iommu = domain_get_iommu(dom);
- iommu_id = iommu->seq_id;
-
- entry_id = flush_data->tables[iommu_id].next;
- ++(flush_data->tables[iommu_id].next);
-
- entry = &flush_data->tables[iommu_id].entries[entry_id];
- entry->domain = dom;
- entry->iova_pfn = iova_pfn;
- entry->nrpages = nrpages;
- entry->freelist = freelist;
-
- if (!flush_data->timer_on) {
- mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
- flush_data->timer_on = 1;
- }
- flush_data->size++;
- spin_unlock_irqrestore(&flush_data->lock, flags);
-}
-
static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
{
struct dmar_domain *domain;
@@ -3791,7 +3689,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
dma_free_pagelist(freelist);
} else {
- add_unmap(domain, iova_pfn, nrpages, freelist);
+ queue_iova(&domain->iovad, iova_pfn, nrpages,
+ (unsigned long)freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
@@ -3945,7 +3844,8 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
if (unlikely(ret)) {
dma_pte_free_pagetable(domain, start_vpfn,
- start_vpfn + size - 1);
+ start_vpfn + size - 1,
+ agaw_to_level(domain->agaw) + 1);
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
return 0;
}
@@ -4728,7 +4628,6 @@ static void free_all_cpu_cached_iovas(unsigned int cpu)
static int intel_iommu_cpu_dead(unsigned int cpu)
{
free_all_cpu_cached_iovas(cpu);
- flush_unmaps_timeout(cpu);
return 0;
}
@@ -5350,7 +5249,8 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
sdev->sid = PCI_DEVID(info->bus, info->devfn);
if (!(ctx_lo & CONTEXT_PASIDE)) {
- context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
+ if (iommu->pasid_state_table)
+ context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
intel_iommu_get_pts(iommu);
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index f620dccec8ee..f6697e55c2d4 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -24,6 +24,7 @@
#include <linux/pci-ats.h>
#include <linux/dmar.h>
#include <linux/interrupt.h>
+#include <asm/page.h>
static irqreturn_t prq_event_thread(int irq, void *d);
@@ -546,6 +547,14 @@ static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req)
return (requested & ~vma->vm_flags) != 0;
}
+static bool is_canonical_address(u64 addr)
+{
+ int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
+ long saddr = (long) addr;
+
+ return (((saddr << shift) >> shift) == saddr);
+}
+
static irqreturn_t prq_event_thread(int irq, void *d)
{
struct intel_iommu *iommu = d;
@@ -603,6 +612,11 @@ static irqreturn_t prq_event_thread(int irq, void *d)
/* If the mm is already defunct, don't handle faults. */
if (!mmget_not_zero(svm->mm))
goto bad_req;
+
+ /* If address is not canonical, return invalid response */
+ if (!is_canonical_address(address))
+ goto bad_req;
+
down_read(&svm->mm->mmap_sem);
vma = find_extend_vma(svm->mm, address);
if (!vma || address < vma->vm_start)
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 3f6ea160afed..3de5c0bcb5cc 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -527,6 +527,8 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group,
}
+ iommu_flush_tlb_all(domain);
+
out:
iommu_put_resv_regions(dev, &mappings);
@@ -1005,11 +1007,10 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
if (group)
return group;
- group = ERR_PTR(-EINVAL);
-
- if (ops && ops->device_group)
- group = ops->device_group(dev);
+ if (!ops)
+ return ERR_PTR(-EINVAL);
+ group = ops->device_group(dev);
if (WARN_ON_ONCE(group == NULL))
return ERR_PTR(-EINVAL);
@@ -1283,6 +1284,10 @@ static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
int ret;
+ if ((domain->ops->is_attach_deferred != NULL) &&
+ domain->ops->is_attach_deferred(domain, dev))
+ return 0;
+
if (unlikely(domain->ops->attach_dev == NULL))
return -ENODEV;
@@ -1298,12 +1303,8 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
int ret;
group = iommu_group_get(dev);
- /* FIXME: Remove this when groups a mandatory for iommu drivers */
- if (group == NULL)
- return __iommu_attach_device(domain, dev);
-
/*
- * We have a group - lock it to make sure the device-count doesn't
+ * Lock the group to make sure the device-count doesn't
* change while we are attaching
*/
mutex_lock(&group->mutex);
@@ -1324,6 +1325,10 @@ EXPORT_SYMBOL_GPL(iommu_attach_device);
static void __iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
+ if ((domain->ops->is_attach_deferred != NULL) &&
+ domain->ops->is_attach_deferred(domain, dev))
+ return;
+
if (unlikely(domain->ops->detach_dev == NULL))
return;
@@ -1336,9 +1341,6 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
struct iommu_group *group;
group = iommu_group_get(dev);
- /* FIXME: Remove this when groups a mandatory for iommu drivers */
- if (group == NULL)
- return __iommu_detach_device(domain, dev);
mutex_lock(&group->mutex);
if (iommu_group_device_count(group) != 1) {
@@ -1360,8 +1362,7 @@ struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
struct iommu_group *group;
group = iommu_group_get(dev);
- /* FIXME: Remove this when groups a mandatory for iommu drivers */
- if (group == NULL)
+ if (!group)
return NULL;
domain = group->domain;
@@ -1556,13 +1557,16 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
}
EXPORT_SYMBOL_GPL(iommu_map);
-size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
+static size_t __iommu_unmap(struct iommu_domain *domain,
+ unsigned long iova, size_t size,
+ bool sync)
{
+ const struct iommu_ops *ops = domain->ops;
size_t unmapped_page, unmapped = 0;
- unsigned int min_pagesz;
unsigned long orig_iova = iova;
+ unsigned int min_pagesz;
- if (unlikely(domain->ops->unmap == NULL ||
+ if (unlikely(ops->unmap == NULL ||
domain->pgsize_bitmap == 0UL))
return -ENODEV;
@@ -1592,10 +1596,13 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
while (unmapped < size) {
size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
- unmapped_page = domain->ops->unmap(domain, iova, pgsize);
+ unmapped_page = ops->unmap(domain, iova, pgsize);
if (!unmapped_page)
break;
+ if (sync && ops->iotlb_range_add)
+ ops->iotlb_range_add(domain, iova, pgsize);
+
pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
iova, unmapped_page);
@@ -1603,11 +1610,27 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
unmapped += unmapped_page;
}
+ if (sync && ops->iotlb_sync)
+ ops->iotlb_sync(domain);
+
trace_unmap(orig_iova, size, unmapped);
return unmapped;
}
+
+size_t iommu_unmap(struct iommu_domain *domain,
+ unsigned long iova, size_t size)
+{
+ return __iommu_unmap(domain, iova, size, true);
+}
EXPORT_SYMBOL_GPL(iommu_unmap);
+size_t iommu_unmap_fast(struct iommu_domain *domain,
+ unsigned long iova, size_t size)
+{
+ return __iommu_unmap(domain, iova, size, false);
+}
+EXPORT_SYMBOL_GPL(iommu_unmap_fast);
+
size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
struct scatterlist *sg, unsigned int nents, int prot)
{
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 246f14c83944..33edfa794ae9 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -32,6 +32,8 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,
unsigned long limit_pfn);
static void init_iova_rcaches(struct iova_domain *iovad);
static void free_iova_rcaches(struct iova_domain *iovad);
+static void fq_destroy_all_entries(struct iova_domain *iovad);
+static void fq_flush_timeout(unsigned long data);
void
init_iova_domain(struct iova_domain *iovad, unsigned long granule,
@@ -50,10 +52,61 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
iovad->granule = granule;
iovad->start_pfn = start_pfn;
iovad->dma_32bit_pfn = pfn_32bit + 1;
+ iovad->flush_cb = NULL;
+ iovad->fq = NULL;
init_iova_rcaches(iovad);
}
EXPORT_SYMBOL_GPL(init_iova_domain);
+static void free_iova_flush_queue(struct iova_domain *iovad)
+{
+ if (!iovad->fq)
+ return;
+
+ if (timer_pending(&iovad->fq_timer))
+ del_timer(&iovad->fq_timer);
+
+ fq_destroy_all_entries(iovad);
+
+ free_percpu(iovad->fq);
+
+ iovad->fq = NULL;
+ iovad->flush_cb = NULL;
+ iovad->entry_dtor = NULL;
+}
+
+int init_iova_flush_queue(struct iova_domain *iovad,
+ iova_flush_cb flush_cb, iova_entry_dtor entry_dtor)
+{
+ int cpu;
+
+ atomic64_set(&iovad->fq_flush_start_cnt, 0);
+ atomic64_set(&iovad->fq_flush_finish_cnt, 0);
+
+ iovad->fq = alloc_percpu(struct iova_fq);
+ if (!iovad->fq)
+ return -ENOMEM;
+
+ iovad->flush_cb = flush_cb;
+ iovad->entry_dtor = entry_dtor;
+
+ for_each_possible_cpu(cpu) {
+ struct iova_fq *fq;
+
+ fq = per_cpu_ptr(iovad->fq, cpu);
+ fq->head = 0;
+ fq->tail = 0;
+
+ spin_lock_init(&fq->lock);
+ }
+
+ setup_timer(&iovad->fq_timer, fq_flush_timeout, (unsigned long)iovad);
+ atomic_set(&iovad->fq_timer_on, 0);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(init_iova_flush_queue);
+
static struct rb_node *
__get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
{
@@ -423,6 +476,135 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
}
EXPORT_SYMBOL_GPL(free_iova_fast);
+#define fq_ring_for_each(i, fq) \
+ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
+
+static inline bool fq_full(struct iova_fq *fq)
+{
+ assert_spin_locked(&fq->lock);
+ return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
+}
+
+static inline unsigned fq_ring_add(struct iova_fq *fq)
+{
+ unsigned idx = fq->tail;
+
+ assert_spin_locked(&fq->lock);
+
+ fq->tail = (idx + 1) % IOVA_FQ_SIZE;
+
+ return idx;
+}
+
+static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
+{
+ u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
+ unsigned idx;
+
+ assert_spin_locked(&fq->lock);
+
+ fq_ring_for_each(idx, fq) {
+
+ if (fq->entries[idx].counter >= counter)
+ break;
+
+ if (iovad->entry_dtor)
+ iovad->entry_dtor(fq->entries[idx].data);
+
+ free_iova_fast(iovad,
+ fq->entries[idx].iova_pfn,
+ fq->entries[idx].pages);
+
+ fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
+ }
+}
+
+static void iova_domain_flush(struct iova_domain *iovad)
+{
+ atomic64_inc(&iovad->fq_flush_start_cnt);
+ iovad->flush_cb(iovad);
+ atomic64_inc(&iovad->fq_flush_finish_cnt);
+}
+
+static void fq_destroy_all_entries(struct iova_domain *iovad)
+{
+ int cpu;
+
+ /*
+ * This code runs when the iova_domain is being detroyed, so don't
+ * bother to free iovas, just call the entry_dtor on all remaining
+ * entries.
+ */
+ if (!iovad->entry_dtor)
+ return;
+
+ for_each_possible_cpu(cpu) {
+ struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
+ int idx;
+
+ fq_ring_for_each(idx, fq)
+ iovad->entry_dtor(fq->entries[idx].data);
+ }
+}
+
+static void fq_flush_timeout(unsigned long data)
+{
+ struct iova_domain *iovad = (struct iova_domain *)data;
+ int cpu;
+
+ atomic_set(&iovad->fq_timer_on, 0);
+ iova_domain_flush(iovad);
+
+ for_each_possible_cpu(cpu) {
+ unsigned long flags;
+ struct iova_fq *fq;
+
+ fq = per_cpu_ptr(iovad->fq, cpu);
+ spin_lock_irqsave(&fq->lock, flags);
+ fq_ring_free(iovad, fq);
+ spin_unlock_irqrestore(&fq->lock, flags);
+ }
+}
+
+void queue_iova(struct iova_domain *iovad,
+ unsigned long pfn, unsigned long pages,
+ unsigned long data)
+{
+ struct iova_fq *fq = get_cpu_ptr(iovad->fq);
+ unsigned long flags;
+ unsigned idx;
+
+ spin_lock_irqsave(&fq->lock, flags);
+
+ /*
+ * First remove all entries from the flush queue that have already been
+ * flushed out on another CPU. This makes the fq_full() check below less
+ * likely to be true.
+ */
+ fq_ring_free(iovad, fq);
+
+ if (fq_full(fq)) {
+ iova_domain_flush(iovad);
+ fq_ring_free(iovad, fq);
+ }
+
+ idx = fq_ring_add(fq);
+
+ fq->entries[idx].iova_pfn = pfn;
+ fq->entries[idx].pages = pages;
+ fq->entries[idx].data = data;
+ fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
+
+ spin_unlock_irqrestore(&fq->lock, flags);
+
+ if (atomic_cmpxchg(&iovad->fq_timer_on, 0, 1) == 0)
+ mod_timer(&iovad->fq_timer,
+ jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
+
+ put_cpu_ptr(iovad->fq);
+}
+EXPORT_SYMBOL_GPL(queue_iova);
+
/**
* put_iova_domain - destroys the iova doamin
* @iovad: - iova domain in question.
@@ -433,6 +615,7 @@ void put_iova_domain(struct iova_domain *iovad)
struct rb_node *node;
unsigned long flags;
+ free_iova_flush_queue(iovad);
free_iova_rcaches(iovad);
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
node = rb_first(&iovad->rbroot);
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 2a38aa15be17..195d6e93ac71 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -19,6 +19,7 @@
#include <linux/iommu.h>
#include <linux/module.h>
#include <linux/of.h>
+#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/sizes.h>
#include <linux/slab.h>
@@ -35,7 +36,7 @@
struct ipmmu_vmsa_device {
struct device *dev;
void __iomem *base;
- struct list_head list;
+ struct iommu_device iommu;
unsigned int num_utlbs;
spinlock_t lock; /* Protects ctx and domains[] */
@@ -58,36 +59,18 @@ struct ipmmu_vmsa_domain {
struct ipmmu_vmsa_iommu_priv {
struct ipmmu_vmsa_device *mmu;
- unsigned int *utlbs;
- unsigned int num_utlbs;
struct device *dev;
struct list_head list;
};
-static DEFINE_SPINLOCK(ipmmu_devices_lock);
-static LIST_HEAD(ipmmu_devices);
-
static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
{
return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
}
-
static struct ipmmu_vmsa_iommu_priv *to_priv(struct device *dev)
{
-#if defined(CONFIG_ARM)
- return dev->archdata.iommu;
-#else
- return dev->iommu_fwspec->iommu_priv;
-#endif
-}
-static void set_priv(struct device *dev, struct ipmmu_vmsa_iommu_priv *p)
-{
-#if defined(CONFIG_ARM)
- dev->archdata.iommu = p;
-#else
- dev->iommu_fwspec->iommu_priv = p;
-#endif
+ return dev->iommu_fwspec ? dev->iommu_fwspec->iommu_priv : NULL;
}
#define TLB_LOOP_TIMEOUT 100 /* 100us */
@@ -312,7 +295,7 @@ static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
/* The hardware doesn't support selective TLB flush. */
}
-static struct iommu_gather_ops ipmmu_gather_ops = {
+static const struct iommu_gather_ops ipmmu_gather_ops = {
.tlb_flush_all = ipmmu_tlb_flush_all,
.tlb_add_flush = ipmmu_tlb_add_flush,
.tlb_sync = ipmmu_tlb_flush_all,
@@ -341,6 +324,19 @@ static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,
return ret;
}
+static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
+ unsigned int context_id)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&mmu->lock, flags);
+
+ clear_bit(context_id, mmu->ctx);
+ mmu->domains[context_id] = NULL;
+
+ spin_unlock_irqrestore(&mmu->lock, flags);
+}
+
static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
{
u64 ttbr;
@@ -370,22 +366,22 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
*/
domain->cfg.iommu_dev = domain->mmu->dev;
- domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
- domain);
- if (!domain->iop)
- return -EINVAL;
-
/*
* Find an unused context.
*/
ret = ipmmu_domain_allocate_context(domain->mmu, domain);
- if (ret == IPMMU_CTX_MAX) {
- free_io_pgtable_ops(domain->iop);
+ if (ret == IPMMU_CTX_MAX)
return -EBUSY;
- }
domain->context_id = ret;
+ domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
+ domain);
+ if (!domain->iop) {
+ ipmmu_domain_free_context(domain->mmu, domain->context_id);
+ return -EINVAL;
+ }
+
/* TTBR0 */
ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];
ipmmu_ctx_write(domain, IMTTLBR0, ttbr);
@@ -426,19 +422,6 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
return 0;
}
-static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
- unsigned int context_id)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&mmu->lock, flags);
-
- clear_bit(context_id, mmu->ctx);
- mmu->domains[context_id] = NULL;
-
- spin_unlock_irqrestore(&mmu->lock, flags);
-}
-
static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
{
/*
@@ -562,13 +545,14 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
struct device *dev)
{
struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct ipmmu_vmsa_device *mmu = priv->mmu;
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
unsigned long flags;
unsigned int i;
int ret = 0;
- if (!mmu) {
+ if (!priv || !priv->mmu) {
dev_err(dev, "Cannot attach to IPMMU\n");
return -ENXIO;
}
@@ -595,8 +579,8 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
if (ret < 0)
return ret;
- for (i = 0; i < priv->num_utlbs; ++i)
- ipmmu_utlb_enable(domain, priv->utlbs[i]);
+ for (i = 0; i < fwspec->num_ids; ++i)
+ ipmmu_utlb_enable(domain, fwspec->ids[i]);
return 0;
}
@@ -604,12 +588,12 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
static void ipmmu_detach_device(struct iommu_domain *io_domain,
struct device *dev)
{
- struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
unsigned int i;
- for (i = 0; i < priv->num_utlbs; ++i)
- ipmmu_utlb_disable(domain, priv->utlbs[i]);
+ for (i = 0; i < fwspec->num_ids; ++i)
+ ipmmu_utlb_disable(domain, fwspec->ids[i]);
/*
* TODO: Optimize by disabling the context when no device is attached.
@@ -645,92 +629,36 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
return domain->iop->iova_to_phys(domain->iop, iova);
}
-static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev,
- unsigned int *utlbs, unsigned int num_utlbs)
-{
- unsigned int i;
-
- for (i = 0; i < num_utlbs; ++i) {
- struct of_phandle_args args;
- int ret;
-
- ret = of_parse_phandle_with_args(dev->of_node, "iommus",
- "#iommu-cells", i, &args);
- if (ret < 0)
- return ret;
-
- of_node_put(args.np);
-
- if (args.np != mmu->dev->of_node || args.args_count != 1)
- return -EINVAL;
-
- utlbs[i] = args.args[0];
- }
-
- return 0;
-}
-
-static int ipmmu_init_platform_device(struct device *dev)
+static int ipmmu_init_platform_device(struct device *dev,
+ struct of_phandle_args *args)
{
+ struct platform_device *ipmmu_pdev;
struct ipmmu_vmsa_iommu_priv *priv;
- struct ipmmu_vmsa_device *mmu;
- unsigned int *utlbs;
- unsigned int i;
- int num_utlbs;
- int ret = -ENODEV;
-
- /* Find the master corresponding to the device. */
- num_utlbs = of_count_phandle_with_args(dev->of_node, "iommus",
- "#iommu-cells");
- if (num_utlbs < 0)
+ ipmmu_pdev = of_find_device_by_node(args->np);
+ if (!ipmmu_pdev)
return -ENODEV;
- utlbs = kcalloc(num_utlbs, sizeof(*utlbs), GFP_KERNEL);
- if (!utlbs)
- return -ENOMEM;
-
- spin_lock(&ipmmu_devices_lock);
-
- list_for_each_entry(mmu, &ipmmu_devices, list) {
- ret = ipmmu_find_utlbs(mmu, dev, utlbs, num_utlbs);
- if (!ret) {
- /*
- * TODO Take a reference to the MMU to protect
- * against device removal.
- */
- break;
- }
- }
-
- spin_unlock(&ipmmu_devices_lock);
-
- if (ret < 0)
- goto error;
-
- for (i = 0; i < num_utlbs; ++i) {
- if (utlbs[i] >= mmu->num_utlbs) {
- ret = -EINVAL;
- goto error;
- }
- }
-
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv) {
- ret = -ENOMEM;
- goto error;
- }
+ if (!priv)
+ return -ENOMEM;
- priv->mmu = mmu;
- priv->utlbs = utlbs;
- priv->num_utlbs = num_utlbs;
+ priv->mmu = platform_get_drvdata(ipmmu_pdev);
priv->dev = dev;
- set_priv(dev, priv);
+ dev->iommu_fwspec->iommu_priv = priv;
return 0;
+}
-error:
- kfree(utlbs);
- return ret;
+static int ipmmu_of_xlate(struct device *dev,
+ struct of_phandle_args *spec)
+{
+ iommu_fwspec_add_ids(dev, spec->args, 1);
+
+ /* Initialize once - xlate() will call multiple times */
+ if (to_priv(dev))
+ return 0;
+
+ return ipmmu_init_platform_device(dev, spec);
}
#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
@@ -749,11 +677,11 @@ static int ipmmu_add_device(struct device *dev)
struct iommu_group *group;
int ret;
- if (to_priv(dev)) {
- dev_warn(dev, "IOMMU driver already assigned to device %s\n",
- dev_name(dev));
- return -EINVAL;
- }
+ /*
+ * Only let through devices that have been verified in xlate()
+ */
+ if (!to_priv(dev))
+ return -ENODEV;
/* Create a device group and add the device to it. */
group = iommu_group_alloc();
@@ -772,10 +700,6 @@ static int ipmmu_add_device(struct device *dev)
goto error;
}
- ret = ipmmu_init_platform_device(dev);
- if (ret < 0)
- goto error;
-
/*
* Create the ARM mapping, used by the ARM DMA mapping core to allocate
* VAs. This will allocate a corresponding IOMMU domain.
@@ -816,24 +740,13 @@ error:
if (!IS_ERR_OR_NULL(group))
iommu_group_remove_device(dev);
- kfree(to_priv(dev)->utlbs);
- kfree(to_priv(dev));
- set_priv(dev, NULL);
-
return ret;
}
static void ipmmu_remove_device(struct device *dev)
{
- struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
-
arm_iommu_detach_device(dev);
iommu_group_remove_device(dev);
-
- kfree(priv->utlbs);
- kfree(priv);
-
- set_priv(dev, NULL);
}
static const struct iommu_ops ipmmu_ops = {
@@ -848,6 +761,7 @@ static const struct iommu_ops ipmmu_ops = {
.add_device = ipmmu_add_device,
.remove_device = ipmmu_remove_device,
.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
+ .of_xlate = ipmmu_of_xlate,
};
#endif /* !CONFIG_ARM && CONFIG_IOMMU_DMA */
@@ -890,14 +804,12 @@ static void ipmmu_domain_free_dma(struct iommu_domain *io_domain)
static int ipmmu_add_device_dma(struct device *dev)
{
- struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct iommu_group *group;
/*
* Only let through devices that have been verified in xlate()
- * We may get called with dev->iommu_fwspec set to NULL.
*/
- if (!fwspec || !fwspec->iommu_priv)
+ if (!to_priv(dev))
return -ENODEV;
group = iommu_group_get_for_dev(dev);
@@ -957,19 +869,6 @@ static struct iommu_group *ipmmu_find_group_dma(struct device *dev)
return group;
}
-static int ipmmu_of_xlate_dma(struct device *dev,
- struct of_phandle_args *spec)
-{
- /* If the IPMMU device is disabled in DT then return error
- * to make sure the of_iommu code does not install ops
- * even though the iommu device is disabled
- */
- if (!of_device_is_available(spec->np))
- return -ENODEV;
-
- return ipmmu_init_platform_device(dev);
-}
-
static const struct iommu_ops ipmmu_ops = {
.domain_alloc = ipmmu_domain_alloc_dma,
.domain_free = ipmmu_domain_free_dma,
@@ -983,7 +882,7 @@ static const struct iommu_ops ipmmu_ops = {
.remove_device = ipmmu_remove_device_dma,
.device_group = ipmmu_find_group_dma,
.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
- .of_xlate = ipmmu_of_xlate_dma,
+ .of_xlate = ipmmu_of_xlate,
};
#endif /* CONFIG_IOMMU_DMA */
@@ -1054,16 +953,24 @@ static int ipmmu_probe(struct platform_device *pdev)
ipmmu_device_reset(mmu);
+ ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,
+ dev_name(&pdev->dev));
+ if (ret)
+ return ret;
+
+ iommu_device_set_ops(&mmu->iommu, &ipmmu_ops);
+ iommu_device_set_fwnode(&mmu->iommu, &pdev->dev.of_node->fwnode);
+
+ ret = iommu_device_register(&mmu->iommu);
+ if (ret)
+ return ret;
+
/*
* We can't create the ARM mapping here as it requires the bus to have
* an IOMMU, which only happens when bus_set_iommu() is called in
* ipmmu_init() after the probe function returns.
*/
- spin_lock(&ipmmu_devices_lock);
- list_add(&mmu->list, &ipmmu_devices);
- spin_unlock(&ipmmu_devices_lock);
-
platform_set_drvdata(pdev, mmu);
return 0;
@@ -1073,9 +980,8 @@ static int ipmmu_remove(struct platform_device *pdev)
{
struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
- spin_lock(&ipmmu_devices_lock);
- list_del(&mmu->list);
- spin_unlock(&ipmmu_devices_lock);
+ iommu_device_sysfs_remove(&mmu->iommu);
+ iommu_device_unregister(&mmu->iommu);
#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
arm_iommu_release_mapping(mmu->mapping);
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index d0448353d501..04f4d51ffacb 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -393,6 +393,7 @@ static struct msm_iommu_dev *find_iommu_for_dev(struct device *dev)
static int msm_iommu_add_device(struct device *dev)
{
struct msm_iommu_dev *iommu;
+ struct iommu_group *group;
unsigned long flags;
int ret = 0;
@@ -406,7 +407,16 @@ static int msm_iommu_add_device(struct device *dev)
spin_unlock_irqrestore(&msm_iommu_lock, flags);
- return ret;
+ if (ret)
+ return ret;
+
+ group = iommu_group_get_for_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ iommu_group_put(group);
+
+ return 0;
}
static void msm_iommu_remove_device(struct device *dev)
@@ -421,6 +431,8 @@ static void msm_iommu_remove_device(struct device *dev)
iommu_device_unlink(&iommu->iommu, dev);
spin_unlock_irqrestore(&msm_iommu_lock, flags);
+
+ iommu_group_remove_device(dev);
}
static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -700,6 +712,7 @@ static struct iommu_ops msm_iommu_ops = {
.iova_to_phys = msm_iommu_iova_to_phys,
.add_device = msm_iommu_add_device,
.remove_device = msm_iommu_remove_device,
+ .device_group = generic_device_group,
.pgsize_bitmap = MSM_IOMMU_PGSIZES,
.of_xlate = qcom_iommu_of_xlate,
};
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index 91c6d367ab35..bd515be5b380 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -31,7 +31,6 @@
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <asm/barrier.h>
-#include <dt-bindings/memory/mt8173-larb-port.h>
#include <soc/mediatek/smi.h>
#include "mtk_iommu.h"
@@ -54,10 +53,16 @@
#define REG_MMU_CTRL_REG 0x110
#define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
-#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5)
+#define F_MMU_TF_PROTECT_SEL_SHIFT(data) \
+ ((data)->m4u_plat == M4U_MT2712 ? 4 : 5)
+/* It's named by F_MMU_TF_PROT_SEL in mt2712. */
+#define F_MMU_TF_PROTECT_SEL(prot, data) \
+ (((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data))
#define REG_MMU_IVRP_PADDR 0x114
#define F_MMU_IVRP_PA_SET(pa, ext) (((pa) >> 1) | ((!!(ext)) << 31))
+#define REG_MMU_VLD_PA_RNG 0x118
+#define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA))
#define REG_MMU_INT_CONTROL0 0x120
#define F_L2_MULIT_HIT_EN BIT(0)
@@ -82,7 +87,6 @@
#define REG_MMU_FAULT_ST1 0x134
#define REG_MMU_FAULT_VA 0x13c
-#define F_MMU_FAULT_VA_MSK 0xfffff000
#define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
#define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
@@ -93,6 +97,13 @@
#define MTK_PROTECT_PA_ALIGN 128
+/*
+ * Get the local arbiter ID and the portid within the larb arbiter
+ * from mtk_m4u_id which is defined by MTK_M4U_ID.
+ */
+#define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf)
+#define MTK_M4U_TO_PORT(id) ((id) & 0x1f)
+
struct mtk_iommu_domain {
spinlock_t pgtlock; /* lock for page table */
@@ -104,6 +115,27 @@ struct mtk_iommu_domain {
static struct iommu_ops mtk_iommu_ops;
+static LIST_HEAD(m4ulist); /* List all the M4U HWs */
+
+#define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list)
+
+/*
+ * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
+ * for the performance.
+ *
+ * Here always return the mtk_iommu_data of the first probed M4U where the
+ * iommu domain information is recorded.
+ */
+static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void)
+{
+ struct mtk_iommu_data *data;
+
+ for_each_m4u(data)
+ return data;
+
+ return NULL;
+}
+
static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
{
return container_of(dom, struct mtk_iommu_domain, domain);
@@ -113,9 +145,12 @@ static void mtk_iommu_tlb_flush_all(void *cookie)
{
struct mtk_iommu_data *data = cookie;
- writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL);
- writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
- wmb(); /* Make sure the tlb flush all done */
+ for_each_m4u(data) {
+ writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
+ data->base + REG_MMU_INV_SEL);
+ writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
+ wmb(); /* Make sure the tlb flush all done */
+ }
}
static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
@@ -124,12 +159,17 @@ static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
{
struct mtk_iommu_data *data = cookie;
- writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL);
+ for_each_m4u(data) {
+ writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
+ data->base + REG_MMU_INV_SEL);
- writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
- writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A);
- writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE);
- data->tlb_flush_active = true;
+ writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
+ writel_relaxed(iova + size - 1,
+ data->base + REG_MMU_INVLD_END_A);
+ writel_relaxed(F_MMU_INV_RANGE,
+ data->base + REG_MMU_INVALIDATE);
+ data->tlb_flush_active = true;
+ }
}
static void mtk_iommu_tlb_sync(void *cookie)
@@ -138,20 +178,22 @@ static void mtk_iommu_tlb_sync(void *cookie)
int ret;
u32 tmp;
- /* Avoid timing out if there's nothing to wait for */
- if (!data->tlb_flush_active)
- return;
+ for_each_m4u(data) {
+ /* Avoid timing out if there's nothing to wait for */
+ if (!data->tlb_flush_active)
+ return;
- ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp,
- tmp != 0, 10, 100000);
- if (ret) {
- dev_warn(data->dev,
- "Partial TLB flush timed out, falling back to full flush\n");
- mtk_iommu_tlb_flush_all(cookie);
+ ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
+ tmp, tmp != 0, 10, 100000);
+ if (ret) {
+ dev_warn(data->dev,
+ "Partial TLB flush timed out, falling back to full flush\n");
+ mtk_iommu_tlb_flush_all(cookie);
+ }
+ /* Clear the CPE status */
+ writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
+ data->tlb_flush_active = false;
}
- /* Clear the CPE status */
- writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
- data->tlb_flush_active = false;
}
static const struct iommu_gather_ops mtk_iommu_gather_ops = {
@@ -173,7 +215,6 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
- fault_iova &= F_MMU_FAULT_VA_MSK;
fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
regval = readl_relaxed(data->base + REG_MMU_INT_ID);
fault_larb = F_MMU0_INT_ID_LARB_ID(regval);
@@ -221,9 +262,9 @@ static void mtk_iommu_config(struct mtk_iommu_data *data,
}
}
-static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
+static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
{
- struct mtk_iommu_domain *dom = data->m4u_dom;
+ struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
spin_lock_init(&dom->pgtlock);
@@ -249,9 +290,6 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
/* Update our support page sizes bitmap */
dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
-
- writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
- data->base + REG_MMU_PT_BASE_ADDR);
return 0;
}
@@ -266,20 +304,30 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
if (!dom)
return NULL;
- if (iommu_get_dma_cookie(&dom->domain)) {
- kfree(dom);
- return NULL;
- }
+ if (iommu_get_dma_cookie(&dom->domain))
+ goto free_dom;
+
+ if (mtk_iommu_domain_finalise(dom))
+ goto put_dma_cookie;
dom->domain.geometry.aperture_start = 0;
dom->domain.geometry.aperture_end = DMA_BIT_MASK(32);
dom->domain.geometry.force_aperture = true;
return &dom->domain;
+
+put_dma_cookie:
+ iommu_put_dma_cookie(&dom->domain);
+free_dom:
+ kfree(dom);
+ return NULL;
}
static void mtk_iommu_domain_free(struct iommu_domain *domain)
{
+ struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+
+ free_io_pgtable_ops(dom->iop);
iommu_put_dma_cookie(domain);
kfree(to_mtk_domain(domain));
}
@@ -289,22 +337,15 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
- int ret;
if (!data)
return -ENODEV;
+ /* Update the pgtable base address register of the M4U HW */
if (!data->m4u_dom) {
data->m4u_dom = dom;
- ret = mtk_iommu_domain_finalise(data);
- if (ret) {
- data->m4u_dom = NULL;
- return ret;
- }
- } else if (data->m4u_dom != dom) {
- /* All the client devices should be in the same m4u domain */
- dev_err(dev, "try to attach into the error iommu domain\n");
- return -EPERM;
+ writel(dom->cfg.arm_v7s_cfg.ttbr[0],
+ data->base + REG_MMU_PT_BASE_ADDR);
}
mtk_iommu_config(data, dev, true);
@@ -354,6 +395,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+ struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
unsigned long flags;
phys_addr_t pa;
@@ -361,6 +403,9 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
pa = dom->iop->iova_to_phys(dom->iop, iova);
spin_unlock_irqrestore(&dom->pgtlock, flags);
+ if (data->enable_4GB)
+ pa |= BIT_ULL(32);
+
return pa;
}
@@ -399,7 +444,7 @@ static void mtk_iommu_remove_device(struct device *dev)
static struct iommu_group *mtk_iommu_device_group(struct device *dev)
{
- struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
+ struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
if (!data)
return ERR_PTR(-ENODEV);
@@ -464,8 +509,9 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
return ret;
}
- regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
- F_MMU_TF_PROTECT_SEL(2);
+ regval = F_MMU_TF_PROTECT_SEL(2, data);
+ if (data->m4u_plat == M4U_MT8173)
+ regval |= F_MMU_PREFETCH_RT_REPLACE_MOD;
writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
regval = F_L2_MULIT_HIT_EN |
@@ -487,9 +533,19 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB),
data->base + REG_MMU_IVRP_PADDR);
-
+ if (data->enable_4GB && data->m4u_plat != M4U_MT8173) {
+ /*
+ * If 4GB mode is enabled, the validate PA range is from
+ * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
+ */
+ regval = F_MMU_VLD_PA_RNG(7, 4);
+ writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG);
+ }
writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
- writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
+
+ /* It's MISC control register whose default value is ok except mt8173.*/
+ if (data->m4u_plat == M4U_MT8173)
+ writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
dev_name(data->dev), (void *)data)) {
@@ -521,6 +577,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (!data)
return -ENOMEM;
data->dev = dev;
+ data->m4u_plat = (enum mtk_iommu_plat)of_device_get_match_data(dev);
/* Protect memory. HW will access here while translation fault.*/
protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
@@ -529,7 +586,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
/* Whether the current dram is over 4GB */
- data->enable_4GB = !!(max_pfn > (0xffffffffUL >> PAGE_SHIFT));
+ data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
data->base = devm_ioremap_resource(dev, res);
@@ -554,6 +611,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
for (i = 0; i < larb_nr; i++) {
struct device_node *larbnode;
struct platform_device *plarbdev;
+ u32 id;
larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
if (!larbnode)
@@ -562,17 +620,14 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (!of_device_is_available(larbnode))
continue;
+ ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
+ if (ret)/* The id is consecutive if there is no this property */
+ id = i;
+
plarbdev = of_find_device_by_node(larbnode);
- if (!plarbdev) {
- plarbdev = of_platform_device_create(
- larbnode, NULL,
- platform_bus_type.dev_root);
- if (!plarbdev) {
- of_node_put(larbnode);
- return -EPROBE_DEFER;
- }
- }
- data->smi_imu.larb_imu[i].dev = &plarbdev->dev;
+ if (!plarbdev)
+ return -EPROBE_DEFER;
+ data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
component_match_add_release(dev, &match, release_of,
compare_of, larbnode);
@@ -596,6 +651,8 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (ret)
return ret;
+ list_add_tail(&data->list, &m4ulist);
+
if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
@@ -612,7 +669,6 @@ static int mtk_iommu_remove(struct platform_device *pdev)
if (iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, NULL);
- free_io_pgtable_ops(data->m4u_dom->iop);
clk_disable_unprepare(data->bclk);
devm_free_irq(&pdev->dev, data->irq, data);
component_master_del(&pdev->dev, &mtk_iommu_com_ops);
@@ -631,6 +687,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev)
reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
+ clk_disable_unprepare(data->bclk);
return 0;
}
@@ -639,9 +696,13 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
struct mtk_iommu_data *data = dev_get_drvdata(dev);
struct mtk_iommu_suspend_reg *reg = &data->reg;
void __iomem *base = data->base;
+ int ret;
- writel_relaxed(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
- base + REG_MMU_PT_BASE_ADDR);
+ ret = clk_prepare_enable(data->bclk);
+ if (ret) {
+ dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
+ return ret;
+ }
writel_relaxed(reg->standard_axi_mode,
base + REG_MMU_STANDARD_AXI_MODE);
writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
@@ -650,15 +711,19 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB),
base + REG_MMU_IVRP_PADDR);
+ if (data->m4u_dom)
+ writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
+ base + REG_MMU_PT_BASE_ADDR);
return 0;
}
-const struct dev_pm_ops mtk_iommu_pm_ops = {
- SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
+static const struct dev_pm_ops mtk_iommu_pm_ops = {
+ SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
};
static const struct of_device_id mtk_iommu_of_ids[] = {
- { .compatible = "mediatek,mt8173-m4u", },
+ { .compatible = "mediatek,mt2712-m4u", .data = (void *)M4U_MT2712},
+ { .compatible = "mediatek,mt8173-m4u", .data = (void *)M4U_MT8173},
{}
};
@@ -667,27 +732,20 @@ static struct platform_driver mtk_iommu_driver = {
.remove = mtk_iommu_remove,
.driver = {
.name = "mtk-iommu",
- .of_match_table = mtk_iommu_of_ids,
+ .of_match_table = of_match_ptr(mtk_iommu_of_ids),
.pm = &mtk_iommu_pm_ops,
}
};
-static int mtk_iommu_init_fn(struct device_node *np)
+static int __init mtk_iommu_init(void)
{
int ret;
- struct platform_device *pdev;
-
- pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
- if (!pdev)
- return -ENOMEM;
ret = platform_driver_register(&mtk_iommu_driver);
- if (ret) {
- pr_err("%s: Failed to register driver\n", __func__);
- return ret;
- }
+ if (ret != 0)
+ pr_err("Failed to register MTK IOMMU driver\n");
- return 0;
+ return ret;
}
-IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn);
+subsys_initcall(mtk_iommu_init)
diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h
index c06cc91b5d9a..b4451a1c7c2f 100644
--- a/drivers/iommu/mtk_iommu.h
+++ b/drivers/iommu/mtk_iommu.h
@@ -34,6 +34,12 @@ struct mtk_iommu_suspend_reg {
u32 int_main_control;
};
+enum mtk_iommu_plat {
+ M4U_MT2701,
+ M4U_MT2712,
+ M4U_MT8173,
+};
+
struct mtk_iommu_domain;
struct mtk_iommu_data {
@@ -50,6 +56,9 @@ struct mtk_iommu_data {
bool tlb_flush_active;
struct iommu_device iommu;
+ enum mtk_iommu_plat m4u_plat;
+
+ struct list_head list;
};
static inline int compare_of(struct device *dev, void *data)
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 8cb60829a7a1..e60e3dba85a0 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -25,6 +25,8 @@
#include <linux/of_pci.h>
#include <linux/slab.h>
+#define NO_IOMMU 1
+
static const struct of_device_id __iommu_of_table_sentinel
__used __section(__iommu_of_table_end);
@@ -109,8 +111,8 @@ static bool of_iommu_driver_present(struct device_node *np)
return of_match_node(&__iommu_of_table, np);
}
-static const struct iommu_ops
-*of_iommu_xlate(struct device *dev, struct of_phandle_args *iommu_spec)
+static int of_iommu_xlate(struct device *dev,
+ struct of_phandle_args *iommu_spec)
{
const struct iommu_ops *ops;
struct fwnode_handle *fwnode = &iommu_spec->np->fwnode;
@@ -120,95 +122,53 @@ static const struct iommu_ops
if ((ops && !ops->of_xlate) ||
!of_device_is_available(iommu_spec->np) ||
(!ops && !of_iommu_driver_present(iommu_spec->np)))
- return NULL;
+ return NO_IOMMU;
err = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
if (err)
- return ERR_PTR(err);
+ return err;
/*
* The otherwise-empty fwspec handily serves to indicate the specific
* IOMMU device we're waiting for, which will be useful if we ever get
* a proper probe-ordering dependency mechanism in future.
*/
if (!ops)
- return ERR_PTR(-EPROBE_DEFER);
-
- err = ops->of_xlate(dev, iommu_spec);
- if (err)
- return ERR_PTR(err);
+ return -EPROBE_DEFER;
- return ops;
+ return ops->of_xlate(dev, iommu_spec);
}
-static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
-{
- struct of_phandle_args *iommu_spec = data;
-
- iommu_spec->args[0] = alias;
- return iommu_spec->np == pdev->bus->dev.of_node;
-}
+struct of_pci_iommu_alias_info {
+ struct device *dev;
+ struct device_node *np;
+};
-static const struct iommu_ops
-*of_pci_iommu_init(struct pci_dev *pdev, struct device_node *bridge_np)
+static int of_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
{
- const struct iommu_ops *ops;
- struct of_phandle_args iommu_spec;
+ struct of_pci_iommu_alias_info *info = data;
+ struct of_phandle_args iommu_spec = { .args_count = 1 };
int err;
- /*
- * Start by tracing the RID alias down the PCI topology as
- * far as the host bridge whose OF node we have...
- * (we're not even attempting to handle multi-alias devices yet)
- */
- iommu_spec.args_count = 1;
- iommu_spec.np = bridge_np;
- pci_for_each_dma_alias(pdev, __get_pci_rid, &iommu_spec);
- /*
- * ...then find out what that becomes once it escapes the PCI
- * bus into the system beyond, and which IOMMU it ends up at.
- */
- iommu_spec.np = NULL;
- err = of_pci_map_rid(bridge_np, iommu_spec.args[0], "iommu-map",
+ err = of_pci_map_rid(info->np, alias, "iommu-map",
"iommu-map-mask", &iommu_spec.np,
iommu_spec.args);
if (err)
- return err == -ENODEV ? NULL : ERR_PTR(err);
-
- ops = of_iommu_xlate(&pdev->dev, &iommu_spec);
+ return err == -ENODEV ? NO_IOMMU : err;
+ err = of_iommu_xlate(info->dev, &iommu_spec);
of_node_put(iommu_spec.np);
- return ops;
-}
-
-static const struct iommu_ops
-*of_platform_iommu_init(struct device *dev, struct device_node *np)
-{
- struct of_phandle_args iommu_spec;
- const struct iommu_ops *ops = NULL;
- int idx = 0;
-
- /*
- * We don't currently walk up the tree looking for a parent IOMMU.
- * See the `Notes:' section of
- * Documentation/devicetree/bindings/iommu/iommu.txt
- */
- while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells",
- idx, &iommu_spec)) {
- ops = of_iommu_xlate(dev, &iommu_spec);
- of_node_put(iommu_spec.np);
- idx++;
- if (IS_ERR_OR_NULL(ops))
- break;
- }
+ if (err)
+ return err;
- return ops;
+ return info->np == pdev->bus->dev.of_node;
}
const struct iommu_ops *of_iommu_configure(struct device *dev,
struct device_node *master_np)
{
- const struct iommu_ops *ops;
+ const struct iommu_ops *ops = NULL;
struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+ int err = NO_IOMMU;
if (!master_np)
return NULL;
@@ -221,25 +181,54 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
iommu_fwspec_free(dev);
}
- if (dev_is_pci(dev))
- ops = of_pci_iommu_init(to_pci_dev(dev), master_np);
- else
- ops = of_platform_iommu_init(dev, master_np);
+ /*
+ * We don't currently walk up the tree looking for a parent IOMMU.
+ * See the `Notes:' section of
+ * Documentation/devicetree/bindings/iommu/iommu.txt
+ */
+ if (dev_is_pci(dev)) {
+ struct of_pci_iommu_alias_info info = {
+ .dev = dev,
+ .np = master_np,
+ };
+
+ err = pci_for_each_dma_alias(to_pci_dev(dev),
+ of_pci_iommu_init, &info);
+ } else {
+ struct of_phandle_args iommu_spec;
+ int idx = 0;
+
+ while (!of_parse_phandle_with_args(master_np, "iommus",
+ "#iommu-cells",
+ idx, &iommu_spec)) {
+ err = of_iommu_xlate(dev, &iommu_spec);
+ of_node_put(iommu_spec.np);
+ idx++;
+ if (err)
+ break;
+ }
+ }
+
+ /*
+ * Two success conditions can be represented by non-negative err here:
+ * >0 : there is no IOMMU, or one was unavailable for non-fatal reasons
+ * 0 : we found an IOMMU, and dev->fwspec is initialised appropriately
+ * <0 : any actual error
+ */
+ if (!err)
+ ops = dev->iommu_fwspec->ops;
/*
* If we have reason to believe the IOMMU driver missed the initial
* add_device callback for dev, replay it to get things in order.
*/
- if (!IS_ERR_OR_NULL(ops) && ops->add_device &&
- dev->bus && !dev->iommu_group) {
- int err = ops->add_device(dev);
-
- if (err)
- ops = ERR_PTR(err);
- }
+ if (ops && ops->add_device && dev->bus && !dev->iommu_group)
+ err = ops->add_device(dev);
/* Ignore all other errors apart from EPROBE_DEFER */
- if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) {
- dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops));
+ if (err == -EPROBE_DEFER) {
+ ops = ERR_PTR(err);
+ } else if (err < 0) {
+ dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
ops = NULL;
}
@@ -255,8 +244,7 @@ static int __init of_iommu_init(void)
const of_iommu_init_fn init_fn = match->data;
if (init_fn && init_fn(np))
- pr_err("Failed to initialise IOMMU %s\n",
- of_node_full_name(np));
+ pr_err("Failed to initialise IOMMU %pOF\n", np);
}
return 0;
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 641e035cf866..bd67e1b2c64e 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -11,6 +11,7 @@
* published by the Free Software Foundation.
*/
+#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
@@ -29,8 +30,6 @@
#include <linux/regmap.h>
#include <linux/mfd/syscon.h>
-#include <asm/cacheflush.h>
-
#include <linux/platform_data/iommu-omap.h>
#include "omap-iopgtable.h"
@@ -454,36 +453,35 @@ static void flush_iotlb_all(struct omap_iommu *obj)
/*
* H/W pagetable operations
*/
-static void flush_iopgd_range(u32 *first, u32 *last)
+static void flush_iopte_range(struct device *dev, dma_addr_t dma,
+ unsigned long offset, int num_entries)
{
- /* FIXME: L2 cache should be taken care of if it exists */
- do {
- asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
- : : "r" (first));
- first += L1_CACHE_BYTES / sizeof(*first);
- } while (first <= last);
-}
+ size_t size = num_entries * sizeof(u32);
-static void flush_iopte_range(u32 *first, u32 *last)
-{
- /* FIXME: L2 cache should be taken care of if it exists */
- do {
- asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
- : : "r" (first));
- first += L1_CACHE_BYTES / sizeof(*first);
- } while (first <= last);
+ dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE);
}
-static void iopte_free(u32 *iopte)
+static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid)
{
+ dma_addr_t pt_dma;
+
/* Note: freed iopte's must be clean ready for re-use */
- if (iopte)
+ if (iopte) {
+ if (dma_valid) {
+ pt_dma = virt_to_phys(iopte);
+ dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE,
+ DMA_TO_DEVICE);
+ }
+
kmem_cache_free(iopte_cachep, iopte);
+ }
}
-static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
+static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd,
+ dma_addr_t *pt_dma, u32 da)
{
u32 *iopte;
+ unsigned long offset = iopgd_index(da) * sizeof(da);
/* a table has already existed */
if (*iopgd)
@@ -500,18 +498,38 @@ static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
if (!iopte)
return ERR_PTR(-ENOMEM);
+ *pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(obj->dev, *pt_dma)) {
+ dev_err(obj->dev, "DMA map error for L2 table\n");
+ iopte_free(obj, iopte, false);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ /*
+ * we rely on dma address and the physical address to be
+ * the same for mapping the L2 table
+ */
+ if (WARN_ON(*pt_dma != virt_to_phys(iopte))) {
+ dev_err(obj->dev, "DMA translation error for L2 table\n");
+ dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE,
+ DMA_TO_DEVICE);
+ iopte_free(obj, iopte, false);
+ return ERR_PTR(-ENOMEM);
+ }
+
*iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
- flush_iopgd_range(iopgd, iopgd);
+ flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
} else {
/* We raced, free the reduniovant table */
- iopte_free(iopte);
+ iopte_free(obj, iopte, false);
}
pte_ready:
iopte = iopte_offset(iopgd, da);
-
+ *pt_dma = virt_to_phys(iopte);
dev_vdbg(obj->dev,
"%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
__func__, da, iopgd, *iopgd, iopte, *iopte);
@@ -522,6 +540,7 @@ pte_ready:
static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
+ unsigned long offset = iopgd_index(da) * sizeof(da);
if ((da | pa) & ~IOSECTION_MASK) {
dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
@@ -530,13 +549,14 @@ static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
}
*iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
- flush_iopgd_range(iopgd, iopgd);
+ flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
return 0;
}
static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
+ unsigned long offset = iopgd_index(da) * sizeof(da);
int i;
if ((da | pa) & ~IOSUPER_MASK) {
@@ -547,20 +567,22 @@ static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
for (i = 0; i < 16; i++)
*(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
- flush_iopgd_range(iopgd, iopgd + 15);
+ flush_iopte_range(obj->dev, obj->pd_dma, offset, 16);
return 0;
}
static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
- u32 *iopte = iopte_alloc(obj, iopgd, da);
+ dma_addr_t pt_dma;
+ u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
+ unsigned long offset = iopte_index(da) * sizeof(da);
if (IS_ERR(iopte))
return PTR_ERR(iopte);
*iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
- flush_iopte_range(iopte, iopte);
+ flush_iopte_range(obj->dev, pt_dma, offset, 1);
dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
__func__, da, pa, iopte, *iopte);
@@ -571,7 +593,9 @@ static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
- u32 *iopte = iopte_alloc(obj, iopgd, da);
+ dma_addr_t pt_dma;
+ u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
+ unsigned long offset = iopte_index(da) * sizeof(da);
int i;
if ((da | pa) & ~IOLARGE_MASK) {
@@ -585,7 +609,7 @@ static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
for (i = 0; i < 16; i++)
*(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
- flush_iopte_range(iopte, iopte + 15);
+ flush_iopte_range(obj->dev, pt_dma, offset, 16);
return 0;
}
@@ -674,6 +698,9 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
size_t bytes;
u32 *iopgd = iopgd_offset(obj, da);
int nent = 1;
+ dma_addr_t pt_dma;
+ unsigned long pd_offset = iopgd_index(da) * sizeof(da);
+ unsigned long pt_offset = iopte_index(da) * sizeof(da);
if (!*iopgd)
return 0;
@@ -690,7 +717,8 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
}
bytes *= nent;
memset(iopte, 0, nent * sizeof(*iopte));
- flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
+ pt_dma = virt_to_phys(iopte);
+ flush_iopte_range(obj->dev, pt_dma, pt_offset, nent);
/*
* do table walk to check if this table is necessary or not
@@ -700,7 +728,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
if (iopte[i])
goto out;
- iopte_free(iopte);
+ iopte_free(obj, iopte, true);
nent = 1; /* for the next L1 entry */
} else {
bytes = IOPGD_SIZE;
@@ -712,7 +740,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
bytes *= nent;
}
memset(iopgd, 0, nent * sizeof(*iopgd));
- flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
+ flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent);
out:
return bytes;
}
@@ -738,6 +766,7 @@ static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
static void iopgtable_clear_entry_all(struct omap_iommu *obj)
{
+ unsigned long offset;
int i;
spin_lock(&obj->page_table_lock);
@@ -748,15 +777,16 @@ static void iopgtable_clear_entry_all(struct omap_iommu *obj)
da = i << IOPGD_SHIFT;
iopgd = iopgd_offset(obj, da);
+ offset = iopgd_index(da) * sizeof(da);
if (!*iopgd)
continue;
if (iopgd_is_table(*iopgd))
- iopte_free(iopte_offset(iopgd, 0));
+ iopte_free(obj, iopte_offset(iopgd, 0), true);
*iopgd = 0;
- flush_iopgd_range(iopgd, iopgd);
+ flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
}
flush_iotlb_all(obj);
@@ -786,7 +816,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data)
if (!report_iommu_fault(domain, obj->dev, da, 0))
return IRQ_HANDLED;
- iommu_disable(obj);
+ iommu_write_reg(obj, 0, MMU_IRQENABLE);
iopgd = iopgd_offset(obj, da);
@@ -815,10 +845,18 @@ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
spin_lock(&obj->iommu_lock);
+ obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(obj->dev, obj->pd_dma)) {
+ dev_err(obj->dev, "DMA map error for L1 table\n");
+ err = -ENOMEM;
+ goto out_err;
+ }
+
obj->iopgd = iopgd;
err = iommu_enable(obj);
if (err)
- goto err_enable;
+ goto out_err;
flush_iotlb_all(obj);
spin_unlock(&obj->iommu_lock);
@@ -827,7 +865,7 @@ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
return 0;
-err_enable:
+out_err:
spin_unlock(&obj->iommu_lock);
return err;
@@ -844,7 +882,10 @@ static void omap_iommu_detach(struct omap_iommu *obj)
spin_lock(&obj->iommu_lock);
+ dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
+ DMA_TO_DEVICE);
iommu_disable(obj);
+ obj->pd_dma = 0;
obj->iopgd = NULL;
spin_unlock(&obj->iommu_lock);
@@ -1008,11 +1049,6 @@ static struct platform_driver omap_iommu_driver = {
},
};
-static void iopte_cachep_ctor(void *iopte)
-{
- clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
-}
-
static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
{
memset(e, 0, sizeof(*e));
@@ -1159,7 +1195,6 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
if (WARN_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)))
goto fail_align;
- clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
spin_lock_init(&omap_domain->lock);
omap_domain->domain.geometry.aperture_start = 0;
@@ -1347,7 +1382,7 @@ static int __init omap_iommu_init(void)
of_node_put(np);
p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
- iopte_cachep_ctor);
+ NULL);
if (!p)
return -ENOMEM;
iopte_cachep = p;
diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
index 6e70515e6038..a675af29a6ec 100644
--- a/drivers/iommu/omap-iommu.h
+++ b/drivers/iommu/omap-iommu.h
@@ -61,6 +61,7 @@ struct omap_iommu {
*/
u32 *iopgd;
spinlock_t page_table_lock; /* protect iopgd */
+ dma_addr_t pd_dma;
int nr_tlb_entries;
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
new file mode 100644
index 000000000000..c8a587d034b0
--- /dev/null
+++ b/drivers/iommu/qcom_iommu.c
@@ -0,0 +1,930 @@
+/*
+ * IOMMU API for QCOM secure IOMMUs. Somewhat based on arm-smmu.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (C) 2013 ARM Limited
+ * Copyright (C) 2017 Red Hat
+ */
+
+#include <linux/atomic.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-iommu.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+#include <linux/iommu.h>
+#include <linux/iopoll.h>
+#include <linux/kconfig.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_iommu.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/qcom_scm.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "io-pgtable.h"
+#include "arm-smmu-regs.h"
+
+#define SMMU_INTR_SEL_NS 0x2000
+
+struct qcom_iommu_ctx;
+
+struct qcom_iommu_dev {
+ /* IOMMU core code handle */
+ struct iommu_device iommu;
+ struct device *dev;
+ struct clk *iface_clk;
+ struct clk *bus_clk;
+ void __iomem *local_base;
+ u32 sec_id;
+ u8 num_ctxs;
+ struct qcom_iommu_ctx *ctxs[0]; /* indexed by asid-1 */
+};
+
+struct qcom_iommu_ctx {
+ struct device *dev;
+ void __iomem *base;
+ bool secure_init;
+ u8 asid; /* asid and ctx bank # are 1:1 */
+};
+
+struct qcom_iommu_domain {
+ struct io_pgtable_ops *pgtbl_ops;
+ spinlock_t pgtbl_lock;
+ struct mutex init_mutex; /* Protects iommu pointer */
+ struct iommu_domain domain;
+ struct qcom_iommu_dev *iommu;
+};
+
+static struct qcom_iommu_domain *to_qcom_iommu_domain(struct iommu_domain *dom)
+{
+ return container_of(dom, struct qcom_iommu_domain, domain);
+}
+
+static const struct iommu_ops qcom_iommu_ops;
+
+static struct qcom_iommu_dev * to_iommu(struct iommu_fwspec *fwspec)
+{
+ if (!fwspec || fwspec->ops != &qcom_iommu_ops)
+ return NULL;
+ return fwspec->iommu_priv;
+}
+
+static struct qcom_iommu_ctx * to_ctx(struct iommu_fwspec *fwspec, unsigned asid)
+{
+ struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+ if (!qcom_iommu)
+ return NULL;
+ return qcom_iommu->ctxs[asid - 1];
+}
+
+static inline void
+iommu_writel(struct qcom_iommu_ctx *ctx, unsigned reg, u32 val)
+{
+ writel_relaxed(val, ctx->base + reg);
+}
+
+static inline void
+iommu_writeq(struct qcom_iommu_ctx *ctx, unsigned reg, u64 val)
+{
+ writeq_relaxed(val, ctx->base + reg);
+}
+
+static inline u32
+iommu_readl(struct qcom_iommu_ctx *ctx, unsigned reg)
+{
+ return readl_relaxed(ctx->base + reg);
+}
+
+static inline u64
+iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg)
+{
+ return readq_relaxed(ctx->base + reg);
+}
+
+static void qcom_iommu_tlb_sync(void *cookie)
+{
+ struct iommu_fwspec *fwspec = cookie;
+ unsigned i;
+
+ for (i = 0; i < fwspec->num_ids; i++) {
+ struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+ unsigned int val, ret;
+
+ iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0);
+
+ ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val,
+ (val & 0x1) == 0, 0, 5000000);
+ if (ret)
+ dev_err(ctx->dev, "timeout waiting for TLB SYNC\n");
+ }
+}
+
+static void qcom_iommu_tlb_inv_context(void *cookie)
+{
+ struct iommu_fwspec *fwspec = cookie;
+ unsigned i;
+
+ for (i = 0; i < fwspec->num_ids; i++) {
+ struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+ iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid);
+ }
+
+ qcom_iommu_tlb_sync(cookie);
+}
+
+static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
+ size_t granule, bool leaf, void *cookie)
+{
+ struct iommu_fwspec *fwspec = cookie;
+ unsigned i, reg;
+
+ reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
+
+ for (i = 0; i < fwspec->num_ids; i++) {
+ struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+ size_t s = size;
+
+ iova &= ~12UL;
+ iova |= ctx->asid;
+ do {
+ iommu_writel(ctx, reg, iova);
+ iova += granule;
+ } while (s -= granule);
+ }
+}
+
+static const struct iommu_gather_ops qcom_gather_ops = {
+ .tlb_flush_all = qcom_iommu_tlb_inv_context,
+ .tlb_add_flush = qcom_iommu_tlb_inv_range_nosync,
+ .tlb_sync = qcom_iommu_tlb_sync,
+};
+
+static irqreturn_t qcom_iommu_fault(int irq, void *dev)
+{
+ struct qcom_iommu_ctx *ctx = dev;
+ u32 fsr, fsynr;
+ u64 iova;
+
+ fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
+
+ if (!(fsr & FSR_FAULT))
+ return IRQ_NONE;
+
+ fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
+ iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
+
+ dev_err_ratelimited(ctx->dev,
+ "Unhandled context fault: fsr=0x%x, "
+ "iova=0x%016llx, fsynr=0x%x, cb=%d\n",
+ fsr, iova, fsynr, ctx->asid);
+
+ iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
+
+ return IRQ_HANDLED;
+}
+
+static int qcom_iommu_init_domain(struct iommu_domain *domain,
+ struct qcom_iommu_dev *qcom_iommu,
+ struct iommu_fwspec *fwspec)
+{
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+ struct io_pgtable_ops *pgtbl_ops;
+ struct io_pgtable_cfg pgtbl_cfg;
+ int i, ret = 0;
+ u32 reg;
+
+ mutex_lock(&qcom_domain->init_mutex);
+ if (qcom_domain->iommu)
+ goto out_unlock;
+
+ pgtbl_cfg = (struct io_pgtable_cfg) {
+ .pgsize_bitmap = qcom_iommu_ops.pgsize_bitmap,
+ .ias = 32,
+ .oas = 40,
+ .tlb = &qcom_gather_ops,
+ .iommu_dev = qcom_iommu->dev,
+ };
+
+ qcom_domain->iommu = qcom_iommu;
+ pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
+ if (!pgtbl_ops) {
+ dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
+ ret = -ENOMEM;
+ goto out_clear_iommu;
+ }
+
+ /* Update the domain's page sizes to reflect the page table format */
+ domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+ domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
+ domain->geometry.force_aperture = true;
+
+ for (i = 0; i < fwspec->num_ids; i++) {
+ struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+ if (!ctx->secure_init) {
+ ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
+ if (ret) {
+ dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
+ goto out_clear_iommu;
+ }
+ ctx->secure_init = true;
+ }
+
+ /* TTBRs */
+ iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
+ pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+ ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+ iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+ pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
+ ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+
+ /* TTBCR */
+ iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
+ (pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+ TTBCR2_SEP_UPSTREAM);
+ iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
+ pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+
+ /* MAIRs (stage-1 only) */
+ iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
+ pgtbl_cfg.arm_lpae_s1_cfg.mair[0]);
+ iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
+ pgtbl_cfg.arm_lpae_s1_cfg.mair[1]);
+
+ /* SCTLR */
+ reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
+ SCTLR_M | SCTLR_S1_ASIDPNE;
+
+ if (IS_ENABLED(CONFIG_BIG_ENDIAN))
+ reg |= SCTLR_E;
+
+ iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
+ }
+
+ mutex_unlock(&qcom_domain->init_mutex);
+
+ /* Publish page table ops for map/unmap */
+ qcom_domain->pgtbl_ops = pgtbl_ops;
+
+ return 0;
+
+out_clear_iommu:
+ qcom_domain->iommu = NULL;
+out_unlock:
+ mutex_unlock(&qcom_domain->init_mutex);
+ return ret;
+}
+
+static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
+{
+ struct qcom_iommu_domain *qcom_domain;
+
+ if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
+ return NULL;
+ /*
+ * Allocate the domain and initialise some of its data structures.
+ * We can't really do anything meaningful until we've added a
+ * master.
+ */
+ qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
+ if (!qcom_domain)
+ return NULL;
+
+ if (type == IOMMU_DOMAIN_DMA &&
+ iommu_get_dma_cookie(&qcom_domain->domain)) {
+ kfree(qcom_domain);
+ return NULL;
+ }
+
+ mutex_init(&qcom_domain->init_mutex);
+ spin_lock_init(&qcom_domain->pgtbl_lock);
+
+ return &qcom_domain->domain;
+}
+
+static void qcom_iommu_domain_free(struct iommu_domain *domain)
+{
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+
+ if (WARN_ON(qcom_domain->iommu)) /* forgot to detach? */
+ return;
+
+ iommu_put_dma_cookie(domain);
+
+ /* NOTE: unmap can be called after client device is powered off,
+ * for example, with GPUs or anything involving dma-buf. So we
+ * cannot rely on the device_link. Make sure the IOMMU is on to
+ * avoid unclocked accesses in the TLB inv path:
+ */
+ pm_runtime_get_sync(qcom_domain->iommu->dev);
+
+ free_io_pgtable_ops(qcom_domain->pgtbl_ops);
+
+ pm_runtime_put_sync(qcom_domain->iommu->dev);
+
+ kfree(qcom_domain);
+}
+
+static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+ struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+ int ret;
+
+ if (!qcom_iommu) {
+ dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
+ return -ENXIO;
+ }
+
+ /* Ensure that the domain is finalized */
+ pm_runtime_get_sync(qcom_iommu->dev);
+ ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
+ pm_runtime_put_sync(qcom_iommu->dev);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Sanity check the domain. We don't support domains across
+ * different IOMMUs.
+ */
+ if (qcom_domain->iommu != qcom_iommu) {
+ dev_err(dev, "cannot attach to IOMMU %s while already "
+ "attached to domain on IOMMU %s\n",
+ dev_name(qcom_domain->iommu->dev),
+ dev_name(qcom_iommu->dev));
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
+{
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+ struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+ unsigned i;
+
+ if (!qcom_domain->iommu)
+ return;
+
+ pm_runtime_get_sync(qcom_iommu->dev);
+ for (i = 0; i < fwspec->num_ids; i++) {
+ struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+ /* Disable the context bank: */
+ iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
+ }
+ pm_runtime_put_sync(qcom_iommu->dev);
+
+ qcom_domain->iommu = NULL;
+}
+
+static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
+ phys_addr_t paddr, size_t size, int prot)
+{
+ int ret;
+ unsigned long flags;
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+ struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+ if (!ops)
+ return -ENODEV;
+
+ spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+ ret = ops->map(ops, iova, paddr, size, prot);
+ spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+ return ret;
+}
+
+static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+ size_t size)
+{
+ size_t ret;
+ unsigned long flags;
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+ struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+ if (!ops)
+ return 0;
+
+ /* NOTE: unmap can be called after client device is powered off,
+ * for example, with GPUs or anything involving dma-buf. So we
+ * cannot rely on the device_link. Make sure the IOMMU is on to
+ * avoid unclocked accesses in the TLB inv path:
+ */
+ pm_runtime_get_sync(qcom_domain->iommu->dev);
+ spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+ ret = ops->unmap(ops, iova, size);
+ spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+ pm_runtime_put_sync(qcom_domain->iommu->dev);
+
+ return ret;
+}
+
+static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
+ dma_addr_t iova)
+{
+ phys_addr_t ret;
+ unsigned long flags;
+ struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+ struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+ if (!ops)
+ return 0;
+
+ spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+ ret = ops->iova_to_phys(ops, iova);
+ spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+
+ return ret;
+}
+
+static bool qcom_iommu_capable(enum iommu_cap cap)
+{
+ switch (cap) {
+ case IOMMU_CAP_CACHE_COHERENCY:
+ /*
+ * Return true here as the SMMU can always send out coherent
+ * requests.
+ */
+ return true;
+ case IOMMU_CAP_NOEXEC:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static int qcom_iommu_add_device(struct device *dev)
+{
+ struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+ struct iommu_group *group;
+ struct device_link *link;
+
+ if (!qcom_iommu)
+ return -ENODEV;
+
+ /*
+ * Establish the link between iommu and master, so that the
+ * iommu gets runtime enabled/disabled as per the master's
+ * needs.
+ */
+ link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
+ if (!link) {
+ dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
+ dev_name(qcom_iommu->dev), dev_name(dev));
+ return -ENODEV;
+ }
+
+ group = iommu_group_get_for_dev(dev);
+ if (IS_ERR_OR_NULL(group))
+ return PTR_ERR_OR_ZERO(group);
+
+ iommu_group_put(group);
+ iommu_device_link(&qcom_iommu->iommu, dev);
+
+ return 0;
+}
+
+static void qcom_iommu_remove_device(struct device *dev)
+{
+ struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+
+ if (!qcom_iommu)
+ return;
+
+ iommu_device_unlink(&qcom_iommu->iommu, dev);
+ iommu_group_remove_device(dev);
+ iommu_fwspec_free(dev);
+}
+
+static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+ struct qcom_iommu_dev *qcom_iommu;
+ struct platform_device *iommu_pdev;
+ unsigned asid = args->args[0];
+
+ if (args->args_count != 1) {
+ dev_err(dev, "incorrect number of iommu params found for %s "
+ "(found %d, expected 1)\n",
+ args->np->full_name, args->args_count);
+ return -EINVAL;
+ }
+
+ iommu_pdev = of_find_device_by_node(args->np);
+ if (WARN_ON(!iommu_pdev))
+ return -EINVAL;
+
+ qcom_iommu = platform_get_drvdata(iommu_pdev);
+
+ /* make sure the asid specified in dt is valid, so we don't have
+ * to sanity check this elsewhere, since 'asid - 1' is used to
+ * index into qcom_iommu->ctxs:
+ */
+ if (WARN_ON(asid < 1) ||
+ WARN_ON(asid > qcom_iommu->num_ctxs))
+ return -EINVAL;
+
+ if (!dev->iommu_fwspec->iommu_priv) {
+ dev->iommu_fwspec->iommu_priv = qcom_iommu;
+ } else {
+ /* make sure devices iommus dt node isn't referring to
+ * multiple different iommu devices. Multiple context
+ * banks are ok, but multiple devices are not:
+ */
+ if (WARN_ON(qcom_iommu != dev->iommu_fwspec->iommu_priv))
+ return -EINVAL;
+ }
+
+ return iommu_fwspec_add_ids(dev, &asid, 1);
+}
+
+static const struct iommu_ops qcom_iommu_ops = {
+ .capable = qcom_iommu_capable,
+ .domain_alloc = qcom_iommu_domain_alloc,
+ .domain_free = qcom_iommu_domain_free,
+ .attach_dev = qcom_iommu_attach_dev,
+ .detach_dev = qcom_iommu_detach_dev,
+ .map = qcom_iommu_map,
+ .unmap = qcom_iommu_unmap,
+ .map_sg = default_iommu_map_sg,
+ .iova_to_phys = qcom_iommu_iova_to_phys,
+ .add_device = qcom_iommu_add_device,
+ .remove_device = qcom_iommu_remove_device,
+ .device_group = generic_device_group,
+ .of_xlate = qcom_iommu_of_xlate,
+ .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+ int ret;
+
+ ret = clk_prepare_enable(qcom_iommu->iface_clk);
+ if (ret) {
+ dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
+ return ret;
+ }
+
+ ret = clk_prepare_enable(qcom_iommu->bus_clk);
+ if (ret) {
+ dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
+ clk_disable_unprepare(qcom_iommu->iface_clk);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+ clk_disable_unprepare(qcom_iommu->bus_clk);
+ clk_disable_unprepare(qcom_iommu->iface_clk);
+}
+
+static int qcom_iommu_sec_ptbl_init(struct device *dev)
+{
+ size_t psize = 0;
+ unsigned int spare = 0;
+ void *cpu_addr;
+ dma_addr_t paddr;
+ unsigned long attrs;
+ static bool allocated = false;
+ int ret;
+
+ if (allocated)
+ return 0;
+
+ ret = qcom_scm_iommu_secure_ptbl_size(spare, &psize);
+ if (ret) {
+ dev_err(dev, "failed to get iommu secure pgtable size (%d)\n",
+ ret);
+ return ret;
+ }
+
+ dev_info(dev, "iommu sec: pgtable size: %zu\n", psize);
+
+ attrs = DMA_ATTR_NO_KERNEL_MAPPING;
+
+ cpu_addr = dma_alloc_attrs(dev, psize, &paddr, GFP_KERNEL, attrs);
+ if (!cpu_addr) {
+ dev_err(dev, "failed to allocate %zu bytes for pgtable\n",
+ psize);
+ return -ENOMEM;
+ }
+
+ ret = qcom_scm_iommu_secure_ptbl_init(paddr, psize, spare);
+ if (ret) {
+ dev_err(dev, "failed to init iommu pgtable (%d)\n", ret);
+ goto free_mem;
+ }
+
+ allocated = true;
+ return 0;
+
+free_mem:
+ dma_free_attrs(dev, psize, cpu_addr, paddr, attrs);
+ return ret;
+}
+
+static int get_asid(const struct device_node *np)
+{
+ u32 reg;
+
+ /* read the "reg" property directly to get the relative address
+ * of the context bank, and calculate the asid from that:
+ */
+ if (of_property_read_u32_index(np, "reg", 0, &reg))
+ return -ENODEV;
+
+ return reg / 0x1000; /* context banks are 0x1000 apart */
+}
+
+static int qcom_iommu_ctx_probe(struct platform_device *pdev)
+{
+ struct qcom_iommu_ctx *ctx;
+ struct device *dev = &pdev->dev;
+ struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
+ struct resource *res;
+ int ret, irq;
+
+ ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+
+ ctx->dev = dev;
+ platform_set_drvdata(pdev, ctx);
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ ctx->base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(ctx->base))
+ return PTR_ERR(ctx->base);
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ dev_err(dev, "failed to get irq\n");
+ return -ENODEV;
+ }
+
+ /* clear IRQs before registering fault handler, just in case the
+ * boot-loader left us a surprise:
+ */
+ iommu_writel(ctx, ARM_SMMU_CB_FSR, iommu_readl(ctx, ARM_SMMU_CB_FSR));
+
+ ret = devm_request_irq(dev, irq,
+ qcom_iommu_fault,
+ IRQF_SHARED,
+ "qcom-iommu-fault",
+ ctx);
+ if (ret) {
+ dev_err(dev, "failed to request IRQ %u\n", irq);
+ return ret;
+ }
+
+ ret = get_asid(dev->of_node);
+ if (ret < 0) {
+ dev_err(dev, "missing reg property\n");
+ return ret;
+ }
+
+ ctx->asid = ret;
+
+ dev_dbg(dev, "found asid %u\n", ctx->asid);
+
+ qcom_iommu->ctxs[ctx->asid - 1] = ctx;
+
+ return 0;
+}
+
+static int qcom_iommu_ctx_remove(struct platform_device *pdev)
+{
+ struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent);
+ struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
+
+ platform_set_drvdata(pdev, NULL);
+
+ qcom_iommu->ctxs[ctx->asid - 1] = NULL;
+
+ return 0;
+}
+
+static const struct of_device_id ctx_of_match[] = {
+ { .compatible = "qcom,msm-iommu-v1-ns" },
+ { .compatible = "qcom,msm-iommu-v1-sec" },
+ { /* sentinel */ }
+};
+
+static struct platform_driver qcom_iommu_ctx_driver = {
+ .driver = {
+ .name = "qcom-iommu-ctx",
+ .of_match_table = of_match_ptr(ctx_of_match),
+ },
+ .probe = qcom_iommu_ctx_probe,
+ .remove = qcom_iommu_ctx_remove,
+};
+
+static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu)
+{
+ struct device_node *child;
+
+ for_each_child_of_node(qcom_iommu->dev->of_node, child)
+ if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec"))
+ return true;
+
+ return false;
+}
+
+static int qcom_iommu_device_probe(struct platform_device *pdev)
+{
+ struct device_node *child;
+ struct qcom_iommu_dev *qcom_iommu;
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+ int ret, sz, max_asid = 0;
+
+ /* find the max asid (which is 1:1 to ctx bank idx), so we know how
+ * many child ctx devices we have:
+ */
+ for_each_child_of_node(dev->of_node, child)
+ max_asid = max(max_asid, get_asid(child));
+
+ sz = sizeof(*qcom_iommu) + (max_asid * sizeof(qcom_iommu->ctxs[0]));
+
+ qcom_iommu = devm_kzalloc(dev, sz, GFP_KERNEL);
+ if (!qcom_iommu)
+ return -ENOMEM;
+ qcom_iommu->num_ctxs = max_asid;
+ qcom_iommu->dev = dev;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (res)
+ qcom_iommu->local_base = devm_ioremap_resource(dev, res);
+
+ qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
+ if (IS_ERR(qcom_iommu->iface_clk)) {
+ dev_err(dev, "failed to get iface clock\n");
+ return PTR_ERR(qcom_iommu->iface_clk);
+ }
+
+ qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
+ if (IS_ERR(qcom_iommu->bus_clk)) {
+ dev_err(dev, "failed to get bus clock\n");
+ return PTR_ERR(qcom_iommu->bus_clk);
+ }
+
+ if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
+ &qcom_iommu->sec_id)) {
+ dev_err(dev, "missing qcom,iommu-secure-id property\n");
+ return -ENODEV;
+ }
+
+ if (qcom_iommu_has_secure_context(qcom_iommu)) {
+ ret = qcom_iommu_sec_ptbl_init(dev);
+ if (ret) {
+ dev_err(dev, "cannot init secure pg table(%d)\n", ret);
+ return ret;
+ }
+ }
+
+ platform_set_drvdata(pdev, qcom_iommu);
+
+ pm_runtime_enable(dev);
+
+ /* register context bank devices, which are child nodes: */
+ ret = devm_of_platform_populate(dev);
+ if (ret) {
+ dev_err(dev, "Failed to populate iommu contexts\n");
+ return ret;
+ }
+
+ ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
+ dev_name(dev));
+ if (ret) {
+ dev_err(dev, "Failed to register iommu in sysfs\n");
+ return ret;
+ }
+
+ iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
+ iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
+
+ ret = iommu_device_register(&qcom_iommu->iommu);
+ if (ret) {
+ dev_err(dev, "Failed to register iommu\n");
+ return ret;
+ }
+
+ bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
+
+ if (qcom_iommu->local_base) {
+ pm_runtime_get_sync(dev);
+ writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
+ pm_runtime_put_sync(dev);
+ }
+
+ return 0;
+}
+
+static int qcom_iommu_device_remove(struct platform_device *pdev)
+{
+ struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+ bus_set_iommu(&platform_bus_type, NULL);
+
+ pm_runtime_force_suspend(&pdev->dev);
+ platform_set_drvdata(pdev, NULL);
+ iommu_device_sysfs_remove(&qcom_iommu->iommu);
+ iommu_device_unregister(&qcom_iommu->iommu);
+
+ return 0;
+}
+
+static int __maybe_unused qcom_iommu_resume(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+ return qcom_iommu_enable_clocks(qcom_iommu);
+}
+
+static int __maybe_unused qcom_iommu_suspend(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+ qcom_iommu_disable_clocks(qcom_iommu);
+
+ return 0;
+}
+
+static const struct dev_pm_ops qcom_iommu_pm_ops = {
+ SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
+ SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+ pm_runtime_force_resume)
+};
+
+static const struct of_device_id qcom_iommu_of_match[] = {
+ { .compatible = "qcom,msm-iommu-v1" },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
+
+static struct platform_driver qcom_iommu_driver = {
+ .driver = {
+ .name = "qcom-iommu",
+ .of_match_table = of_match_ptr(qcom_iommu_of_match),
+ .pm = &qcom_iommu_pm_ops,
+ },
+ .probe = qcom_iommu_device_probe,
+ .remove = qcom_iommu_device_remove,
+};
+
+static int __init qcom_iommu_init(void)
+{
+ int ret;
+
+ ret = platform_driver_register(&qcom_iommu_ctx_driver);
+ if (ret)
+ return ret;
+
+ ret = platform_driver_register(&qcom_iommu_driver);
+ if (ret)
+ platform_driver_unregister(&qcom_iommu_ctx_driver);
+
+ return ret;
+}
+
+static void __exit qcom_iommu_exit(void)
+{
+ platform_driver_unregister(&qcom_iommu_driver);
+ platform_driver_unregister(&qcom_iommu_ctx_driver);
+}
+
+module_init(qcom_iommu_init);
+module_exit(qcom_iommu_exit);
+
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index 4ba48a26b389..9d991c2d8767 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -90,7 +90,9 @@ struct rk_iommu {
struct device *dev;
void __iomem **bases;
int num_mmu;
- int irq;
+ int *irq;
+ int num_irq;
+ bool reset_disabled;
struct iommu_device iommu;
struct list_head node; /* entry in rk_iommu_domain.iommus */
struct iommu_domain *domain; /* domain to which iommu is attached */
@@ -414,6 +416,9 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu)
int ret, i;
u32 dte_addr;
+ if (iommu->reset_disabled)
+ return 0;
+
/*
* Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY
* and verifying that upper 5 nybbles are read back.
@@ -825,10 +830,12 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
iommu->domain = domain;
- ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq,
- IRQF_SHARED, dev_name(dev), iommu);
- if (ret)
- return ret;
+ for (i = 0; i < iommu->num_irq; i++) {
+ ret = devm_request_irq(iommu->dev, iommu->irq[i], rk_iommu_irq,
+ IRQF_SHARED, dev_name(dev), iommu);
+ if (ret)
+ return ret;
+ }
for (i = 0; i < iommu->num_mmu; i++) {
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
@@ -878,7 +885,8 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
}
rk_iommu_disable_stall(iommu);
- devm_free_irq(iommu->dev, iommu->irq, iommu);
+ for (i = 0; i < iommu->num_irq; i++)
+ devm_free_irq(iommu->dev, iommu->irq[i], iommu);
iommu->domain = NULL;
@@ -1008,20 +1016,20 @@ static int rk_iommu_group_set_iommudata(struct iommu_group *group,
ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0,
&args);
if (ret) {
- dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n",
- np->full_name, ret);
+ dev_err(dev, "of_parse_phandle_with_args(%pOF) => %d\n",
+ np, ret);
return ret;
}
if (args.args_count != 0) {
- dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n",
- args.np->full_name, args.args_count);
+ dev_err(dev, "incorrect number of iommu params found for %pOF (found %d, expected 0)\n",
+ args.np, args.args_count);
return -EINVAL;
}
pd = of_find_device_by_node(args.np);
of_node_put(args.np);
if (!pd) {
- dev_err(dev, "iommu %s not found\n", args.np->full_name);
+ dev_err(dev, "iommu %pOF not found\n", args.np);
return -EPROBE_DEFER;
}
@@ -1157,12 +1165,28 @@ static int rk_iommu_probe(struct platform_device *pdev)
if (iommu->num_mmu == 0)
return PTR_ERR(iommu->bases[0]);
- iommu->irq = platform_get_irq(pdev, 0);
- if (iommu->irq < 0) {
- dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq);
+ iommu->num_irq = platform_irq_count(pdev);
+ if (iommu->num_irq < 0)
+ return iommu->num_irq;
+ if (iommu->num_irq == 0)
return -ENXIO;
+
+ iommu->irq = devm_kcalloc(dev, iommu->num_irq, sizeof(*iommu->irq),
+ GFP_KERNEL);
+ if (!iommu->irq)
+ return -ENOMEM;
+
+ for (i = 0; i < iommu->num_irq; i++) {
+ iommu->irq[i] = platform_get_irq(pdev, i);
+ if (iommu->irq[i] < 0) {
+ dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq[i]);
+ return -ENXIO;
+ }
}
+ iommu->reset_disabled = device_property_read_bool(dev,
+ "rockchip,disable-mmu-reset");
+
err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
if (err)
return err;
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 8788640756a7..0e2f31f9032b 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -18,6 +18,8 @@
*/
#define S390_IOMMU_PGSIZES (~0xFFFUL)
+static const struct iommu_ops s390_iommu_ops;
+
struct s390_domain {
struct iommu_domain domain;
struct list_head devices;
@@ -166,11 +168,13 @@ static void s390_iommu_detach_device(struct iommu_domain *domain,
static int s390_iommu_add_device(struct device *dev)
{
struct iommu_group *group = iommu_group_get_for_dev(dev);
+ struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
+ iommu_device_link(&zdev->iommu_dev, dev);
return 0;
}
@@ -197,6 +201,7 @@ static void s390_iommu_remove_device(struct device *dev)
s390_iommu_detach_device(domain, dev);
}
+ iommu_device_unlink(&zdev->iommu_dev, dev);
iommu_group_remove_device(dev);
}
@@ -327,7 +332,37 @@ static size_t s390_iommu_unmap(struct iommu_domain *domain,
return size;
}
-static struct iommu_ops s390_iommu_ops = {
+int zpci_init_iommu(struct zpci_dev *zdev)
+{
+ int rc = 0;
+
+ rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
+ "s390-iommu.%08x", zdev->fid);
+ if (rc)
+ goto out_err;
+
+ iommu_device_set_ops(&zdev->iommu_dev, &s390_iommu_ops);
+
+ rc = iommu_device_register(&zdev->iommu_dev);
+ if (rc)
+ goto out_sysfs;
+
+ return 0;
+
+out_sysfs:
+ iommu_device_sysfs_remove(&zdev->iommu_dev);
+
+out_err:
+ return rc;
+}
+
+void zpci_destroy_iommu(struct zpci_dev *zdev)
+{
+ iommu_device_unregister(&zdev->iommu_dev);
+ iommu_device_sysfs_remove(&zdev->iommu_dev);
+}
+
+static const struct iommu_ops s390_iommu_ops = {
.capable = s390_iommu_capable,
.domain_alloc = s390_domain_alloc,
.domain_free = s390_domain_free,
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index 37e708fdbb5a..b62f790ad1ba 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -61,6 +61,8 @@ struct gart_device {
struct list_head client;
spinlock_t client_lock; /* for client list */
struct device *dev;
+
+ struct iommu_device iommu; /* IOMMU Core handle */
};
struct gart_domain {
@@ -334,12 +336,35 @@ static bool gart_iommu_capable(enum iommu_cap cap)
return false;
}
+static int gart_iommu_add_device(struct device *dev)
+{
+ struct iommu_group *group = iommu_group_get_for_dev(dev);
+
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ iommu_group_put(group);
+
+ iommu_device_link(&gart_handle->iommu, dev);
+
+ return 0;
+}
+
+static void gart_iommu_remove_device(struct device *dev)
+{
+ iommu_group_remove_device(dev);
+ iommu_device_unlink(&gart_handle->iommu, dev);
+}
+
static const struct iommu_ops gart_iommu_ops = {
.capable = gart_iommu_capable,
.domain_alloc = gart_iommu_domain_alloc,
.domain_free = gart_iommu_domain_free,
.attach_dev = gart_iommu_attach_dev,
.detach_dev = gart_iommu_detach_dev,
+ .add_device = gart_iommu_add_device,
+ .remove_device = gart_iommu_remove_device,
+ .device_group = generic_device_group,
.map = gart_iommu_map,
.map_sg = default_iommu_map_sg,
.unmap = gart_iommu_unmap,
@@ -378,6 +403,7 @@ static int tegra_gart_probe(struct platform_device *pdev)
struct resource *res, *res_remap;
void __iomem *gart_regs;
struct device *dev = &pdev->dev;
+ int ret;
if (gart_handle)
return -EIO;
@@ -404,6 +430,22 @@ static int tegra_gart_probe(struct platform_device *pdev)
return -ENXIO;
}
+ ret = iommu_device_sysfs_add(&gart->iommu, &pdev->dev, NULL,
+ dev_name(&pdev->dev));
+ if (ret) {
+ dev_err(dev, "Failed to register IOMMU in sysfs\n");
+ return ret;
+ }
+
+ iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
+
+ ret = iommu_device_register(&gart->iommu);
+ if (ret) {
+ dev_err(dev, "Failed to register IOMMU\n");
+ iommu_device_sysfs_remove(&gart->iommu);
+ return ret;
+ }
+
gart->dev = &pdev->dev;
spin_lock_init(&gart->pte_lock);
spin_lock_init(&gart->client_lock);
@@ -430,6 +472,9 @@ static int tegra_gart_remove(struct platform_device *pdev)
{
struct gart_device *gart = platform_get_drvdata(pdev);
+ iommu_device_unregister(&gart->iommu);
+ iommu_device_sysfs_remove(&gart->iommu);
+
writel(0, gart->regs + GART_CONFIG);
if (gart->savedata)
vfree(gart->savedata);
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index eeb19f560a05..3b6449e2cbf1 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -36,6 +36,8 @@ struct tegra_smmu {
struct list_head list;
struct dentry *debugfs;
+
+ struct iommu_device iommu; /* IOMMU Core code handle */
};
struct tegra_smmu_as {
@@ -704,6 +706,7 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
static int tegra_smmu_add_device(struct device *dev)
{
struct device_node *np = dev->of_node;
+ struct iommu_group *group;
struct of_phandle_args args;
unsigned int index = 0;
@@ -719,18 +722,33 @@ static int tegra_smmu_add_device(struct device *dev)
* first match.
*/
dev->archdata.iommu = smmu;
+
+ iommu_device_link(&smmu->iommu, dev);
+
break;
}
index++;
}
+ group = iommu_group_get_for_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ iommu_group_put(group);
+
return 0;
}
static void tegra_smmu_remove_device(struct device *dev)
{
+ struct tegra_smmu *smmu = dev->archdata.iommu;
+
+ if (smmu)
+ iommu_device_unlink(&smmu->iommu, dev);
+
dev->archdata.iommu = NULL;
+ iommu_group_remove_device(dev);
}
static const struct iommu_ops tegra_smmu_ops = {
@@ -741,6 +759,7 @@ static const struct iommu_ops tegra_smmu_ops = {
.detach_dev = tegra_smmu_detach_dev,
.add_device = tegra_smmu_add_device,
.remove_device = tegra_smmu_remove_device,
+ .device_group = generic_device_group,
.map = tegra_smmu_map,
.unmap = tegra_smmu_unmap,
.map_sg = default_iommu_map_sg,
@@ -930,9 +949,24 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
tegra_smmu_ahb_enable();
+ err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
+ if (err)
+ return ERR_PTR(err);
+
+ iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
+
+ err = iommu_device_register(&smmu->iommu);
+ if (err) {
+ iommu_device_sysfs_remove(&smmu->iommu);
+ return ERR_PTR(err);
+ }
+
err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
- if (err < 0)
+ if (err < 0) {
+ iommu_device_unregister(&smmu->iommu);
+ iommu_device_sysfs_remove(&smmu->iommu);
return ERR_PTR(err);
+ }
if (IS_ENABLED(CONFIG_DEBUG_FS))
tegra_smmu_debugfs_init(smmu);
@@ -942,6 +976,9 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
void tegra_smmu_remove(struct tegra_smmu *smmu)
{
+ iommu_device_unregister(&smmu->iommu);
+ iommu_device_sysfs_remove(&smmu->iommu);
+
if (IS_ENABLED(CONFIG_DEBUG_FS))
tegra_smmu_debugfs_exit(smmu);
}