From 6e8b566806b7a8788dfde59b88ed79faa422fbaf Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 23 Apr 2019 15:50:08 +0800 Subject: iommu/omap: Use dev_get_drvdata() Using dev_get_drvdata directly. Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Signed-off-by: Kefeng Wang Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index d2fb347aa4ff..b16c711fc5fc 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -38,8 +38,7 @@ static const struct iommu_ops omap_iommu_ops; -#define to_iommu(dev) \ - ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) +#define to_iommu(dev) ((struct omap_iommu *)dev_get_drvdata(dev)) /* bitmap of the page sizes currently supported */ #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) -- cgit From d7af4d987dfa31c5d715e940a447ea959f89e575 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Wed, 8 May 2019 12:22:46 -0700 Subject: iommu/vt-d: Fix bind svm with multiple devices If multiple devices try to bind to the same mm/PASID, we need to set up first level PASID entries for all the devices. The current code does not consider this case which results in failed DMA for devices after the first bind. Signed-off-by: Jacob Pan Reported-by: Mike Campin Signed-off-by: Joerg Roedel --- drivers/iommu/intel-svm.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 8f87304f915c..421e4145a823 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -374,6 +374,21 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ } list_add_tail(&svm->list, &global_svm_list); + } else { + /* + * Binding a new device with existing PASID, need to setup + * the PASID entry. + */ + spin_lock(&iommu->lock); + ret = intel_pasid_setup_first_level(iommu, dev, + mm ? mm->pgd : init_mm.pgd, + svm->pasid, FLPT_DEFAULT_DID, + mm ? 0 : PASID_FLAG_SUPERVISOR_MODE); + spin_unlock(&iommu->lock); + if (ret) { + kfree(sdev); + goto out; + } } list_add_rcu(&sdev->list, &svm->devs); -- cgit From 2dbbcce1f8dd3cd3cbb95717fd73d771a6c60718 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sat, 11 May 2019 13:41:35 +0100 Subject: iommu/amd: Remove redundant assignment to variable npages The variable npages is being initialized however this is never read and later it is being reassigned to a new value. The initialization is redundant and hence can be removed. Addresses-Coverity: ("Unused Value") Signed-off-by: Colin Ian King Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 09c9e45f7fa2..c0b5b9298e8e 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2609,7 +2609,7 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, struct protection_domain *domain; struct dma_ops_domain *dma_dom; unsigned long startaddr; - int npages = 2; + int npages; domain = get_domain(dev); if (IS_ERR(domain)) -- cgit From f780a8dc196db1c41b5da21ecfa27e83ee5fb776 Mon Sep 17 00:00:00 2001 From: Lukasz Odzioba Date: Mon, 20 May 2019 15:41:28 +0200 Subject: iommu/vt-d: Remove unnecessary rcu_read_locks We use RCU's for rarely updated lists like iommus, rmrr, atsr units. I'm not sure why domain_remove_dev_info() in domain_exit() was surrounded by rcu_read_lock. Lock was present before refactoring in d160aca527, but it was related to rcu list, not domain_remove_dev_info function. dmar_remove_one_dev_info() doesn't touch any of those lists, so it doesn't require a lock. In fact it is called 6 times without it anyway. Fixes: d160aca5276d ("iommu/vt-d: Unify domain->iommu attach/detachment") Signed-off-by: Lukasz Odzioba Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a209199f3af6..1b7ad80c0537 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1911,9 +1911,7 @@ static void domain_exit(struct dmar_domain *domain) struct page *freelist; /* Remove associated devices and clear attached or cached domains */ - rcu_read_lock(); domain_remove_dev_info(domain); - rcu_read_unlock(); /* destroy iovas */ put_iova_domain(&domain->iovad); @@ -5254,9 +5252,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, old_domain = find_domain(dev); if (old_domain) { - rcu_read_lock(); dmar_remove_one_dev_info(dev); - rcu_read_unlock(); if (!domain_type_is_vm_or_si(old_domain) && list_empty(&old_domain->devices)) -- cgit From ea09506c363e90ef0b7532fdcf187bb67281a5e6 Mon Sep 17 00:00:00 2001 From: Sai Praneeth Prakhya Date: Fri, 24 May 2019 16:40:15 -0700 Subject: iommu/vt-d: Modify the format of intel DMAR tables dump Presently, "/sys/kernel/debug/iommu/intel/dmar_translation_struct" file dumps DMAR tables in the below format IOMMU dmar2: Root Table Address:4362cc000 Root Table Entries: Bus: 0 H: 0 L: 4362f0001 Context Table Entries for Bus: 0 Entry B:D.F High Low 160 00:14.0 102 4362ef001 184 00:17.0 302 435ec4001 248 00:1f.0 202 436300001 This format has few short comings like 1. When extended for dumping scalable mode DMAR table it will quickly be very clumsy, making it unreadable. 2. It has information like the Bus number and Entry which are basically part of B:D.F, hence are a repetition and are not so useful. So, change it to a new format which could be easily extended to dump scalable mode DMAR table. The new format looks as below: IOMMU dmar2: Root Table Address: 0x436f7d000 B.D.F Root_entry Context_entry 00:14.0 0x0000000000000000:0x0000000436fbd001 0x0000000000000102:0x0000000436fbc001 00:17.0 0x0000000000000000:0x0000000436fbd001 0x0000000000000302:0x0000000436af4001 00:1f.0 0x0000000000000000:0x0000000436fbd001 0x0000000000000202:0x0000000436fcd001 Cc: Joerg Roedel Cc: Ashok Raj Cc: Lu Baolu Cc: Sohil Mehta Cc: David Woodhouse Cc: Jacob Pan Cc: Andy Shevchenko Reviewed-by: Lu Baolu Reviewed-by: Andy Shevchenko Signed-off-by: Sai Praneeth Prakhya Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu-debugfs.c | 65 +++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu-debugfs.c b/drivers/iommu/intel-iommu-debugfs.c index 7fabf9b1c2dc..3f5399b5e6c0 100644 --- a/drivers/iommu/intel-iommu-debugfs.c +++ b/drivers/iommu/intel-iommu-debugfs.c @@ -14,6 +14,13 @@ #include +struct tbl_walk { + u16 bus; + u16 devfn; + struct root_entry *rt_entry; + struct context_entry *ctx_entry; +}; + struct iommu_regset { int offset; const char *regs; @@ -131,16 +138,25 @@ out: } DEFINE_SHOW_ATTRIBUTE(iommu_regset); -static void ctx_tbl_entry_show(struct seq_file *m, struct intel_iommu *iommu, - int bus) +static inline void print_tbl_walk(struct seq_file *m) { - struct context_entry *context; - int devfn; + struct tbl_walk *tbl_wlk = m->private; - seq_printf(m, " Context Table Entries for Bus: %d\n", bus); - seq_puts(m, " Entry\tB:D.F\tHigh\tLow\n"); + seq_printf(m, "%02x:%02x.%x\t0x%016llx:0x%016llx\t0x%016llx:0x%016llx\n", + tbl_wlk->bus, PCI_SLOT(tbl_wlk->devfn), + PCI_FUNC(tbl_wlk->devfn), tbl_wlk->rt_entry->hi, + tbl_wlk->rt_entry->lo, tbl_wlk->ctx_entry->hi, + tbl_wlk->ctx_entry->lo); +} + +static void ctx_tbl_walk(struct seq_file *m, struct intel_iommu *iommu, u16 bus) +{ + struct context_entry *context; + u16 devfn; for (devfn = 0; devfn < 256; devfn++) { + struct tbl_walk tbl_wlk = {0}; + context = iommu_context_addr(iommu, bus, devfn, 0); if (!context) return; @@ -148,33 +164,34 @@ static void ctx_tbl_entry_show(struct seq_file *m, struct intel_iommu *iommu, if (!context_present(context)) continue; - seq_printf(m, " %-5d\t%02x:%02x.%x\t%-6llx\t%llx\n", devfn, - bus, PCI_SLOT(devfn), PCI_FUNC(devfn), - context[0].hi, context[0].lo); + tbl_wlk.bus = bus; + tbl_wlk.devfn = devfn; + tbl_wlk.rt_entry = &iommu->root_entry[bus]; + tbl_wlk.ctx_entry = context; + m->private = &tbl_wlk; + + print_tbl_walk(m); } } -static void root_tbl_entry_show(struct seq_file *m, struct intel_iommu *iommu) +static void root_tbl_walk(struct seq_file *m, struct intel_iommu *iommu) { unsigned long flags; - int bus; + u16 bus; spin_lock_irqsave(&iommu->lock, flags); - seq_printf(m, "IOMMU %s: Root Table Address:%llx\n", iommu->name, + seq_printf(m, "IOMMU %s: Root Table Address: 0x%llx\n", iommu->name, (u64)virt_to_phys(iommu->root_entry)); - seq_puts(m, "Root Table Entries:\n"); - - for (bus = 0; bus < 256; bus++) { - if (!(iommu->root_entry[bus].lo & 1)) - continue; + seq_puts(m, "B.D.F\tRoot_entry\t\t\t\tContext_entry\n"); - seq_printf(m, " Bus: %d H: %llx L: %llx\n", bus, - iommu->root_entry[bus].hi, - iommu->root_entry[bus].lo); + /* + * No need to check if the root entry is present or not because + * iommu_context_addr() performs the same check before returning + * context entry. + */ + for (bus = 0; bus < 256; bus++) + ctx_tbl_walk(m, iommu, bus); - ctx_tbl_entry_show(m, iommu, bus); - seq_putc(m, '\n'); - } spin_unlock_irqrestore(&iommu->lock, flags); } @@ -185,7 +202,7 @@ static int dmar_translation_struct_show(struct seq_file *m, void *unused) rcu_read_lock(); for_each_active_iommu(iommu, drhd) { - root_tbl_entry_show(m, iommu); + root_tbl_walk(m, iommu); seq_putc(m, '\n'); } rcu_read_unlock(); -- cgit From cdd3a2499d30695730b22dc025c00b9b28884c6b Mon Sep 17 00:00:00 2001 From: Sai Praneeth Prakhya Date: Fri, 24 May 2019 16:40:16 -0700 Subject: iommu/vt-d: Introduce macros useful for dumping DMAR table A scalable mode DMAR table walk would involve looking at bits in each stage of walk, like, 1. Is PASID enabled in the context entry? 2. What's the size of PASID directory? 3. Is the PASID directory entry present? 4. Is the PASID table entry present? 5. Number of PASID table entries? Hence, add these macros that will later be used during this walk. Apart from adding new macros, move existing macros (like pasid_pde_is_present(), get_pasid_table_from_pde() and pasid_supported()) to appropriate header files so that they could be reused. Cc: Joerg Roedel Cc: Ashok Raj Cc: Lu Baolu Cc: Sohil Mehta Cc: David Woodhouse Cc: Jacob Pan Cc: Andy Shevchenko Reviewed-by: Lu Baolu Reviewed-by: Andy Shevchenko Signed-off-by: Sai Praneeth Prakhya Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 6 +----- drivers/iommu/intel-pasid.c | 17 ----------------- drivers/iommu/intel-pasid.h | 26 ++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 22 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1b7ad80c0537..9722c2ffe428 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -357,6 +357,7 @@ int dmar_disabled = 0; int dmar_disabled = 1; #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/ +int intel_iommu_sm; int intel_iommu_enabled = 0; EXPORT_SYMBOL_GPL(intel_iommu_enabled); @@ -364,17 +365,12 @@ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; -static int intel_iommu_sm; static int iommu_identity_mapping; #define IDENTMAP_ALL 1 #define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 -#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap)) -#define pasid_supported(iommu) (sm_supported(iommu) && \ - ecap_pasid((iommu)->ecap)) - int intel_iommu_gfx_mapped; EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c index 2fefeafda437..6895a23b2157 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel-pasid.c @@ -169,23 +169,6 @@ attach_out: return 0; } -/* Get PRESENT bit of a PASID directory entry. */ -static inline bool -pasid_pde_is_present(struct pasid_dir_entry *pde) -{ - return READ_ONCE(pde->val) & PASID_PTE_PRESENT; -} - -/* Get PASID table from a PASID directory entry. */ -static inline struct pasid_entry * -get_pasid_table_from_pde(struct pasid_dir_entry *pde) -{ - if (!pasid_pde_is_present(pde)) - return NULL; - - return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK); -} - void intel_pasid_free_table(struct device *dev) { struct device_domain_info *info; diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 23537b3f34e3..fc8cd8f17de1 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -18,6 +18,10 @@ #define PDE_PFN_MASK PAGE_MASK #define PASID_PDE_SHIFT 6 #define MAX_NR_PASID_BITS 20 +#define PASID_TBL_ENTRIES BIT(PASID_PDE_SHIFT) + +#define is_pasid_enabled(entry) (((entry)->lo >> 3) & 0x1) +#define get_pasid_dir_size(entry) (1 << ((((entry)->lo >> 9) & 0x7) + 7)) /* * Domain ID reserved for pasid entries programmed for first-level @@ -49,6 +53,28 @@ struct pasid_table { struct list_head dev; /* device list */ }; +/* Get PRESENT bit of a PASID directory entry. */ +static inline bool pasid_pde_is_present(struct pasid_dir_entry *pde) +{ + return READ_ONCE(pde->val) & PASID_PTE_PRESENT; +} + +/* Get PASID table from a PASID directory entry. */ +static inline struct pasid_entry * +get_pasid_table_from_pde(struct pasid_dir_entry *pde) +{ + if (!pasid_pde_is_present(pde)) + return NULL; + + return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK); +} + +/* Get PRESENT bit of a PASID table entry. */ +static inline bool pasid_pte_is_present(struct pasid_entry *pte) +{ + return READ_ONCE(pte->val[0]) & PASID_PTE_PRESENT; +} + extern u32 intel_pasid_max_id; int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp); void intel_pasid_free_id(int pasid); -- cgit From dd5142ca5d24bcdb22e40add842d50a59ffa769e Mon Sep 17 00:00:00 2001 From: Sai Praneeth Prakhya Date: Fri, 24 May 2019 16:40:17 -0700 Subject: iommu/vt-d: Add debugfs support to show scalable mode DMAR table internals A DMAR table walk would typically follow the below process. 1. Bus number is used to index into root table which points to a context table. 2. Device number and Function number are used together to index into context table which then points to a pasid directory. 3. PASID[19:6] is used to index into PASID directory which points to a PASID table. 4. PASID[5:0] is used to index into PASID table which points to all levels of page tables. Whenever a user opens the file "/sys/kernel/debug/iommu/intel/dmar_translation_struct", the above described DMAR table walk is performed and the contents of the table are dumped into the file. The dump could be handy while dealing with devices that use PASID. Example of such dump: cat /sys/kernel/debug/iommu/intel/dmar_translation_struct (Please note that because of 80 char limit, entries that should have been in the same line are broken into different lines) IOMMU dmar0: Root Table Address: 0x436f7c000 B.D.F Root_entry Context_entry PASID PASID_table_entry 00:0a.0 0x0000000000000000:0x000000044dd3f001 0x0000000000100000:0x0000000435460e1d 0 0x000000044d6e1089:0x0000000000000003:0x0000000000000001 00:0a.0 0x0000000000000000:0x000000044dd3f001 0x0000000000100000:0x0000000435460e1d 1 0x0000000000000049:0x0000000000000001:0x0000000003c0e001 Note that the above format is followed even for legacy DMAR table dump which doesn't support PASID and hence in such cases PASID is defaulted to -1 indicating that PASID and it's related fields are invalid. Cc: Joerg Roedel Cc: Ashok Raj Cc: Lu Baolu Cc: Sohil Mehta Cc: David Woodhouse Cc: Jacob Pan Cc: Andy Shevchenko Reviewed-by: Lu Baolu Reviewed-by: Andy Shevchenko Signed-off-by: Sai Praneeth Prakhya Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu-debugfs.c | 78 +++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu-debugfs.c b/drivers/iommu/intel-iommu-debugfs.c index 3f5399b5e6c0..73a552914455 100644 --- a/drivers/iommu/intel-iommu-debugfs.c +++ b/drivers/iommu/intel-iommu-debugfs.c @@ -14,11 +14,15 @@ #include +#include "intel-pasid.h" + struct tbl_walk { u16 bus; u16 devfn; + u32 pasid; struct root_entry *rt_entry; struct context_entry *ctx_entry; + struct pasid_entry *pasid_tbl_entry; }; struct iommu_regset { @@ -142,21 +146,82 @@ static inline void print_tbl_walk(struct seq_file *m) { struct tbl_walk *tbl_wlk = m->private; - seq_printf(m, "%02x:%02x.%x\t0x%016llx:0x%016llx\t0x%016llx:0x%016llx\n", + seq_printf(m, "%02x:%02x.%x\t0x%016llx:0x%016llx\t0x%016llx:0x%016llx\t", tbl_wlk->bus, PCI_SLOT(tbl_wlk->devfn), PCI_FUNC(tbl_wlk->devfn), tbl_wlk->rt_entry->hi, tbl_wlk->rt_entry->lo, tbl_wlk->ctx_entry->hi, tbl_wlk->ctx_entry->lo); + + /* + * A legacy mode DMAR doesn't support PASID, hence default it to -1 + * indicating that it's invalid. Also, default all PASID related fields + * to 0. + */ + if (!tbl_wlk->pasid_tbl_entry) + seq_printf(m, "%-6d\t0x%016llx:0x%016llx:0x%016llx\n", -1, + (u64)0, (u64)0, (u64)0); + else + seq_printf(m, "%-6d\t0x%016llx:0x%016llx:0x%016llx\n", + tbl_wlk->pasid, tbl_wlk->pasid_tbl_entry->val[0], + tbl_wlk->pasid_tbl_entry->val[1], + tbl_wlk->pasid_tbl_entry->val[2]); +} + +static void pasid_tbl_walk(struct seq_file *m, struct pasid_entry *tbl_entry, + u16 dir_idx) +{ + struct tbl_walk *tbl_wlk = m->private; + u8 tbl_idx; + + for (tbl_idx = 0; tbl_idx < PASID_TBL_ENTRIES; tbl_idx++) { + if (pasid_pte_is_present(tbl_entry)) { + tbl_wlk->pasid_tbl_entry = tbl_entry; + tbl_wlk->pasid = (dir_idx << PASID_PDE_SHIFT) + tbl_idx; + print_tbl_walk(m); + } + + tbl_entry++; + } +} + +static void pasid_dir_walk(struct seq_file *m, u64 pasid_dir_ptr, + u16 pasid_dir_size) +{ + struct pasid_dir_entry *dir_entry = phys_to_virt(pasid_dir_ptr); + struct pasid_entry *pasid_tbl; + u16 dir_idx; + + for (dir_idx = 0; dir_idx < pasid_dir_size; dir_idx++) { + pasid_tbl = get_pasid_table_from_pde(dir_entry); + if (pasid_tbl) + pasid_tbl_walk(m, pasid_tbl, dir_idx); + + dir_entry++; + } } static void ctx_tbl_walk(struct seq_file *m, struct intel_iommu *iommu, u16 bus) { struct context_entry *context; - u16 devfn; + u16 devfn, pasid_dir_size; + u64 pasid_dir_ptr; for (devfn = 0; devfn < 256; devfn++) { struct tbl_walk tbl_wlk = {0}; + /* + * Scalable mode root entry points to upper scalable mode + * context table and lower scalable mode context table. Each + * scalable mode context table has 128 context entries where as + * legacy mode context table has 256 context entries. So in + * scalable mode, the context entries for former 128 devices are + * in the lower scalable mode context table, while the latter + * 128 devices are in the upper scalable mode context table. + * In scalable mode, when devfn > 127, iommu_context_addr() + * automatically refers to upper scalable mode context table and + * hence the caller doesn't have to worry about differences + * between scalable mode and non scalable mode. + */ context = iommu_context_addr(iommu, bus, devfn, 0); if (!context) return; @@ -170,6 +235,13 @@ static void ctx_tbl_walk(struct seq_file *m, struct intel_iommu *iommu, u16 bus) tbl_wlk.ctx_entry = context; m->private = &tbl_wlk; + if (pasid_supported(iommu) && is_pasid_enabled(context)) { + pasid_dir_ptr = context->lo & VTD_PAGE_MASK; + pasid_dir_size = get_pasid_dir_size(context); + pasid_dir_walk(m, pasid_dir_ptr, pasid_dir_size); + continue; + } + print_tbl_walk(m); } } @@ -182,7 +254,7 @@ static void root_tbl_walk(struct seq_file *m, struct intel_iommu *iommu) spin_lock_irqsave(&iommu->lock, flags); seq_printf(m, "IOMMU %s: Root Table Address: 0x%llx\n", iommu->name, (u64)virt_to_phys(iommu->root_entry)); - seq_puts(m, "B.D.F\tRoot_entry\t\t\t\tContext_entry\n"); + seq_puts(m, "B.D.F\tRoot_entry\t\t\t\tContext_entry\t\t\t\tPASID\tPASID_table_entry\n"); /* * No need to check if the root entry is present or not because -- cgit From 57274ea25736496ee019a5c40479855b21888839 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 21 May 2019 15:27:35 +0800 Subject: iommu: Use right function to get group for device The iommu_group_get_for_dev() will allocate a group for a device if it isn't in any group. This isn't the use case in iommu_request_dm_for_dev(). Let's use iommu_group_get() instead. Fixes: d290f1e70d85a ("iommu: Introduce iommu_request_dm_for_dev()") Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 67ee6623f9b2..3fa025f849e9 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1915,9 +1915,9 @@ int iommu_request_dm_for_dev(struct device *dev) int ret; /* Device must already be in a group before calling this function */ - group = iommu_group_get_for_dev(dev); - if (IS_ERR(group)) - return PTR_ERR(group); + group = iommu_group_get(dev); + if (!group) + return -EINVAL; mutex_lock(&group->mutex); -- cgit From 7423e01741dd6a5f1255f589145313f0fb1c8cbe Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:22 +0800 Subject: iommu: Add API to request DMA domain for device Normally during iommu probing a device, a default doamin will be allocated and attached to the device. The domain type of the default domain is statically defined, which results in a situation where the allocated default domain isn't suitable for the device due to some limitations. We already have API iommu_request_dm_for_dev() to replace a DMA domain with an identity one. This adds iommu_request_dma_domain_for_dev() to request a dma domain if an allocated identity domain isn't suitable for the device in question. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 67ee6623f9b2..2fca04c3dbaf 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1907,10 +1907,10 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, return region; } -/* Request that a device is direct mapped by the IOMMU */ -int iommu_request_dm_for_dev(struct device *dev) +static int +request_default_domain_for_dev(struct device *dev, unsigned long type) { - struct iommu_domain *dm_domain; + struct iommu_domain *domain; struct iommu_group *group; int ret; @@ -1923,8 +1923,7 @@ int iommu_request_dm_for_dev(struct device *dev) /* Check if the default domain is already direct mapped */ ret = 0; - if (group->default_domain && - group->default_domain->type == IOMMU_DOMAIN_IDENTITY) + if (group->default_domain && group->default_domain->type == type) goto out; /* Don't change mappings of existing devices */ @@ -1934,23 +1933,26 @@ int iommu_request_dm_for_dev(struct device *dev) /* Allocate a direct mapped domain */ ret = -ENOMEM; - dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY); - if (!dm_domain) + domain = __iommu_domain_alloc(dev->bus, type); + if (!domain) goto out; /* Attach the device to the domain */ - ret = __iommu_attach_group(dm_domain, group); + ret = __iommu_attach_group(domain, group); if (ret) { - iommu_domain_free(dm_domain); + iommu_domain_free(domain); goto out; } + iommu_group_create_direct_mappings(group, dev); + /* Make the direct mapped domain the default for this group */ if (group->default_domain) iommu_domain_free(group->default_domain); - group->default_domain = dm_domain; + group->default_domain = domain; - dev_info(dev, "Using iommu direct mapping\n"); + dev_info(dev, "Using iommu %s mapping\n", + type == IOMMU_DOMAIN_DMA ? "dma" : "direct"); ret = 0; out: @@ -1960,6 +1962,18 @@ out: return ret; } +/* Request that a device is direct mapped by the IOMMU */ +int iommu_request_dm_for_dev(struct device *dev) +{ + return request_default_domain_for_dev(dev, IOMMU_DOMAIN_IDENTITY); +} + +/* Request that a device can't be direct mapped by the IOMMU */ +int iommu_request_dma_domain_for_dev(struct device *dev) +{ + return request_default_domain_for_dev(dev, IOMMU_DOMAIN_DMA); +} + const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) { const struct iommu_ops *ops = NULL; -- cgit From 73bcbdc9fa8d88db2d26d06885fb180cfe478f80 Mon Sep 17 00:00:00 2001 From: James Sewart Date: Sat, 25 May 2019 13:41:23 +0800 Subject: iommu/vt-d: Implement apply_resv_region iommu ops entry Used by iommu.c before creating identity mappings for reserved ranges to ensure dma-ops won't ever remap these ranges. Signed-off-by: James Sewart Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 9722c2ffe428..3995afc59c28 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5549,6 +5549,19 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev) return ret; } +static void intel_iommu_apply_resv_region(struct device *dev, + struct iommu_domain *domain, + struct iommu_resv_region *region) +{ + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + unsigned long start, end; + + start = IOVA_PFN(region->start); + end = IOVA_PFN(region->start + region->length - 1); + + WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end)); +} + #ifdef CONFIG_INTEL_IOMMU_SVM struct intel_iommu *intel_svm_device_to_iommu(struct device *dev) { @@ -5714,6 +5727,7 @@ const struct iommu_ops intel_iommu_ops = { .remove_device = intel_iommu_remove_device, .get_resv_regions = intel_iommu_get_resv_regions, .put_resv_regions = intel_iommu_put_resv_regions, + .apply_resv_region = intel_iommu_apply_resv_region, .device_group = pci_device_group, .dev_has_feat = intel_iommu_dev_has_feat, .dev_feat_enabled = intel_iommu_dev_feat_enabled, -- cgit From af751d4308a7c80434b5f40fd44288d33dc1962f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:27 +0200 Subject: iommu/dma: Remove the flush_page callback We now have a arch_dma_prep_coherent architecture hook that is used for the generic DMA remap allocator, and we should use the same interface for the dma-iommu code. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Acked-by: Catalin Marinas Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 129c4badf9ae..aac12433ffef 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -560,8 +561,6 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * @attrs: DMA attributes for this allocation * @prot: IOMMU mapping flags * @handle: Out argument for allocated DMA handle - * @flush_page: Arch callback which must ensure PAGE_SIZE bytes from the - * given VA/PA are visible to the given non-coherent device. * * If @size is less than PAGE_SIZE, then a full CPU page will be allocated, * but an IOMMU which supports smaller pages might not map the whole thing. @@ -570,8 +569,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * or NULL on failure. */ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, - unsigned long attrs, int prot, dma_addr_t *handle, - void (*flush_page)(struct device *, const void *, phys_addr_t)) + unsigned long attrs, int prot, dma_addr_t *handle) { struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; @@ -615,7 +613,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, */ sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, SG_MITER_FROM_SG); while (sg_miter_next(&miter)) - flush_page(dev, miter.addr, page_to_phys(miter.page)); + arch_dma_prep_coherent(miter.page, PAGE_SIZE); sg_miter_stop(&miter); } -- cgit From 23f88e0a7e9f084e91a40cc90a15968fb1e5d506 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:28 +0200 Subject: iommu/dma: Use for_each_sg in iommu_dma_alloc arch_dma_prep_coherent can handle physically contiguous ranges larger than PAGE_SIZE just fine, which means we don't need a page-based iterator. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index aac12433ffef..9b7f120d7381 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -606,15 +606,11 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, goto out_free_iova; if (!(prot & IOMMU_CACHE)) { - struct sg_mapping_iter miter; - /* - * The CPU-centric flushing implied by SG_MITER_TO_SG isn't - * sufficient here, so skip it by using the "wrong" direction. - */ - sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, SG_MITER_FROM_SG); - while (sg_miter_next(&miter)) - arch_dma_prep_coherent(miter.page, PAGE_SIZE); - sg_miter_stop(&miter); + struct scatterlist *sg; + int i; + + for_each_sg(sgt.sgl, sg, sgt.orig_nents, i) + arch_dma_prep_coherent(sg_page(sg), sg->length); } if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, prot) -- cgit From 06d60728ff5c01795ac0bad66a5c42e3e78dcb6b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:29 +0200 Subject: iommu/dma: move the arm64 wrappers to common code There is nothing really arm64 specific in the iommu_dma_ops implementation, so move it to dma-iommu.c and keep a lot of symbols self-contained. Note the implementation does depend on the DMA_DIRECT_REMAP infrastructure for now, so we'll have to make the DMA_IOMMU support depend on it, but this will be relaxed soon. Signed-off-by: Christoph Hellwig Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 + drivers/iommu/dma-iommu.c | 398 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 371 insertions(+), 28 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 83664db5221d..d6d063160dd6 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -97,6 +97,7 @@ config IOMMU_DMA select IOMMU_IOVA select IRQ_MSI_IOMMU select NEED_SG_DMA_LENGTH + depends on DMA_DIRECT_REMAP config FSL_PAMU bool "Freescale IOMMU support" diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 9b7f120d7381..e34ba23353cb 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -79,11 +80,6 @@ static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) return cookie; } -int iommu_dma_init(void) -{ - return iova_cache_get(); -} - /** * iommu_get_dma_cookie - Acquire DMA-API resources for a domain * @domain: IOMMU domain to prepare for DMA-API usage @@ -314,7 +310,7 @@ static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but * any change which could make prior IOVAs invalid will fail. */ -int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, +static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size, struct device *dev) { struct iommu_dma_cookie *cookie = domain->iova_cookie; @@ -365,7 +361,6 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, return iova_reserve_iommu_regions(dev, domain); } -EXPORT_SYMBOL(iommu_dma_init_domain); /** * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API @@ -376,7 +371,7 @@ EXPORT_SYMBOL(iommu_dma_init_domain); * * Return: corresponding IOMMU API page protection flags */ -int dma_info_to_prot(enum dma_data_direction dir, bool coherent, +static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, unsigned long attrs) { int prot = coherent ? IOMMU_CACHE : 0; @@ -535,17 +530,17 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, } /** - * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc() + * iommu_dma_free - Free a buffer allocated by __iommu_dma_alloc() * @dev: Device which owns this buffer - * @pages: Array of buffer pages as returned by iommu_dma_alloc() + * @pages: Array of buffer pages as returned by __iommu_dma_alloc() * @size: Size of buffer in bytes * @handle: DMA address of buffer * * Frees both the pages associated with the buffer, and the array * describing them */ -void iommu_dma_free(struct device *dev, struct page **pages, size_t size, - dma_addr_t *handle) +static void __iommu_dma_free(struct device *dev, struct page **pages, + size_t size, dma_addr_t *handle) { __iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size); __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); @@ -553,7 +548,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, } /** - * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space + * __iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space * @dev: Device to allocate memory for. Must be a real device * attached to an iommu_dma_domain * @size: Size of buffer in bytes @@ -568,8 +563,8 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * Return: Array of struct page pointers describing the buffer, * or NULL on failure. */ -struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, - unsigned long attrs, int prot, dma_addr_t *handle) +static struct page **__iommu_dma_alloc(struct device *dev, size_t size, + gfp_t gfp, unsigned long attrs, int prot, dma_addr_t *handle) { struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; @@ -631,20 +626,72 @@ out_free_pages: } /** - * iommu_dma_mmap - Map a buffer into provided user VMA - * @pages: Array representing buffer from iommu_dma_alloc() + * __iommu_dma_mmap - Map a buffer into provided user VMA + * @pages: Array representing buffer from __iommu_dma_alloc() * @size: Size of buffer in bytes * @vma: VMA describing requested userspace mapping * * Maps the pages of the buffer in @pages into @vma. The caller is responsible * for verifying the correct size and protection of @vma beforehand. */ - -int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma) +static int __iommu_dma_mmap(struct page **pages, size_t size, + struct vm_area_struct *vma) { return vm_map_pages(vma, pages, PAGE_ALIGN(size) >> PAGE_SHIFT); } +static void iommu_dma_sync_single_for_cpu(struct device *dev, + dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) +{ + phys_addr_t phys; + + if (dev_is_dma_coherent(dev)) + return; + + phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); + arch_sync_dma_for_cpu(dev, phys, size, dir); +} + +static void iommu_dma_sync_single_for_device(struct device *dev, + dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) +{ + phys_addr_t phys; + + if (dev_is_dma_coherent(dev)) + return; + + phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); + arch_sync_dma_for_device(dev, phys, size, dir); +} + +static void iommu_dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + if (dev_is_dma_coherent(dev)) + return; + + for_each_sg(sgl, sg, nelems, i) + arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); +} + +static void iommu_dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + if (dev_is_dma_coherent(dev)) + return; + + for_each_sg(sgl, sg, nelems, i) + arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); +} + static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, size_t size, int prot, struct iommu_domain *domain) { @@ -668,19 +715,44 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, return iova + iova_off; } -dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, +static dma_addr_t __iommu_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, int prot) { return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot, iommu_get_dma_domain(dev)); } -void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir, unsigned long attrs) +static void __iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) { __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); } +static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + phys_addr_t phys = page_to_phys(page) + offset; + bool coherent = dev_is_dma_coherent(dev); + dma_addr_t dma_handle; + + dma_handle =__iommu_dma_map(dev, phys, size, + dma_info_to_prot(dir, coherent, attrs), + iommu_get_dma_domain(dev)); + if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + dma_handle != DMA_MAPPING_ERROR) + arch_sync_dma_for_device(dev, phys, size, dir); + return dma_handle; +} + +static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); + __iommu_dma_unmap(iommu_get_dma_domain(dev), dma_handle, size); +} + /* * Prepare a successfully-mapped scatterlist to give back to the caller. * @@ -763,18 +835,22 @@ static void __invalidate_sg(struct scatterlist *sg, int nents) * impedance-matching, to be able to hand off a suitably-aligned list, * but still preserve the original offsets and sizes for the caller. */ -int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, int prot) +static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; struct scatterlist *s, *prev = NULL; + int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); dma_addr_t iova; size_t iova_len = 0; unsigned long mask = dma_get_seg_boundary(dev); int i; + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_sg_for_device(dev, sg, nents, dir); + /* * Work out how much IOVA space we need, and align the segments to * IOVA granules for the IOMMU driver to handle. With some clever @@ -834,12 +910,16 @@ out_restore_sg: return 0; } -void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) +static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { dma_addr_t start, end; struct scatterlist *tmp; int i; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); + /* * The scatterlist segments are mapped into a single * contiguous IOVA allocation, so this is incredibly easy. @@ -854,7 +934,7 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, __iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start); } -dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, +static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs) { return __iommu_dma_map(dev, phys, size, @@ -862,12 +942,268 @@ dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, iommu_get_dma_domain(dev)); } -void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, +static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); } +static void *iommu_dma_alloc(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +{ + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + size_t iosize = size; + void *addr; + + size = PAGE_ALIGN(size); + gfp |= __GFP_ZERO; + + if (!gfpflags_allow_blocking(gfp)) { + struct page *page; + /* + * In atomic context we can't remap anything, so we'll only + * get the virtually contiguous buffer we need by way of a + * physically contiguous allocation. + */ + if (coherent) { + page = alloc_pages(gfp, get_order(size)); + addr = page ? page_address(page) : NULL; + } else { + addr = dma_alloc_from_pool(size, &page, gfp); + } + if (!addr) + return NULL; + + *handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot); + if (*handle == DMA_MAPPING_ERROR) { + if (coherent) + __free_pages(page, get_order(size)); + else + dma_free_from_pool(addr, size); + addr = NULL; + } + } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); + struct page *page; + + page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, + get_order(size), gfp & __GFP_NOWARN); + if (!page) + return NULL; + + *handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot); + if (*handle == DMA_MAPPING_ERROR) { + dma_release_from_contiguous(dev, page, + size >> PAGE_SHIFT); + return NULL; + } + addr = dma_common_contiguous_remap(page, size, VM_USERMAP, + prot, + __builtin_return_address(0)); + if (addr) { + if (!coherent) + arch_dma_prep_coherent(page, iosize); + memset(addr, 0, size); + } else { + __iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs); + dma_release_from_contiguous(dev, page, + size >> PAGE_SHIFT); + } + } else { + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); + struct page **pages; + + pages = __iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot, + handle); + if (!pages) + return NULL; + + addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, + __builtin_return_address(0)); + if (!addr) + __iommu_dma_free(dev, pages, iosize, handle); + } + return addr; +} + +static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) +{ + size_t iosize = size; + + size = PAGE_ALIGN(size); + /* + * @cpu_addr will be one of 4 things depending on how it was allocated: + * - A remapped array of pages for contiguous allocations. + * - A remapped array of pages from __iommu_dma_alloc(), for all + * non-atomic allocations. + * - A non-cacheable alias from the atomic pool, for atomic + * allocations by non-coherent devices. + * - A normal lowmem address, for atomic allocations by + * coherent devices. + * Hence how dodgy the below logic looks... + */ + if (dma_in_atomic_pool(cpu_addr, size)) { + __iommu_dma_unmap_page(dev, handle, iosize, 0, 0); + dma_free_from_pool(cpu_addr, size); + } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { + struct page *page = vmalloc_to_page(cpu_addr); + + __iommu_dma_unmap_page(dev, handle, iosize, 0, attrs); + dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); + dma_common_free_remap(cpu_addr, size, VM_USERMAP); + } else if (is_vmalloc_addr(cpu_addr)){ + struct vm_struct *area = find_vm_area(cpu_addr); + + if (WARN_ON(!area || !area->pages)) + return; + __iommu_dma_free(dev, area->pages, iosize, &handle); + dma_common_free_remap(cpu_addr, size, VM_USERMAP); + } else { + __iommu_dma_unmap_page(dev, handle, iosize, 0, 0); + __free_pages(virt_to_page(cpu_addr), get_order(size)); + } +} + +static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, + unsigned long pfn, size_t size) +{ + int ret = -ENXIO; + unsigned long nr_vma_pages = vma_pages(vma); + unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long off = vma->vm_pgoff; + + if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { + ret = remap_pfn_range(vma, vma->vm_start, + pfn + off, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); + } + + return ret; +} + +static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long off = vma->vm_pgoff; + struct vm_struct *area; + int ret; + + vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); + + if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (off >= nr_pages || vma_pages(vma) > nr_pages - off) + return -ENXIO; + + if (!is_vmalloc_addr(cpu_addr)) { + unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); + return __iommu_dma_mmap_pfn(vma, pfn, size); + } + + if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { + /* + * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, + * hence in the vmalloc space. + */ + unsigned long pfn = vmalloc_to_pfn(cpu_addr); + return __iommu_dma_mmap_pfn(vma, pfn, size); + } + + area = find_vm_area(cpu_addr); + if (WARN_ON(!area || !area->pages)) + return -ENXIO; + + return __iommu_dma_mmap(area->pages, size, vma); +} + +static int __iommu_dma_get_sgtable_page(struct sg_table *sgt, struct page *page, + size_t size) +{ + int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + + if (!ret) + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return ret; +} + +static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + struct vm_struct *area = find_vm_area(cpu_addr); + + if (!is_vmalloc_addr(cpu_addr)) { + struct page *page = virt_to_page(cpu_addr); + return __iommu_dma_get_sgtable_page(sgt, page, size); + } + + if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { + /* + * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, + * hence in the vmalloc space. + */ + struct page *page = vmalloc_to_page(cpu_addr); + return __iommu_dma_get_sgtable_page(sgt, page, size); + } + + if (WARN_ON(!area || !area->pages)) + return -ENXIO; + + return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size, + GFP_KERNEL); +} + +static const struct dma_map_ops iommu_dma_ops = { + .alloc = iommu_dma_alloc, + .free = iommu_dma_free, + .mmap = iommu_dma_mmap, + .get_sgtable = iommu_dma_get_sgtable, + .map_page = iommu_dma_map_page, + .unmap_page = iommu_dma_unmap_page, + .map_sg = iommu_dma_map_sg, + .unmap_sg = iommu_dma_unmap_sg, + .sync_single_for_cpu = iommu_dma_sync_single_for_cpu, + .sync_single_for_device = iommu_dma_sync_single_for_device, + .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu, + .sync_sg_for_device = iommu_dma_sync_sg_for_device, + .map_resource = iommu_dma_map_resource, + .unmap_resource = iommu_dma_unmap_resource, +}; + +/* + * The IOMMU core code allocates the default DMA domain, which the underlying + * IOMMU driver needs to support via the dma-iommu layer. + */ +void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + + if (!domain) + goto out_err; + + /* + * The IOMMU core code allocates the default DMA domain, which the + * underlying IOMMU driver needs to support via the dma-iommu layer. + */ + if (domain->type == IOMMU_DOMAIN_DMA) { + if (iommu_dma_init_domain(domain, dma_base, size, dev)) + goto out_err; + dev->dma_ops = &iommu_dma_ops; + } + + return; +out_err: + pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", + dev_name(dev)); +} + static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, phys_addr_t msi_addr, struct iommu_domain *domain) { @@ -948,3 +1284,9 @@ void iommu_dma_compose_msi_msg(struct msi_desc *desc, msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1; msg->address_lo += lower_32_bits(msi_page->iova); } + +static int iommu_dma_init(void) +{ + return iova_cache_get(); +} +arch_initcall(iommu_dma_init); -- cgit From 92aec09cc87975f66d3a84001876693d9ac6073b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:30 +0200 Subject: iommu/dma: Move __iommu_dma_map Moving this function up to its unmap counterpart helps to keep related code together for the following changes. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e34ba23353cb..c406abe3be01 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -464,6 +464,29 @@ static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr, iommu_dma_free_iova(cookie, dma_addr, size); } +static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, + size_t size, int prot, struct iommu_domain *domain) +{ + struct iommu_dma_cookie *cookie = domain->iova_cookie; + size_t iova_off = 0; + dma_addr_t iova; + + if (cookie->type == IOMMU_DMA_IOVA_COOKIE) { + iova_off = iova_offset(&cookie->iovad, phys); + size = iova_align(&cookie->iovad, size + iova_off); + } + + iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); + if (!iova) + return DMA_MAPPING_ERROR; + + if (iommu_map(domain, iova, phys - iova_off, size, prot)) { + iommu_dma_free_iova(cookie, iova, size); + return DMA_MAPPING_ERROR; + } + return iova + iova_off; +} + static void __iommu_dma_free_pages(struct page **pages, int count) { while (count--) @@ -692,29 +715,6 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); } -static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, - size_t size, int prot, struct iommu_domain *domain) -{ - struct iommu_dma_cookie *cookie = domain->iova_cookie; - size_t iova_off = 0; - dma_addr_t iova; - - if (cookie->type == IOMMU_DMA_IOVA_COOKIE) { - iova_off = iova_offset(&cookie->iovad, phys); - size = iova_align(&cookie->iovad, size + iova_off); - } - - iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); - if (!iova) - return DMA_MAPPING_ERROR; - - if (iommu_map(domain, iova, phys - iova_off, size, prot)) { - iommu_dma_free_iova(cookie, iova, size); - return DMA_MAPPING_ERROR; - } - return iova + iova_off; -} - static dma_addr_t __iommu_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, int prot) { -- cgit From b61d271e59d7fd679ad9922ce5f16c116c0f8e94 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:31 +0200 Subject: iommu/dma: Move domain lookup into __iommu_dma_{map,unmap} Most of the callers don't care, and the couple that do already have the domain to hand for other reasons are in slow paths where the (trivial) overhead of a repeated lookup will be utterly immaterial. Signed-off-by: Robin Murphy [hch: dropped the hunk touching iommu_dma_get_msi_page to avoid a conflict with another series] Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c406abe3be01..6ece8f477fc8 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -448,9 +448,10 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, size >> iova_shift(iovad)); } -static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr, +static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t size) { + struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; size_t iova_off = iova_offset(iovad, dma_addr); @@ -465,8 +466,9 @@ static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr, } static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, - size_t size, int prot, struct iommu_domain *domain) + size_t size, int prot) { + struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; size_t iova_off = 0; dma_addr_t iova; @@ -565,7 +567,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, static void __iommu_dma_free(struct device *dev, struct page **pages, size_t size, dma_addr_t *handle) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size); + __iommu_dma_unmap(dev, *handle, size); __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); *handle = DMA_MAPPING_ERROR; } @@ -718,14 +720,13 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, static dma_addr_t __iommu_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, int prot) { - return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot, - iommu_get_dma_domain(dev)); + return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot); } static void __iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); + __iommu_dma_unmap(dev, handle, size); } static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, @@ -734,11 +735,10 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, { phys_addr_t phys = page_to_phys(page) + offset; bool coherent = dev_is_dma_coherent(dev); + int prot = dma_info_to_prot(dir, coherent, attrs); dma_addr_t dma_handle; - dma_handle =__iommu_dma_map(dev, phys, size, - dma_info_to_prot(dir, coherent, attrs), - iommu_get_dma_domain(dev)); + dma_handle =__iommu_dma_map(dev, phys, size, prot); if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && dma_handle != DMA_MAPPING_ERROR) arch_sync_dma_for_device(dev, phys, size, dir); @@ -750,7 +750,7 @@ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, { if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); - __iommu_dma_unmap(iommu_get_dma_domain(dev), dma_handle, size); + __iommu_dma_unmap(dev, dma_handle, size); } /* @@ -931,21 +931,20 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, sg = tmp; } end = sg_dma_address(sg) + sg_dma_len(sg); - __iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start); + __iommu_dma_unmap(dev, start, end - start); } static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs) { return __iommu_dma_map(dev, phys, size, - dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, - iommu_get_dma_domain(dev)); + dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO); } static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); + __iommu_dma_unmap(dev, handle, size); } static void *iommu_dma_alloc(struct device *dev, size_t size, @@ -1222,7 +1221,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, if (!msi_page) return NULL; - iova = __iommu_dma_map(dev, msi_addr, size, prot, domain); + iova = __iommu_dma_map(dev, msi_addr, size, prot); if (iova == DMA_MAPPING_ERROR) goto out_free_page; -- cgit From 796a08cf16054ef93783ca8ab4aec2469c3307c1 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:32 +0200 Subject: iommu/dma: Squash __iommu_dma_{map,unmap}_page helpers The remaining internal callsites don't care about having prototypes compatible with the relevant dma_map_ops callbacks, so the extra level of indirection just wastes space and complictaes things. Signed-off-by: Robin Murphy Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 6ece8f477fc8..498e319d6607 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -717,18 +717,6 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); } -static dma_addr_t __iommu_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, int prot) -{ - return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot); -} - -static void __iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - __iommu_dma_unmap(dev, handle, size); -} - static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) @@ -974,7 +962,8 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, if (!addr) return NULL; - *handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot); + *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, + ioprot); if (*handle == DMA_MAPPING_ERROR) { if (coherent) __free_pages(page, get_order(size)); @@ -991,7 +980,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, if (!page) return NULL; - *handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot); + *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, ioprot); if (*handle == DMA_MAPPING_ERROR) { dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); @@ -1005,7 +994,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, arch_dma_prep_coherent(page, iosize); memset(addr, 0, size); } else { - __iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs); + __iommu_dma_unmap(dev, *handle, iosize); dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); } @@ -1044,12 +1033,12 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, * Hence how dodgy the below logic looks... */ if (dma_in_atomic_pool(cpu_addr, size)) { - __iommu_dma_unmap_page(dev, handle, iosize, 0, 0); + __iommu_dma_unmap(dev, handle, iosize); dma_free_from_pool(cpu_addr, size); } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { struct page *page = vmalloc_to_page(cpu_addr); - __iommu_dma_unmap_page(dev, handle, iosize, 0, attrs); + __iommu_dma_unmap(dev, handle, iosize); dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); dma_common_free_remap(cpu_addr, size, VM_USERMAP); } else if (is_vmalloc_addr(cpu_addr)){ @@ -1060,7 +1049,7 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, __iommu_dma_free(dev, area->pages, iosize, &handle); dma_common_free_remap(cpu_addr, size, VM_USERMAP); } else { - __iommu_dma_unmap_page(dev, handle, iosize, 0, 0); + __iommu_dma_unmap(dev, handle, iosize); __free_pages(virt_to_page(cpu_addr), get_order(size)); } } -- cgit From 4c360acee9298d048921bec3e21c4183d06cb43a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:33 +0200 Subject: iommu/dma: Factor out remapped pages lookup Since we duplicate the find_vm_area() logic a few times in places where we only care aboute the pages, factor out a helper to abstract it. Signed-off-by: Robin Murphy [hch: don't warn when not finding a region, as we'll rely on that later] Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 498e319d6607..5e81165e6755 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -554,6 +554,15 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, return pages; } +static struct page **__iommu_dma_get_pages(void *cpu_addr) +{ + struct vm_struct *area = find_vm_area(cpu_addr); + + if (!area || !area->pages) + return NULL; + return area->pages; +} + /** * iommu_dma_free - Free a buffer allocated by __iommu_dma_alloc() * @dev: Device which owns this buffer @@ -1042,11 +1051,11 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); dma_common_free_remap(cpu_addr, size, VM_USERMAP); } else if (is_vmalloc_addr(cpu_addr)){ - struct vm_struct *area = find_vm_area(cpu_addr); + struct page **pages = __iommu_dma_get_pages(cpu_addr); - if (WARN_ON(!area || !area->pages)) + if (!pages) return; - __iommu_dma_free(dev, area->pages, iosize, &handle); + __iommu_dma_free(dev, pages, iosize, &handle); dma_common_free_remap(cpu_addr, size, VM_USERMAP); } else { __iommu_dma_unmap(dev, handle, iosize); @@ -1078,7 +1087,7 @@ static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, { unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long off = vma->vm_pgoff; - struct vm_struct *area; + struct page **pages; int ret; vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); @@ -1103,11 +1112,10 @@ static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, return __iommu_dma_mmap_pfn(vma, pfn, size); } - area = find_vm_area(cpu_addr); - if (WARN_ON(!area || !area->pages)) + pages = __iommu_dma_get_pages(cpu_addr); + if (!pages) return -ENXIO; - - return __iommu_dma_mmap(area->pages, size, vma); + return __iommu_dma_mmap(pages, size, vma); } static int __iommu_dma_get_sgtable_page(struct sg_table *sgt, struct page *page, @@ -1125,7 +1133,7 @@ static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - struct vm_struct *area = find_vm_area(cpu_addr); + struct page **pages; if (!is_vmalloc_addr(cpu_addr)) { struct page *page = virt_to_page(cpu_addr); @@ -1141,10 +1149,10 @@ static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, return __iommu_dma_get_sgtable_page(sgt, page, size); } - if (WARN_ON(!area || !area->pages)) + pages = __iommu_dma_get_pages(cpu_addr); + if (!pages) return -ENXIO; - - return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size, + return sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL); } -- cgit From 21b95aaf5f2212764d37b7569f3829a295e4042c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:34 +0200 Subject: iommu/dma: Refactor the page array remapping allocator Move the call to dma_common_pages_remap into __iommu_dma_alloc and rename it to iommu_dma_alloc_remap. This creates a self-contained helper for remapped pages allocation and mapping. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 54 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 28 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 5e81165e6755..0ffb7805de77 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -564,9 +564,9 @@ static struct page **__iommu_dma_get_pages(void *cpu_addr) } /** - * iommu_dma_free - Free a buffer allocated by __iommu_dma_alloc() + * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc_remap() * @dev: Device which owns this buffer - * @pages: Array of buffer pages as returned by __iommu_dma_alloc() + * @pages: Array of buffer pages as returned by __iommu_dma_alloc_remap() * @size: Size of buffer in bytes * @handle: DMA address of buffer * @@ -582,33 +582,35 @@ static void __iommu_dma_free(struct device *dev, struct page **pages, } /** - * __iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space + * iommu_dma_alloc_remap - Allocate and map a buffer contiguous in IOVA space * @dev: Device to allocate memory for. Must be a real device * attached to an iommu_dma_domain * @size: Size of buffer in bytes + * @dma_handle: Out argument for allocated DMA handle * @gfp: Allocation flags * @attrs: DMA attributes for this allocation - * @prot: IOMMU mapping flags - * @handle: Out argument for allocated DMA handle * * If @size is less than PAGE_SIZE, then a full CPU page will be allocated, * but an IOMMU which supports smaller pages might not map the whole thing. * - * Return: Array of struct page pointers describing the buffer, - * or NULL on failure. + * Return: Mapped virtual address, or NULL on failure. */ -static struct page **__iommu_dma_alloc(struct device *dev, size_t size, - gfp_t gfp, unsigned long attrs, int prot, dma_addr_t *handle) +static void *iommu_dma_alloc_remap(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); + unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; struct page **pages; struct sg_table sgt; dma_addr_t iova; - unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; + void *vaddr; - *handle = DMA_MAPPING_ERROR; + *dma_handle = DMA_MAPPING_ERROR; min_size = alloc_sizes & -alloc_sizes; if (min_size < PAGE_SIZE) { @@ -634,7 +636,7 @@ static struct page **__iommu_dma_alloc(struct device *dev, size_t size, if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, GFP_KERNEL)) goto out_free_iova; - if (!(prot & IOMMU_CACHE)) { + if (!(ioprot & IOMMU_CACHE)) { struct scatterlist *sg; int i; @@ -642,14 +644,21 @@ static struct page **__iommu_dma_alloc(struct device *dev, size_t size, arch_dma_prep_coherent(sg_page(sg), sg->length); } - if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, prot) + if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, ioprot) < size) goto out_free_sg; - *handle = iova; + vaddr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, + __builtin_return_address(0)); + if (!vaddr) + goto out_unmap; + + *dma_handle = iova; sg_free_table(&sgt); - return pages; + return vaddr; +out_unmap: + __iommu_dma_unmap(dev, iova, size); out_free_sg: sg_free_table(&sgt); out_free_iova: @@ -1008,18 +1017,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, size >> PAGE_SHIFT); } } else { - pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); - struct page **pages; - - pages = __iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot, - handle); - if (!pages) - return NULL; - - addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, - __builtin_return_address(0)); - if (!addr) - __iommu_dma_free(dev, pages, iosize, handle); + addr = iommu_dma_alloc_remap(dev, iosize, handle, gfp, attrs); } return addr; } @@ -1033,7 +1031,7 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, /* * @cpu_addr will be one of 4 things depending on how it was allocated: * - A remapped array of pages for contiguous allocations. - * - A remapped array of pages from __iommu_dma_alloc(), for all + * - A remapped array of pages from iommu_dma_alloc_remap(), for all * non-atomic allocations. * - A non-cacheable alias from the atomic pool, for atomic * allocations by non-coherent devices. -- cgit From aa8ba2275705aa47df9f52e13c5126688c478966 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:35 +0200 Subject: iommu/dma: Remove __iommu_dma_free We only have a single caller of this function left, so open code it there. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0ffb7805de77..5e0c8450fa0b 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -563,24 +563,6 @@ static struct page **__iommu_dma_get_pages(void *cpu_addr) return area->pages; } -/** - * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc_remap() - * @dev: Device which owns this buffer - * @pages: Array of buffer pages as returned by __iommu_dma_alloc_remap() - * @size: Size of buffer in bytes - * @handle: DMA address of buffer - * - * Frees both the pages associated with the buffer, and the array - * describing them - */ -static void __iommu_dma_free(struct device *dev, struct page **pages, - size_t size, dma_addr_t *handle) -{ - __iommu_dma_unmap(dev, *handle, size); - __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); - *handle = DMA_MAPPING_ERROR; -} - /** * iommu_dma_alloc_remap - Allocate and map a buffer contiguous in IOVA space * @dev: Device to allocate memory for. Must be a real device @@ -1053,7 +1035,8 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, if (!pages) return; - __iommu_dma_free(dev, pages, iosize, &handle); + __iommu_dma_unmap(dev, handle, iosize); + __iommu_dma_free_pages(pages, size >> PAGE_SHIFT); dma_common_free_remap(cpu_addr, size, VM_USERMAP); } else { __iommu_dma_unmap(dev, handle, iosize); -- cgit From bcf4b9c4c2ee0f00d9e273b19419416a20cce9a4 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:36 +0200 Subject: iommu/dma: Refactor iommu_dma_free The freeing logic was made particularly horrible by part of it being opaque to the arch wrapper, which led to a lot of convoluted repetition to ensure each path did everything in the right order. Now that it's all private, we can pick apart and consolidate the logically-distinct steps of freeing the IOMMU mapping, the underlying pages, and the CPU remap (if necessary) into something much more manageable. Signed-off-by: Robin Murphy [various cosmetic changes to the code flow] Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 73 +++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 40 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 5e0c8450fa0b..a288b3d366ae 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -935,6 +935,39 @@ static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, __iommu_dma_unmap(dev, handle, size); } +static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) +{ + size_t alloc_size = PAGE_ALIGN(size); + int count = alloc_size >> PAGE_SHIFT; + struct page *page = NULL, **pages = NULL; + + __iommu_dma_unmap(dev, handle, size); + + /* Non-coherent atomic allocation? Easy */ + if (dma_free_from_pool(cpu_addr, alloc_size)) + return; + + if (is_vmalloc_addr(cpu_addr)) { + /* + * If it the address is remapped, then it's either non-coherent + * or highmem CMA, or an iommu_dma_alloc_remap() construction. + */ + pages = __iommu_dma_get_pages(cpu_addr); + if (!pages) + page = vmalloc_to_page(cpu_addr); + dma_common_free_remap(cpu_addr, alloc_size, VM_USERMAP); + } else { + /* Lowmem means a coherent atomic or CMA allocation */ + page = virt_to_page(cpu_addr); + } + + if (pages) + __iommu_dma_free_pages(pages, count); + if (page && !dma_release_from_contiguous(dev, page, count)) + __free_pages(page, get_order(alloc_size)); +} + static void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { @@ -1004,46 +1037,6 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, return addr; } -static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) -{ - size_t iosize = size; - - size = PAGE_ALIGN(size); - /* - * @cpu_addr will be one of 4 things depending on how it was allocated: - * - A remapped array of pages for contiguous allocations. - * - A remapped array of pages from iommu_dma_alloc_remap(), for all - * non-atomic allocations. - * - A non-cacheable alias from the atomic pool, for atomic - * allocations by non-coherent devices. - * - A normal lowmem address, for atomic allocations by - * coherent devices. - * Hence how dodgy the below logic looks... - */ - if (dma_in_atomic_pool(cpu_addr, size)) { - __iommu_dma_unmap(dev, handle, iosize); - dma_free_from_pool(cpu_addr, size); - } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { - struct page *page = vmalloc_to_page(cpu_addr); - - __iommu_dma_unmap(dev, handle, iosize); - dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); - dma_common_free_remap(cpu_addr, size, VM_USERMAP); - } else if (is_vmalloc_addr(cpu_addr)){ - struct page **pages = __iommu_dma_get_pages(cpu_addr); - - if (!pages) - return; - __iommu_dma_unmap(dev, handle, iosize); - __iommu_dma_free_pages(pages, size >> PAGE_SHIFT); - dma_common_free_remap(cpu_addr, size, VM_USERMAP); - } else { - __iommu_dma_unmap(dev, handle, iosize); - __free_pages(virt_to_page(cpu_addr), get_order(size)); - } -} - static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, unsigned long pfn, size_t size) { -- cgit From 072bebc0691737a6b44c648b4c59501a0cd25357 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:37 +0200 Subject: iommu/dma: Refactor iommu_dma_alloc Shuffle around the self-contained atomic and non-contiguous cases to return early and get out of the way of the CMA case that we're about to work on next. Signed-off-by: Robin Murphy [hch: slight changes to the code flow] Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 60 +++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index a288b3d366ae..4134f13b5529 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -973,14 +973,19 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, { bool coherent = dev_is_dma_coherent(dev); int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); size_t iosize = size; + struct page *page; void *addr; size = PAGE_ALIGN(size); gfp |= __GFP_ZERO; + if (gfpflags_allow_blocking(gfp) && + !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) + return iommu_dma_alloc_remap(dev, iosize, handle, gfp, attrs); + if (!gfpflags_allow_blocking(gfp)) { - struct page *page; /* * In atomic context we can't remap anything, so we'll only * get the virtually contiguous buffer we need by way of a @@ -1002,39 +1007,34 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, __free_pages(page, get_order(size)); else dma_free_from_pool(addr, size); - addr = NULL; - } - } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { - pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); - struct page *page; - - page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, - get_order(size), gfp & __GFP_NOWARN); - if (!page) return NULL; - - *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, ioprot); - if (*handle == DMA_MAPPING_ERROR) { - dma_release_from_contiguous(dev, page, - size >> PAGE_SHIFT); - return NULL; - } - addr = dma_common_contiguous_remap(page, size, VM_USERMAP, - prot, - __builtin_return_address(0)); - if (addr) { - if (!coherent) - arch_dma_prep_coherent(page, iosize); - memset(addr, 0, size); - } else { - __iommu_dma_unmap(dev, *handle, iosize); - dma_release_from_contiguous(dev, page, - size >> PAGE_SHIFT); } - } else { - addr = iommu_dma_alloc_remap(dev, iosize, handle, gfp, attrs); + return addr; } + + page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, + get_order(size), gfp & __GFP_NOWARN); + if (!page) + return NULL; + + *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, ioprot); + if (*handle == DMA_MAPPING_ERROR) + goto out_free_pages; + + addr = dma_common_contiguous_remap(page, size, VM_USERMAP, prot, + __builtin_return_address(0)); + if (!addr) + goto out_unmap; + + if (!coherent) + arch_dma_prep_coherent(page, iosize); + memset(addr, 0, size); return addr; +out_unmap: + __iommu_dma_unmap(dev, *handle, iosize); +out_free_pages: + dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); + return NULL; } static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, -- cgit From 8680aa5a58abfe6087a3d8248c02232d3e05dc80 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:38 +0200 Subject: iommu/dma: Don't remap CMA unnecessarily Always remapping CMA allocations was largely a bodge to keep the freeing logic manageable when it was split between here and an arch wrapper. Now that it's all together and streamlined, we can relax that limitation. Signed-off-by: Robin Murphy Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 4134f13b5529..cffd30810d41 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -973,7 +973,6 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, { bool coherent = dev_is_dma_coherent(dev); int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); - pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); size_t iosize = size; struct page *page; void *addr; @@ -1021,13 +1020,19 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, if (*handle == DMA_MAPPING_ERROR) goto out_free_pages; - addr = dma_common_contiguous_remap(page, size, VM_USERMAP, prot, - __builtin_return_address(0)); - if (!addr) - goto out_unmap; + if (!coherent || PageHighMem(page)) { + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); - if (!coherent) - arch_dma_prep_coherent(page, iosize); + addr = dma_common_contiguous_remap(page, size, VM_USERMAP, prot, + __builtin_return_address(0)); + if (!addr) + goto out_unmap; + + if (!coherent) + arch_dma_prep_coherent(page, iosize); + } else { + addr = page_address(page); + } memset(addr, 0, size); return addr; out_unmap: -- cgit From 9a4ab94afcc4738d65ecfdff726a4a14abde1b98 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:39 +0200 Subject: iommu/dma: Merge the CMA and alloc_pages allocation paths Instead of having a separate code path for the non-blocking alloc_pages and CMA allocations paths merge them into one. There is a slight behavior change here in that we try the page allocator if CMA fails. This matches what dma-direct and other iommu drivers do and will be needed to use the dma-iommu code on architectures without DMA remapping later on. Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index cffd30810d41..ee7dcf03c304 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -974,7 +974,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, bool coherent = dev_is_dma_coherent(dev); int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); size_t iosize = size; - struct page *page; + struct page *page = NULL; void *addr; size = PAGE_ALIGN(size); @@ -984,35 +984,26 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) return iommu_dma_alloc_remap(dev, iosize, handle, gfp, attrs); - if (!gfpflags_allow_blocking(gfp)) { - /* - * In atomic context we can't remap anything, so we'll only - * get the virtually contiguous buffer we need by way of a - * physically contiguous allocation. - */ - if (coherent) { - page = alloc_pages(gfp, get_order(size)); - addr = page ? page_address(page) : NULL; - } else { - addr = dma_alloc_from_pool(size, &page, gfp); - } + if (!gfpflags_allow_blocking(gfp) && !coherent) { + addr = dma_alloc_from_pool(size, &page, gfp); if (!addr) return NULL; *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, ioprot); if (*handle == DMA_MAPPING_ERROR) { - if (coherent) - __free_pages(page, get_order(size)); - else - dma_free_from_pool(addr, size); + dma_free_from_pool(addr, size); return NULL; } return addr; } - page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, - get_order(size), gfp & __GFP_NOWARN); + if (gfpflags_allow_blocking(gfp)) + page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, + get_order(size), + gfp & __GFP_NOWARN); + if (!page) + page = alloc_pages(gfp, get_order(size)); if (!page) return NULL; @@ -1038,7 +1029,8 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, out_unmap: __iommu_dma_unmap(dev, *handle, iosize); out_free_pages: - dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); + if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) + __free_pages(page, get_order(size)); return NULL; } -- cgit From 8553f6e65240ea0c2ae970eeba18e4d3a47230aa Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:40 +0200 Subject: iommu/dma: Split iommu_dma_free Most of it can double up to serve the failure cleanup path for iommu_dma_alloc(). Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index ee7dcf03c304..f0cd35fd11dd 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -935,15 +935,12 @@ static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, __iommu_dma_unmap(dev, handle, size); } -static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) +static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) { size_t alloc_size = PAGE_ALIGN(size); int count = alloc_size >> PAGE_SHIFT; struct page *page = NULL, **pages = NULL; - __iommu_dma_unmap(dev, handle, size); - /* Non-coherent atomic allocation? Easy */ if (dma_free_from_pool(cpu_addr, alloc_size)) return; @@ -968,6 +965,13 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, __free_pages(page, get_order(alloc_size)); } +static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) +{ + __iommu_dma_unmap(dev, handle, size); + __iommu_dma_free(dev, size, cpu_addr); +} + static void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { -- cgit From 9ad5d6eddcb0fa7c227c0612011221e715e8ef49 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 20 May 2019 09:29:41 +0200 Subject: iommu/dma: Cleanup variable naming in iommu_dma_alloc Most importantly clear up the size / iosize confusion. Also rename addr to cpu_addr to match the surrounding code and make the intention a little more clear. Signed-off-by: Robin Murphy [hch: split from a larger patch] Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index f0cd35fd11dd..4e27a29f4458 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -977,64 +977,63 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, { bool coherent = dev_is_dma_coherent(dev); int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); - size_t iosize = size; + size_t alloc_size = PAGE_ALIGN(size); struct page *page = NULL; - void *addr; + void *cpu_addr; - size = PAGE_ALIGN(size); gfp |= __GFP_ZERO; if (gfpflags_allow_blocking(gfp) && !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) - return iommu_dma_alloc_remap(dev, iosize, handle, gfp, attrs); + return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); if (!gfpflags_allow_blocking(gfp) && !coherent) { - addr = dma_alloc_from_pool(size, &page, gfp); - if (!addr) + cpu_addr = dma_alloc_from_pool(alloc_size, &page, gfp); + if (!cpu_addr) return NULL; - *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, + *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot); if (*handle == DMA_MAPPING_ERROR) { - dma_free_from_pool(addr, size); + dma_free_from_pool(cpu_addr, alloc_size); return NULL; } - return addr; + return cpu_addr; } if (gfpflags_allow_blocking(gfp)) - page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, - get_order(size), + page = dma_alloc_from_contiguous(dev, alloc_size >> PAGE_SHIFT, + get_order(alloc_size), gfp & __GFP_NOWARN); if (!page) - page = alloc_pages(gfp, get_order(size)); + page = alloc_pages(gfp, get_order(alloc_size)); if (!page) return NULL; - *handle = __iommu_dma_map(dev, page_to_phys(page), iosize, ioprot); + *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot); if (*handle == DMA_MAPPING_ERROR) goto out_free_pages; if (!coherent || PageHighMem(page)) { pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); - addr = dma_common_contiguous_remap(page, size, VM_USERMAP, prot, - __builtin_return_address(0)); - if (!addr) + cpu_addr = dma_common_contiguous_remap(page, alloc_size, + VM_USERMAP, prot, __builtin_return_address(0)); + if (!cpu_addr) goto out_unmap; if (!coherent) - arch_dma_prep_coherent(page, iosize); + arch_dma_prep_coherent(page, size); } else { - addr = page_address(page); + cpu_addr = page_address(page); } - memset(addr, 0, size); - return addr; + memset(cpu_addr, 0, alloc_size); + return cpu_addr; out_unmap: - __iommu_dma_unmap(dev, *handle, iosize); + __iommu_dma_unmap(dev, *handle, size); out_free_pages: - if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) - __free_pages(page, get_order(size)); + if (!dma_release_from_contiguous(dev, page, alloc_size >> PAGE_SHIFT)) + __free_pages(page, get_order(alloc_size)); return NULL; } -- cgit From ee1ef05d02b03118e4c197e1193329f5b64246b7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:42 +0200 Subject: iommu/dma: Refactor iommu_dma_alloc, part 2 All the logic in iommu_dma_alloc that deals with page allocation from the CMA or page allocators can be split into a self-contained helper, and we can than map the result of that or the atomic pool allocation with the iommu later. This also allows reusing __iommu_dma_free to tear down the allocations and MMU mappings when the IOMMU mapping fails. Based on a patch from Robin Murphy. Signed-off-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 65 +++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 30 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 4e27a29f4458..84761adbb1d4 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -972,35 +972,14 @@ static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, __iommu_dma_free(dev, size, cpu_addr); } -static void *iommu_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +static void *iommu_dma_alloc_pages(struct device *dev, size_t size, + struct page **pagep, gfp_t gfp, unsigned long attrs) { bool coherent = dev_is_dma_coherent(dev); - int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); size_t alloc_size = PAGE_ALIGN(size); struct page *page = NULL; void *cpu_addr; - gfp |= __GFP_ZERO; - - if (gfpflags_allow_blocking(gfp) && - !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) - return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); - - if (!gfpflags_allow_blocking(gfp) && !coherent) { - cpu_addr = dma_alloc_from_pool(alloc_size, &page, gfp); - if (!cpu_addr) - return NULL; - - *handle = __iommu_dma_map(dev, page_to_phys(page), size, - ioprot); - if (*handle == DMA_MAPPING_ERROR) { - dma_free_from_pool(cpu_addr, alloc_size); - return NULL; - } - return cpu_addr; - } - if (gfpflags_allow_blocking(gfp)) page = dma_alloc_from_contiguous(dev, alloc_size >> PAGE_SHIFT, get_order(alloc_size), @@ -1010,33 +989,59 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, if (!page) return NULL; - *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot); - if (*handle == DMA_MAPPING_ERROR) - goto out_free_pages; - if (!coherent || PageHighMem(page)) { pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); cpu_addr = dma_common_contiguous_remap(page, alloc_size, VM_USERMAP, prot, __builtin_return_address(0)); if (!cpu_addr) - goto out_unmap; + goto out_free_pages; if (!coherent) arch_dma_prep_coherent(page, size); } else { cpu_addr = page_address(page); } + + *pagep = page; memset(cpu_addr, 0, alloc_size); return cpu_addr; -out_unmap: - __iommu_dma_unmap(dev, *handle, size); out_free_pages: if (!dma_release_from_contiguous(dev, page, alloc_size >> PAGE_SHIFT)) __free_pages(page, get_order(alloc_size)); return NULL; } +static void *iommu_dma_alloc(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +{ + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + struct page *page = NULL; + void *cpu_addr; + + gfp |= __GFP_ZERO; + + if (gfpflags_allow_blocking(gfp) && + !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) + return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); + + if (!gfpflags_allow_blocking(gfp) && !coherent) + cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp); + else + cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); + if (!cpu_addr) + return NULL; + + *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot); + if (*handle == DMA_MAPPING_ERROR) { + __iommu_dma_free(dev, size, cpu_addr); + return NULL; + } + + return cpu_addr; +} + static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, unsigned long pfn, size_t size) { -- cgit From 3fb3378bb1eb2905065211ab51a7bcc2c5e7bd29 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:43 +0200 Subject: iommu/dma: Refactor iommu_dma_get_sgtable Inline __iommu_dma_get_sgtable_page into the main function, and use the fact that __iommu_dma_get_pages return NULL for remapped contigous allocations to simplify the code flow a bit. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 45 +++++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 84761adbb1d4..fa95794868a4 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1097,42 +1097,31 @@ static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, return __iommu_dma_mmap(pages, size, vma); } -static int __iommu_dma_get_sgtable_page(struct sg_table *sgt, struct page *page, - size_t size) -{ - int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); - - if (!ret) - sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); - return ret; -} - static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - struct page **pages; + struct page *page; + int ret; - if (!is_vmalloc_addr(cpu_addr)) { - struct page *page = virt_to_page(cpu_addr); - return __iommu_dma_get_sgtable_page(sgt, page, size); - } + if (is_vmalloc_addr(cpu_addr)) { + struct page **pages = __iommu_dma_get_pages(cpu_addr); - if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { - /* - * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, - * hence in the vmalloc space. - */ - struct page *page = vmalloc_to_page(cpu_addr); - return __iommu_dma_get_sgtable_page(sgt, page, size); + if (pages) { + return sg_alloc_table_from_pages(sgt, pages, + PAGE_ALIGN(size) >> PAGE_SHIFT, + 0, size, GFP_KERNEL); + } + + page = vmalloc_to_page(cpu_addr); + } else { + page = virt_to_page(cpu_addr); } - pages = __iommu_dma_get_pages(cpu_addr); - if (!pages) - return -ENXIO; - return sg_alloc_table_from_pages(sgt, pages, count, 0, size, - GFP_KERNEL); + ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (!ret) + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return ret; } static const struct dma_map_ops iommu_dma_ops = { -- cgit From efd9f10b70689fdeacadc95b6e0ea6dc311fa64f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:44 +0200 Subject: iommu/dma: Refactor iommu_dma_mmap Inline __iommu_dma_mmap_pfn into the main function, and use the fact that __iommu_dma_get_pages return NULL for remapped contigous allocations to simplify the code flow a bit. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 46 +++++++++++----------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index fa95794868a4..84150ca7b572 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1042,31 +1042,12 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, return cpu_addr; } -static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, - unsigned long pfn, size_t size) -{ - int ret = -ENXIO; - unsigned long nr_vma_pages = vma_pages(vma); - unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long off = vma->vm_pgoff; - - if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); - } - - return ret; -} - static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long off = vma->vm_pgoff; - struct page **pages; + unsigned long pfn, off = vma->vm_pgoff; int ret; vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); @@ -1077,24 +1058,19 @@ static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, if (off >= nr_pages || vma_pages(vma) > nr_pages - off) return -ENXIO; - if (!is_vmalloc_addr(cpu_addr)) { - unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); - return __iommu_dma_mmap_pfn(vma, pfn, size); - } + if (is_vmalloc_addr(cpu_addr)) { + struct page **pages = __iommu_dma_get_pages(cpu_addr); - if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { - /* - * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, - * hence in the vmalloc space. - */ - unsigned long pfn = vmalloc_to_pfn(cpu_addr); - return __iommu_dma_mmap_pfn(vma, pfn, size); + if (pages) + return __iommu_dma_mmap(pages, size, vma); + pfn = vmalloc_to_pfn(cpu_addr); + } else { + pfn = page_to_pfn(virt_to_page(cpu_addr)); } - pages = __iommu_dma_get_pages(cpu_addr); - if (!pages) - return -ENXIO; - return __iommu_dma_mmap(pages, size, vma); + return remap_pfn_range(vma, vma->vm_start, pfn + off, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); } static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, -- cgit From e6475eb010fae28a2550faf9f2f12dc13ab2931a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:45 +0200 Subject: iommu/dma: Don't depend on CONFIG_DMA_DIRECT_REMAP For entirely dma coherent architectures there is no requirement to ever remap dma coherent allocation. Move all the remap and pool code under IS_ENABLED() checks and drop the Kconfig dependency. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 - drivers/iommu/dma-iommu.c | 16 +++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d6d063160dd6..83664db5221d 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -97,7 +97,6 @@ config IOMMU_DMA select IOMMU_IOVA select IRQ_MSI_IOMMU select NEED_SG_DMA_LENGTH - depends on DMA_DIRECT_REMAP config FSL_PAMU bool "Freescale IOMMU support" diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 84150ca7b572..0aff220c4aed 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -942,10 +942,11 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) struct page *page = NULL, **pages = NULL; /* Non-coherent atomic allocation? Easy */ - if (dma_free_from_pool(cpu_addr, alloc_size)) + if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && + dma_free_from_pool(cpu_addr, alloc_size)) return; - if (is_vmalloc_addr(cpu_addr)) { + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { /* * If it the address is remapped, then it's either non-coherent * or highmem CMA, or an iommu_dma_alloc_remap() construction. @@ -989,7 +990,7 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size, if (!page) return NULL; - if (!coherent || PageHighMem(page)) { + if (IS_ENABLED(CONFIG_DMA_REMAP) && (!coherent || PageHighMem(page))) { pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); cpu_addr = dma_common_contiguous_remap(page, alloc_size, @@ -1022,11 +1023,12 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, gfp |= __GFP_ZERO; - if (gfpflags_allow_blocking(gfp) && + if (IS_ENABLED(CONFIG_DMA_REMAP) && gfpflags_allow_blocking(gfp) && !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); - if (!gfpflags_allow_blocking(gfp) && !coherent) + if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && + !gfpflags_allow_blocking(gfp) && !coherent) cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp); else cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); @@ -1058,7 +1060,7 @@ static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, if (off >= nr_pages || vma_pages(vma) > nr_pages - off) return -ENXIO; - if (is_vmalloc_addr(cpu_addr)) { + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { struct page **pages = __iommu_dma_get_pages(cpu_addr); if (pages) @@ -1080,7 +1082,7 @@ static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, struct page *page; int ret; - if (is_vmalloc_addr(cpu_addr)) { + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { struct page **pages = __iommu_dma_get_pages(cpu_addr); if (pages) { -- cgit From a9f4d93dbeb6f5ccb50c6362ba944afe34cb8f12 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 May 2019 09:29:46 +0200 Subject: iommu/dma: Switch copyright boilerplace to SPDX Signed-off-by: Christoph Hellwig Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0aff220c4aed..0cd49c2d3770 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * A fairly generic DMA-API to IOMMU-API glue layer. * @@ -5,18 +6,6 @@ * * based in part on arch/arm/mm/dma-mapping.c: * Copyright (C) 2000-2004 Russell King - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . */ #include -- cgit From d850c2ee5fe2259968e3889624ad22ea15cb4a38 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:24 +0800 Subject: iommu/vt-d: Expose ISA direct mapping region via iommu_get_resv_regions To support mapping ISA region via iommu_group_create_direct_mappings, make sure its exposed by iommu_get_resv_regions. Signed-off-by: James Sewart Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 3995afc59c28..bc7cbe0a6ac6 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5479,6 +5479,19 @@ static void intel_iommu_get_resv_regions(struct device *device, } rcu_read_unlock(); +#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA + if (dev_is_pci(device)) { + struct pci_dev *pdev = to_pci_dev(device); + + if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) { + reg = iommu_alloc_resv_region(0, 1UL << 24, 0, + IOMMU_RESV_DIRECT); + if (reg) + list_add_tail(®->list, head); + } + } +#endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */ + reg = iommu_alloc_resv_region(IOAPIC_RANGE_START, IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1, 0, IOMMU_RESV_MSI); -- cgit From d8190dc638866fe5811d4b45b3af6f34b3f5870f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:25 +0800 Subject: iommu/vt-d: Enable DMA remapping after rmrr mapped The rmrr devices require identity map of the rmrr regions before enabling DMA remapping. Otherwise, there will be a window during which DMA from/to the rmrr regions will be blocked. In order to alleviate this, we move enabling DMA remapping after all rmrr regions get mapped. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index bc7cbe0a6ac6..1bd22511a542 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3510,11 +3510,6 @@ domains_done: ret = dmar_set_interrupt(iommu); if (ret) goto free_iommu; - - if (!translation_pre_enabled(iommu)) - iommu_enable_translation(iommu); - - iommu_disable_protect_mem_regions(iommu); } return 0; @@ -4902,7 +4897,6 @@ int __init intel_iommu_init(void) goto out_free_reserved_range; } up_write(&dmar_global_lock); - pr_info("Intel(R) Virtualization Technology for Directed I/O\n"); #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB) swiotlb = 0; @@ -4925,6 +4919,16 @@ int __init intel_iommu_init(void) register_memory_notifier(&intel_iommu_memory_nb); cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, intel_iommu_cpu_dead); + + /* Finally, we enable the DMA remapping hardware. */ + for_each_iommu(iommu, drhd) { + if (!translation_pre_enabled(iommu)) + iommu_enable_translation(iommu); + + iommu_disable_protect_mem_regions(iommu); + } + pr_info("Intel(R) Virtualization Technology for Directed I/O\n"); + intel_iommu_enabled = 1; intel_iommu_debugfs_init(); -- cgit From f273a453b740772beef4ccb7220cb930176ab38b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:26 +0800 Subject: iommu/vt-d: Add device_def_domain_type() helper This helper returns the default domain type that the device requires. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1bd22511a542..9db0f7f7f1e2 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2930,29 +2930,37 @@ static bool device_is_rmrr_locked(struct device *dev) return true; } -static int iommu_should_identity_map(struct device *dev, int startup) +/* + * Return the required default domain type for a specific device. + * + * @dev: the device in query + * @startup: true if this is during early boot + * + * Returns: + * - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain + * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain + * - 0: both identity and dynamic domains work for this device + */ +static int device_def_domain_type(struct device *dev, int startup) { if (dev_is_pci(dev)) { struct pci_dev *pdev = to_pci_dev(dev); if (device_is_rmrr_locked(dev)) - return 0; + return IOMMU_DOMAIN_DMA; /* * Prevent any device marked as untrusted from getting * placed into the statically identity mapping domain. */ if (pdev->untrusted) - return 0; + return IOMMU_DOMAIN_DMA; if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev)) - return 1; + return IOMMU_DOMAIN_IDENTITY; if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev)) - return 1; - - if (!(iommu_identity_mapping & IDENTMAP_ALL)) - return 0; + return IOMMU_DOMAIN_IDENTITY; /* * We want to start off with all devices in the 1:1 domain, and @@ -2973,14 +2981,14 @@ static int iommu_should_identity_map(struct device *dev, int startup) */ if (!pci_is_pcie(pdev)) { if (!pci_is_root_bus(pdev->bus)) - return 0; + return IOMMU_DOMAIN_DMA; if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI) - return 0; + return IOMMU_DOMAIN_DMA; } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE) - return 0; + return IOMMU_DOMAIN_DMA; } else { if (device_has_rmrr(dev)) - return 0; + return IOMMU_DOMAIN_DMA; } /* @@ -3002,7 +3010,13 @@ static int iommu_should_identity_map(struct device *dev, int startup) return dma_mask >= dma_get_required_mask(dev); } - return 1; + return (iommu_identity_mapping & IDENTMAP_ALL) ? + IOMMU_DOMAIN_IDENTITY : 0; +} + +static inline int iommu_should_identity_map(struct device *dev, int startup) +{ + return device_def_domain_type(dev, startup) == IOMMU_DOMAIN_IDENTITY; } static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw) -- cgit From 4de354ec2f0c0529980ea4aa9fc4f35b296d3de8 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:27 +0800 Subject: iommu/vt-d: Delegate the identity domain to upper layer This allows the iommu generic layer to allocate an identity domain and attach it to a device. Hence, the identity domain is delegated to upper layer. As a side effect, iommu_identity_mapping can't be used to check the existence of identity domains any more. Signed-off-by: James Sewart Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 90 +++++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 32 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 9db0f7f7f1e2..dc7d376a03aa 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -350,6 +350,7 @@ static void domain_context_clear(struct intel_iommu *iommu, struct device *dev); static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu); +static bool device_is_rmrr_locked(struct device *dev); #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON int dmar_disabled = 0; @@ -2808,7 +2809,9 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width); static int __init si_domain_init(int hw) { - int nid, ret; + struct dmar_rmrr_unit *rmrr; + struct device *dev; + int i, nid, ret; si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY); if (!si_domain) @@ -2819,8 +2822,6 @@ static int __init si_domain_init(int hw) return -EFAULT; } - pr_debug("Identity mapping domain allocated\n"); - if (hw) return 0; @@ -2836,6 +2837,31 @@ static int __init si_domain_init(int hw) } } + /* + * Normally we use DMA domains for devices which have RMRRs. But we + * loose this requirement for graphic and usb devices. Identity map + * the RMRRs for graphic and USB devices so that they could use the + * si_domain. + */ + for_each_rmrr_units(rmrr) { + for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, + i, dev) { + unsigned long long start = rmrr->base_address; + unsigned long long end = rmrr->end_address; + + if (device_is_rmrr_locked(dev)) + continue; + + if (WARN_ON(end < start || + end >> agaw_to_width(si_domain->agaw))) + continue; + + ret = iommu_domain_identity_map(si_domain, start, end); + if (ret) + return ret; + } + } + return 0; } @@ -2843,9 +2869,6 @@ static int identity_mapping(struct device *dev) { struct device_domain_info *info; - if (likely(!iommu_identity_mapping)) - return 0; - info = dev->archdata.iommu; if (info && info != DUMMY_DEVICE_DOMAIN_INFO) return (info->domain == si_domain); @@ -3431,11 +3454,9 @@ static int __init init_dmars(void) check_tylersburg_isoch(); - if (iommu_identity_mapping) { - ret = si_domain_init(hw_pass_through); - if (ret) - goto free_iommu; - } + ret = si_domain_init(hw_pass_through); + if (ret) + goto free_iommu; /* @@ -3628,9 +3649,6 @@ static bool iommu_need_mapping(struct device *dev) if (iommu_dummy(dev)) return false; - if (!iommu_identity_mapping) - return true; - found = identity_mapping(dev); if (found) { if (iommu_should_identity_map(dev, 0)) @@ -5051,32 +5069,40 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) struct dmar_domain *dmar_domain; struct iommu_domain *domain; - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE); + if (!dmar_domain) { + pr_err("Can't allocate dmar_domain\n"); + return NULL; + } + if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { + pr_err("Domain initialization failed\n"); + domain_exit(dmar_domain); + return NULL; + } + domain_update_iommu_cap(dmar_domain); - dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE); - if (!dmar_domain) { - pr_err("Can't allocate dmar_domain\n"); - return NULL; - } - if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { - pr_err("Domain initialization failed\n"); - domain_exit(dmar_domain); + domain = &dmar_domain->domain; + domain->geometry.aperture_start = 0; + domain->geometry.aperture_end = + __DOMAIN_MAX_ADDR(dmar_domain->gaw); + domain->geometry.force_aperture = true; + + return domain; + case IOMMU_DOMAIN_IDENTITY: + return &si_domain->domain; + default: return NULL; } - domain_update_iommu_cap(dmar_domain); - - domain = &dmar_domain->domain; - domain->geometry.aperture_start = 0; - domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw); - domain->geometry.force_aperture = true; - return domain; + return NULL; } static void intel_iommu_domain_free(struct iommu_domain *domain) { - domain_exit(to_dmar_domain(domain)); + if (domain != &si_domain->domain) + domain_exit(to_dmar_domain(domain)); } /* -- cgit From fa954e683178949e3c50a7d40c176e7b951bb22d Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:28 +0800 Subject: iommu/vt-d: Delegate the dma domain to upper layer This allows the iommu generic layer to allocate a dma domain and attach it to a device through the iommu api's. With all types of domains being delegated to upper layer, we can remove an internal flag which was used to distinguish domains mananged internally or externally. Signed-off-by: James Sewart Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 74 ++++++++++++--------------------------------- 1 file changed, 19 insertions(+), 55 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index dc7d376a03aa..479fb60aaad0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -302,14 +302,8 @@ static inline void context_clear_entry(struct context_entry *context) static struct dmar_domain *si_domain; static int hw_pass_through = 1; -/* - * Domain represents a virtual machine, more than one devices - * across iommus may be owned in one domain, e.g. kvm guest. - */ -#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0) - /* si_domain contains mulitple devices */ -#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1) +#define DOMAIN_FLAG_STATIC_IDENTITY BIT(0) #define for_each_domain_iommu(idx, domain) \ for (idx = 0; idx < g_num_of_iommus; idx++) \ @@ -540,22 +534,11 @@ static inline void free_devinfo_mem(void *vaddr) kmem_cache_free(iommu_devinfo_cache, vaddr); } -static inline int domain_type_is_vm(struct dmar_domain *domain) -{ - return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE; -} - static inline int domain_type_is_si(struct dmar_domain *domain) { return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY; } -static inline int domain_type_is_vm_or_si(struct dmar_domain *domain) -{ - return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE | - DOMAIN_FLAG_STATIC_IDENTITY); -} - static inline int domain_pfn_supported(struct dmar_domain *domain, unsigned long pfn) { @@ -603,7 +586,9 @@ struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) int iommu_id; /* si_domain and vm domain should not get here. */ - BUG_ON(domain_type_is_vm_or_si(domain)); + if (WARN_ON(domain->domain.type != IOMMU_DOMAIN_DMA)) + return NULL; + for_each_domain_iommu(iommu_id, domain) break; @@ -1651,7 +1636,6 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) if (!iommu->domains || !iommu->domain_ids) return; -again: spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry_safe(info, tmp, &device_domain_list, global) { struct dmar_domain *domain; @@ -1665,18 +1649,6 @@ again: domain = info->domain; __dmar_remove_one_dev_info(info); - - if (!domain_type_is_vm_or_si(domain)) { - /* - * The domain_exit() function can't be called under - * device_domain_lock, as it takes this lock itself. - * So release the lock here and re-run the loop - * afterwards. - */ - spin_unlock_irqrestore(&device_domain_lock, flags); - domain_exit(domain); - goto again; - } } spin_unlock_irqrestore(&device_domain_lock, flags); @@ -2339,7 +2311,7 @@ static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, struct scatterlist *sg, unsigned long phys_pfn, unsigned long nr_pages, int prot) { - int ret; + int iommu_id, ret; struct intel_iommu *iommu; /* Do the real mapping first */ @@ -2347,18 +2319,8 @@ static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, if (ret) return ret; - /* Notify about the new mapping */ - if (domain_type_is_vm(domain)) { - /* VM typed domains can have more than one IOMMUs */ - int iommu_id; - - for_each_domain_iommu(iommu_id, domain) { - iommu = g_iommus[iommu_id]; - __mapping_notify_one(iommu, domain, iov_pfn, nr_pages); - } - } else { - /* General domains only have one IOMMU */ - iommu = domain_get_iommu(domain); + for_each_domain_iommu(iommu_id, domain) { + iommu = g_iommus[iommu_id]; __mapping_notify_one(iommu, domain, iov_pfn, nr_pages); } @@ -4599,9 +4561,6 @@ static int device_notifier(struct notifier_block *nb, return 0; dmar_remove_one_dev_info(dev); - if (!domain_type_is_vm_or_si(domain) && - list_empty(&domain->devices)) - domain_exit(domain); } else if (action == BUS_NOTIFY_ADD_DEVICE) { if (iommu_should_identity_map(dev, 1)) domain_add_dev_info(si_domain, dev); @@ -5070,8 +5029,10 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) struct iommu_domain *domain; switch (type) { + case IOMMU_DOMAIN_DMA: + /* fallthrough */ case IOMMU_DOMAIN_UNMANAGED: - dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE); + dmar_domain = alloc_domain(0); if (!dmar_domain) { pr_err("Can't allocate dmar_domain\n"); return NULL; @@ -5081,6 +5042,14 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) domain_exit(dmar_domain); return NULL; } + + if (type == IOMMU_DOMAIN_DMA && + init_iova_flush_queue(&dmar_domain->iovad, + iommu_flush_iova, iova_entry_free)) { + pr_warn("iova flush queue initialization failed\n"); + intel_iommu_strict = 1; + } + domain_update_iommu_cap(dmar_domain); domain = &dmar_domain->domain; @@ -5291,13 +5260,8 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, struct dmar_domain *old_domain; old_domain = find_domain(dev); - if (old_domain) { + if (old_domain) dmar_remove_one_dev_info(dev); - - if (!domain_type_is_vm_or_si(old_domain) && - list_empty(&old_domain->devices)) - domain_exit(old_domain); - } } ret = prepare_domain_attach_device(domain, dev); -- cgit From 942067f1b6b97b1e97cd073b0e2815bc499656db Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:29 +0800 Subject: iommu/vt-d: Identify default domains replaced with private When we put a device into an iommu group, the group's default domain will be attached to the device. There are some corner cases where the type (identity or dma) of the default domain doesn't work for the device and the request of a new default domain results in failure (e.x. multiple devices have already existed in the group). In order to be compatible with the past, we used a private domain. Mark the private domains and disallow some iommu apis (map/unmap/iova_to_phys) on them. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 64 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 479fb60aaad0..5dae957531f3 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -305,6 +305,14 @@ static int hw_pass_through = 1; /* si_domain contains mulitple devices */ #define DOMAIN_FLAG_STATIC_IDENTITY BIT(0) +/* + * This is a DMA domain allocated through the iommu domain allocation + * interface. But one or more devices belonging to this domain have + * been chosen to use a private domain. We should avoid to use the + * map/unmap/iova_to_phys APIs on it. + */ +#define DOMAIN_FLAG_LOSE_CHILDREN BIT(1) + #define for_each_domain_iommu(idx, domain) \ for (idx = 0; idx < g_num_of_iommus; idx++) \ if (domain->iommu_refcnt[idx]) @@ -4958,6 +4966,7 @@ static void domain_context_clear(struct intel_iommu *iommu, struct device *dev) static void __dmar_remove_one_dev_info(struct device_domain_info *info) { + struct dmar_domain *domain; struct intel_iommu *iommu; unsigned long flags; @@ -4967,6 +4976,7 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info) return; iommu = info->iommu; + domain = info->domain; if (info->dev) { if (dev_is_pci(info->dev) && sm_supported(iommu)) @@ -4981,9 +4991,14 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info) unlink_domain_info(info); spin_lock_irqsave(&iommu->lock, flags); - domain_detach_iommu(info->domain, iommu); + domain_detach_iommu(domain, iommu); spin_unlock_irqrestore(&iommu->lock, flags); + /* free the private domain */ + if (domain->flags & DOMAIN_FLAG_LOSE_CHILDREN && + !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) + domain_exit(info->domain); + free_devinfo_mem(info); } @@ -5307,6 +5322,9 @@ static int intel_iommu_map(struct iommu_domain *domain, int prot = 0; int ret; + if (dmar_domain->flags & DOMAIN_FLAG_LOSE_CHILDREN) + return -EINVAL; + if (iommu_prot & IOMMU_READ) prot |= DMA_PTE_READ; if (iommu_prot & IOMMU_WRITE) @@ -5348,6 +5366,8 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, /* Cope with horrid API which requires us to unmap more than the size argument if it happens to be a large-page mapping. */ BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level)); + if (dmar_domain->flags & DOMAIN_FLAG_LOSE_CHILDREN) + return 0; if (size < VTD_PAGE_SIZE << level_to_offset_bits(level)) size = VTD_PAGE_SIZE << level_to_offset_bits(level); @@ -5379,6 +5399,9 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, int level = 0; u64 phys = 0; + if (dmar_domain->flags & DOMAIN_FLAG_LOSE_CHILDREN) + return 0; + pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level); if (pte) phys = dma_pte_addr(pte); @@ -5434,9 +5457,12 @@ static bool intel_iommu_capable(enum iommu_cap cap) static int intel_iommu_add_device(struct device *dev) { + struct dmar_domain *dmar_domain; + struct iommu_domain *domain; struct intel_iommu *iommu; struct iommu_group *group; u8 bus, devfn; + int ret; iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) @@ -5450,6 +5476,42 @@ static int intel_iommu_add_device(struct device *dev) return PTR_ERR(group); iommu_group_put(group); + + domain = iommu_get_domain_for_dev(dev); + dmar_domain = to_dmar_domain(domain); + if (domain->type == IOMMU_DOMAIN_DMA) { + if (device_def_domain_type(dev, 1) == IOMMU_DOMAIN_IDENTITY) { + ret = iommu_request_dm_for_dev(dev); + if (ret) { + dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; + domain_add_dev_info(si_domain, dev); + dev_info(dev, + "Device uses a private identity domain.\n"); + return 0; + } + + return -ENODEV; + } + } else { + if (device_def_domain_type(dev, 1) == IOMMU_DOMAIN_DMA) { + ret = iommu_request_dma_domain_for_dev(dev); + if (ret) { + dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; + if (!get_valid_domain_for_dev(dev)) { + dev_warn(dev, + "Failed to get a private domain.\n"); + return -ENOMEM; + } + + dev_info(dev, + "Device uses a private dma domain.\n"); + return 0; + } + + return -ENODEV; + } + } + return 0; } -- cgit From 98b2fffb5e274d73e47d495da2caf213cfa28fc8 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:30 +0800 Subject: iommu/vt-d: Handle 32bit device with identity default domain The iommu driver doesn't know whether the bit width of a PCI device is sufficient for access to the whole system memory. Hence, the driver checks this when the driver calls into the dma APIs. If a device is using an identity domain, but the bit width is less than the system requirement, we need to use a dma domain instead. This also applies after we delegated the domain life cycle management to the upper layer. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 55 ++++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5dae957531f3..87f9f2237238 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2984,25 +2984,6 @@ static int device_def_domain_type(struct device *dev, int startup) return IOMMU_DOMAIN_DMA; } - /* - * At boot time, we don't yet know if devices will be 64-bit capable. - * Assume that they will — if they turn out not to be, then we can - * take them out of the 1:1 domain later. - */ - if (!startup) { - /* - * If the device's dma_mask is less than the system's memory - * size then this is not a candidate for identity mapping. - */ - u64 dma_mask = *dev->dma_mask; - - if (dev->coherent_dma_mask && - dev->coherent_dma_mask < dma_mask) - dma_mask = dev->coherent_dma_mask; - - return dma_mask >= dma_get_required_mask(dev); - } - return (iommu_identity_mapping & IDENTMAP_ALL) ? IOMMU_DOMAIN_IDENTITY : 0; } @@ -3614,14 +3595,19 @@ out: /* Check if the dev needs to go through non-identity map and unmap process.*/ static bool iommu_need_mapping(struct device *dev) { - int found; + int ret; if (iommu_dummy(dev)) return false; - found = identity_mapping(dev); - if (found) { - if (iommu_should_identity_map(dev, 0)) + ret = identity_mapping(dev); + if (ret) { + u64 dma_mask = *dev->dma_mask; + + if (dev->coherent_dma_mask && dev->coherent_dma_mask < dma_mask) + dma_mask = dev->coherent_dma_mask; + + if (dma_mask >= dma_get_required_mask(dev)) return false; /* @@ -3629,17 +3615,20 @@ static bool iommu_need_mapping(struct device *dev) * non-identity mapping. */ dmar_remove_one_dev_info(dev); - dev_info(dev, "32bit DMA uses non-identity mapping\n"); - } else { - /* - * In case of a detached 64 bit DMA device from vm, the device - * is put into si_domain for identity mapping. - */ - if (iommu_should_identity_map(dev, 0) && - !domain_add_dev_info(si_domain, dev)) { - dev_info(dev, "64bit DMA uses identity mapping\n"); - return false; + ret = iommu_request_dma_domain_for_dev(dev); + if (ret) { + struct iommu_domain *domain; + struct dmar_domain *dmar_domain; + + domain = iommu_get_domain_for_dev(dev); + if (domain) { + dmar_domain = to_dmar_domain(domain); + dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; + } + get_valid_domain_for_dev(dev); } + + dev_info(dev, "32bit DMA uses non-identity mapping\n"); } return true; -- cgit From fa212a97f3a366adcb2046c565e7c978ab067c73 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:31 +0800 Subject: iommu/vt-d: Probe DMA-capable ACPI name space devices Some platforms may support ACPI name-space enumerated devices that are capable of generating DMA requests. Platforms which support DMA remapping explicitly declares any such DMA-capable ACPI name-space devices in the platform through ACPI Name-space Device Declaration (ANDD) structure and enumerate them through the Device Scope of the appropriate remapping hardware unit. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 87f9f2237238..0bae0b73076c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4796,6 +4796,48 @@ static int __init platform_optin_force_iommu(void) return 1; } +static int __init probe_acpi_namespace_devices(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu; + struct device *dev; + int i, ret = 0; + + for_each_active_iommu(iommu, drhd) { + for_each_active_dev_scope(drhd->devices, + drhd->devices_cnt, i, dev) { + struct acpi_device_physical_node *pn; + struct iommu_group *group; + struct acpi_device *adev; + + if (dev->bus != &acpi_bus_type) + continue; + + adev = to_acpi_device(dev); + mutex_lock(&adev->physical_node_lock); + list_for_each_entry(pn, + &adev->physical_node_list, node) { + group = iommu_group_get(pn->dev); + if (group) { + iommu_group_put(group); + continue; + } + + pn->dev->bus->iommu_ops = &intel_iommu_ops; + ret = iommu_probe_device(pn->dev); + if (ret) + break; + } + mutex_unlock(&adev->physical_node_lock); + + if (ret) + return ret; + } + } + + return 0; +} + int __init intel_iommu_init(void) { int ret = -ENODEV; @@ -4908,6 +4950,9 @@ int __init intel_iommu_init(void) cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, intel_iommu_cpu_dead); + if (probe_acpi_namespace_devices()) + pr_warn("ACPI name space devices didn't probe correctly\n"); + /* Finally, we enable the DMA remapping hardware. */ for_each_iommu(iommu, drhd) { if (!translation_pre_enabled(iommu)) -- cgit From 8af46c784ecfe8929f66b5eaae987f6874953226 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:32 +0800 Subject: iommu/vt-d: Implement is_attach_deferred iommu ops entry As a domain is now attached to a device earlier, we should implement the is_attach_deferred call-back and use it to defer the domain attach from iommu driver init to device driver init when iommu is pre-enabled in kdump kernel. Suggested-by: Tom Murphy Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0bae0b73076c..c8b73802f0e0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -353,6 +353,8 @@ static void domain_context_clear(struct intel_iommu *iommu, static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu); static bool device_is_rmrr_locked(struct device *dev); +static int intel_iommu_attach_device(struct iommu_domain *domain, + struct device *dev); #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON int dmar_disabled = 0; @@ -378,6 +380,7 @@ int intel_iommu_gfx_mapped; EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) +#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2)) static DEFINE_SPINLOCK(device_domain_lock); static LIST_HEAD(device_domain_list); @@ -2408,8 +2411,18 @@ static struct dmar_domain *find_domain(struct device *dev) { struct device_domain_info *info; + if (unlikely(dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO)) { + struct iommu_domain *domain; + + dev->archdata.iommu = NULL; + domain = iommu_get_domain_for_dev(dev); + if (domain) + intel_iommu_attach_device(domain, dev); + } + /* No lock here, assumes no domain exit in normal case */ info = dev->archdata.iommu; + if (likely(info)) return info->domain; return NULL; @@ -5504,6 +5517,9 @@ static int intel_iommu_add_device(struct device *dev) iommu_device_link(&iommu->iommu, dev); + if (translation_pre_enabled(iommu)) + dev->archdata.iommu = DEFER_DEVICE_DOMAIN_INFO; + group = iommu_group_get_for_dev(dev); if (IS_ERR(group)) @@ -5828,6 +5844,12 @@ intel_iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) dmar_domain->default_pasid : -EINVAL; } +static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain, + struct device *dev) +{ + return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO; +} + const struct iommu_ops intel_iommu_ops = { .capable = intel_iommu_capable, .domain_alloc = intel_iommu_domain_alloc, @@ -5850,6 +5872,7 @@ const struct iommu_ops intel_iommu_ops = { .dev_feat_enabled = intel_iommu_dev_feat_enabled, .dev_enable_feat = intel_iommu_dev_enable_feat, .dev_disable_feat = intel_iommu_dev_disable_feat, + .is_attach_deferred = intel_iommu_is_attach_deferred, .pgsize_bitmap = INTEL_IOMMU_PGSIZES, }; -- cgit From 4ec066c7b1476e0ca66a7acdb575627a5d1a1ee6 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:33 +0800 Subject: iommu/vt-d: Cleanup get_valid_domain_for_dev() Previously, get_valid_domain_for_dev() is used to retrieve the DMA domain which has been attached to the device or allocate one if no domain has been attached yet. As we have delegated the DMA domain management to upper layer, this function is used purely to allocate a private DMA domain if the default domain doesn't work for ths device. Cleanup the code for readability. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index c8b73802f0e0..ebc06ee79dce 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2609,7 +2609,6 @@ static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw) } out: - return domain; } @@ -3558,16 +3557,17 @@ static unsigned long intel_alloc_iova(struct device *dev, return iova_pfn; } -struct dmar_domain *get_valid_domain_for_dev(struct device *dev) +static struct dmar_domain *get_private_domain_for_dev(struct device *dev) { struct dmar_domain *domain, *tmp; struct dmar_rmrr_unit *rmrr; struct device *i_dev; int i, ret; + /* Device shouldn't be attached by any domains. */ domain = find_domain(dev); if (domain) - goto out; + return NULL; domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); if (!domain) @@ -3597,11 +3597,9 @@ struct dmar_domain *get_valid_domain_for_dev(struct device *dev) } out: - if (!domain) dev_err(dev, "Allocating domain failed\n"); - return domain; } @@ -3638,7 +3636,7 @@ static bool iommu_need_mapping(struct device *dev) dmar_domain = to_dmar_domain(domain); dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; } - get_valid_domain_for_dev(dev); + get_private_domain_for_dev(dev); } dev_info(dev, "32bit DMA uses non-identity mapping\n"); @@ -3660,7 +3658,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, BUG_ON(dir == DMA_NONE); - domain = get_valid_domain_for_dev(dev); + domain = find_domain(dev); if (!domain) return DMA_MAPPING_ERROR; @@ -3875,7 +3873,7 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele if (!iommu_need_mapping(dev)) return dma_direct_map_sg(dev, sglist, nelems, dir, attrs); - domain = get_valid_domain_for_dev(dev); + domain = find_domain(dev); if (!domain) return 0; @@ -5547,7 +5545,7 @@ static int intel_iommu_add_device(struct device *dev) ret = iommu_request_dma_domain_for_dev(dev); if (ret) { dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; - if (!get_valid_domain_for_dev(dev)) { + if (!get_private_domain_for_dev(dev)) { dev_warn(dev, "Failed to get a private domain.\n"); return -ENOMEM; @@ -5640,7 +5638,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev) u64 ctx_lo; int ret; - domain = get_valid_domain_for_dev(dev); + domain = find_domain(dev); if (!domain) return -EINVAL; -- cgit From 0e31a7266508487bd48bfc5507a3369b797300b1 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:34 +0800 Subject: iommu/vt-d: Remove startup parameter from device_def_domain_type() It isn't used anywhere. Remove it to make code concise. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ebc06ee79dce..ff46227b3409 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2946,7 +2946,7 @@ static bool device_is_rmrr_locked(struct device *dev) * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain * - 0: both identity and dynamic domains work for this device */ -static int device_def_domain_type(struct device *dev, int startup) +static int device_def_domain_type(struct device *dev) { if (dev_is_pci(dev)) { struct pci_dev *pdev = to_pci_dev(dev); @@ -3000,16 +3000,16 @@ static int device_def_domain_type(struct device *dev, int startup) IOMMU_DOMAIN_IDENTITY : 0; } -static inline int iommu_should_identity_map(struct device *dev, int startup) +static inline int iommu_should_identity_map(struct device *dev) { - return device_def_domain_type(dev, startup) == IOMMU_DOMAIN_IDENTITY; + return device_def_domain_type(dev) == IOMMU_DOMAIN_IDENTITY; } static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw) { int ret; - if (!iommu_should_identity_map(dev, 1)) + if (!iommu_should_identity_map(dev)) return 0; ret = domain_add_dev_info(si_domain, dev); @@ -4570,7 +4570,7 @@ static int device_notifier(struct notifier_block *nb, dmar_remove_one_dev_info(dev); } else if (action == BUS_NOTIFY_ADD_DEVICE) { - if (iommu_should_identity_map(dev, 1)) + if (iommu_should_identity_map(dev)) domain_add_dev_info(si_domain, dev); } @@ -5528,7 +5528,7 @@ static int intel_iommu_add_device(struct device *dev) domain = iommu_get_domain_for_dev(dev); dmar_domain = to_dmar_domain(domain); if (domain->type == IOMMU_DOMAIN_DMA) { - if (device_def_domain_type(dev, 1) == IOMMU_DOMAIN_IDENTITY) { + if (device_def_domain_type(dev) == IOMMU_DOMAIN_IDENTITY) { ret = iommu_request_dm_for_dev(dev); if (ret) { dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; @@ -5541,7 +5541,7 @@ static int intel_iommu_add_device(struct device *dev) return -ENODEV; } } else { - if (device_def_domain_type(dev, 1) == IOMMU_DOMAIN_DMA) { + if (device_def_domain_type(dev) == IOMMU_DOMAIN_DMA) { ret = iommu_request_dma_domain_for_dev(dev); if (ret) { dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; -- cgit From b7297783c2bb6fa7bd557842657bed7e95220978 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:35 +0800 Subject: iommu/vt-d: Remove duplicated code for device hotplug The iommu generic code has handled the device hotplug cases. Remove the duplicated code. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 34 ---------------------------------- 1 file changed, 34 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ff46227b3409..007aac554b37 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4548,39 +4548,6 @@ int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) return 0; } -/* - * Here we only respond to action of unbound device from driver. - * - * Added device is not attached to its DMAR domain here yet. That will happen - * when mapping the device to iova. - */ -static int device_notifier(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct device *dev = data; - struct dmar_domain *domain; - - if (iommu_dummy(dev)) - return 0; - - if (action == BUS_NOTIFY_REMOVED_DEVICE) { - domain = find_domain(dev); - if (!domain) - return 0; - - dmar_remove_one_dev_info(dev); - } else if (action == BUS_NOTIFY_ADD_DEVICE) { - if (iommu_should_identity_map(dev)) - domain_add_dev_info(si_domain, dev); - } - - return 0; -} - -static struct notifier_block device_nb = { - .notifier_call = device_notifier, -}; - static int intel_iommu_memory_notifier(struct notifier_block *nb, unsigned long val, void *v) { @@ -4955,7 +4922,6 @@ int __init intel_iommu_init(void) } bus_set_iommu(&pci_bus_type, &intel_iommu_ops); - bus_register_notifier(&pci_bus_type, &device_nb); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, -- cgit From df4f3c603aeb6dd40b74b93bf2db6d9c3213d4e1 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 25 May 2019 13:41:36 +0800 Subject: iommu/vt-d: Remove static identity map code The code to prepare the static identity map for various reserved memory ranges in intel_iommu_init() is duplicated with the default domain mechanism now. Remove it to avoid duplication. Signed-off-by: James Sewart Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 144 +------------------------------------------- 1 file changed, 1 insertion(+), 143 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 007aac554b37..d9d7d669de81 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2762,31 +2762,6 @@ static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr, rmrr->end_address); } -#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA -static inline void iommu_prepare_isa(void) -{ - struct pci_dev *pdev; - int ret; - - pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL); - if (!pdev) - return; - - pr_info("Prepare 0-16MiB unity mapping for LPC\n"); - ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1); - - if (ret) - pr_err("Failed to create 0-16MiB identity map - floppy might not work\n"); - - pci_dev_put(pdev); -} -#else -static inline void iommu_prepare_isa(void) -{ - return; -} -#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */ - static int md_domain_init(struct dmar_domain *domain, int guest_width); static int __init si_domain_init(int hw) @@ -3000,68 +2975,6 @@ static int device_def_domain_type(struct device *dev) IOMMU_DOMAIN_IDENTITY : 0; } -static inline int iommu_should_identity_map(struct device *dev) -{ - return device_def_domain_type(dev) == IOMMU_DOMAIN_IDENTITY; -} - -static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw) -{ - int ret; - - if (!iommu_should_identity_map(dev)) - return 0; - - ret = domain_add_dev_info(si_domain, dev); - if (!ret) - dev_info(dev, "%s identity mapping\n", - hw ? "Hardware" : "Software"); - else if (ret == -ENODEV) - /* device not associated with an iommu */ - ret = 0; - - return ret; -} - - -static int __init iommu_prepare_static_identity_mapping(int hw) -{ - struct pci_dev *pdev = NULL; - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - struct device *dev; - int i; - int ret = 0; - - for_each_pci_dev(pdev) { - ret = dev_prepare_static_identity_mapping(&pdev->dev, hw); - if (ret) - return ret; - } - - for_each_active_iommu(iommu, drhd) - for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) { - struct acpi_device_physical_node *pn; - struct acpi_device *adev; - - if (dev->bus != &acpi_bus_type) - continue; - - adev= to_acpi_device(dev); - mutex_lock(&adev->physical_node_lock); - list_for_each_entry(pn, &adev->physical_node_list, node) { - ret = dev_prepare_static_identity_mapping(pn->dev, hw); - if (ret) - break; - } - mutex_unlock(&adev->physical_node_lock); - if (ret) - return ret; - } - - return 0; -} - static void intel_iommu_init_qi(struct intel_iommu *iommu) { /* @@ -3284,11 +3197,8 @@ out_unmap: static int __init init_dmars(void) { struct dmar_drhd_unit *drhd; - struct dmar_rmrr_unit *rmrr; - bool copied_tables = false; - struct device *dev; struct intel_iommu *iommu; - int i, ret; + int ret; /* * for each drhd @@ -3381,7 +3291,6 @@ static int __init init_dmars(void) } else { pr_info("Copied translation tables from previous kernel for %s\n", iommu->name); - copied_tables = true; } } @@ -3421,57 +3330,6 @@ static int __init init_dmars(void) if (ret) goto free_iommu; - - /* - * If we copied translations from a previous kernel in the kdump - * case, we can not assign the devices to domains now, as that - * would eliminate the old mappings. So skip this part and defer - * the assignment to device driver initialization time. - */ - if (copied_tables) - goto domains_done; - - /* - * If pass through is not set or not enabled, setup context entries for - * identity mappings for rmrr, gfx, and isa and may fall back to static - * identity mapping if iommu_identity_mapping is set. - */ - if (iommu_identity_mapping) { - ret = iommu_prepare_static_identity_mapping(hw_pass_through); - if (ret) { - pr_crit("Failed to setup IOMMU pass-through\n"); - goto free_iommu; - } - } - /* - * For each rmrr - * for each dev attached to rmrr - * do - * locate drhd for dev, alloc domain for dev - * allocate free domain - * allocate page table entries for rmrr - * if context not allocated for bus - * allocate and init context - * set present in root table for this bus - * init context with domain, translation etc - * endfor - * endfor - */ - pr_info("Setting RMRR:\n"); - for_each_rmrr_units(rmrr) { - /* some BIOS lists non-exist devices in DMAR table. */ - for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, - i, dev) { - ret = iommu_prepare_rmrr_dev(rmrr, dev); - if (ret) - pr_err("Mapping reserved region failed\n"); - } - } - - iommu_prepare_isa(); - -domains_done: - /* * for each drhd * enable fault log -- cgit From c1ddcf1cc9add03d775ba787b942729a4b87e7a6 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Thu, 8 Nov 2018 11:57:33 +0000 Subject: iommu/amd: Add missed 'tag' to error msg in iommu_print_event Fixes gcc '-Wunused-but-set-variable' warning: drivers/iommu/amd_iommu.c: In function 'iommu_print_event': drivers/iommu/amd_iommu.c:550:33: warning: variable 'tag' set but not used [-Wunused-but-set-variable] It was introduced in e7f63ffc1bf1 ("iommu/amd: Update logging information for new event type") seems just missed in the error message, add it as suggested by Joerg. Signed-off-by: YueHaibing Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c0b5b9298e8e..86613f139618 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -631,9 +631,9 @@ retry: pasid = ((event[0] >> 16) & 0xFFFF) | ((event[1] << 6) & 0xF0000); tag = event[1] & 0x03FF; - dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", + dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), - pasid, address, flags); + pasid, address, flags, tag); break; default: dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", -- cgit From e314a7c92b3217e893cbe03b944c7afc9cbd3e4c Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Fri, 31 May 2019 16:16:02 -0400 Subject: iommu/vt-d: Fix a variable set but not used The commit "iommu/vt-d: Delegate the dma domain to upper layer" left an unused variable, drivers/iommu/intel-iommu.c: In function 'disable_dmar_iommu': drivers/iommu/intel-iommu.c:1652:23: warning: variable 'domain' set but not used [-Wunused-but-set-variable] Signed-off-by: Qian Cai Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d9d7d669de81..876096c1f91b 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1649,16 +1649,12 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry_safe(info, tmp, &device_domain_list, global) { - struct dmar_domain *domain; - if (info->iommu != iommu) continue; if (!info->dev || !info->domain) continue; - domain = info->domain; - __dmar_remove_one_dev_info(info); } spin_unlock_irqrestore(&device_domain_lock, flags); -- cgit From 1b961423158caaae49d3900b7c9c37477bbfa9b3 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 29 May 2019 01:15:32 -0700 Subject: iommu/dma: Fix condition check in iommu_dma_unmap_sg Clang warns: drivers/iommu/dma-iommu.c:897:6: warning: logical not is only applied to the left hand side of this comparison [-Wlogical-not-parentheses] if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) ^ ~~ drivers/iommu/dma-iommu.c:897:6: note: add parentheses after the '!' to evaluate the comparison first if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) ^ ( ) drivers/iommu/dma-iommu.c:897:6: note: add parentheses around left hand side expression to silence this warning if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) ^ ( ) 1 warning generated. Judging from the rest of the commit and the conditional in iommu_dma_map_sg, either if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) or if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) was intended, not a combination of the two. I personally think that the former is easier to understand so use that. Fixes: 06d60728ff5c ("iommu/dma: move the arm64 wrappers to common code") Link: https://github.com/ClangBuiltLinux/linux/issues/497 Signed-off-by: Nathan Chancellor Reviewed-by: Robin Murphy Reviewed-by: Christoph Hellwig Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0cd49c2d3770..0dee374fc64a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -894,7 +894,7 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, struct scatterlist *tmp; int i; - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); /* -- cgit From 80eaa9f558134a31c10dddb156d347b9c983290e Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 27 May 2019 13:52:48 +0200 Subject: iommu/ipmmu-vmsa: Link IOMMUs and devices in sysfs As of commit 7af9a5fdb9e0ca33 ("iommu/ipmmu-vmsa: Use iommu_device_sysfs_add()/remove()"), IOMMU devices show up under /sys/class/iommu/, but their "devices" subdirectories are empty. Likewise, devices tied to an IOMMU do not have an "iommu" backlink. Make sure all links are created, on both arm32 and arm64. Signed-off-by: Geert Uytterhoeven Reviewed-by: Laurent Pinchart Reviewed-by: Yoshihiro Shimoda Tested-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 9a380c10655e..9f2b781e20a0 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -885,27 +885,37 @@ error: static int ipmmu_add_device(struct device *dev) { + struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); struct iommu_group *group; + int ret; /* * Only let through devices that have been verified in xlate() */ - if (!to_ipmmu(dev)) + if (!mmu) return -ENODEV; - if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) - return ipmmu_init_arm_mapping(dev); + if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) { + ret = ipmmu_init_arm_mapping(dev); + if (ret) + return ret; + } else { + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); - group = iommu_group_get_for_dev(dev); - if (IS_ERR(group)) - return PTR_ERR(group); + iommu_group_put(group); + } - iommu_group_put(group); + iommu_device_link(&mmu->iommu, dev); return 0; } static void ipmmu_remove_device(struct device *dev) { + struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); + + iommu_device_unlink(&mmu->iommu, dev); arm_iommu_detach_device(dev); iommu_group_remove_device(dev); } -- cgit From 82576aa8af49a1aa31909a8500b972084582a118 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 27 May 2019 13:52:49 +0200 Subject: iommu/ipmmu-vmsa: Prepare to handle 40-bit error addresses On R-Car Gen3, the faulting virtual address is a 40-bit address, and comprised of two registers. Read the upper address part, and combine both parts, when running on a 64-bit system. Signed-off-by: Geert Uytterhoeven Reviewed-by: Simon Horman Reviewed-by: Yoshihiro Shimoda Tested-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 9f2b781e20a0..f2061bd1dc7b 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -186,7 +186,8 @@ static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) #define IMMAIR_ATTR_IDX_WBRWA 1 #define IMMAIR_ATTR_IDX_DEV 2 -#define IMEAR 0x0030 +#define IMELAR 0x0030 /* IMEAR on R-Car Gen2 */ +#define IMEUAR 0x0034 /* R-Car Gen3 only */ #define IMPCTR 0x0200 #define IMPSTR 0x0208 @@ -522,14 +523,16 @@ static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) { const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; struct ipmmu_vmsa_device *mmu = domain->mmu; + unsigned long iova; u32 status; - u32 iova; status = ipmmu_ctx_read_root(domain, IMSTR); if (!(status & err_mask)) return IRQ_NONE; - iova = ipmmu_ctx_read_root(domain, IMEAR); + iova = ipmmu_ctx_read_root(domain, IMELAR); + if (IS_ENABLED(CONFIG_64BIT)) + iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32; /* * Clear the error status flags. Unlike traditional interrupt flag @@ -541,10 +544,10 @@ static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) /* Log fatal errors. */ if (status & IMSTR_MHIT) - dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n", + dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n", iova); if (status & IMSTR_ABORT) - dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n", + dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n", iova); if (!(status & (IMSTR_PF | IMSTR_TF))) @@ -560,7 +563,7 @@ static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) return IRQ_HANDLED; dev_err_ratelimited(mmu->dev, - "Unhandled fault: status 0x%08x iova 0x%08x\n", + "Unhandled fault: status 0x%08x iova 0x%lx\n", status, iova); return IRQ_HANDLED; -- cgit From b43e0d8a458cf267c98f914a4e0a4cafe2e8fd76 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 27 May 2019 13:52:50 +0200 Subject: iommu/ipmmu-vmsa: Make IPMMU_CTX_MAX unsigned Make the IPMMU_CTX_MAX constant unsigned, to match the type of ipmmu_features.number_of_contexts. This allows to use plain min() instead of type-casting min_t(). Signed-off-by: Geert Uytterhoeven Reviewed-by: Laurent Pinchart Reviewed-by: Simon Horman Reviewed-by: Yoshihiro Shimoda Tested-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index f2061bd1dc7b..87acf86f295f 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -36,7 +36,7 @@ #define arm_iommu_detach_device(...) do {} while (0) #endif -#define IPMMU_CTX_MAX 8 +#define IPMMU_CTX_MAX 8U struct ipmmu_features { bool use_ns_alias_offset; @@ -1060,8 +1060,7 @@ static int ipmmu_probe(struct platform_device *pdev) if (mmu->features->use_ns_alias_offset) mmu->base += IM_NS_ALIAS_OFFSET; - mmu->num_ctx = min_t(unsigned int, IPMMU_CTX_MAX, - mmu->features->number_of_contexts); + mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts); irq = platform_get_irq(pdev, 0); -- cgit From b7f3f047aea47d82aeb251fb38a694e6d890d139 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 27 May 2019 13:52:51 +0200 Subject: iommu/ipmmu-vmsa: Move num_utlbs to SoC-specific features The maximum number of micro-TLBs per IPMMU instance is not fixed, but depends on the SoC type. Hence move it from struct ipmmu_vmsa_device to struct ipmmu_features, and set up the correct value for both R-Car Gen2 and Gen3 SoCs. Note that currently no code uses this value. Signed-off-by: Geert Uytterhoeven Reviewed-by: Laurent Pinchart Reviewed-by: Simon Horman Reviewed-by: Yoshihiro Shimoda Tested-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 87acf86f295f..3fa57627b1e3 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -42,6 +42,7 @@ struct ipmmu_features { bool use_ns_alias_offset; bool has_cache_leaf_nodes; unsigned int number_of_contexts; + unsigned int num_utlbs; bool setup_imbuscr; bool twobit_imttbcr_sl0; bool reserved_context; @@ -53,7 +54,6 @@ struct ipmmu_vmsa_device { struct iommu_device iommu; struct ipmmu_vmsa_device *root; const struct ipmmu_features *features; - unsigned int num_utlbs; unsigned int num_ctx; spinlock_t lock; /* Protects ctx and domains[] */ DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); @@ -972,6 +972,7 @@ static const struct ipmmu_features ipmmu_features_default = { .use_ns_alias_offset = true, .has_cache_leaf_nodes = false, .number_of_contexts = 1, /* software only tested with one context */ + .num_utlbs = 32, .setup_imbuscr = true, .twobit_imttbcr_sl0 = false, .reserved_context = false, @@ -981,6 +982,7 @@ static const struct ipmmu_features ipmmu_features_rcar_gen3 = { .use_ns_alias_offset = false, .has_cache_leaf_nodes = true, .number_of_contexts = 8, + .num_utlbs = 48, .setup_imbuscr = false, .twobit_imttbcr_sl0 = true, .reserved_context = true, @@ -1033,7 +1035,6 @@ static int ipmmu_probe(struct platform_device *pdev) } mmu->dev = &pdev->dev; - mmu->num_utlbs = 48; spin_lock_init(&mmu->lock); bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); mmu->features = of_device_get_match_data(&pdev->dev); -- cgit From 892db541cc68eb39d7813af34f33ce74a0014a1d Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 27 May 2019 13:52:52 +0200 Subject: iommu/ipmmu-vmsa: Extract hardware context initialization ipmmu_domain_init_context() takes care of (1) initializing the software domain, and (2) initializing the hardware context for the domain. Extract the code to initialize the hardware context into a new subroutine ipmmu_domain_setup_context(), to prepare for later reuse. Signed-off-by: Geert Uytterhoeven Reviewed-by: Simon Horman Reviewed-by: Yoshihiro Shimoda Tested-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 91 ++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 43 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 3fa57627b1e3..56e84bcc9532 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -404,52 +404,10 @@ static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu, spin_unlock_irqrestore(&mmu->lock, flags); } -static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) +static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) { u64 ttbr; u32 tmp; - int ret; - - /* - * Allocate the page table operations. - * - * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory - * access, Long-descriptor format" that the NStable bit being set in a - * table descriptor will result in the NStable and NS bits of all child - * entries being ignored and considered as being set. The IPMMU seems - * not to comply with this, as it generates a secure access page fault - * if any of the NStable and NS bits isn't set when running in - * non-secure mode. - */ - domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; - domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; - domain->cfg.ias = 32; - domain->cfg.oas = 40; - domain->cfg.tlb = &ipmmu_gather_ops; - domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); - domain->io_domain.geometry.force_aperture = true; - /* - * TODO: Add support for coherent walk through CCI with DVM and remove - * cache handling. For now, delegate it to the io-pgtable code. - */ - domain->cfg.iommu_dev = domain->mmu->root->dev; - - /* - * Find an unused context. - */ - ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); - if (ret < 0) - return ret; - - domain->context_id = ret; - - domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, - domain); - if (!domain->iop) { - ipmmu_domain_free_context(domain->mmu->root, - domain->context_id); - return -EINVAL; - } /* TTBR0 */ ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; @@ -495,7 +453,54 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) */ ipmmu_ctx_write_all(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); +} + +static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) +{ + int ret; + + /* + * Allocate the page table operations. + * + * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory + * access, Long-descriptor format" that the NStable bit being set in a + * table descriptor will result in the NStable and NS bits of all child + * entries being ignored and considered as being set. The IPMMU seems + * not to comply with this, as it generates a secure access page fault + * if any of the NStable and NS bits isn't set when running in + * non-secure mode. + */ + domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; + domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; + domain->cfg.ias = 32; + domain->cfg.oas = 40; + domain->cfg.tlb = &ipmmu_gather_ops; + domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); + domain->io_domain.geometry.force_aperture = true; + /* + * TODO: Add support for coherent walk through CCI with DVM and remove + * cache handling. For now, delegate it to the io-pgtable code. + */ + domain->cfg.iommu_dev = domain->mmu->root->dev; + + /* + * Find an unused context. + */ + ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); + if (ret < 0) + return ret; + + domain->context_id = ret; + + domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, + domain); + if (!domain->iop) { + ipmmu_domain_free_context(domain->mmu->root, + domain->context_id); + return -EINVAL; + } + ipmmu_domain_setup_context(domain); return 0; } -- cgit From da38e9ec9c2d0ebec1499b5961baffd08f8ca062 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 27 May 2019 13:52:53 +0200 Subject: iommu/ipmmu-vmsa: Add suspend/resume support During PSCI system suspend, R-Car Gen3 SoCs are powered down, and all IPMMU state is lost. Hence after s2ram, devices wired behind an IPMMU, and configured to use it, will see their DMA operations hang. To fix this, restore all IPMMU contexts, and re-enable all active micro-TLBs during system resume. Signed-off-by: Geert Uytterhoeven Reviewed-by: Yoshihiro Shimoda Tested-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 47 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 56e84bcc9532..408ad0b25919 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -36,7 +36,10 @@ #define arm_iommu_detach_device(...) do {} while (0) #endif -#define IPMMU_CTX_MAX 8U +#define IPMMU_CTX_MAX 8U +#define IPMMU_CTX_INVALID -1 + +#define IPMMU_UTLB_MAX 48U struct ipmmu_features { bool use_ns_alias_offset; @@ -58,6 +61,7 @@ struct ipmmu_vmsa_device { spinlock_t lock; /* Protects ctx and domains[] */ DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX]; + s8 utlb_ctx[IPMMU_UTLB_MAX]; struct iommu_group *group; struct dma_iommu_mapping *mapping; @@ -335,6 +339,7 @@ static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, ipmmu_write(mmu, IMUCTR(utlb), IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | IMUCTR_MMUEN); + mmu->utlb_ctx[utlb] = domain->context_id; } /* @@ -346,6 +351,7 @@ static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, struct ipmmu_vmsa_device *mmu = domain->mmu; ipmmu_write(mmu, IMUCTR(utlb), 0); + mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; } static void ipmmu_tlb_flush_all(void *cookie) @@ -1043,6 +1049,7 @@ static int ipmmu_probe(struct platform_device *pdev) spin_lock_init(&mmu->lock); bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); mmu->features = of_device_get_match_data(&pdev->dev); + memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs); dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); /* Map I/O memory and request IRQ. */ @@ -1158,10 +1165,48 @@ static int ipmmu_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int ipmmu_resume_noirq(struct device *dev) +{ + struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); + unsigned int i; + + /* Reset root MMU and restore contexts */ + if (ipmmu_is_root(mmu)) { + ipmmu_device_reset(mmu); + + for (i = 0; i < mmu->num_ctx; i++) { + if (!mmu->domains[i]) + continue; + + ipmmu_domain_setup_context(mmu->domains[i]); + } + } + + /* Re-enable active micro-TLBs */ + for (i = 0; i < mmu->features->num_utlbs; i++) { + if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID) + continue; + + ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i); + } + + return 0; +} + +static const struct dev_pm_ops ipmmu_pm = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) +}; +#define DEV_PM_OPS &ipmmu_pm +#else +#define DEV_PM_OPS NULL +#endif /* CONFIG_PM_SLEEP */ + static struct platform_driver ipmmu_driver = { .driver = { .name = "ipmmu-vmsa", .of_match_table = of_match_ptr(ipmmu_of_ids), + .pm = DEV_PM_OPS, }, .probe = ipmmu_probe, .remove = ipmmu_remove, -- cgit From 0c830e6b32826311fc2b9ea1f4679be0f4ef0933 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Mon, 3 Jun 2019 15:57:48 +0100 Subject: iommu: Introduce device fault report API Traditionally, device specific faults are detected and handled within their own device drivers. When IOMMU is enabled, faults such as DMA related transactions are detected by IOMMU. There is no generic reporting mechanism to report faults back to the in-kernel device driver or the guest OS in case of assigned devices. This patch introduces a registration API for device specific fault handlers. This differs from the existing iommu_set_fault_handler/ report_iommu_fault infrastructures in several ways: - it allows to report more sophisticated fault events (both unrecoverable faults and page request faults) due to the nature of the iommu_fault struct - it is device specific and not domain specific. The current iommu_report_device_fault() implementation only handles the "shoot and forget" unrecoverable fault case. Handling of page request faults or stalled faults will come later. Signed-off-by: Jacob Pan Signed-off-by: Ashok Raj Signed-off-by: Jean-Philippe Brucker Signed-off-by: Eric Auger Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 146 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3fa025f849e9..293a6fa716e0 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -107,15 +107,43 @@ void iommu_device_unregister(struct iommu_device *iommu) spin_unlock(&iommu_device_lock); } +static struct iommu_param *iommu_get_dev_param(struct device *dev) +{ + struct iommu_param *param = dev->iommu_param; + + if (param) + return param; + + param = kzalloc(sizeof(*param), GFP_KERNEL); + if (!param) + return NULL; + + mutex_init(¶m->lock); + dev->iommu_param = param; + return param; +} + +static void iommu_free_dev_param(struct device *dev) +{ + kfree(dev->iommu_param); + dev->iommu_param = NULL; +} + int iommu_probe_device(struct device *dev) { const struct iommu_ops *ops = dev->bus->iommu_ops; - int ret = -EINVAL; + int ret; WARN_ON(dev->iommu_group); + if (!ops) + return -EINVAL; - if (ops) - ret = ops->add_device(dev); + if (!iommu_get_dev_param(dev)) + return -ENOMEM; + + ret = ops->add_device(dev); + if (ret) + iommu_free_dev_param(dev); return ret; } @@ -126,6 +154,8 @@ void iommu_release_device(struct device *dev) if (dev->iommu_group) ops->remove_device(dev); + + iommu_free_dev_param(dev); } static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, @@ -854,6 +884,116 @@ int iommu_group_unregister_notifier(struct iommu_group *group, } EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); +/** + * iommu_register_device_fault_handler() - Register a device fault handler + * @dev: the device + * @handler: the fault handler + * @data: private data passed as argument to the handler + * + * When an IOMMU fault event is received, this handler gets called with the + * fault event and data as argument. The handler should return 0 on success. + * + * Return 0 if the fault handler was installed successfully, or an error. + */ +int iommu_register_device_fault_handler(struct device *dev, + iommu_dev_fault_handler_t handler, + void *data) +{ + struct iommu_param *param = dev->iommu_param; + int ret = 0; + + if (!param) + return -EINVAL; + + mutex_lock(¶m->lock); + /* Only allow one fault handler registered for each device */ + if (param->fault_param) { + ret = -EBUSY; + goto done_unlock; + } + + get_device(dev); + param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); + if (!param->fault_param) { + put_device(dev); + ret = -ENOMEM; + goto done_unlock; + } + param->fault_param->handler = handler; + param->fault_param->data = data; + +done_unlock: + mutex_unlock(¶m->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); + +/** + * iommu_unregister_device_fault_handler() - Unregister the device fault handler + * @dev: the device + * + * Remove the device fault handler installed with + * iommu_register_device_fault_handler(). + * + * Return 0 on success, or an error. + */ +int iommu_unregister_device_fault_handler(struct device *dev) +{ + struct iommu_param *param = dev->iommu_param; + int ret = 0; + + if (!param) + return -EINVAL; + + mutex_lock(¶m->lock); + + if (!param->fault_param) + goto unlock; + + kfree(param->fault_param); + param->fault_param = NULL; + put_device(dev); +unlock: + mutex_unlock(¶m->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); + +/** + * iommu_report_device_fault() - Report fault event to device driver + * @dev: the device + * @evt: fault event data + * + * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ + * handler. + * + * Return 0 on success, or an error. + */ +int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) +{ + struct iommu_param *param = dev->iommu_param; + struct iommu_fault_param *fparam; + int ret = 0; + + if (!param || !evt) + return -EINVAL; + + /* we only report device fault if there is a handler registered */ + mutex_lock(¶m->lock); + fparam = param->fault_param; + if (!fparam || !fparam->handler) { + ret = -EINVAL; + goto done_unlock; + } + ret = fparam->handler(&evt->fault, fparam->data); +done_unlock: + mutex_unlock(¶m->lock); + return ret; +} +EXPORT_SYMBOL_GPL(iommu_report_device_fault); + /** * iommu_group_id - Return ID for a group * @group: the group to ID -- cgit From bf3255b3cfe2d06280340dbac3f44b65d3ee6da3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Mon, 3 Jun 2019 15:57:49 +0100 Subject: iommu: Add recoverable fault reporting Some IOMMU hardware features, for example PCI PRI and Arm SMMU Stall, enable recoverable I/O page faults. Allow IOMMU drivers to report PRI Page Requests and Stall events through the new fault reporting API. The consumer of the fault can be either an I/O page fault handler in the host, or a guest OS. Once handled, the fault must be completed by sending a page response back to the IOMMU. Add an iommu_page_response() function to complete a page fault. There are two ways to extend the userspace API: * Add a field to iommu_page_response and a flag to iommu_page_response::flags describing the validity of this field. * Introduce a new iommu_page_response_X structure with a different version number. The kernel must then support both versions. Signed-off-by: Jacob Pan Signed-off-by: Jean-Philippe Brucker Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 293a6fa716e0..ac1f29c19e59 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -891,7 +891,14 @@ EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); * @data: private data passed as argument to the handler * * When an IOMMU fault event is received, this handler gets called with the - * fault event and data as argument. The handler should return 0 on success. + * fault event and data as argument. The handler should return 0 on success. If + * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also + * complete the fault by calling iommu_page_response() with one of the following + * response code: + * - IOMMU_PAGE_RESP_SUCCESS: retry the translation + * - IOMMU_PAGE_RESP_INVALID: terminate the fault + * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting + * page faults if possible. * * Return 0 if the fault handler was installed successfully, or an error. */ @@ -921,6 +928,8 @@ int iommu_register_device_fault_handler(struct device *dev, } param->fault_param->handler = handler; param->fault_param->data = data; + mutex_init(¶m->fault_param->lock); + INIT_LIST_HEAD(¶m->fault_param->faults); done_unlock: mutex_unlock(¶m->lock); @@ -951,6 +960,12 @@ int iommu_unregister_device_fault_handler(struct device *dev) if (!param->fault_param) goto unlock; + /* we cannot unregister handler if there are pending faults */ + if (!list_empty(¶m->fault_param->faults)) { + ret = -EBUSY; + goto unlock; + } + kfree(param->fault_param); param->fault_param = NULL; put_device(dev); @@ -967,13 +982,15 @@ EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); * @evt: fault event data * * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. + * handler. When this function fails and the fault is recoverable, it is the + * caller's responsibility to complete the fault. * * Return 0 on success, or an error. */ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) { struct iommu_param *param = dev->iommu_param; + struct iommu_fault_event *evt_pending = NULL; struct iommu_fault_param *fparam; int ret = 0; @@ -987,13 +1004,86 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) ret = -EINVAL; goto done_unlock; } + + if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && + (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), + GFP_KERNEL); + if (!evt_pending) { + ret = -ENOMEM; + goto done_unlock; + } + mutex_lock(&fparam->lock); + list_add_tail(&evt_pending->list, &fparam->faults); + mutex_unlock(&fparam->lock); + } + ret = fparam->handler(&evt->fault, fparam->data); + if (ret && evt_pending) { + mutex_lock(&fparam->lock); + list_del(&evt_pending->list); + mutex_unlock(&fparam->lock); + kfree(evt_pending); + } done_unlock: mutex_unlock(¶m->lock); return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); +int iommu_page_response(struct device *dev, + struct iommu_page_response *msg) +{ + bool pasid_valid; + int ret = -EINVAL; + struct iommu_fault_event *evt; + struct iommu_fault_page_request *prm; + struct iommu_param *param = dev->iommu_param; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + + if (!domain || !domain->ops->page_response) + return -ENODEV; + + if (!param || !param->fault_param) + return -EINVAL; + + if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || + msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) + return -EINVAL; + + /* Only send response if there is a fault report pending */ + mutex_lock(¶m->fault_param->lock); + if (list_empty(¶m->fault_param->faults)) { + dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); + goto done_unlock; + } + /* + * Check if we have a matching page request pending to respond, + * otherwise return -EINVAL + */ + list_for_each_entry(evt, ¶m->fault_param->faults, list) { + prm = &evt->fault.prm; + pasid_valid = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + + if ((pasid_valid && prm->pasid != msg->pasid) || + prm->grpid != msg->grpid) + continue; + + /* Sanitize the reply */ + msg->flags = pasid_valid ? IOMMU_PAGE_RESP_PASID_VALID : 0; + + ret = domain->ops->page_response(dev, evt, msg); + list_del(&evt->list); + kfree(evt); + break; + } + +done_unlock: + mutex_unlock(¶m->fault_param->lock); + return ret; +} +EXPORT_SYMBOL_GPL(iommu_page_response); + /** * iommu_group_id - Return ID for a group * @group: the group to ID -- cgit From ad0834dedaa15c3a176f783c0373f836e44b4700 Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:30 +0200 Subject: iommu: Fix a leak in iommu_insert_resv_region In case we expand an existing region, we unlink this latter and insert the larger one. In that case we should free the original region after the insertion. Also we can immediately return. Fixes: 6c65fb318e8b ("iommu: iommu_get_group_resv_regions") Signed-off-by: Eric Auger Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 2fca04c3dbaf..ba0661744a3d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -237,18 +237,21 @@ static int iommu_insert_resv_region(struct iommu_resv_region *new, pos = pos->next; } else if ((start >= a) && (end <= b)) { if (new->type == type) - goto done; + return 0; else pos = pos->next; } else { if (new->type == type) { phys_addr_t new_start = min(a, start); phys_addr_t new_end = max(b, end); + int ret; list_del(&entry->list); entry->start = new_start; entry->length = new_end - new_start + 1; - iommu_insert_resv_region(entry, regions); + ret = iommu_insert_resv_region(entry, regions); + kfree(entry); + return ret; } else { pos = pos->next; } @@ -261,7 +264,6 @@ insert: return -ENOMEM; list_add_tail(®ion->list, pos); -done: return 0; } -- cgit From 5f64ce5411b467f1cfea6c63e2494c22b773582b Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:31 +0200 Subject: iommu/vt-d: Duplicate iommu_resv_region objects per device list intel_iommu_get_resv_regions() aims to return the list of reserved regions accessible by a given @device. However several devices can access the same reserved memory region and when building the list it is not safe to use a single iommu_resv_region object, whose container is the RMRR. This iommu_resv_region must be duplicated per device reserved region list. Let's remove the struct iommu_resv_region from the RMRR unit and allocate the iommu_resv_region directly in intel_iommu_get_resv_regions(). We hold the dmar_global_lock instead of the rcu-lock to allow sleeping. Fixes: 0659b8dc45a6 ("iommu/vt-d: Implement reserved region get/put callbacks") Signed-off-by: Eric Auger Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 876096c1f91b..082fbb1bdeaf 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -324,7 +324,6 @@ struct dmar_rmrr_unit { u64 end_address; /* reserved end address */ struct dmar_dev_scope *devices; /* target devices */ int devices_cnt; /* target device count */ - struct iommu_resv_region *resv; /* reserved region handle */ }; struct dmar_atsr_unit { @@ -4050,7 +4049,6 @@ static inline void init_iommu_pm_ops(void) {} int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg) { struct acpi_dmar_reserved_memory *rmrr; - int prot = DMA_PTE_READ|DMA_PTE_WRITE; struct dmar_rmrr_unit *rmrru; size_t length; @@ -4064,22 +4062,16 @@ int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg) rmrru->end_address = rmrr->end_address; length = rmrr->end_address - rmrr->base_address + 1; - rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot, - IOMMU_RESV_DIRECT); - if (!rmrru->resv) - goto free_rmrru; rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1), ((void *)rmrr) + rmrr->header.length, &rmrru->devices_cnt); if (rmrru->devices_cnt && rmrru->devices == NULL) - goto free_all; + goto free_rmrru; list_add(&rmrru->list, &dmar_rmrr_units); return 0; -free_all: - kfree(rmrru->resv); free_rmrru: kfree(rmrru); out: @@ -4297,7 +4289,6 @@ static void intel_iommu_free_dmars(void) list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) { list_del(&rmrru->list); dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt); - kfree(rmrru->resv); kfree(rmrru); } @@ -5400,22 +5391,33 @@ static void intel_iommu_remove_device(struct device *dev) static void intel_iommu_get_resv_regions(struct device *device, struct list_head *head) { + int prot = DMA_PTE_READ | DMA_PTE_WRITE; struct iommu_resv_region *reg; struct dmar_rmrr_unit *rmrr; struct device *i_dev; int i; - rcu_read_lock(); + down_read(&dmar_global_lock); for_each_rmrr_units(rmrr) { for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, i_dev) { + struct iommu_resv_region *resv; + size_t length; + if (i_dev != device) continue; - list_add_tail(&rmrr->resv->list, head); + length = rmrr->end_address - rmrr->base_address + 1; + resv = iommu_alloc_resv_region(rmrr->base_address, + length, prot, + IOMMU_RESV_DIRECT); + if (!resv) + break; + + list_add_tail(&resv->list, head); } } - rcu_read_unlock(); + up_read(&dmar_global_lock); #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA if (dev_is_pci(device)) { @@ -5443,10 +5445,8 @@ static void intel_iommu_put_resv_regions(struct device *dev, { struct iommu_resv_region *entry, *next; - list_for_each_entry_safe(entry, next, head, list) { - if (entry->type == IOMMU_RESV_MSI) - kfree(entry); - } + list_for_each_entry_safe(entry, next, head, list) + kfree(entry); } int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev) -- cgit From b9a7f9816483b19360b92e9b8c91bea2f9f30308 Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:32 +0200 Subject: iommu/vt-d: Introduce is_downstream_to_pci_bridge helper Several call sites are about to check whether a device belongs to the PCI sub-hierarchy of a candidate PCI-PCI bridge. Introduce an helper to perform that check. Signed-off-by: Eric Auger Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 082fbb1bdeaf..6747fec46cfb 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -729,12 +729,39 @@ static int iommu_dummy(struct device *dev) return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO; } +/** + * is_downstream_to_pci_bridge - test if a device belongs to the PCI + * sub-hierarchy of a candidate PCI-PCI bridge + * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy + * @bridge: the candidate PCI-PCI bridge + * + * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false. + */ +static bool +is_downstream_to_pci_bridge(struct device *dev, struct device *bridge) +{ + struct pci_dev *pdev, *pbridge; + + if (!dev_is_pci(dev) || !dev_is_pci(bridge)) + return false; + + pdev = to_pci_dev(dev); + pbridge = to_pci_dev(bridge); + + if (pbridge->subordinate && + pbridge->subordinate->number <= pdev->bus->number && + pbridge->subordinate->busn_res.end >= pdev->bus->number) + return true; + + return false; +} + static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) { struct dmar_drhd_unit *drhd = NULL; struct intel_iommu *iommu; struct device *tmp; - struct pci_dev *ptmp, *pdev = NULL; + struct pci_dev *pdev = NULL; u16 segment = 0; int i; @@ -780,13 +807,7 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf goto out; } - if (!pdev || !dev_is_pci(tmp)) - continue; - - ptmp = to_pci_dev(tmp); - if (ptmp->subordinate && - ptmp->subordinate->number <= pdev->bus->number && - ptmp->subordinate->busn_res.end >= pdev->bus->number) + if (is_downstream_to_pci_bridge(dev, tmp)) goto got_pdev; } -- cgit From e143fd4598ddf7401d3699957cd59ceb5426d53d Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:33 +0200 Subject: iommu/vt-d: Handle RMRR with PCI bridge device scopes When reading the vtd specification and especially the Reserved Memory Region Reporting Structure chapter, it is not obvious a device scope element cannot be a PCI-PCI bridge, in which case all downstream ports are likely to access the reserved memory region. Let's handle this case in device_has_rmrr. Fixes: ea2447f700ca ("intel-iommu: Prevent devices with RMRRs from being placed into SI Domain") Signed-off-by: Eric Auger Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 6747fec46cfb..0d7d177673dd 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2880,7 +2880,8 @@ static bool device_has_rmrr(struct device *dev) */ for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, tmp) - if (tmp == dev) { + if (tmp == dev || + is_downstream_to_pci_bridge(dev, tmp)) { rcu_read_unlock(); return true; } -- cgit From 3855ba2d834d8f7727b7f992ff781fa66cc09f96 Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:34 +0200 Subject: iommu/vt-d: Handle PCI bridge RMRR device scopes in intel_iommu_get_resv_regions In the case the RMRR device scope is a PCI-PCI bridge, let's check the device belongs to the PCI sub-hierarchy. Fixes: 0659b8dc45a6 ("iommu/vt-d: Implement reserved region get/put callbacks") Signed-off-by: Eric Auger Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0d7d177673dd..5de48ed1f763 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5426,7 +5426,8 @@ static void intel_iommu_get_resv_regions(struct device *device, struct iommu_resv_region *resv; size_t length; - if (i_dev != device) + if (i_dev != device && + !is_downstream_to_pci_bridge(device, i_dev)) continue; length = rmrr->end_address - rmrr->base_address + 1; -- cgit From adfd373820906d376c8b643f1a279ac809605b6b Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:35 +0200 Subject: iommu: Introduce IOMMU_RESV_DIRECT_RELAXABLE reserved memory regions Introduce a new type for reserved region. This corresponds to directly mapped regions which are known to be relaxable in some specific conditions, such as device assignment use case. Well known examples are those used by USB controllers providing PS/2 keyboard emulation for pre-boot BIOS and early BOOT or RMRRs associated to IGD working in legacy mode. Since commit c875d2c1b808 ("iommu/vt-d: Exclude devices using RMRRs from IOMMU API domains") and commit 18436afdc11a ("iommu/vt-d: Allow RMRR on graphics devices too"), those regions are currently considered "safe" with respect to device assignment use case which requires a non direct mapping at IOMMU physical level (RAM GPA -> HPA mapping). Those RMRRs currently exist and sometimes the device is attempting to access it but this has not been considered an issue until now. However at the moment, iommu_get_group_resv_regions() is not able to make any difference between directly mapped regions: those which must be absolutely enforced and those like above ones which are known as relaxable. This is a blocker for reporting severe conflicts between non relaxable RMRRs (like MSI doorbells) and guest GPA space. With this new reserved region type we will be able to use iommu_get_group_resv_regions() to enumerate the IOVA space that is usable through the IOMMU API without introducing regressions with respect to existing device assignment use cases (USB and IGD). Signed-off-by: Eric Auger Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index ba0661744a3d..46a06ff46e47 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -73,10 +73,11 @@ struct iommu_group_attribute { }; static const char * const iommu_group_resv_type_string[] = { - [IOMMU_RESV_DIRECT] = "direct", - [IOMMU_RESV_RESERVED] = "reserved", - [IOMMU_RESV_MSI] = "msi", - [IOMMU_RESV_SW_MSI] = "msi", + [IOMMU_RESV_DIRECT] = "direct", + [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable", + [IOMMU_RESV_RESERVED] = "reserved", + [IOMMU_RESV_MSI] = "msi", + [IOMMU_RESV_SW_MSI] = "msi", }; #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ @@ -575,7 +576,8 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group, start = ALIGN(entry->start, pg_size); end = ALIGN(entry->start + entry->length, pg_size); - if (entry->type != IOMMU_RESV_DIRECT) + if (entry->type != IOMMU_RESV_DIRECT && + entry->type != IOMMU_RESV_DIRECT_RELAXABLE) continue; for (addr = start; addr < end; addr += pg_size) { -- cgit From 1c5c59fbad20a63954de07687e4a29af18d1be12 Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Mon, 3 Jun 2019 08:53:36 +0200 Subject: iommu/vt-d: Differentiate relaxable and non relaxable RMRRs Now we have a new IOMMU_RESV_DIRECT_RELAXABLE reserved memory region type, let's report USB and GFX RMRRs as relaxable ones. We introduce a new device_rmrr_is_relaxable() helper to check whether the rmrr belongs to the relaxable category. This allows to have a finer reporting at IOMMU API level of reserved memory regions. This will be exploitable by VFIO to define the usable IOVA range and detect potential conflicts between the guest physical address space and host reserved regions. Signed-off-by: Eric Auger Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 54 ++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5de48ed1f763..10bdf7ea9564 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2890,6 +2890,35 @@ static bool device_has_rmrr(struct device *dev) return false; } +/** + * device_rmrr_is_relaxable - Test whether the RMRR of this device + * is relaxable (ie. is allowed to be not enforced under some conditions) + * @dev: device handle + * + * We assume that PCI USB devices with RMRRs have them largely + * for historical reasons and that the RMRR space is not actively used post + * boot. This exclusion may change if vendors begin to abuse it. + * + * The same exception is made for graphics devices, with the requirement that + * any use of the RMRR regions will be torn down before assigning the device + * to a guest. + * + * Return: true if the RMRR is relaxable, false otherwise + */ +static bool device_rmrr_is_relaxable(struct device *dev) +{ + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return false; + + pdev = to_pci_dev(dev); + if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev)) + return true; + else + return false; +} + /* * There are a couple cases where we need to restrict the functionality of * devices associated with RMRRs. The first is when evaluating a device for @@ -2904,25 +2933,16 @@ static bool device_has_rmrr(struct device *dev) * We therefore prevent devices associated with an RMRR from participating in * the IOMMU API, which eliminates them from device assignment. * - * In both cases we assume that PCI USB devices with RMRRs have them largely - * for historical reasons and that the RMRR space is not actively used post - * boot. This exclusion may change if vendors begin to abuse it. - * - * The same exception is made for graphics devices, with the requirement that - * any use of the RMRR regions will be torn down before assigning the device - * to a guest. + * In both cases, devices which have relaxable RMRRs are not concerned by this + * restriction. See device_rmrr_is_relaxable comment. */ static bool device_is_rmrr_locked(struct device *dev) { if (!device_has_rmrr(dev)) return false; - if (dev_is_pci(dev)) { - struct pci_dev *pdev = to_pci_dev(dev); - - if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev)) - return false; - } + if (device_rmrr_is_relaxable(dev)) + return false; return true; } @@ -5424,6 +5444,7 @@ static void intel_iommu_get_resv_regions(struct device *device, for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, i_dev) { struct iommu_resv_region *resv; + enum iommu_resv_type type; size_t length; if (i_dev != device && @@ -5431,9 +5452,12 @@ static void intel_iommu_get_resv_regions(struct device *device, continue; length = rmrr->end_address - rmrr->base_address + 1; + + type = device_rmrr_is_relaxable(device) ? + IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT; + resv = iommu_alloc_resv_region(rmrr->base_address, - length, prot, - IOMMU_RESV_DIRECT); + length, prot, type); if (!resv) break; -- cgit From f4c63ea91c6f6d9a1a3062071ff6cc4910ebb8a9 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 12 Jun 2019 08:28:45 +0800 Subject: iommu/vt-d: Don't return error when device gets right domain If a device gets a right domain in add_device ops, it shouldn't return error. Fixes: 942067f1b6b97 ("iommu/vt-d: Identify default domains replaced with private") Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 10bdf7ea9564..60ec58bf6701 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5388,10 +5388,7 @@ static int intel_iommu_add_device(struct device *dev) domain_add_dev_info(si_domain, dev); dev_info(dev, "Device uses a private identity domain.\n"); - return 0; } - - return -ENODEV; } } else { if (device_def_domain_type(dev) == IOMMU_DOMAIN_DMA) { @@ -5406,10 +5403,7 @@ static int intel_iommu_add_device(struct device *dev) dev_info(dev, "Device uses a private dma domain.\n"); - return 0; } - - return -ENODEV; } } -- cgit From c57b260a7d7d60dfbcf794dd9836c1d9fdbf5434 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 12 Jun 2019 08:28:46 +0800 Subject: iommu/vt-d: Set domain type for a private domain Otherwise, domain_get_iommu() will be broken. Fixes: 942067f1b6b97 ("iommu/vt-d: Identify default domains replaced with private") Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 60ec58bf6701..862c978d3eb4 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3494,6 +3494,8 @@ static struct dmar_domain *get_private_domain_for_dev(struct device *dev) out: if (!domain) dev_err(dev, "Allocating domain failed\n"); + else + domain->domain.type = IOMMU_DOMAIN_DMA; return domain; } -- cgit From 6a8c6748b94b0968a2f72d4bd2231eae703c0b13 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 12 Jun 2019 08:28:47 +0800 Subject: iommu/vt-d: Don't enable iommu's which have been ignored The iommu driver will ignore some iommu units if there's no device under its scope or those devices have been explicitly set to bypass the DMA translation. Don't enable those iommu units, otherwise the devices under its scope won't work. Fixes: d8190dc638866 ("iommu/vt-d: Enable DMA remapping after rmrr mapped") Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 862c978d3eb4..03946dc4941f 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3268,7 +3268,12 @@ static int __init init_dmars(void) goto error; } - for_each_active_iommu(iommu, drhd) { + for_each_iommu(iommu, drhd) { + if (drhd->ignored) { + iommu_disable_translation(iommu); + continue; + } + /* * Find the max pasid size of all IOMMU's in the system. * We need to ensure the system pasid table is no bigger @@ -4821,7 +4826,7 @@ int __init intel_iommu_init(void) /* Finally, we enable the DMA remapping hardware. */ for_each_iommu(iommu, drhd) { - if (!translation_pre_enabled(iommu)) + if (!drhd->ignored && !translation_pre_enabled(iommu)) iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); -- cgit From 5679582c2fad11733ba65c12d127f9d703390e55 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 12 Jun 2019 08:28:48 +0800 Subject: iommu/vt-d: Allow DMA domain attaching to rmrr locked device We don't allow a device to be assigned to user level when it is locked by any RMRR's. Hence, intel_iommu_attach_device() will return error if a domain of type IOMMU_DOMAIN_UNMANAGED is about to attach to a device locked by rmrr. But this doesn't apply to a domain of type other than IOMMU_DOMAIN_UNMANAGED. This adds a check to fix this. Fixes: fa954e6831789 ("iommu/vt-d: Delegate the dma domain to upper layer") Signed-off-by: Lu Baolu Reported-and-tested-by: Qian Cai Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 03946dc4941f..c31fbe5790c7 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5167,7 +5167,8 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, { int ret; - if (device_is_rmrr_locked(dev)) { + if (domain->type == IOMMU_DOMAIN_UNMANAGED && + device_is_rmrr_locked(dev)) { dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n"); return -EPERM; } -- cgit From d5692d4af08cde48ee98968aeb62077e2f6c50d5 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 12 Jun 2019 08:28:49 +0800 Subject: iommu/vt-d: Fix suspicious RCU usage in probe_acpi_namespace_devices() The drhd and device scope list should be iterated with the iommu global lock held. Otherwise, a suspicious RCU usage message will be displayed. [ 3.695886] ============================= [ 3.695917] WARNING: suspicious RCU usage [ 3.695950] 5.2.0-rc2+ #2467 Not tainted [ 3.695981] ----------------------------- [ 3.696014] drivers/iommu/intel-iommu.c:4569 suspicious rcu_dereference_check() usage! [ 3.696069] other info that might help us debug this: [ 3.696126] rcu_scheduler_active = 2, debug_locks = 1 [ 3.696173] no locks held by swapper/0/1. [ 3.696204] stack backtrace: [ 3.696241] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.2.0-rc2+ #2467 [ 3.696370] Call Trace: [ 3.696404] dump_stack+0x85/0xcb [ 3.696441] intel_iommu_init+0x128c/0x13ce [ 3.696478] ? kmem_cache_free+0x16b/0x2c0 [ 3.696516] ? __fput+0x14b/0x270 [ 3.696550] ? __call_rcu+0xb7/0x300 [ 3.696583] ? get_max_files+0x10/0x10 [ 3.696631] ? set_debug_rodata+0x11/0x11 [ 3.696668] ? e820__memblock_setup+0x60/0x60 [ 3.696704] ? pci_iommu_init+0x16/0x3f [ 3.696737] ? set_debug_rodata+0x11/0x11 [ 3.696770] pci_iommu_init+0x16/0x3f [ 3.696805] do_one_initcall+0x5d/0x2e4 [ 3.696844] ? set_debug_rodata+0x11/0x11 [ 3.696880] ? rcu_read_lock_sched_held+0x6b/0x80 [ 3.696924] kernel_init_freeable+0x1f0/0x27c [ 3.696961] ? rest_init+0x260/0x260 [ 3.696997] kernel_init+0xa/0x110 [ 3.697028] ret_from_fork+0x3a/0x50 Fixes: fa212a97f3a36 ("iommu/vt-d: Probe DMA-capable ACPI name space devices") Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index c31fbe5790c7..9761ac366e3f 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4821,8 +4821,10 @@ int __init intel_iommu_init(void) cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, intel_iommu_cpu_dead); + down_read(&dmar_global_lock); if (probe_acpi_namespace_devices()) pr_warn("ACPI name space devices didn't probe correctly\n"); + up_read(&dmar_global_lock); /* Finally, we enable the DMA remapping hardware. */ for_each_iommu(iommu, drhd) { -- cgit From 16c9e29e12e8edbdf437001f46cf3f557aca80e8 Mon Sep 17 00:00:00 2001 From: Sai Praneeth Prakhya Date: Wed, 12 Jun 2019 08:28:50 +0800 Subject: iommu/vt-d: Cleanup after delegating DMA domain to generic iommu [No functional changes] 1. Starting with commit df4f3c603aeb ("iommu/vt-d: Remove static identity map code") there are no callers for iommu_prepare_rmrr_dev() but the implementation of the function still exists, so remove it. Also, as a ripple effect remove get_domain_for_dev() and iommu_prepare_identity_map() because they aren't being used either. 2. Remove extra new line in couple of places. Signed-off-by: Sai Praneeth Prakhya Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 55 --------------------------------------------- 1 file changed, 55 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 9761ac366e3f..8c6ed39dec01 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -929,7 +929,6 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, return pte; } - /* return address's pte at specific level */ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain, unsigned long pfn, @@ -1598,7 +1597,6 @@ static void iommu_disable_translation(struct intel_iommu *iommu) raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } - static int iommu_init_domains(struct intel_iommu *iommu) { u32 ndomains, nlongs; @@ -1636,8 +1634,6 @@ static int iommu_init_domains(struct intel_iommu *iommu) return -ENOMEM; } - - /* * If Caching mode is set, then invalid translations are tagged * with domain-id 0, hence we need to pre-allocate it. We also @@ -2664,29 +2660,6 @@ static struct dmar_domain *set_domain_for_dev(struct device *dev, return domain; } -static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) -{ - struct dmar_domain *domain, *tmp; - - domain = find_domain(dev); - if (domain) - goto out; - - domain = find_or_alloc_domain(dev, gaw); - if (!domain) - goto out; - - tmp = set_domain_for_dev(dev, domain); - if (!tmp || domain != tmp) { - domain_exit(domain); - domain = tmp; - } - -out: - - return domain; -} - static int iommu_domain_identity_map(struct dmar_domain *domain, unsigned long long start, unsigned long long end) @@ -2751,33 +2724,6 @@ static int domain_prepare_identity_map(struct device *dev, return iommu_domain_identity_map(domain, start, end); } -static int iommu_prepare_identity_map(struct device *dev, - unsigned long long start, - unsigned long long end) -{ - struct dmar_domain *domain; - int ret; - - domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); - if (!domain) - return -ENOMEM; - - ret = domain_prepare_identity_map(dev, domain, start, end); - if (ret) - domain_exit(domain); - - return ret; -} - -static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr, - struct device *dev) -{ - if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO) - return 0; - return iommu_prepare_identity_map(dev, rmrr->base_address, - rmrr->end_address); -} - static int md_domain_init(struct dmar_domain *domain, int guest_width); static int __init si_domain_init(int hw) @@ -4094,7 +4040,6 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ - int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg) { struct acpi_dmar_reserved_memory *rmrr; -- cgit From 123b2ffc376e1b3e9e015c75175b61e88a8b8518 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 12 Jun 2019 08:28:51 +0800 Subject: iommu/vt-d: Consolidate domain_init() to avoid duplication The domain_init() and md_domain_init() do almost the same job. Consolidate them to avoid duplication. Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 123 +++++++++++++------------------------------- 1 file changed, 36 insertions(+), 87 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 8c6ed39dec01..466129e7e50c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1841,63 +1841,6 @@ static inline int guestwidth_to_adjustwidth(int gaw) return agaw; } -static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu, - int guest_width) -{ - int adjust_width, agaw; - unsigned long sagaw; - int err; - - init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN); - - err = init_iova_flush_queue(&domain->iovad, - iommu_flush_iova, iova_entry_free); - if (err) - return err; - - domain_reserve_special_ranges(domain); - - /* calculate AGAW */ - if (guest_width > cap_mgaw(iommu->cap)) - guest_width = cap_mgaw(iommu->cap); - domain->gaw = guest_width; - adjust_width = guestwidth_to_adjustwidth(guest_width); - agaw = width_to_agaw(adjust_width); - sagaw = cap_sagaw(iommu->cap); - if (!test_bit(agaw, &sagaw)) { - /* hardware doesn't support it, choose a bigger one */ - pr_debug("Hardware doesn't support agaw %d\n", agaw); - agaw = find_next_bit(&sagaw, 5, agaw); - if (agaw >= 5) - return -ENODEV; - } - domain->agaw = agaw; - - if (ecap_coherent(iommu->ecap)) - domain->iommu_coherency = 1; - else - domain->iommu_coherency = 0; - - if (ecap_sc_support(iommu->ecap)) - domain->iommu_snooping = 1; - else - domain->iommu_snooping = 0; - - if (intel_iommu_superpage) - domain->iommu_superpage = fls(cap_super_page_val(iommu->cap)); - else - domain->iommu_superpage = 0; - - domain->nid = iommu->node; - - /* always allocate the top pgd */ - domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid); - if (!domain->pgd) - return -ENOMEM; - __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE); - return 0; -} - static void domain_exit(struct dmar_domain *domain) { struct page *freelist; @@ -2578,6 +2521,31 @@ static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque) return 0; } +static int domain_init(struct dmar_domain *domain, int guest_width) +{ + int adjust_width; + + init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN); + domain_reserve_special_ranges(domain); + + /* calculate AGAW */ + domain->gaw = guest_width; + adjust_width = guestwidth_to_adjustwidth(guest_width); + domain->agaw = width_to_agaw(adjust_width); + + domain->iommu_coherency = 0; + domain->iommu_snooping = 0; + domain->iommu_superpage = 0; + domain->max_addr = 0; + + /* always allocate the top pgd */ + domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid); + if (!domain->pgd) + return -ENOMEM; + domain_flush_cache(domain, domain->pgd, PAGE_SIZE); + return 0; +} + static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw) { struct device_domain_info *info; @@ -2615,11 +2583,19 @@ static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw) domain = alloc_domain(0); if (!domain) return NULL; - if (domain_init(domain, iommu, gaw)) { + + if (domain_init(domain, gaw)) { domain_exit(domain); return NULL; } + if (init_iova_flush_queue(&domain->iovad, + iommu_flush_iova, + iova_entry_free)) { + pr_warn("iova flush queue initialization failed\n"); + intel_iommu_strict = 1; + } + out: return domain; } @@ -2724,8 +2700,6 @@ static int domain_prepare_identity_map(struct device *dev, return iommu_domain_identity_map(domain, start, end); } -static int md_domain_init(struct dmar_domain *domain, int guest_width); - static int __init si_domain_init(int hw) { struct dmar_rmrr_unit *rmrr; @@ -2736,7 +2710,7 @@ static int __init si_domain_init(int hw) if (!si_domain) return -EFAULT; - if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { + if (domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { domain_exit(si_domain); return -EFAULT; } @@ -4865,31 +4839,6 @@ static void dmar_remove_one_dev_info(struct device *dev) spin_unlock_irqrestore(&device_domain_lock, flags); } -static int md_domain_init(struct dmar_domain *domain, int guest_width) -{ - int adjust_width; - - init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN); - domain_reserve_special_ranges(domain); - - /* calculate AGAW */ - domain->gaw = guest_width; - adjust_width = guestwidth_to_adjustwidth(guest_width); - domain->agaw = width_to_agaw(adjust_width); - - domain->iommu_coherency = 0; - domain->iommu_snooping = 0; - domain->iommu_superpage = 0; - domain->max_addr = 0; - - /* always allocate the top pgd */ - domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid); - if (!domain->pgd) - return -ENOMEM; - domain_flush_cache(domain, domain->pgd, PAGE_SIZE); - return 0; -} - static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) { struct dmar_domain *dmar_domain; @@ -4904,7 +4853,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) pr_err("Can't allocate dmar_domain\n"); return NULL; } - if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { + if (domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { pr_err("Domain initialization failed\n"); domain_exit(dmar_domain); return NULL; -- cgit From 29fcea8ce7f3c16ebd267f8cb2f4a07887530aa9 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 17 Jun 2019 15:30:54 +0200 Subject: iommu: Fix integer truncation On 32-bit architectures, phys_addr_t may be different from dma_add_t, both smaller and bigger. This can lead to an overflow during an assignment that clang warns about: drivers/iommu/dma-iommu.c:230:10: error: implicit conversion from 'dma_addr_t' (aka 'unsigned long long') to 'phys_addr_t' (aka 'unsigned int') changes value from 18446744073709551615 to 4294967295 [-Werror,-Wconstant-conversion] Use phys_addr_t here because that is the type that the variable was declared as. Fixes: aadad097cd46 ("iommu/dma: Reserve IOVA for PCIe inaccessible DMA address") Signed-off-by: Arnd Bergmann Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 129c4badf9ae..749e3251ee85 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -240,8 +240,8 @@ resv_iova: start = window->res->end - window->offset + 1; /* If window is last entry */ if (window->node.next == &bridge->dma_ranges && - end != ~(dma_addr_t)0) { - end = ~(dma_addr_t)0; + end != ~(phys_addr_t)0) { + end = ~(phys_addr_t)0; goto resv_iova; } } -- cgit From 42db7c6a404b21dafd3288ffa43136da562490ac Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Mon, 17 Jun 2019 09:20:27 -0400 Subject: iommu/vt-d: Remove an unused variable "length" The linux-next commit "iommu/vt-d: Duplicate iommu_resv_region objects per device list" [1] left out an unused variable, drivers/iommu/intel-iommu.c: In function 'dmar_parse_one_rmrr': drivers/iommu/intel-iommu.c:4014:9: warning: variable 'length' set but not used [-Wunused-but-set-variable] [1] https://lore.kernel.org/patchwork/patch/1083073/ Signed-off-by: Qian Cai Reviewed-by: Eric Auger Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 466129e7e50c..1b89c3407251 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4018,7 +4018,6 @@ int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg) { struct acpi_dmar_reserved_memory *rmrr; struct dmar_rmrr_unit *rmrru; - size_t length; rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); if (!rmrru) @@ -4029,8 +4028,6 @@ int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg) rmrru->base_address = rmrr->base_address; rmrru->end_address = rmrr->end_address; - length = rmrr->end_address - rmrr->base_address + 1; - rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1), ((void *)rmrr) + rmrr->header.length, &rmrru->devices_cnt); -- cgit From af88ec3962010e3ac96bcfe5476c9bd68b42477f Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Mon, 3 Jun 2019 10:05:19 -0400 Subject: iommu/vt-d: Silence a variable set but not used The commit "iommu/vt-d: Probe DMA-capable ACPI name space devices" introduced a compilation warning due to the "iommu" variable in for_each_active_iommu() but never used the for each element, i.e, "drhd->iommu". drivers/iommu/intel-iommu.c: In function 'probe_acpi_namespace_devices': drivers/iommu/intel-iommu.c:4639:22: warning: variable 'iommu' set but not used [-Wunused-but-set-variable] struct intel_iommu *iommu; Silence the warning the same way as in the commit d3ed71e5cc50 ("drivers/iommu/intel-iommu.c: fix variable 'iommu' set but not used") Signed-off-by: Qian Cai Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1b89c3407251..ca0a1d5d2983 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4587,7 +4587,8 @@ static int __init platform_optin_force_iommu(void) static int __init probe_acpi_namespace_devices(void) { struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; + /* To avoid a -Wunused-but-set-variable warning. */ + struct intel_iommu *iommu __maybe_unused; struct device *dev; int i, ret = 0; -- cgit From d25f6ead162eab3f51b6616be23691ac42e141b5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 16 May 2019 16:08:47 +0100 Subject: iommu/arm-smmu-v3: Increase maximum size of queues We've been artificially limiting the size of our queues to 4k so that we don't end up allocating huge amounts of physically-contiguous memory at probe time. However, 4k is only enough for 256 commands in the command queue, so instead let's try to allocate the largest queue that the SMMU supports, retrying with a smaller size if the allocation fails. The caveat here is that we have to limit our upper bound based on CONFIG_CMA_ALIGNMENT to ensure that our queue allocations remain natually aligned, which is required by the SMMU architecture. Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 54 +++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4d5a694f02c2..65de2458999f 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -191,6 +191,7 @@ #define Q_BASE_RWA (1UL << 62) #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) #define Q_BASE_LOG2SIZE GENMASK(4, 0) +#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) /* * Stream table. @@ -289,8 +290,9 @@ FIELD_GET(ARM64_TCR_##fld, tcr)) /* Command queue */ -#define CMDQ_ENT_DWORDS 2 -#define CMDQ_MAX_SZ_SHIFT 8 +#define CMDQ_ENT_SZ_SHIFT 4 +#define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3) +#define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT) #define CMDQ_CONS_ERR GENMASK(30, 24) #define CMDQ_ERR_CERROR_NONE_IDX 0 @@ -336,14 +338,16 @@ #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2) /* Event queue */ -#define EVTQ_ENT_DWORDS 4 -#define EVTQ_MAX_SZ_SHIFT 7 +#define EVTQ_ENT_SZ_SHIFT 5 +#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) +#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT) #define EVTQ_0_ID GENMASK_ULL(7, 0) /* PRI queue */ -#define PRIQ_ENT_DWORDS 2 -#define PRIQ_MAX_SZ_SHIFT 8 +#define PRIQ_ENT_SZ_SHIFT 4 +#define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3) +#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT) #define PRIQ_0_SID GENMASK_ULL(31, 0) #define PRIQ_0_SSID GENMASK_ULL(51, 32) @@ -798,7 +802,7 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) /* High-level queue accessors */ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) { - memset(cmd, 0, CMDQ_ENT_DWORDS << 3); + memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT); cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); switch (ent->opcode) { @@ -2270,17 +2274,32 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, struct arm_smmu_queue *q, unsigned long prod_off, unsigned long cons_off, - size_t dwords) + size_t dwords, const char *name) { - size_t qsz = ((1 << q->max_n_shift) * dwords) << 3; + size_t qsz; + + do { + qsz = ((1 << q->max_n_shift) * dwords) << 3; + q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, + GFP_KERNEL); + if (q->base || qsz < PAGE_SIZE) + break; + + q->max_n_shift--; + } while (1); - q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL); if (!q->base) { - dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n", - qsz); + dev_err(smmu->dev, + "failed to allocate queue (0x%zx bytes) for %s\n", + qsz, name); return -ENOMEM; } + if (!WARN_ON(q->base_dma & (qsz - 1))) { + dev_info(smmu->dev, "allocated %u entries for %s\n", + 1 << q->max_n_shift, name); + } + q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu); q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu); q->ent_dwords = dwords; @@ -2300,13 +2319,15 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) /* cmdq */ spin_lock_init(&smmu->cmdq.lock); ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD, - ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS); + ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS, + "cmdq"); if (ret) return ret; /* evtq */ ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD, - ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS); + ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS, + "evtq"); if (ret) return ret; @@ -2315,7 +2336,8 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) return 0; return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, - ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS); + ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS, + "priq"); } static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) @@ -2879,7 +2901,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) return -ENXIO; } - /* Queue sizes, capped at 4k */ + /* Queue sizes, capped to ensure natural alignment */ smmu->cmdq.q.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, reg)); if (!smmu->cmdq.q.max_n_shift) { -- cgit From 90ec7a76cc4ba65bfedeb8621cba09cd5a317d8f Mon Sep 17 00:00:00 2001 From: Vivek Gautam Date: Thu, 16 May 2019 15:00:20 +0530 Subject: iommu/io-pgtable-arm: Add support to use system cache Few Qualcomm platforms such as, sdm845 have an additional outer cache called as System cache, aka. Last level cache (LLC) that allows non-coherent devices to upgrade to using caching. This cache sits right before the DDR, and is tightly coupled with the memory controller. The clients using this cache request their slices from this system cache, make it active, and can then start using it. There is a fundamental assumption that non-coherent devices can't access caches. This change adds an exception where they *can* use some level of cache despite still being non-coherent overall. The coherent devices that use cacheable memory, and CPU make use of this system cache by default. Looking at memory types, we have following - a) Normal uncached :- MAIR 0x44, inner non-cacheable, outer non-cacheable; b) Normal cached :- MAIR 0xff, inner read write-back non-transient, outer read write-back non-transient; attribute setting for coherenet I/O devices. and, for non-coherent i/o devices that can allocate in system cache another type gets added - c) Normal sys-cached :- MAIR 0xf4, inner non-cacheable, outer read write-back non-transient Coherent I/O devices use system cache by marking the memory as normal cached. Non-coherent I/O devices should mark the memory as normal sys-cached in page tables to use system cache. Acked-by: Robin Murphy Signed-off-by: Vivek Gautam Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 4e21efbc4459..2454ac11aa97 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -167,10 +167,12 @@ #define ARM_LPAE_MAIR_ATTR_MASK 0xff #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 #define ARM_LPAE_MAIR_ATTR_NC 0x44 +#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 +#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0) #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2) @@ -470,6 +472,9 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, else if (prot & IOMMU_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_QCOM_SYS_CACHE) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE + << ARM_LPAE_PTE_ATTRINDX_SHIFT); } if (prot & IOMMU_NOEXEC) @@ -857,7 +862,9 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) (ARM_LPAE_MAIR_ATTR_WBRWA << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | (ARM_LPAE_MAIR_ATTR_DEVICE - << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) | + (ARM_LPAE_MAIR_ATTR_INC_OWBRWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE)); cfg->arm_lpae_s1_cfg.mair[0] = reg; cfg->arm_lpae_s1_cfg.mair[1] = 0; -- cgit From 4f41845b340783eaec9cc2840fe3cb9a00574054 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 25 Jun 2019 12:51:25 +0100 Subject: iommu/io-pgtable: Replace IO_PGTABLE_QUIRK_NO_DMA with specific flag IO_PGTABLE_QUIRK_NO_DMA is a bit of a misnomer, since it's really just an indication of whether or not the page-table walker for the IOMMU is coherent with the CPU caches. Since cache coherency is more than just a quirk, replace the flag with its own field in the io_pgtable_cfg structure. Cc: Bjorn Andersson Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 4 +--- drivers/iommu/arm-smmu.c | 4 +--- drivers/iommu/io-pgtable-arm-v7s.c | 10 +++++----- drivers/iommu/io-pgtable-arm.c | 19 ++++++++----------- drivers/iommu/ipmmu-vmsa.c | 1 + 5 files changed, 16 insertions(+), 22 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 65de2458999f..8ff8f61d9e1c 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1789,13 +1789,11 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, + .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, .tlb = &arm_smmu_gather_ops, .iommu_dev = smmu->dev, }; - if (smmu->features & ARM_SMMU_FEAT_COHERENCY) - pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; - if (smmu_domain->non_strict) pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 5e54cc0a28b3..009156bb6d42 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -895,13 +895,11 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, + .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, .tlb = smmu_domain->tlb_ops, .iommu_dev = smmu->dev, }; - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) - pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; - if (smmu_domain->non_strict) pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 9a8a8870e267..8454de93e356 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -215,7 +215,7 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, dev_err(dev, "Page table does not fit in PTE: %pa", &phys); goto out_free; } - if (table && !(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { + if (table && !cfg->coherent_walk) { dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) goto out_free; @@ -249,7 +249,7 @@ static void __arm_v7s_free_table(void *table, int lvl, struct device *dev = cfg->iommu_dev; size_t size = ARM_V7S_TABLE_SIZE(lvl); - if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) + if (!cfg->coherent_walk) dma_unmap_single(dev, __arm_v7s_dma_addr(table), size, DMA_TO_DEVICE); if (lvl == 1) @@ -261,7 +261,7 @@ static void __arm_v7s_free_table(void *table, int lvl, static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, struct io_pgtable_cfg *cfg) { - if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) + if (cfg->coherent_walk) return; dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), @@ -727,7 +727,6 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, IO_PGTABLE_QUIRK_NO_PERMS | IO_PGTABLE_QUIRK_TLBI_ON_MAP | IO_PGTABLE_QUIRK_ARM_MTK_4GB | - IO_PGTABLE_QUIRK_NO_DMA | IO_PGTABLE_QUIRK_NON_STRICT)) return NULL; @@ -846,7 +845,8 @@ static int __init arm_v7s_do_selftests(void) .tlb = &dummy_tlb_ops, .oas = 32, .ias = 32, - .quirks = IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA, + .coherent_walk = true, + .quirks = IO_PGTABLE_QUIRK_ARM_NS, .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, }; unsigned int iova, size, iova_start; diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 2454ac11aa97..91d0a4228b58 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -252,7 +252,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, return NULL; pages = page_address(p); - if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { + if (!cfg->coherent_walk) { dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) goto out_free; @@ -278,7 +278,7 @@ out_free: static void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg) { - if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) + if (!cfg->coherent_walk) dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), size, DMA_TO_DEVICE); free_pages((unsigned long)pages, get_order(size)); @@ -296,7 +296,7 @@ static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, { *ptep = pte; - if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) + if (!cfg->coherent_walk) __arm_lpae_sync_pte(ptep, cfg); } @@ -374,8 +374,7 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, old = cmpxchg64_relaxed(ptep, curr, new); - if ((cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) || - (old & ARM_LPAE_PTE_SW_SYNC)) + if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) return old; /* Even if it's not ours, there's no point waiting; just kick it */ @@ -416,8 +415,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, pte = arm_lpae_install_table(cptep, ptep, 0, cfg); if (pte) __arm_lpae_free_pages(cptep, tblsz, cfg); - } else if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) && - !(pte & ARM_LPAE_PTE_SW_SYNC)) { + } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { __arm_lpae_sync_pte(ptep, cfg); } @@ -799,7 +797,7 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) u64 reg; struct arm_lpae_io_pgtable *data; - if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA | + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NON_STRICT)) return NULL; @@ -894,8 +892,7 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) struct arm_lpae_io_pgtable *data; /* The NS quirk doesn't apply at stage 2 */ - if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NO_DMA | - IO_PGTABLE_QUIRK_NON_STRICT)) + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -1230,7 +1227,7 @@ static int __init arm_lpae_do_selftests(void) struct io_pgtable_cfg cfg = { .tlb = &dummy_tlb_ops, .oas = 48, - .quirks = IO_PGTABLE_QUIRK_NO_DMA, + .coherent_walk = true, }; for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 9a380c10655e..12bcb95bdaa8 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -431,6 +431,7 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) * TODO: Add support for coherent walk through CCI with DVM and remove * cache handling. For now, delegate it to the io-pgtable code. */ + domain->cfg.coherent_walk = false; domain->cfg.iommu_dev = domain->mmu->root->dev; /* -- cgit From 9e6ea59f3ff37192fd7aec7821dca6ece629b7d0 Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Wed, 15 May 2019 16:32:34 -0700 Subject: iommu/io-pgtable: Support non-coherent page tables Describe the memory related to page table walks as non-cacheable for iommu instances that are not DMA coherent. Signed-off-by: Bjorn Andersson [will: Use cfg->coherent_walk, fix arm-v7s, ensure outer-shareable for NC] Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 7 +++++-- drivers/iommu/io-pgtable-arm.c | 12 +++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 8454de93e356..35de9ebb500c 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -789,8 +789,11 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, /* TTBRs */ cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) | ARM_V7S_TTBR_S | ARM_V7S_TTBR_NOS | - ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | - ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA); + (cfg->coherent_walk ? + (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) : + (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC))); cfg->arm_v7s_cfg.ttbr[1] = 0; return &data->iop; diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 91d0a4228b58..b4e624afd1bb 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -806,9 +806,15 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) return NULL; /* TCR */ - reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + if (cfg->coherent_walk) { + reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + } else { + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT); + } switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: -- cgit From 3ddbe913e55516d3e2165d43d4d5570761769878 Mon Sep 17 00:00:00 2001 From: Kevin Mitchell Date: Wed, 12 Jun 2019 14:52:03 -0700 Subject: iommu/amd: Make iommu_disable safer Make it safe to call iommu_disable during early init error conditions before mmio_base is set, but after the struct amd_iommu has been added to the amd_iommu_list. For example, this happens if firmware fails to fill in mmio_phys in the ACPI table leading to a NULL pointer dereference in iommu_feature_disable. Fixes: 2c0ae1720c09c ('iommu/amd: Convert iommu initialization to state machine') Signed-off-by: Kevin Mitchell Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index f977df90d2a4..df144dc6d8ac 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -418,6 +418,9 @@ static void iommu_enable(struct amd_iommu *iommu) static void iommu_disable(struct amd_iommu *iommu) { + if (!iommu->mmio_base) + return; + /* Disable command buffer */ iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); -- cgit From bf4bff46eac151c3fd299741d71c4216e45b5d8b Mon Sep 17 00:00:00 2001 From: Kevin Mitchell Date: Wed, 12 Jun 2019 14:52:04 -0700 Subject: iommu/amd: Move gart fallback to amd_iommu_init The fallback to the GART driver in the case amd_iommu doesn't work was executed in a function called free_iommu_resources, which didn't really make sense. This was even being called twice if amd_iommu=off was specified on the command line. The only complication is that it needs to be verified that amd_iommu has fully relinquished control by calling free_iommu_resources and emptying the amd_iommu_list. Signed-off-by: Kevin Mitchell Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index df144dc6d8ac..b90e26effe0a 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -2340,15 +2340,6 @@ static void __init free_iommu_resources(void) amd_iommu_dev_table = NULL; free_iommu_all(); - -#ifdef CONFIG_GART_IOMMU - /* - * We failed to initialize the AMD IOMMU - try fallback to GART - * if possible. - */ - gart_iommu_init(); - -#endif } /* SB IOAPIC is always on this device in AMD systems */ @@ -2767,6 +2758,16 @@ static int __init amd_iommu_init(void) } } +#ifdef CONFIG_GART_IOMMU + if (ret && list_empty(&amd_iommu_list)) { + /* + * We failed to initialize the AMD IOMMU - try fallback + * to GART if possible. + */ + gart_iommu_init(); + } +#endif + for_each_iommu(iommu) amd_iommu_debugfs_setup(iommu); -- cgit From 5c90501a7290e8066a5530f491e764730791945a Mon Sep 17 00:00:00 2001 From: Kevin Mitchell Date: Wed, 12 Jun 2019 14:52:05 -0700 Subject: iommu/amd: Only free resources once on init error When amd_iommu=off was specified on the command line, free_X_resources functions were called immediately after early_amd_iommu_init. They were then called again when amd_iommu_init also failed (as expected). Instead, call them only once: at the end of state_next() whenever there's an error. These functions should be safe to call any time and any number of times. However, since state_next is never called again in an error state, the cleanup will only ever be run once. This also ensures that cleanup code is run as soon as possible after an error is detected rather than waiting for amd_iommu_init() to be called. Signed-off-by: Kevin Mitchell Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index b90e26effe0a..e308fb6a2908 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -2631,8 +2631,6 @@ static int __init state_next(void) init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; if (init_state == IOMMU_ACPI_FINISHED && amd_iommu_disabled) { pr_info("AMD IOMMU disabled on kernel command-line\n"); - free_dma_resources(); - free_iommu_resources(); init_state = IOMMU_CMDLINE_DISABLED; ret = -EINVAL; } @@ -2673,6 +2671,19 @@ static int __init state_next(void) BUG(); } + if (ret) { + free_dma_resources(); + if (!irq_remapping_enabled) { + disable_iommus(); + free_iommu_resources(); + } else { + struct amd_iommu *iommu; + + uninit_device_table_dma(); + for_each_iommu(iommu) + iommu_flush_all_caches(iommu); + } + } return ret; } @@ -2746,18 +2757,6 @@ static int __init amd_iommu_init(void) int ret; ret = iommu_go_to_state(IOMMU_INITIALIZED); - if (ret) { - free_dma_resources(); - if (!irq_remapping_enabled) { - disable_iommus(); - free_iommu_resources(); - } else { - uninit_device_table_dma(); - for_each_iommu(iommu) - iommu_flush_all_caches(iommu); - } - } - #ifdef CONFIG_GART_IOMMU if (ret && list_empty(&amd_iommu_list)) { /* -- cgit From 5cd3f2e98ccaa668210ed2e3a1d6bde3189fe366 Mon Sep 17 00:00:00 2001 From: Tom Murphy Date: Thu, 13 Jun 2019 23:04:55 +0100 Subject: iommu/amd: Flush not present cache in iommu_map_page check if there is a not-present cache present and flush it if there is. Signed-off-by: Tom Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 86613f139618..a91732f170ea 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1307,6 +1307,16 @@ static void domain_flush_complete(struct protection_domain *domain) } } +/* Flush the not present cache if it exists */ +static void domain_flush_np_cache(struct protection_domain *domain, + dma_addr_t iova, size_t size) +{ + if (unlikely(amd_iommu_np_cache)) { + domain_flush_pages(domain, iova, size); + domain_flush_complete(domain); + } +} + /* * This function flushes the DTEs for all devices in domain @@ -2389,10 +2399,7 @@ static dma_addr_t __map_single(struct device *dev, } address += offset; - if (unlikely(amd_iommu_np_cache)) { - domain_flush_pages(&dma_dom->domain, address, size); - domain_flush_complete(&dma_dom->domain); - } + domain_flush_np_cache(&dma_dom->domain, address, size); out: return address; @@ -2571,6 +2578,9 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, s->dma_length = s->length; } + if (s) + domain_flush_np_cache(domain, s->dma_address, s->dma_length); + return nelems; out_unmap: @@ -3051,6 +3061,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, ret = iommu_map_page(domain, iova, paddr, page_size, prot, GFP_KERNEL); mutex_unlock(&domain->api_lock); + domain_flush_np_cache(domain, iova, page_size); + return ret; } -- cgit From 0bcfa628f8a303f86876424475ba5390d229e736 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Mon, 24 Jun 2019 13:17:42 -0700 Subject: iommu/vt-d: Cleanup unused variable Linux IRQ number virq is not used in IRTE allocation. Remove it. Signed-off-by: Jacob Pan Signed-off-by: Joerg Roedel --- drivers/iommu/intel_irq_remapping.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 4160aa9f3f80..4786ca061e31 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -101,7 +101,7 @@ static void init_ir_status(struct intel_iommu *iommu) iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; } -static int alloc_irte(struct intel_iommu *iommu, int irq, +static int alloc_irte(struct intel_iommu *iommu, struct irq_2_iommu *irq_iommu, u16 count) { struct ir_table *table = iommu->ir_table; @@ -1374,7 +1374,7 @@ static int intel_irq_remapping_alloc(struct irq_domain *domain, goto out_free_parent; down_read(&dmar_global_lock); - index = alloc_irte(iommu, virq, &data->irq_2_iommu, nr_irqs); + index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); up_read(&dmar_global_lock); if (index < 0) { pr_warn("Failed to allocate IRTE\n"); -- cgit From 900a85ca43f4ae88c325a468cb8827e4852b9141 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 2 Jul 2019 12:53:18 +0100 Subject: iommu/arm-smmu-v3: Fix compilation when CONFIG_CMA=n MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When compiling a kernel without support for CMA, CONFIG_CMA_ALIGNMENT is not defined which results in the following build failure: In file included from ./include/linux/list.h:9:0 from ./include/linux/kobject.h:19, from ./include/linux/of.h:17 from ./include/linux/irqdomain.h:35, from ./include/linux/acpi.h:13, from drivers/iommu/arm-smmu-v3.c:12: drivers/iommu/arm-smmu-v3.c: In function ‘arm_smmu_device_hw_probe’: drivers/iommu/arm-smmu-v3.c:194:40: error: ‘CONFIG_CMA_ALIGNMENT’ undeclared (first use in this function) #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) Fix the breakage by capping the maximum queue size based on MAX_ORDER when CMA is not enabled. Reported-by: Zhangshaokun Signed-off-by: Will Deacon Tested-by: Shaokun Zhang Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 8ff8f61d9e1c..2734beb709e0 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -191,7 +191,13 @@ #define Q_BASE_RWA (1UL << 62) #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) #define Q_BASE_LOG2SIZE GENMASK(4, 0) + +/* Ensure DMA allocations are naturally aligned */ +#ifdef CONFIG_CMA_ALIGNMENT #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) +#else +#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1) +#endif /* * Stream table. -- cgit From 8dd8f005bdd45823fc153ef490239558caf6ff20 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 3 Jul 2019 12:19:20 +0100 Subject: iommu/arm-smmu-v3: Invalidate ATC when detaching a device We make the invalid assumption in arm_smmu_detach_dev() that the ATC is clear after calling pci_disable_ats(). For one thing, only enabling the PCIe ATS capability constitutes an implicit invalidation event, so the comment was wrong. More importantly, the ATS capability isn't necessarily disabled by pci_disable_ats() in a PF, if the associated VFs have ATS enabled. Explicitly invalidate all ATC entries in arm_smmu_detach_dev(). The endpoint cannot form new ATC entries because STE.EATS is clear. Fixes: 9ce27afc0830 ("iommu/arm-smmu-v3: Add support for PCI ATS") Reported-by: Manoj Kumar Reported-by: Robin Murphy Signed-off-by: Jean-Philippe Brucker Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 2734beb709e0..2d96cf0023dd 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1892,9 +1892,13 @@ static int arm_smmu_enable_ats(struct arm_smmu_master *master) static void arm_smmu_disable_ats(struct arm_smmu_master *master) { + struct arm_smmu_cmdq_ent cmd; + if (!master->ats_enabled || !dev_is_pci(master->dev)) return; + arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); + arm_smmu_atc_inv_master(master, &cmd); pci_disable_ats(to_pci_dev(master->dev)); master->ats_enabled = false; } @@ -1914,7 +1918,6 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) master->domain = NULL; arm_smmu_install_ste_for_dev(master); - /* Disabling ATS invalidates all ATC entries */ arm_smmu_disable_ats(master); } -- cgit From 9378bfeaafcbbdf234daeee93db31a3f2c815513 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 4 Jul 2019 16:36:49 +0200 Subject: iommu/omap: No need to check return value of debugfs_create functions When calling debugfs functions, there is no need to ever check the return value. The function can work or not, but the code logic should never do something different based on this. Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Signed-off-by: Greg Kroah-Hartman Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu-debug.c | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index 4abc0ef522a8..f87a72821bad 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -239,17 +239,6 @@ DEBUG_FOPS_RO(regs); DEFINE_SHOW_ATTRIBUTE(tlb); DEFINE_SHOW_ATTRIBUTE(pagetable); -#define __DEBUG_ADD_FILE(attr, mode) \ - { \ - struct dentry *dent; \ - dent = debugfs_create_file(#attr, mode, obj->debug_dir, \ - obj, &attr##_fops); \ - if (!dent) \ - goto err; \ - } - -#define DEBUG_ADD_FILE_RO(name) __DEBUG_ADD_FILE(name, 0400) - void omap_iommu_debugfs_add(struct omap_iommu *obj) { struct dentry *d; @@ -257,23 +246,13 @@ void omap_iommu_debugfs_add(struct omap_iommu *obj) if (!iommu_debug_root) return; - obj->debug_dir = debugfs_create_dir(obj->name, iommu_debug_root); - if (!obj->debug_dir) - return; + d = debugfs_create_dir(obj->name, iommu_debug_root); + obj->debug_dir = d; - d = debugfs_create_u32("nr_tlb_entries", 0400, obj->debug_dir, - &obj->nr_tlb_entries); - if (!d) - return; - - DEBUG_ADD_FILE_RO(regs); - DEBUG_ADD_FILE_RO(tlb); - DEBUG_ADD_FILE_RO(pagetable); - - return; - -err: - debugfs_remove_recursive(obj->debug_dir); + debugfs_create_u32("nr_tlb_entries", 0400, d, &obj->nr_tlb_entries); + debugfs_create_file("regs", 0400, d, obj, ®s_fops); + debugfs_create_file("tlb", 0400, d, obj, &tlb_fops); + debugfs_create_file("pagetable", 0400, d, obj, &pagetable_fops); } void omap_iommu_debugfs_remove(struct omap_iommu *obj) @@ -287,8 +266,6 @@ void omap_iommu_debugfs_remove(struct omap_iommu *obj) void __init omap_iommu_debugfs_init(void) { iommu_debug_root = debugfs_create_dir("omap_iommu", NULL); - if (!iommu_debug_root) - pr_err("can't create debugfs dir\n"); } void __exit omap_iommu_debugfs_exit(void) -- cgit