1 files changed, 135 insertions, 92 deletions

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 2cd5800e6888..75470ffee358 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -71,6 +71,19 @@ static u64 *fetch_pte(struct protection_domain *domain,
 		      unsigned long address, int map_size);
 static void update_domain(struct protection_domain *domain);
 
+/****************************************************************************
+ *
+ * Helper functions
+ *
+ ****************************************************************************/
+
+static inline u16 get_device_id(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return calc_devid(pdev->bus->number, pdev->devfn);
+}
+
 #ifdef CONFIG_AMD_IOMMU_STATS
 
 /*
@@ -1174,26 +1187,13 @@ static bool dma_ops_domain(struct protection_domain *domain)
 	return domain->flags & PD_DMA_OPS_MASK;
 }
 
-/*
- * Find out the protection domain structure for a given PCI device. This
- * will give us the pointer to the page table root for example.
- */
-static struct protection_domain *domain_for_device(u16 devid)
-{
-	struct protection_domain *dom;
-	unsigned long flags;
-
-	read_lock_irqsave(&amd_iommu_devtable_lock, flags);
-	dom = amd_iommu_pd_table[devid];
-	read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
-
-	return dom;
-}
-
 static void set_dte_entry(u16 devid, struct protection_domain *domain)
 {
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
 	u64 pte_root = virt_to_phys(domain->pt_root);
 
+	BUG_ON(amd_iommu_pd_table[devid] != NULL);
+
 	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
 		    << DEV_ENTRY_MODE_SHIFT;
 	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
@@ -1203,42 +1203,87 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain)
 	amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
 
 	amd_iommu_pd_table[devid] = domain;
+
+	/* Do reference counting */
+	domain->dev_iommu[iommu->index] += 1;
+	domain->dev_cnt                 += 1;
+
+	/* Flush the changes DTE entry */
+	iommu_queue_inv_dev_entry(iommu, devid);
+}
+
+static void clear_dte_entry(u16 devid)
+{
+	struct protection_domain *domain = amd_iommu_pd_table[devid];
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+
+	BUG_ON(domain == NULL);
+
+	/* remove domain from the lookup table */
+	amd_iommu_pd_table[devid] = NULL;
+
+	/* remove entry from the device table seen by the hardware */
+	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
+	amd_iommu_dev_table[devid].data[1] = 0;
+	amd_iommu_dev_table[devid].data[2] = 0;
+
+	amd_iommu_apply_erratum_63(devid);
+
+	/* decrease reference counters */
+	domain->dev_iommu[iommu->index] -= 1;
+	domain->dev_cnt                 -= 1;
+
+	iommu_queue_inv_dev_entry(iommu, devid);
 }
 
 /*
  * If a device is not yet associated with a domain, this function does
  * assigns it visible for the hardware
  */
-static void __attach_device(struct amd_iommu *iommu,
-			    struct protection_domain *domain,
-			    u16 devid)
+static int __attach_device(struct device *dev,
+			   struct protection_domain *domain)
 {
+	u16 devid = get_device_id(dev);
+	u16 alias = amd_iommu_alias_table[devid];
+
 	/* lock domain */
 	spin_lock(&domain->lock);
 
-	/* update DTE entry */
-	set_dte_entry(devid, domain);
+	/* Some sanity checks */
+	if (amd_iommu_pd_table[alias] != NULL &&
+	    amd_iommu_pd_table[alias] != domain)
+		return -EBUSY;
 
-	/* Do reference counting */
-	domain->dev_iommu[iommu->index] += 1;
-	domain->dev_cnt                 += 1;
+	if (amd_iommu_pd_table[devid] != NULL &&
+	    amd_iommu_pd_table[devid] != domain)
+		return -EBUSY;
+
+	/* Do real assignment */
+	if (alias != devid &&
+	    amd_iommu_pd_table[alias] == NULL)
+		set_dte_entry(alias, domain);
+
+	if (amd_iommu_pd_table[devid] == NULL)
+		set_dte_entry(devid, domain);
 
 	/* ready */
 	spin_unlock(&domain->lock);
+
+	return 0;
 }
 
 /*
  * If a device is not yet associated with a domain, this function does
  * assigns it visible for the hardware
  */
-static void attach_device(struct amd_iommu *iommu,
-			  struct protection_domain *domain,
-			  u16 devid)
+static int attach_device(struct device *dev,
+			 struct protection_domain *domain)
 {
 	unsigned long flags;
+	int ret;
 
 	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
-	__attach_device(iommu, domain, devid);
+	ret = __attach_device(dev, domain);
 	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 
 	/*
@@ -1246,62 +1291,70 @@ static void attach_device(struct amd_iommu *iommu,
 	 * left the caches in the IOMMU dirty. So we have to flush
 	 * here to evict all dirty stuff.
 	 */
-	iommu_queue_inv_dev_entry(iommu, devid);
 	iommu_flush_tlb_pde(domain);
+
+	return ret;
 }
 
 /*
  * Removes a device from a protection domain (unlocked)
  */
-static void __detach_device(struct protection_domain *domain, u16 devid)
+static void __detach_device(struct device *dev)
 {
+	u16 devid = get_device_id(dev);
 	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
 
 	BUG_ON(!iommu);
 
-	/* lock domain */
-	spin_lock(&domain->lock);
-
-	/* remove domain from the lookup table */
-	amd_iommu_pd_table[devid] = NULL;
-
-	/* remove entry from the device table seen by the hardware */
-	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
-	amd_iommu_dev_table[devid].data[1] = 0;
-	amd_iommu_dev_table[devid].data[2] = 0;
-
-	amd_iommu_apply_erratum_63(devid);
-
-	/* decrease reference counters */
-	domain->dev_iommu[iommu->index] -= 1;
-	domain->dev_cnt                 -= 1;
-
-	/* ready */
-	spin_unlock(&domain->lock);
+	clear_dte_entry(devid);
 
 	/*
 	 * If we run in passthrough mode the device must be assigned to the
 	 * passthrough domain if it is detached from any other domain
 	 */
-	if (iommu_pass_through) {
-		struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
-		__attach_device(iommu, pt_domain, devid);
-	}
+	if (iommu_pass_through)
+		__attach_device(dev, pt_domain);
 }
 
 /*
  * Removes a device from a protection domain (with devtable_lock held)
  */
-static void detach_device(struct protection_domain *domain, u16 devid)
+static void detach_device(struct device *dev)
 {
 	unsigned long flags;
 
 	/* lock device table */
 	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
-	__detach_device(domain, devid);
+	__detach_device(dev);
 	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 }
 
+/*
+ * Find out the protection domain structure for a given PCI device. This
+ * will give us the pointer to the page table root for example.
+ */
+static struct protection_domain *domain_for_device(struct device *dev)
+{
+	struct protection_domain *dom;
+	unsigned long flags;
+	u16 devid, alias;
+
+	devid = get_device_id(dev);
+	alias = amd_iommu_alias_table[devid];
+
+	read_lock_irqsave(&amd_iommu_devtable_lock, flags);
+	dom = amd_iommu_pd_table[devid];
+	if (dom == NULL &&
+	    amd_iommu_pd_table[alias] != NULL) {
+		__attach_device(dev, amd_iommu_pd_table[alias]);
+		dom = amd_iommu_pd_table[devid];
+	}
+
+	read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+
+	return dom;
+}
+
 static int device_change_notifier(struct notifier_block *nb,
 				  unsigned long action, void *data)
 {
@@ -1322,7 +1375,7 @@ static int device_change_notifier(struct notifier_block *nb,
 	if (iommu == NULL)
 		goto out;
 
-	domain = domain_for_device(devid);
+	domain = domain_for_device(dev);
 
 	if (domain && !dma_ops_domain(domain))
 		WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
@@ -1334,7 +1387,7 @@ static int device_change_notifier(struct notifier_block *nb,
 			goto out;
 		if (iommu_pass_through)
 			break;
-		detach_device(domain, devid);
+		detach_device(dev);
 		break;
 	case BUS_NOTIFY_ADD_DEVICE:
 		/* allocate a protection domain if a device is added */
@@ -1441,30 +1494,25 @@ static bool get_device_resources(struct device *dev,
 {
 	struct dma_ops_domain *dma_dom;
 	struct amd_iommu *iommu;
-	struct pci_dev *pcidev;
-	u16 _bdf;
 
 	if (!check_device(dev))
 		return false;
 
-	pcidev  = to_pci_dev(dev);
-	_bdf    = calc_devid(pcidev->bus->number, pcidev->devfn);
-	*bdf    = amd_iommu_alias_table[_bdf];
+	*bdf    = get_device_id(dev);
+	*domain = domain_for_device(dev);
 	iommu   = amd_iommu_rlookup_table[*bdf];
-	*domain = domain_for_device(*bdf);
 
-	if (*domain == NULL) {
-		dma_dom = find_protection_domain(*bdf);
-		if (!dma_dom)
-			dma_dom = iommu->default_dom;
-		*domain = &dma_dom->domain;
-		attach_device(iommu, *domain, *bdf);
-		DUMP_printk("Using protection domain %d for device %s\n",
-			    (*domain)->id, dev_name(dev));
-	}
+	if (*domain != NULL)
+		return true;
 
-	if (domain_for_device(_bdf) == NULL)
-		attach_device(iommu, *domain, _bdf);
+	/* Device not bount yet - bind it */
+	dma_dom = find_protection_domain(*bdf);
+	if (!dma_dom)
+		dma_dom = iommu->default_dom;
+	*domain = &dma_dom->domain;
+	attach_device(dev, *domain);
+	DUMP_printk("Using protection domain %d for device %s\n",
+		    (*domain)->id, dev_name(dev));
 
 	return true;
 }
@@ -2068,7 +2116,7 @@ static void prealloc_protection_domains(void)
 		if (devid > amd_iommu_last_bdf)
 			continue;
 		devid = amd_iommu_alias_table[devid];
-		if (domain_for_device(devid))
+		if (domain_for_device(&dev->dev))
 			continue;
 		iommu = amd_iommu_rlookup_table[devid];
 		if (!iommu)
@@ -2079,9 +2127,7 @@ static void prealloc_protection_domains(void)
 		init_unity_mappings_for_device(dma_dom, devid);
 		dma_dom->target_dev = devid;
 
-		attach_device(iommu, &dma_dom->domain, devid);
-		if (__devid != devid)
-			attach_device(iommu, &dma_dom->domain, __devid);
+		attach_device(&dev->dev, &dma_dom->domain);
 
 		list_add_tail(&dma_dom->list, &iommu_pd_list);
 	}
@@ -2174,7 +2220,7 @@ static void cleanup_domain(struct protection_domain *domain)
 
 	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
 		if (amd_iommu_pd_table[devid] == domain)
-			__detach_device(domain, devid);
+			clear_dte_entry(devid);
 
 	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 }
@@ -2262,7 +2308,6 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom)
 static void amd_iommu_detach_device(struct iommu_domain *dom,
 				    struct device *dev)
 {
-	struct protection_domain *domain = dom->priv;
 	struct amd_iommu *iommu;
 	struct pci_dev *pdev;
 	u16 devid;
@@ -2275,7 +2320,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
 	devid = calc_devid(pdev->bus->number, pdev->devfn);
 
 	if (devid > 0)
-		detach_device(domain, devid);
+		detach_device(dev);
 
 	iommu = amd_iommu_rlookup_table[devid];
 	if (!iommu)
@@ -2292,6 +2337,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 	struct protection_domain *old_domain;
 	struct amd_iommu *iommu;
 	struct pci_dev *pdev;
+	int ret;
 	u16 devid;
 
 	if (dev->bus != &pci_bus_type)
@@ -2309,15 +2355,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 	if (!iommu)
 		return -EINVAL;
 
-	old_domain = domain_for_device(devid);
+	old_domain = amd_iommu_pd_table[devid];
 	if (old_domain)
-		detach_device(old_domain, devid);
+		detach_device(dev);
 
-	attach_device(iommu, domain, devid);
+	ret = attach_device(dev, domain);
 
 	iommu_completion_wait(iommu);
 
-	return 0;
+	return ret;
 }
 
 static int amd_iommu_map_range(struct iommu_domain *dom,
@@ -2414,8 +2460,9 @@ static struct iommu_ops amd_iommu_ops = {
 
 int __init amd_iommu_init_passthrough(void)
 {
+	struct amd_iommu *iommu;
 	struct pci_dev *dev = NULL;
-	u16 devid, devid2;
+	u16 devid;
 
 	/* allocate passthroug domain */
 	pt_domain = protection_domain_alloc();
@@ -2425,20 +2472,16 @@ int __init amd_iommu_init_passthrough(void)
 	pt_domain->mode |= PAGE_MODE_NONE;
 
 	while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
-		struct amd_iommu *iommu;
 
 		devid = calc_devid(dev->bus->number, dev->devfn);
 		if (devid > amd_iommu_last_bdf)
 			continue;
 
-		devid2 = amd_iommu_alias_table[devid];
-
-		iommu = amd_iommu_rlookup_table[devid2];
+		iommu = amd_iommu_rlookup_table[devid];
 		if (!iommu)
 			continue;
 
-		__attach_device(iommu, pt_domain, devid);
-		__attach_device(iommu, pt_domain, devid2);
+		attach_device(&dev->dev, pt_domain);
 	}
 
 	pr_info("AMD-Vi: Initialized for Passthrough Mode\n");