diff options
Diffstat (limited to 'drivers/pci/pci-driver.c')
| -rw-r--r-- | drivers/pci/pci-driver.c | 236 |
1 files changed, 156 insertions, 80 deletions
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 588588cfda48..7c2d9d596258 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -20,6 +20,7 @@ #include <linux/of_device.h> #include <linux/acpi.h> #include <linux/dma-map-ops.h> +#include <linux/iommu.h> #include "pci.h" #include "pcie/portdrv.h" @@ -192,7 +193,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, u32 vendor, device, subvendor = PCI_ANY_ID, subdevice = PCI_ANY_ID, class = 0, class_mask = 0; unsigned long driver_data = 0; - int fields = 0; + int fields; int retval = 0; fields = sscanf(buf, "%x %x %x %x %x %x %lx", @@ -259,7 +260,7 @@ static ssize_t remove_id_store(struct device_driver *driver, const char *buf, struct pci_driver *pdrv = to_pci_driver(driver); u32 vendor, device, subvendor = PCI_ANY_ID, subdevice = PCI_ANY_ID, class = 0, class_mask = 0; - int fields = 0; + int fields; size_t retval = -ENODEV; fields = sscanf(buf, "%x %x %x %x %x %x", @@ -350,7 +351,6 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev, const struct pci_device_id *id) { int error, node, cpu; - int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ; struct drv_dev_and_id ddi = { drv, dev, id }; /* @@ -368,17 +368,29 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev, * device is probed from work_on_cpu() of the Physical device. */ if (node < 0 || node >= MAX_NUMNODES || !node_online(node) || - pci_physfn_is_probed(dev)) + pci_physfn_is_probed(dev)) { cpu = nr_cpu_ids; - else + } else { + cpumask_var_t wq_domain_mask; + + if (!zalloc_cpumask_var(&wq_domain_mask, GFP_KERNEL)) { + error = -ENOMEM; + goto out; + } + cpumask_and(wq_domain_mask, + housekeeping_cpumask(HK_TYPE_WQ), + housekeeping_cpumask(HK_TYPE_DOMAIN)); + cpu = cpumask_any_and(cpumask_of_node(node), - housekeeping_cpumask(hk_flags)); + wq_domain_mask); + free_cpumask_var(wq_domain_mask); + } if (cpu < nr_cpu_ids) error = work_on_cpu(cpu, local_pci_probe, &ddi); else error = local_pci_probe(&ddi); - +out: dev->is_probed = 0; cpu_hotplug_enable(); return error; @@ -407,15 +419,6 @@ static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) return error; } -int __weak pcibios_alloc_irq(struct pci_dev *dev) -{ - return 0; -} - -void __weak pcibios_free_irq(struct pci_dev *dev) -{ -} - #ifdef CONFIG_PCI_IOV static inline bool pci_device_can_probe(struct pci_dev *pdev) { @@ -461,6 +464,13 @@ static void pci_device_remove(struct device *dev) if (drv->remove) { pm_runtime_get_sync(dev); + /* + * If the driver provides a .runtime_idle() callback and it has + * started to run already, it may continue to run in parallel + * with the code below, so wait until all of the runtime PM + * activity has completed. + */ + pm_runtime_barrier(dev); drv->remove(pci_dev); pm_runtime_put_noidle(dev); } @@ -511,9 +521,9 @@ static void pci_device_shutdown(struct device *dev) pci_clear_master(pci_dev); } -#ifdef CONFIG_PM +#ifdef CONFIG_PM_SLEEP -/* Auxiliary functions used for system resume and run-time resume. */ +/* Auxiliary functions used for system resume */ /** * pci_restore_standard_config - restore standard config registers of PCI device @@ -533,6 +543,11 @@ static int pci_restore_standard_config(struct pci_dev *pci_dev) pci_pme_restore(pci_dev); return 0; } +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_PM + +/* Auxiliary functions used for system resume and run-time resume */ static void pci_pm_default_resume(struct pci_dev *pci_dev) { @@ -540,18 +555,41 @@ static void pci_pm_default_resume(struct pci_dev *pci_dev) pci_enable_wake(pci_dev, PCI_D0, false); } -#endif - -#ifdef CONFIG_PM_SLEEP - static void pci_pm_default_resume_early(struct pci_dev *pci_dev) { - pci_power_up(pci_dev); - pci_update_current_state(pci_dev, PCI_D0); + pci_pm_power_up_and_verify_state(pci_dev); pci_restore_state(pci_dev); pci_pme_restore(pci_dev); } +static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev) +{ + int ret; + + ret = pci_bridge_wait_for_secondary_bus(pci_dev, "resume"); + if (ret) { + /* + * The downstream link failed to come up, so mark the + * devices below as disconnected to make sure we don't + * attempt to resume them. + */ + pci_walk_bus(pci_dev->subordinate, pci_dev_set_disconnected, + NULL); + return; + } + + /* + * When powering on a bridge from D3cold, the whole hierarchy may be + * powered on into D0uninitialized state, resume them to give them a + * chance to suspend again + */ + pci_resume_bus(pci_dev->subordinate); +} + +#endif /* CONFIG_PM */ + +#ifdef CONFIG_PM_SLEEP + /* * Default "suspend" method for devices that have no driver provided suspend, * or not even a driver at all (second part). @@ -591,12 +629,14 @@ static int pci_legacy_suspend(struct device *dev, pm_message_t state) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = pci_dev->driver; + pci_dev->state_saved = false; + if (drv && drv->suspend) { pci_power_t prev = pci_dev->current_state; int error; error = drv->suspend(pci_dev, state); - suspend_report_result(drv->suspend, error); + suspend_report_result(dev, drv->suspend, error); if (error) return error; @@ -613,7 +653,7 @@ static int pci_legacy_suspend(struct device *dev, pm_message_t state) return 0; } -static int pci_legacy_suspend_late(struct device *dev, pm_message_t state) +static int pci_legacy_suspend_late(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); @@ -670,6 +710,8 @@ static int pci_pm_prepare(struct device *dev) struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; + dev_pm_set_strict_midlayer(dev, true); + if (pm && pm->prepare) { int error = pm->prepare(dev); if (error < 0) @@ -711,6 +753,8 @@ static void pci_pm_complete(struct device *dev) if (pci_dev->current_state < pre_sleep_state) pm_request_resume(dev); } + + dev_pm_set_strict_midlayer(dev, false); } #else /* !CONFIG_PM_SLEEP */ @@ -741,6 +785,12 @@ static int pci_pm_suspend(struct device *dev) pci_dev->skip_bus_pm = false; + /* + * Disabling PTM allows some systems, e.g., Intel mobile chips + * since Coffee Lake, to enter a lower-power PM state. + */ + pci_suspend_ptm(pci_dev); + if (pci_has_legacy_pm_support(pci_dev)) return pci_legacy_suspend(dev, PMSG_SUSPEND); @@ -762,8 +812,7 @@ static int pci_pm_suspend(struct device *dev) * suspend callbacks can cope with runtime-suspended devices, it is * better to resume the device from runtime suspend here. */ - if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) || - pci_dev_need_resume(pci_dev)) { + if (!dev_pm_smart_suspend(dev) || pci_dev_need_resume(pci_dev)) { pm_runtime_resume(dev); pci_dev->state_saved = false; } else { @@ -775,7 +824,7 @@ static int pci_pm_suspend(struct device *dev) int error; error = pm->suspend(dev); - suspend_report_result(pm->suspend, error); + suspend_report_result(dev, pm->suspend, error); if (error) return error; @@ -809,7 +858,7 @@ static int pci_pm_suspend_noirq(struct device *dev) return 0; if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_suspend_late(dev, PMSG_SUSPEND); + return pci_legacy_suspend_late(dev); if (!pm) { pci_save_state(pci_dev); @@ -821,7 +870,7 @@ static int pci_pm_suspend_noirq(struct device *dev) int error; error = pm->suspend_noirq(dev); - suspend_report_result(pm->suspend_noirq, error); + suspend_report_result(dev, pm->suspend_noirq, error); if (error) return error; @@ -834,20 +883,15 @@ static int pci_pm_suspend_noirq(struct device *dev) } } - if (pci_dev->skip_bus_pm) { + if (!pci_dev->state_saved) { + pci_save_state(pci_dev); + /* - * Either the device is a bridge with a child in D0 below it, or - * the function is running for the second time in a row without - * going through full resume, which is possible only during - * suspend-to-idle in a spurious wakeup case. The device should - * be in D0 at this point, but if it is a bridge, it may be - * necessary to save its state. + * If the device is a bridge with a child in D0 below it, + * it needs to stay in D0, so check skip_bus_pm to avoid + * putting it into a low-power state in that case. */ - if (!pci_dev->state_saved) - pci_save_state(pci_dev); - } else if (!pci_dev->state_saved) { - pci_save_state(pci_dev); - if (pci_power_manageable(pci_dev)) + if (!pci_dev->skip_bus_pm && pci_power_manageable(pci_dev)) pci_prepare_to_sleep(pci_dev); } @@ -923,7 +967,7 @@ static int pci_pm_resume_noirq(struct device *dev) pcie_pme_root_status_cleanup(pci_dev); if (!skip_bus_pm && prev_state == PCI_D3cold) - pci_bridge_wait_for_secondary_bus(pci_dev); + pci_pm_bridge_power_up_actions(pci_dev); if (pci_has_legacy_pm_support(pci_dev)) return 0; @@ -954,6 +998,8 @@ static int pci_pm_resume(struct device *dev) if (pci_dev->state_saved) pci_restore_standard_config(pci_dev); + pci_resume_ptm(pci_dev); + if (pci_has_legacy_pm_support(pci_dev)) return pci_legacy_resume(dev); @@ -992,6 +1038,8 @@ static int pci_pm_freeze(struct device *dev) if (!pm) { pci_pm_default_suspend(pci_dev); + if (!pm_runtime_suspended(dev)) + pci_dev->state_saved = false; return 0; } @@ -1010,7 +1058,7 @@ static int pci_pm_freeze(struct device *dev) int error; error = pm->freeze(dev); - suspend_report_result(pm->freeze, error); + suspend_report_result(dev, pm->freeze, error); if (error) return error; } @@ -1024,13 +1072,13 @@ static int pci_pm_freeze_noirq(struct device *dev) const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_suspend_late(dev, PMSG_FREEZE); + return pci_legacy_suspend_late(dev); if (pm && pm->freeze_noirq) { int error; error = pm->freeze_noirq(dev); - suspend_report_result(pm->freeze_noirq, error); + suspend_report_result(dev, pm->freeze_noirq, error); if (error) return error; } @@ -1057,7 +1105,7 @@ static int pci_pm_thaw_noirq(struct device *dev) * in case the driver's "freeze" callbacks put it into a low-power * state. */ - pci_set_power_state(pci_dev, PCI_D0); + pci_pm_power_up_and_verify_state(pci_dev); pci_restore_state(pci_dev); if (pci_has_legacy_pm_support(pci_dev)) @@ -1085,8 +1133,6 @@ static int pci_pm_thaw(struct device *dev) pci_pm_reenable_device(pci_dev); } - pci_dev->state_saved = false; - return error; } @@ -1104,8 +1150,7 @@ static int pci_pm_poweroff(struct device *dev) } /* The reason to do that is the same as in pci_pm_suspend(). */ - if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) || - pci_dev_need_resume(pci_dev)) { + if (!dev_pm_smart_suspend(dev) || pci_dev_need_resume(pci_dev)) { pm_runtime_resume(dev); pci_dev->state_saved = false; } else { @@ -1116,7 +1161,7 @@ static int pci_pm_poweroff(struct device *dev) int error; error = pm->poweroff(dev); - suspend_report_result(pm->poweroff, error); + suspend_report_result(dev, pm->poweroff, error); if (error) return error; } @@ -1143,7 +1188,7 @@ static int pci_pm_poweroff_noirq(struct device *dev) return 0; if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_suspend_late(dev, PMSG_HIBERNATE); + return pci_legacy_suspend_late(dev); if (!pm) { pci_fixup_device(pci_fixup_suspend_late, pci_dev); @@ -1154,7 +1199,7 @@ static int pci_pm_poweroff_noirq(struct device *dev) int error; error = pm->poweroff_noirq(dev); - suspend_report_result(pm->poweroff_noirq, error); + suspend_report_result(dev, pm->poweroff_noirq, error); if (error) return error; } @@ -1241,6 +1286,8 @@ static int pci_pm_runtime_suspend(struct device *dev) pci_power_t prev = pci_dev->current_state; int error; + pci_suspend_ptm(pci_dev); + /* * If pci_dev->driver is not set (unbound), we leave the device in D0, * but it may go to D3cold when the bridge above it runtime suspends. @@ -1301,7 +1348,8 @@ static int pci_pm_runtime_resume(struct device *dev) * to a driver because although we left it in D0, it may have gone to * D3cold when the bridge above it runtime suspended. */ - pci_restore_standard_config(pci_dev); + pci_pm_default_resume_early(pci_dev); + pci_resume_ptm(pci_dev); if (!pci_dev->driver) return 0; @@ -1310,13 +1358,11 @@ static int pci_pm_runtime_resume(struct device *dev) pci_pm_default_resume(pci_dev); if (prev_state == PCI_D3cold) - pci_bridge_wait_for_secondary_bus(pci_dev); + pci_pm_bridge_power_up_actions(pci_dev); if (pm && pm->runtime_resume) error = pm->runtime_resume(dev); - pci_dev->runtime_d3cold = false; - return error; } @@ -1332,10 +1378,7 @@ static int pci_pm_runtime_idle(struct device *dev) if (!pci_dev->driver) return 0; - if (!pm) - return -ENOSYS; - - if (pm->runtime_idle) + if (pm && pm->runtime_idle) return pm->runtime_idle(dev); return 0; @@ -1436,14 +1479,15 @@ static struct pci_driver pci_compat_driver = { */ struct pci_driver *pci_dev_driver(const struct pci_dev *dev) { + int i; + if (dev->driver) return dev->driver; - else { - int i; - for (i = 0; i <= PCI_ROM_RESOURCE; i++) - if (dev->resource[i].flags & IORESOURCE_BUSY) - return &pci_compat_driver; - } + + for (i = 0; i <= PCI_ROM_RESOURCE; i++) + if (dev->resource[i].flags & IORESOURCE_BUSY) + return &pci_compat_driver; + return NULL; } EXPORT_SYMBOL(pci_dev_driver); @@ -1457,16 +1501,16 @@ EXPORT_SYMBOL(pci_dev_driver); * system is in its list of supported devices. Returns the matching * pci_device_id structure or %NULL if there is no match. */ -static int pci_bus_match(struct device *dev, struct device_driver *drv) +static int pci_bus_match(struct device *dev, const struct device_driver *drv) { struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *pci_drv; const struct pci_device_id *found_id; - if (!pci_dev->match_driver) + if (pci_dev_binding_disallowed(pci_dev)) return 0; - pci_drv = to_pci_driver(drv); + pci_drv = (struct pci_driver *)to_pci_driver(drv); found_id = pci_match_device(pci_drv, pci_dev); if (found_id) return 1; @@ -1508,9 +1552,9 @@ void pci_dev_put(struct pci_dev *dev) } EXPORT_SYMBOL(pci_dev_put); -static int pci_uevent(struct device *dev, struct kobj_uevent_env *env) +static int pci_uevent(const struct device *dev, struct kobj_uevent_env *env) { - struct pci_dev *pdev; + const struct pci_dev *pdev; if (!dev) return -ENODEV; @@ -1540,7 +1584,7 @@ static int pci_uevent(struct device *dev, struct kobj_uevent_env *env) return 0; } -#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH) +#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH) || defined(CONFIG_S390) /** * pci_uevent_ers - emit a uevent during recovery path of PCI device * @pdev: PCI device undergoing error recovery @@ -1554,6 +1598,7 @@ void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type) switch (err_type) { case PCI_ERS_RESULT_NONE: case PCI_ERS_RESULT_CAN_RECOVER: + case PCI_ERS_RESULT_NEED_RESET: envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY"; envp[idx++] = "DEVICE_ONLINE=0"; break; @@ -1590,6 +1635,7 @@ static int pci_bus_num_vf(struct device *dev) */ static int pci_dma_configure(struct device *dev) { + const struct device_driver *drv = READ_ONCE(dev->driver); struct device *bridge; int ret = 0; @@ -1605,30 +1651,61 @@ static int pci_dma_configure(struct device *dev) } pci_put_host_bridge_device(bridge); + + /* @drv may not be valid when we're called from the IOMMU layer */ + if (!ret && drv && !to_pci_driver(drv)->driver_managed_dma) { + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); + } + return ret; } -struct bus_type pci_bus_type = { +static void pci_dma_cleanup(struct device *dev) +{ + struct pci_driver *driver = to_pci_driver(dev->driver); + + if (!driver->driver_managed_dma) + iommu_device_unuse_default_domain(dev); +} + +/* + * pci_device_irq_get_affinity - get IRQ affinity mask for device + * @dev: ptr to dev structure + * @irq_vec: interrupt vector number + * + * Return the CPU affinity mask for @dev and @irq_vec. + */ +static const struct cpumask *pci_device_irq_get_affinity(struct device *dev, + unsigned int irq_vec) +{ + return pci_irq_get_affinity(to_pci_dev(dev), irq_vec); +} + +const struct bus_type pci_bus_type = { .name = "pci", .match = pci_bus_match, .uevent = pci_uevent, .probe = pci_device_probe, .remove = pci_device_remove, .shutdown = pci_device_shutdown, + .irq_get_affinity = pci_device_irq_get_affinity, .dev_groups = pci_dev_groups, .bus_groups = pci_bus_groups, .drv_groups = pci_drv_groups, .pm = PCI_PM_OPS_PTR, .num_vf = pci_bus_num_vf, .dma_configure = pci_dma_configure, + .dma_cleanup = pci_dma_cleanup, }; EXPORT_SYMBOL(pci_bus_type); #ifdef CONFIG_PCIEPORTBUS -static int pcie_port_bus_match(struct device *dev, struct device_driver *drv) +static int pcie_port_bus_match(struct device *dev, const struct device_driver *drv) { struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; + const struct pcie_port_service_driver *driver; if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type) return 0; @@ -1646,11 +1723,10 @@ static int pcie_port_bus_match(struct device *dev, struct device_driver *drv) return 1; } -struct bus_type pcie_port_bus_type = { +const struct bus_type pcie_port_bus_type = { .name = "pci_express", .match = pcie_port_bus_match, }; -EXPORT_SYMBOL_GPL(pcie_port_bus_type); #endif static int __init pci_driver_init(void) |