diff options
Diffstat (limited to 'drivers/pci/pci-driver.c')
| -rw-r--r-- | drivers/pci/pci-driver.c | 763 |
1 files changed, 421 insertions, 342 deletions
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 79b1610a8beb..7c2d9d596258 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -12,12 +12,15 @@ #include <linux/string.h> #include <linux/slab.h> #include <linux/sched.h> +#include <linux/sched/isolation.h> #include <linux/cpu.h> #include <linux/pm_runtime.h> #include <linux/suspend.h> #include <linux/kexec.h> #include <linux/of_device.h> #include <linux/acpi.h> +#include <linux/dma-map-ops.h> +#include <linux/iommu.h> #include "pci.h" #include "pcie/portdrv.h" @@ -88,7 +91,94 @@ static void pci_free_dynids(struct pci_driver *drv) } /** - * store_new_id - sysfs frontend to pci_add_dynid() + * pci_match_id - See if a PCI device matches a given pci_id table + * @ids: array of PCI device ID structures to search in + * @dev: the PCI device structure to match against. + * + * Used by a driver to check whether a PCI device is in its list of + * supported devices. Returns the matching pci_device_id structure or + * %NULL if there is no match. + * + * Deprecated; don't use this as it will not catch any dynamic IDs + * that a driver might want to check for. + */ +const struct pci_device_id *pci_match_id(const struct pci_device_id *ids, + struct pci_dev *dev) +{ + if (ids) { + while (ids->vendor || ids->subvendor || ids->class_mask) { + if (pci_match_one_device(ids, dev)) + return ids; + ids++; + } + } + return NULL; +} +EXPORT_SYMBOL(pci_match_id); + +static const struct pci_device_id pci_device_id_any = { + .vendor = PCI_ANY_ID, + .device = PCI_ANY_ID, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, +}; + +/** + * pci_match_device - See if a device matches a driver's list of IDs + * @drv: the PCI driver to match against + * @dev: the PCI device structure to match against + * + * Used by a driver to check whether a PCI device is in its list of + * supported devices or in the dynids list, which may have been augmented + * via the sysfs "new_id" file. Returns the matching pci_device_id + * structure or %NULL if there is no match. + */ +static const struct pci_device_id *pci_match_device(struct pci_driver *drv, + struct pci_dev *dev) +{ + struct pci_dynid *dynid; + const struct pci_device_id *found_id = NULL, *ids; + + /* When driver_override is set, only bind to the matching driver */ + if (dev->driver_override && strcmp(dev->driver_override, drv->name)) + return NULL; + + /* Look at the dynamic ids first, before the static ones */ + spin_lock(&drv->dynids.lock); + list_for_each_entry(dynid, &drv->dynids.list, node) { + if (pci_match_one_device(&dynid->id, dev)) { + found_id = &dynid->id; + break; + } + } + spin_unlock(&drv->dynids.lock); + + if (found_id) + return found_id; + + for (ids = drv->id_table; (found_id = pci_match_id(ids, dev)); + ids = found_id + 1) { + /* + * The match table is split based on driver_override. + * In case override_only was set, enforce driver_override + * matching. + */ + if (found_id->override_only) { + if (dev->driver_override) + return found_id; + } else { + return found_id; + } + } + + /* driver_override will always match, send a dummy id */ + if (dev->driver_override) + return &pci_device_id_any; + return NULL; +} + +/** + * new_id_store - sysfs frontend to pci_add_dynid() * @driver: target device driver * @buf: buffer for scanning device ID data * @count: input size @@ -100,10 +190,10 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, { struct pci_driver *pdrv = to_pci_driver(driver); const struct pci_device_id *ids = pdrv->id_table; - __u32 vendor, device, subvendor = PCI_ANY_ID, + u32 vendor, device, subvendor = PCI_ANY_ID, subdevice = PCI_ANY_ID, class = 0, class_mask = 0; unsigned long driver_data = 0; - int fields = 0; + int fields; int retval = 0; fields = sscanf(buf, "%x %x %x %x %x %x %lx", @@ -123,7 +213,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, pdev->subsystem_device = subdevice; pdev->class = class; - if (pci_match_id(pdrv->id_table, pdev)) + if (pci_match_device(pdrv, pdev)) retval = -EEXIST; kfree(pdev); @@ -156,7 +246,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, static DRIVER_ATTR_WO(new_id); /** - * store_remove_id - remove a PCI device ID from this driver + * remove_id_store - remove a PCI device ID from this driver * @driver: target device driver * @buf: buffer for scanning device ID data * @count: input size @@ -168,9 +258,9 @@ static ssize_t remove_id_store(struct device_driver *driver, const char *buf, { struct pci_dynid *dynid, *n; struct pci_driver *pdrv = to_pci_driver(driver); - __u32 vendor, device, subvendor = PCI_ANY_ID, + u32 vendor, device, subvendor = PCI_ANY_ID, subdevice = PCI_ANY_ID, class = 0, class_mask = 0; - int fields = 0; + int fields; size_t retval = -ENODEV; fields = sscanf(buf, "%x %x %x %x %x %x", @@ -206,78 +296,6 @@ static struct attribute *pci_drv_attrs[] = { }; ATTRIBUTE_GROUPS(pci_drv); -/** - * pci_match_id - See if a pci device matches a given pci_id table - * @ids: array of PCI device id structures to search in - * @dev: the PCI device structure to match against. - * - * Used by a driver to check whether a PCI device present in the - * system is in its list of supported devices. Returns the matching - * pci_device_id structure or %NULL if there is no match. - * - * Deprecated, don't use this as it will not catch any dynamic ids - * that a driver might want to check for. - */ -const struct pci_device_id *pci_match_id(const struct pci_device_id *ids, - struct pci_dev *dev) -{ - if (ids) { - while (ids->vendor || ids->subvendor || ids->class_mask) { - if (pci_match_one_device(ids, dev)) - return ids; - ids++; - } - } - return NULL; -} -EXPORT_SYMBOL(pci_match_id); - -static const struct pci_device_id pci_device_id_any = { - .vendor = PCI_ANY_ID, - .device = PCI_ANY_ID, - .subvendor = PCI_ANY_ID, - .subdevice = PCI_ANY_ID, -}; - -/** - * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure - * @drv: the PCI driver to match against - * @dev: the PCI device structure to match against - * - * Used by a driver to check whether a PCI device present in the - * system is in its list of supported devices. Returns the matching - * pci_device_id structure or %NULL if there is no match. - */ -static const struct pci_device_id *pci_match_device(struct pci_driver *drv, - struct pci_dev *dev) -{ - struct pci_dynid *dynid; - const struct pci_device_id *found_id = NULL; - - /* When driver_override is set, only bind to the matching driver */ - if (dev->driver_override && strcmp(dev->driver_override, drv->name)) - return NULL; - - /* Look at the dynamic ids first, before the static ones */ - spin_lock(&drv->dynids.lock); - list_for_each_entry(dynid, &drv->dynids.list, node) { - if (pci_match_one_device(&dynid->id, dev)) { - found_id = &dynid->id; - break; - } - } - spin_unlock(&drv->dynids.lock); - - if (!found_id) - found_id = pci_match_id(drv->id_table, dev); - - /* driver_override will always match, send a dummy id */ - if (!found_id && dev->driver_override) - found_id = &pci_device_id_any; - - return found_id; -} - struct drv_dev_and_id { struct pci_driver *drv; struct pci_dev *dev; @@ -315,7 +333,8 @@ static long local_pci_probe(void *_ddi) * Probe function should return < 0 for failure, 0 for success * Treat values > 0 as success, but warn. */ - dev_warn(dev, "Driver probe function unexpectedly returned %d\n", rc); + pci_warn(pci_dev, "Driver probe function unexpectedly returned %d\n", + rc); return 0; } @@ -349,16 +368,29 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev, * device is probed from work_on_cpu() of the Physical device. */ if (node < 0 || node >= MAX_NUMNODES || !node_online(node) || - pci_physfn_is_probed(dev)) + pci_physfn_is_probed(dev)) { cpu = nr_cpu_ids; - else - cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask); + } else { + cpumask_var_t wq_domain_mask; + + if (!zalloc_cpumask_var(&wq_domain_mask, GFP_KERNEL)) { + error = -ENOMEM; + goto out; + } + cpumask_and(wq_domain_mask, + housekeeping_cpumask(HK_TYPE_WQ), + housekeeping_cpumask(HK_TYPE_DOMAIN)); + + cpu = cpumask_any_and(cpumask_of_node(node), + wq_domain_mask); + free_cpumask_var(wq_domain_mask); + } if (cpu < nr_cpu_ids) error = work_on_cpu(cpu, local_pci_probe, &ddi); else error = local_pci_probe(&ddi); - +out: dev->is_probed = 0; cpu_hotplug_enable(); return error; @@ -377,7 +409,7 @@ static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) const struct pci_device_id *id; int error = 0; - if (!pci_dev->driver && drv->probe) { + if (drv->probe) { error = -ENODEV; id = pci_match_device(drv, pci_dev); @@ -387,19 +419,11 @@ static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) return error; } -int __weak pcibios_alloc_irq(struct pci_dev *dev) -{ - return 0; -} - -void __weak pcibios_free_irq(struct pci_dev *dev) -{ -} - #ifdef CONFIG_PCI_IOV static inline bool pci_device_can_probe(struct pci_dev *pdev) { - return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe); + return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe || + pdev->driver_override); } #else static inline bool pci_device_can_probe(struct pci_dev *pdev) @@ -414,6 +438,9 @@ static int pci_device_probe(struct device *dev) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = to_pci_driver(dev->driver); + if (!pci_device_can_probe(pci_dev)) + return -ENODEV; + pci_assign_irq(pci_dev); error = pcibios_alloc_irq(pci_dev); @@ -421,32 +448,35 @@ static int pci_device_probe(struct device *dev) return error; pci_dev_get(pci_dev); - if (pci_device_can_probe(pci_dev)) { - error = __pci_device_probe(drv, pci_dev); - if (error) { - pcibios_free_irq(pci_dev); - pci_dev_put(pci_dev); - } + error = __pci_device_probe(drv, pci_dev); + if (error) { + pcibios_free_irq(pci_dev); + pci_dev_put(pci_dev); } return error; } -static int pci_device_remove(struct device *dev) +static void pci_device_remove(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = pci_dev->driver; - if (drv) { - if (drv->remove) { - pm_runtime_get_sync(dev); - drv->remove(pci_dev); - pm_runtime_put_noidle(dev); - } - pcibios_free_irq(pci_dev); - pci_dev->driver = NULL; - pci_iov_remove(pci_dev); + if (drv->remove) { + pm_runtime_get_sync(dev); + /* + * If the driver provides a .runtime_idle() callback and it has + * started to run already, it may continue to run in parallel + * with the code below, so wait until all of the runtime PM + * activity has completed. + */ + pm_runtime_barrier(dev); + drv->remove(pci_dev); + pm_runtime_put_noidle(dev); } + pcibios_free_irq(pci_dev); + pci_dev->driver = NULL; + pci_iov_remove(pci_dev); /* Undo the runtime PM settings in local_pci_probe() */ pm_runtime_put_sync(dev); @@ -468,7 +498,6 @@ static int pci_device_remove(struct device *dev) */ pci_dev_put(pci_dev); - return 0; } static void pci_device_shutdown(struct device *dev) @@ -492,9 +521,9 @@ static void pci_device_shutdown(struct device *dev) pci_clear_master(pci_dev); } -#ifdef CONFIG_PM +#ifdef CONFIG_PM_SLEEP -/* Auxiliary functions used for system resume and run-time resume. */ +/* Auxiliary functions used for system resume */ /** * pci_restore_standard_config - restore standard config registers of PCI device @@ -514,19 +543,53 @@ static int pci_restore_standard_config(struct pci_dev *pci_dev) pci_pme_restore(pci_dev); return 0; } +#endif /* CONFIG_PM_SLEEP */ -#endif +#ifdef CONFIG_PM -#ifdef CONFIG_PM_SLEEP +/* Auxiliary functions used for system resume and run-time resume */ + +static void pci_pm_default_resume(struct pci_dev *pci_dev) +{ + pci_fixup_device(pci_fixup_resume, pci_dev); + pci_enable_wake(pci_dev, PCI_D0, false); +} static void pci_pm_default_resume_early(struct pci_dev *pci_dev) { - pci_power_up(pci_dev); + pci_pm_power_up_and_verify_state(pci_dev); pci_restore_state(pci_dev); pci_pme_restore(pci_dev); - pci_fixup_device(pci_fixup_resume_early, pci_dev); } +static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev) +{ + int ret; + + ret = pci_bridge_wait_for_secondary_bus(pci_dev, "resume"); + if (ret) { + /* + * The downstream link failed to come up, so mark the + * devices below as disconnected to make sure we don't + * attempt to resume them. + */ + pci_walk_bus(pci_dev->subordinate, pci_dev_set_disconnected, + NULL); + return; + } + + /* + * When powering on a bridge from D3cold, the whole hierarchy may be + * powered on into D0uninitialized state, resume them to give them a + * chance to suspend again + */ + pci_resume_bus(pci_dev->subordinate); +} + +#endif /* CONFIG_PM */ + +#ifdef CONFIG_PM_SLEEP + /* * Default "suspend" method for devices that have no driver provided suspend, * or not even a driver at all (second part). @@ -549,7 +612,7 @@ static int pci_pm_reenable_device(struct pci_dev *pci_dev) { int retval; - /* if the device was enabled before suspend, reenable */ + /* if the device was enabled before suspend, re-enable */ retval = pci_reenable_device(pci_dev); /* * if the device was busmaster before the suspend, make it busmaster @@ -566,20 +629,22 @@ static int pci_legacy_suspend(struct device *dev, pm_message_t state) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = pci_dev->driver; + pci_dev->state_saved = false; + if (drv && drv->suspend) { pci_power_t prev = pci_dev->current_state; int error; error = drv->suspend(pci_dev, state); - suspend_report_result(drv->suspend, error); + suspend_report_result(dev, drv->suspend, error); if (error) return error; if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0 && pci_dev->current_state != PCI_UNKNOWN) { - WARN_ONCE(pci_dev->current_state != prev, - "PCI PM: Device state not saved by %pF\n", - drv->suspend); + pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev, + "PCI PM: Device state not saved by %pS\n", + drv->suspend); } } @@ -588,49 +653,20 @@ static int pci_legacy_suspend(struct device *dev, pm_message_t state) return 0; } -static int pci_legacy_suspend_late(struct device *dev, pm_message_t state) +static int pci_legacy_suspend_late(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct pci_driver *drv = pci_dev->driver; - - if (drv && drv->suspend_late) { - pci_power_t prev = pci_dev->current_state; - int error; - - error = drv->suspend_late(pci_dev, state); - suspend_report_result(drv->suspend_late, error); - if (error) - return error; - - if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0 - && pci_dev->current_state != PCI_UNKNOWN) { - WARN_ONCE(pci_dev->current_state != prev, - "PCI PM: Device state not saved by %pF\n", - drv->suspend_late); - goto Fixup; - } - } if (!pci_dev->state_saved) pci_save_state(pci_dev); pci_pm_set_unknown_state(pci_dev); -Fixup: pci_fixup_device(pci_fixup_suspend_late, pci_dev); return 0; } -static int pci_legacy_resume_early(struct device *dev) -{ - struct pci_dev *pci_dev = to_pci_dev(dev); - struct pci_driver *drv = pci_dev->driver; - - return drv && drv->resume_early ? - drv->resume_early(pci_dev) : 0; -} - static int pci_legacy_resume(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); @@ -644,12 +680,6 @@ static int pci_legacy_resume(struct device *dev) /* Auxiliary functions used by the new power management framework */ -static void pci_pm_default_resume(struct pci_dev *pci_dev) -{ - pci_fixup_device(pci_fixup_resume, pci_dev); - pci_enable_wake(pci_dev, PCI_D0, false); -} - static void pci_pm_default_suspend(struct pci_dev *pci_dev) { /* Disable non-bridge devices without PM support */ @@ -660,16 +690,15 @@ static void pci_pm_default_suspend(struct pci_dev *pci_dev) static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev) { struct pci_driver *drv = pci_dev->driver; - bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume - || drv->resume_early); + bool ret = drv && (drv->suspend || drv->resume); /* * Legacy PM support is used by default, so warn if the new framework is * supported as well. Drivers are supposed to support either the * former, or the latter, but not both at the same time. */ - WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n", - drv->name, pci_dev->vendor, pci_dev->device); + pci_WARN(pci_dev, ret && drv->driver.pm, "device %04x:%04x\n", + pci_dev->vendor, pci_dev->device); return ret; } @@ -678,17 +707,28 @@ static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev) static int pci_pm_prepare(struct device *dev) { - struct device_driver *drv = dev->driver; + struct pci_dev *pci_dev = to_pci_dev(dev); + const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; + + dev_pm_set_strict_midlayer(dev, true); - if (drv && drv->pm && drv->pm->prepare) { - int error = drv->pm->prepare(dev); + if (pm && pm->prepare) { + int error = pm->prepare(dev); if (error < 0) return error; if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE)) return 0; } - return pci_dev_keep_suspended(to_pci_dev(dev)); + if (pci_dev_need_resume(pci_dev)) + return 0; + + /* + * The PME setting needs to be adjusted here in case the direct-complete + * optimization is used with respect to this device. + */ + pci_dev_adjust_pme(pci_dev); + return 1; } static void pci_pm_complete(struct device *dev) @@ -702,10 +742,19 @@ static void pci_pm_complete(struct device *dev) if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) { pci_power_t pre_sleep_state = pci_dev->current_state; - pci_update_current_state(pci_dev, pci_dev->current_state); + pci_refresh_power_state(pci_dev); + /* + * On platforms with ACPI this check may also trigger for + * devices sharing power resources if one of those power + * resources has been activated as a result of a change of the + * power state of another device sharing it. However, in that + * case it is also better to resume the device, in general. + */ if (pci_dev->current_state < pre_sleep_state) pm_request_resume(dev); } + + dev_pm_set_strict_midlayer(dev, false); } #else /* !CONFIG_PM_SLEEP */ @@ -734,6 +783,14 @@ static int pci_pm_suspend(struct device *dev) struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; + pci_dev->skip_bus_pm = false; + + /* + * Disabling PTM allows some systems, e.g., Intel mobile chips + * since Coffee Lake, to enter a lower-power PM state. + */ + pci_suspend_ptm(pci_dev); + if (pci_has_legacy_pm_support(pci_dev)) return pci_legacy_suspend(dev, PMSG_SUSPEND); @@ -755,10 +812,11 @@ static int pci_pm_suspend(struct device *dev) * suspend callbacks can cope with runtime-suspended devices, it is * better to resume the device from runtime suspend here. */ - if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) || - !pci_dev_keep_suspended(pci_dev)) { + if (!dev_pm_smart_suspend(dev) || pci_dev_need_resume(pci_dev)) { pm_runtime_resume(dev); pci_dev->state_saved = false; + } else { + pci_dev_adjust_pme(pci_dev); } if (pm->suspend) { @@ -766,15 +824,15 @@ static int pci_pm_suspend(struct device *dev) int error; error = pm->suspend(dev); - suspend_report_result(pm->suspend, error); + suspend_report_result(dev, pm->suspend, error); if (error) return error; if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0 && pci_dev->current_state != PCI_UNKNOWN) { - WARN_ONCE(pci_dev->current_state != prev, - "PCI PM: State of device not saved by %pF\n", - pm->suspend); + pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev, + "PCI PM: State of device not saved by %pS\n", + pm->suspend); } } @@ -783,7 +841,7 @@ static int pci_pm_suspend(struct device *dev) static int pci_pm_suspend_late(struct device *dev) { - if (dev_pm_smart_suspend_and_suspended(dev)) + if (dev_pm_skip_suspend(dev)) return 0; pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev)); @@ -796,13 +854,11 @@ static int pci_pm_suspend_noirq(struct device *dev) struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - if (dev_pm_smart_suspend_and_suspended(dev)) { - dev->power.may_skip_resume = true; + if (dev_pm_skip_suspend(dev)) return 0; - } if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_suspend_late(dev, PMSG_SUSPEND); + return pci_legacy_suspend_late(dev); if (!pm) { pci_save_state(pci_dev); @@ -814,28 +870,50 @@ static int pci_pm_suspend_noirq(struct device *dev) int error; error = pm->suspend_noirq(dev); - suspend_report_result(pm->suspend_noirq, error); + suspend_report_result(dev, pm->suspend_noirq, error); if (error) return error; if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0 && pci_dev->current_state != PCI_UNKNOWN) { - WARN_ONCE(pci_dev->current_state != prev, - "PCI PM: State of device not saved by %pF\n", - pm->suspend_noirq); + pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev, + "PCI PM: State of device not saved by %pS\n", + pm->suspend_noirq); goto Fixup; } } if (!pci_dev->state_saved) { pci_save_state(pci_dev); - if (pci_power_manageable(pci_dev)) + + /* + * If the device is a bridge with a child in D0 below it, + * it needs to stay in D0, so check skip_bus_pm to avoid + * putting it into a low-power state in that case. + */ + if (!pci_dev->skip_bus_pm && pci_power_manageable(pci_dev)) pci_prepare_to_sleep(pci_dev); } - dev_dbg(dev, "PCI PM: Suspend power state: %s\n", + pci_dbg(pci_dev, "PCI PM: Suspend power state: %s\n", pci_power_name(pci_dev->current_state)); + if (pci_dev->current_state == PCI_D0) { + pci_dev->skip_bus_pm = true; + /* + * Per PCI PM r1.2, table 6-1, a bridge must be in D0 if any + * downstream device is in D0, so avoid changing the power state + * of the parent bridge by setting the skip_bus_pm flag for it. + */ + if (pci_dev->bus->self) + pci_dev->bus->self->skip_bus_pm = true; + } + + if (pci_dev->skip_bus_pm && pm_suspend_no_platform()) { + pci_dbg(pci_dev, "PCI PM: Skipped\n"); + goto Fixup; + } + pci_pm_set_unknown_state(pci_dev); /* @@ -860,8 +938,8 @@ Fixup: * pci_pm_complete() to take care of fixing up the device's state * anyway, if need be. */ - dev->power.may_skip_resume = device_may_wakeup(dev) || - !device_can_wakeup(dev); + if (device_can_wakeup(dev) && !device_may_wakeup(dev)) + dev->power.may_skip_resume = false; return 0; } @@ -869,38 +947,49 @@ Fixup: static int pci_pm_resume_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct device_driver *drv = dev->driver; - int error = 0; + const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; + pci_power_t prev_state = pci_dev->current_state; + bool skip_bus_pm = pci_dev->skip_bus_pm; - if (dev_pm_may_skip_resume(dev)) + if (dev_pm_skip_resume(dev)) return 0; /* - * Devices with DPM_FLAG_SMART_SUSPEND may be left in runtime suspend - * during system suspend, so update their runtime PM status to "active" - * as they are going to be put into D0 shortly. + * In the suspend-to-idle case, devices left in D0 during suspend will + * stay in D0, so it is not necessary to restore or update their + * configuration here and attempting to put them into D0 again is + * pointless, so avoid doing that. */ - if (dev_pm_smart_suspend_and_suspended(dev)) - pm_runtime_set_active(dev); + if (!(skip_bus_pm && pm_suspend_no_platform())) + pci_pm_default_resume_early(pci_dev); - pci_pm_default_resume_early(pci_dev); + pci_fixup_device(pci_fixup_resume_early, pci_dev); + pcie_pme_root_status_cleanup(pci_dev); + + if (!skip_bus_pm && prev_state == PCI_D3cold) + pci_pm_bridge_power_up_actions(pci_dev); if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_resume_early(dev); + return 0; - pcie_pme_root_status_cleanup(pci_dev); + if (pm && pm->resume_noirq) + return pm->resume_noirq(dev); - if (drv && drv->pm && drv->pm->resume_noirq) - error = drv->pm->resume_noirq(dev); + return 0; +} - return error; +static int pci_pm_resume_early(struct device *dev) +{ + if (dev_pm_skip_resume(dev)) + return 0; + + return pm_generic_resume_early(dev); } static int pci_pm_resume(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int error = 0; /* * This is necessary for the suspend error path in which resume is @@ -909,6 +998,8 @@ static int pci_pm_resume(struct device *dev) if (pci_dev->state_saved) pci_restore_standard_config(pci_dev); + pci_resume_ptm(pci_dev); + if (pci_has_legacy_pm_support(pci_dev)) return pci_legacy_resume(dev); @@ -916,12 +1007,12 @@ static int pci_pm_resume(struct device *dev) if (pm) { if (pm->resume) - error = pm->resume(dev); + return pm->resume(dev); } else { pci_pm_reenable_device(pci_dev); } - return error; + return 0; } #else /* !CONFIG_SUSPEND */ @@ -930,19 +1021,13 @@ static int pci_pm_resume(struct device *dev) #define pci_pm_suspend_late NULL #define pci_pm_suspend_noirq NULL #define pci_pm_resume NULL +#define pci_pm_resume_early NULL #define pci_pm_resume_noirq NULL #endif /* !CONFIG_SUSPEND */ #ifdef CONFIG_HIBERNATE_CALLBACKS - -/* - * pcibios_pm_ops - provide arch-specific hooks when a PCI device is doing - * a hibernate transition - */ -struct dev_pm_ops __weak pcibios_pm_ops; - static int pci_pm_freeze(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); @@ -953,25 +1038,27 @@ static int pci_pm_freeze(struct device *dev) if (!pm) { pci_pm_default_suspend(pci_dev); + if (!pm_runtime_suspended(dev)) + pci_dev->state_saved = false; return 0; } /* - * This used to be done in pci_pm_prepare() for all devices and some - * drivers may depend on it, so do it here. Ideally, runtime-suspended - * devices should not be touched during freeze/thaw transitions, - * however. + * Resume all runtime-suspended devices before creating a snapshot + * image of system memory, because the restore kernel generally cannot + * be expected to always handle them consistently and they need to be + * put into the runtime-active metastate during system resume anyway, + * so it is better to ensure that the state saved in the image will be + * always consistent with that. */ - if (!dev_pm_smart_suspend_and_suspended(dev)) { - pm_runtime_resume(dev); - pci_dev->state_saved = false; - } + pm_runtime_resume(dev); + pci_dev->state_saved = false; if (pm->freeze) { int error; error = pm->freeze(dev); - suspend_report_result(pm->freeze, error); + suspend_report_result(dev, pm->freeze, error); if (error) return error; } @@ -979,30 +1066,19 @@ static int pci_pm_freeze(struct device *dev) return 0; } -static int pci_pm_freeze_late(struct device *dev) -{ - if (dev_pm_smart_suspend_and_suspended(dev)) - return 0; - - return pm_generic_freeze_late(dev); -} - static int pci_pm_freeze_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct device_driver *drv = dev->driver; - - if (dev_pm_smart_suspend_and_suspended(dev)) - return 0; + const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_suspend_late(dev, PMSG_FREEZE); + return pci_legacy_suspend_late(dev); - if (drv && drv->pm && drv->pm->freeze_noirq) { + if (pm && pm->freeze_noirq) { int error; - error = drv->pm->freeze_noirq(dev); - suspend_report_result(drv->pm->freeze_noirq, error); + error = pm->freeze_noirq(dev); + suspend_report_result(dev, pm->freeze_noirq, error); if (error) return error; } @@ -1012,49 +1088,33 @@ static int pci_pm_freeze_noirq(struct device *dev) pci_pm_set_unknown_state(pci_dev); - if (pcibios_pm_ops.freeze_noirq) - return pcibios_pm_ops.freeze_noirq(dev); - return 0; } static int pci_pm_thaw_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct device_driver *drv = dev->driver; - int error = 0; + const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; /* - * If the device is in runtime suspend, the code below may not work - * correctly with it, so skip that code and make the PM core skip all of - * the subsequent "thaw" callbacks for the device. + * The pm->thaw_noirq() callback assumes the device has been + * returned to D0 and its config state has been restored. + * + * In addition, pci_restore_state() restores MSI-X state in MMIO + * space, which requires the device to be in D0, so return it to D0 + * in case the driver's "freeze" callbacks put it into a low-power + * state. */ - if (dev_pm_smart_suspend_and_suspended(dev)) { - dev_pm_skip_next_resume_phases(dev); - return 0; - } - - if (pcibios_pm_ops.thaw_noirq) { - error = pcibios_pm_ops.thaw_noirq(dev); - if (error) - return error; - } + pci_pm_power_up_and_verify_state(pci_dev); + pci_restore_state(pci_dev); if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_resume_early(dev); - - /* - * pci_restore_state() requires the device to be in D0 (because of MSI - * restoration among other things), so force it into D0 in case the - * driver's "freeze" callbacks put it into a low-power state directly. - */ - pci_set_power_state(pci_dev, PCI_D0); - pci_restore_state(pci_dev); + return 0; - if (drv && drv->pm && drv->pm->thaw_noirq) - error = drv->pm->thaw_noirq(dev); + if (pm && pm->thaw_noirq) + return pm->thaw_noirq(dev); - return error; + return 0; } static int pci_pm_thaw(struct device *dev) @@ -1073,8 +1133,6 @@ static int pci_pm_thaw(struct device *dev) pci_pm_reenable_device(pci_dev); } - pci_dev->state_saved = false; - return error; } @@ -1092,16 +1150,18 @@ static int pci_pm_poweroff(struct device *dev) } /* The reason to do that is the same as in pci_pm_suspend(). */ - if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) || - !pci_dev_keep_suspended(pci_dev)) + if (!dev_pm_smart_suspend(dev) || pci_dev_need_resume(pci_dev)) { pm_runtime_resume(dev); + pci_dev->state_saved = false; + } else { + pci_dev_adjust_pme(pci_dev); + } - pci_dev->state_saved = false; if (pm->poweroff) { int error; error = pm->poweroff(dev); - suspend_report_result(pm->poweroff, error); + suspend_report_result(dev, pm->poweroff, error); if (error) return error; } @@ -1111,7 +1171,7 @@ static int pci_pm_poweroff(struct device *dev) static int pci_pm_poweroff_late(struct device *dev) { - if (dev_pm_smart_suspend_and_suspended(dev)) + if (dev_pm_skip_suspend(dev)) return 0; pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev)); @@ -1122,24 +1182,24 @@ static int pci_pm_poweroff_late(struct device *dev) static int pci_pm_poweroff_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct device_driver *drv = dev->driver; + const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - if (dev_pm_smart_suspend_and_suspended(dev)) + if (dev_pm_skip_suspend(dev)) return 0; - if (pci_has_legacy_pm_support(to_pci_dev(dev))) - return pci_legacy_suspend_late(dev, PMSG_HIBERNATE); + if (pci_has_legacy_pm_support(pci_dev)) + return pci_legacy_suspend_late(dev); - if (!drv || !drv->pm) { + if (!pm) { pci_fixup_device(pci_fixup_suspend_late, pci_dev); return 0; } - if (drv->pm->poweroff_noirq) { + if (pm->poweroff_noirq) { int error; - error = drv->pm->poweroff_noirq(dev); - suspend_report_result(drv->pm->poweroff_noirq, error); + error = pm->poweroff_noirq(dev); + suspend_report_result(dev, pm->poweroff_noirq, error); if (error) return error; } @@ -1156,44 +1216,30 @@ static int pci_pm_poweroff_noirq(struct device *dev) pci_fixup_device(pci_fixup_suspend_late, pci_dev); - if (pcibios_pm_ops.poweroff_noirq) - return pcibios_pm_ops.poweroff_noirq(dev); - return 0; } static int pci_pm_restore_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct device_driver *drv = dev->driver; - int error = 0; - - /* This is analogous to the pci_pm_resume_noirq() case. */ - if (dev_pm_smart_suspend_and_suspended(dev)) - pm_runtime_set_active(dev); - - if (pcibios_pm_ops.restore_noirq) { - error = pcibios_pm_ops.restore_noirq(dev); - if (error) - return error; - } + const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; pci_pm_default_resume_early(pci_dev); + pci_fixup_device(pci_fixup_resume_early, pci_dev); if (pci_has_legacy_pm_support(pci_dev)) - return pci_legacy_resume_early(dev); + return 0; - if (drv && drv->pm && drv->pm->restore_noirq) - error = drv->pm->restore_noirq(dev); + if (pm && pm->restore_noirq) + return pm->restore_noirq(dev); - return error; + return 0; } static int pci_pm_restore(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int error = 0; /* * This is necessary for the hibernation error path in which restore is @@ -1209,18 +1255,17 @@ static int pci_pm_restore(struct device *dev) if (pm) { if (pm->restore) - error = pm->restore(dev); + return pm->restore(dev); } else { pci_pm_reenable_device(pci_dev); } - return error; + return 0; } #else /* !CONFIG_HIBERNATE_CALLBACKS */ #define pci_pm_freeze NULL -#define pci_pm_freeze_late NULL #define pci_pm_freeze_noirq NULL #define pci_pm_thaw NULL #define pci_pm_thaw_noirq NULL @@ -1241,6 +1286,8 @@ static int pci_pm_runtime_suspend(struct device *dev) pci_power_t prev = pci_dev->current_state; int error; + pci_suspend_ptm(pci_dev); + /* * If pci_dev->driver is not set (unbound), we leave the device in D0, * but it may go to D3cold when the bridge above it runtime suspends. @@ -1260,11 +1307,11 @@ static int pci_pm_runtime_suspend(struct device *dev) * log level. */ if (error == -EBUSY || error == -EAGAIN) { - dev_dbg(dev, "can't suspend now (%pf returned %d)\n", + pci_dbg(pci_dev, "can't suspend now (%ps returned %d)\n", pm->runtime_suspend, error); return error; } else if (error) { - dev_err(dev, "can't suspend (%pf returned %d)\n", + pci_err(pci_dev, "can't suspend (%ps returned %d)\n", pm->runtime_suspend, error); return error; } @@ -1275,9 +1322,9 @@ static int pci_pm_runtime_suspend(struct device *dev) if (pm && pm->runtime_suspend && !pci_dev->state_saved && pci_dev->current_state != PCI_D0 && pci_dev->current_state != PCI_UNKNOWN) { - WARN_ONCE(pci_dev->current_state != prev, - "PCI PM: State of device not saved by %pF\n", - pm->runtime_suspend); + pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev, + "PCI PM: State of device not saved by %pS\n", + pm->runtime_suspend); return 0; } @@ -1291,37 +1338,38 @@ static int pci_pm_runtime_suspend(struct device *dev) static int pci_pm_runtime_resume(struct device *dev) { - int rc = 0; struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; + pci_power_t prev_state = pci_dev->current_state; + int error = 0; /* * Restoring config space is necessary even if the device is not bound * to a driver because although we left it in D0, it may have gone to * D3cold when the bridge above it runtime suspended. */ - pci_restore_standard_config(pci_dev); + pci_pm_default_resume_early(pci_dev); + pci_resume_ptm(pci_dev); if (!pci_dev->driver) return 0; pci_fixup_device(pci_fixup_resume_early, pci_dev); - pci_enable_wake(pci_dev, PCI_D0, false); - pci_fixup_device(pci_fixup_resume, pci_dev); + pci_pm_default_resume(pci_dev); - if (pm && pm->runtime_resume) - rc = pm->runtime_resume(dev); + if (prev_state == PCI_D3cold) + pci_pm_bridge_power_up_actions(pci_dev); - pci_dev->runtime_d3cold = false; + if (pm && pm->runtime_resume) + error = pm->runtime_resume(dev); - return rc; + return error; } static int pci_pm_runtime_idle(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret = 0; /* * If pci_dev->driver is not set (unbound), the device should @@ -1330,13 +1378,10 @@ static int pci_pm_runtime_idle(struct device *dev) if (!pci_dev->driver) return 0; - if (!pm) - return -ENOSYS; - - if (pm->runtime_idle) - ret = pm->runtime_idle(dev); + if (pm && pm->runtime_idle) + return pm->runtime_idle(dev); - return ret; + return 0; } static const struct dev_pm_ops pci_dev_pm_ops = { @@ -1345,8 +1390,8 @@ static const struct dev_pm_ops pci_dev_pm_ops = { .suspend = pci_pm_suspend, .suspend_late = pci_pm_suspend_late, .resume = pci_pm_resume, + .resume_early = pci_pm_resume_early, .freeze = pci_pm_freeze, - .freeze_late = pci_pm_freeze_late, .thaw = pci_pm_thaw, .poweroff = pci_pm_poweroff, .poweroff_late = pci_pm_poweroff_late, @@ -1394,6 +1439,7 @@ int __pci_register_driver(struct pci_driver *drv, struct module *owner, drv->driver.owner = owner; drv->driver.mod_name = mod_name; drv->driver.groups = drv->groups; + drv->driver.dev_groups = drv->dev_groups; spin_lock_init(&drv->dynids.lock); INIT_LIST_HEAD(&drv->dynids.list); @@ -1433,14 +1479,15 @@ static struct pci_driver pci_compat_driver = { */ struct pci_driver *pci_dev_driver(const struct pci_dev *dev) { + int i; + if (dev->driver) return dev->driver; - else { - int i; - for (i = 0; i <= PCI_ROM_RESOURCE; i++) - if (dev->resource[i].flags & IORESOURCE_BUSY) - return &pci_compat_driver; - } + + for (i = 0; i <= PCI_ROM_RESOURCE; i++) + if (dev->resource[i].flags & IORESOURCE_BUSY) + return &pci_compat_driver; + return NULL; } EXPORT_SYMBOL(pci_dev_driver); @@ -1454,16 +1501,16 @@ EXPORT_SYMBOL(pci_dev_driver); * system is in its list of supported devices. Returns the matching * pci_device_id structure or %NULL if there is no match. */ -static int pci_bus_match(struct device *dev, struct device_driver *drv) +static int pci_bus_match(struct device *dev, const struct device_driver *drv) { struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *pci_drv; const struct pci_device_id *found_id; - if (!pci_dev->match_driver) + if (pci_dev_binding_disallowed(pci_dev)) return 0; - pci_drv = to_pci_driver(drv); + pci_drv = (struct pci_driver *)to_pci_driver(drv); found_id = pci_match_device(pci_drv, pci_dev); if (found_id) return 1; @@ -1505,9 +1552,9 @@ void pci_dev_put(struct pci_dev *dev) } EXPORT_SYMBOL(pci_dev_put); -static int pci_uevent(struct device *dev, struct kobj_uevent_env *env) +static int pci_uevent(const struct device *dev, struct kobj_uevent_env *env) { - struct pci_dev *pdev; + const struct pci_dev *pdev; if (!dev) return -ENODEV; @@ -1537,7 +1584,7 @@ static int pci_uevent(struct device *dev, struct kobj_uevent_env *env) return 0; } -#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH) +#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH) || defined(CONFIG_S390) /** * pci_uevent_ers - emit a uevent during recovery path of PCI device * @pdev: PCI device undergoing error recovery @@ -1551,6 +1598,7 @@ void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type) switch (err_type) { case PCI_ERS_RESULT_NONE: case PCI_ERS_RESULT_CAN_RECOVER: + case PCI_ERS_RESULT_NEED_RESET: envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY"; envp[idx++] = "DEVICE_ONLINE=0"; break; @@ -1587,6 +1635,7 @@ static int pci_bus_num_vf(struct device *dev) */ static int pci_dma_configure(struct device *dev) { + const struct device_driver *drv = READ_ONCE(dev->driver); struct device *bridge; int ret = 0; @@ -1602,30 +1651,61 @@ static int pci_dma_configure(struct device *dev) } pci_put_host_bridge_device(bridge); + + /* @drv may not be valid when we're called from the IOMMU layer */ + if (!ret && drv && !to_pci_driver(drv)->driver_managed_dma) { + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); + } + return ret; } -struct bus_type pci_bus_type = { +static void pci_dma_cleanup(struct device *dev) +{ + struct pci_driver *driver = to_pci_driver(dev->driver); + + if (!driver->driver_managed_dma) + iommu_device_unuse_default_domain(dev); +} + +/* + * pci_device_irq_get_affinity - get IRQ affinity mask for device + * @dev: ptr to dev structure + * @irq_vec: interrupt vector number + * + * Return the CPU affinity mask for @dev and @irq_vec. + */ +static const struct cpumask *pci_device_irq_get_affinity(struct device *dev, + unsigned int irq_vec) +{ + return pci_irq_get_affinity(to_pci_dev(dev), irq_vec); +} + +const struct bus_type pci_bus_type = { .name = "pci", .match = pci_bus_match, .uevent = pci_uevent, .probe = pci_device_probe, .remove = pci_device_remove, .shutdown = pci_device_shutdown, + .irq_get_affinity = pci_device_irq_get_affinity, .dev_groups = pci_dev_groups, .bus_groups = pci_bus_groups, .drv_groups = pci_drv_groups, .pm = PCI_PM_OPS_PTR, .num_vf = pci_bus_num_vf, .dma_configure = pci_dma_configure, + .dma_cleanup = pci_dma_cleanup, }; EXPORT_SYMBOL(pci_bus_type); #ifdef CONFIG_PCIEPORTBUS -static int pcie_port_bus_match(struct device *dev, struct device_driver *drv) +static int pcie_port_bus_match(struct device *dev, const struct device_driver *drv) { struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; + const struct pcie_port_service_driver *driver; if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type) return 0; @@ -1643,11 +1723,10 @@ static int pcie_port_bus_match(struct device *dev, struct device_driver *drv) return 1; } -struct bus_type pcie_port_bus_type = { +const struct bus_type pcie_port_bus_type = { .name = "pci_express", .match = pcie_port_bus_match, }; -EXPORT_SYMBOL_GPL(pcie_port_bus_type); #endif static int __init pci_driver_init(void) |