diff options
Diffstat (limited to 'drivers/base/power')
| -rw-r--r-- | drivers/base/power/Makefile | 2 | ||||
| -rw-r--r-- | drivers/base/power/clock_ops.c | 75 | ||||
| -rw-r--r-- | drivers/base/power/common.c | 233 | ||||
| -rw-r--r-- | drivers/base/power/domain.c | 3407 | ||||
| -rw-r--r-- | drivers/base/power/domain_governor.c | 411 | ||||
| -rw-r--r-- | drivers/base/power/generic_ops.c | 109 | ||||
| -rw-r--r-- | drivers/base/power/main.c | 881 | ||||
| -rw-r--r-- | drivers/base/power/power.h | 1 | ||||
| -rw-r--r-- | drivers/base/power/qos.c | 3 | ||||
| -rw-r--r-- | drivers/base/power/runtime-test.c | 249 | ||||
| -rw-r--r-- | drivers/base/power/runtime.c | 526 | ||||
| -rw-r--r-- | drivers/base/power/sysfs.c | 33 | ||||
| -rw-r--r-- | drivers/base/power/trace.c | 6 | ||||
| -rw-r--r-- | drivers/base/power/wakeirq.c | 91 | ||||
| -rw-r--r-- | drivers/base/power/wakeup.c | 60 | ||||
| -rw-r--r-- | drivers/base/power/wakeup_stats.c | 4 |
16 files changed, 1546 insertions, 4545 deletions
diff --git a/drivers/base/power/Makefile b/drivers/base/power/Makefile index 8fdd0073eeeb..2989e42d0161 100644 --- a/drivers/base/power/Makefile +++ b/drivers/base/power/Makefile @@ -2,8 +2,8 @@ obj-$(CONFIG_PM) += sysfs.o generic_ops.o common.o qos.o runtime.o wakeirq.o obj-$(CONFIG_PM_SLEEP) += main.o wakeup.o wakeup_stats.o obj-$(CONFIG_PM_TRACE_RTC) += trace.o -obj-$(CONFIG_PM_GENERIC_DOMAINS) += domain.o domain_governor.o obj-$(CONFIG_HAVE_CLK) += clock_ops.o obj-$(CONFIG_PM_QOS_KUNIT_TEST) += qos-test.o +obj-$(CONFIG_PM_RUNTIME_KUNIT_TEST) += runtime-test.o ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index 4110c19c08dc..b69bcb37c830 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -259,39 +259,6 @@ int pm_clk_add_clk(struct device *dev, struct clk *clk) } EXPORT_SYMBOL_GPL(pm_clk_add_clk); - -/** - * of_pm_clk_add_clk - Start using a device clock for power management. - * @dev: Device whose clock is going to be used for power management. - * @name: Name of clock that is going to be used for power management. - * - * Add the clock described in the 'clocks' device-tree node that matches - * with the 'name' provided, to the list of clocks used for the power - * management of @dev. On success, returns 0. Returns a negative error - * code if the clock is not found or cannot be added. - */ -int of_pm_clk_add_clk(struct device *dev, const char *name) -{ - struct clk *clk; - int ret; - - if (!dev || !dev->of_node || !name) - return -EINVAL; - - clk = of_clk_get_by_name(dev->of_node, name); - if (IS_ERR(clk)) - return PTR_ERR(clk); - - ret = pm_clk_add_clk(dev, clk); - if (ret) { - clk_put(clk); - return ret; - } - - return 0; -} -EXPORT_SYMBOL_GPL(of_pm_clk_add_clk); - /** * of_pm_clk_add_clks - Start using device clock(s) for power management. * @dev: Device whose clock(s) is going to be used for power management. @@ -377,46 +344,6 @@ static void __pm_clk_remove(struct pm_clock_entry *ce) } /** - * pm_clk_remove - Stop using a device clock for power management. - * @dev: Device whose clock should not be used for PM any more. - * @con_id: Connection ID of the clock. - * - * Remove the clock represented by @con_id from the list of clocks used for - * the power management of @dev. - */ -void pm_clk_remove(struct device *dev, const char *con_id) -{ - struct pm_subsys_data *psd = dev_to_psd(dev); - struct pm_clock_entry *ce; - - if (!psd) - return; - - pm_clk_list_lock(psd); - - list_for_each_entry(ce, &psd->clock_list, node) { - if (!con_id && !ce->con_id) - goto remove; - else if (!con_id || !ce->con_id) - continue; - else if (!strcmp(con_id, ce->con_id)) - goto remove; - } - - pm_clk_list_unlock(psd); - return; - - remove: - list_del(&ce->node); - if (ce->enabled_when_prepared) - psd->clock_op_might_sleep--; - pm_clk_list_unlock(psd); - - __pm_clk_remove(ce); -} -EXPORT_SYMBOL_GPL(pm_clk_remove); - -/** * pm_clk_remove_clk - Stop using a device clock for power management. * @dev: Device whose clock should not be used for PM any more. * @clk: Clock pointer @@ -793,7 +720,7 @@ static int pm_clk_notify(struct notifier_block *nb, * the remaining members of @clknb should be populated prior to calling this * routine. */ -void pm_clk_add_notifier(struct bus_type *bus, +void pm_clk_add_notifier(const struct bus_type *bus, struct pm_clk_notifier_block *clknb) { if (!bus || !clknb) diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index 72115917e0bd..6ecf9ce4a4e6 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -11,6 +11,7 @@ #include <linux/pm_clock.h> #include <linux/acpi.h> #include <linux/pm_domain.h> +#include <linux/pm_opp.h> #include "power.h" @@ -82,7 +83,7 @@ EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data); /** * dev_pm_domain_attach - Attach a device to its PM domain. * @dev: Device to attach. - * @power_on: Used to indicate whether we should power on the device. + * @flags: indicate whether we should power on/off the device on attach/detach * * The @dev may only be attached to a single PM domain. By iterating through * the available alternatives we try to find a valid PM domain for the device. @@ -99,17 +100,20 @@ EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data); * Returns 0 on successfully attached PM domain, or when it is found that the * device doesn't need a PM domain, else a negative error code. */ -int dev_pm_domain_attach(struct device *dev, bool power_on) +int dev_pm_domain_attach(struct device *dev, u32 flags) { int ret; if (dev->pm_domain) return 0; - ret = acpi_dev_pm_attach(dev, power_on); + ret = acpi_dev_pm_attach(dev, !!(flags & PD_FLAG_ATTACH_POWER_ON)); if (!ret) ret = genpd_dev_pm_attach(dev); + if (dev->pm_domain) + dev->power.detach_power_off = !!(flags & PD_FLAG_DETACH_POWER_OFF); + return ret < 0 ? ret : 0; } EXPORT_SYMBOL_GPL(dev_pm_domain_attach); @@ -168,6 +172,179 @@ struct device *dev_pm_domain_attach_by_name(struct device *dev, EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_name); /** + * dev_pm_domain_attach_list - Associate a device with its PM domains. + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * This function helps to attach a device to its multiple PM domains. The + * caller, which is typically a driver's probe function, may provide a list of + * names for the PM domains that we should try to attach the device to, but it + * may also provide an empty list, in case the attach should be done for all of + * the available PM domains. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. Note that, to detach the list of PM domains, the driver shall call + * dev_pm_domain_detach_list(), typically during the remove phase. + */ +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + struct device_node *np = dev->of_node; + struct dev_pm_domain_list *pds; + struct device *pd_dev = NULL; + int ret, i, num_pds = 0; + bool by_id = true; + size_t size; + u32 pd_flags = data ? data->pd_flags : 0; + u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 : + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME; + + if (dev->pm_domain) + return -EEXIST; + + /* For now this is limited to OF based platforms. */ + if (!np) + return 0; + + if (data && data->pd_names) { + num_pds = data->num_pd_names; + by_id = false; + } else { + num_pds = of_count_phandle_with_args(np, "power-domains", + "#power-domain-cells"); + } + + if (num_pds <= 0) + return 0; + + pds = kzalloc(sizeof(*pds), GFP_KERNEL); + if (!pds) + return -ENOMEM; + + size = sizeof(*pds->pd_devs) + sizeof(*pds->pd_links) + + sizeof(*pds->opp_tokens); + pds->pd_devs = kcalloc(num_pds, size, GFP_KERNEL); + if (!pds->pd_devs) { + ret = -ENOMEM; + goto free_pds; + } + pds->pd_links = (void *)(pds->pd_devs + num_pds); + pds->opp_tokens = (void *)(pds->pd_links + num_pds); + + if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON) + link_flags |= DL_FLAG_RPM_ACTIVE; + + for (i = 0; i < num_pds; i++) { + if (by_id) + pd_dev = dev_pm_domain_attach_by_id(dev, i); + else + pd_dev = dev_pm_domain_attach_by_name(dev, + data->pd_names[i]); + if (IS_ERR_OR_NULL(pd_dev)) { + ret = pd_dev ? PTR_ERR(pd_dev) : -ENODEV; + goto err_attach; + } + + if (pd_flags & PD_FLAG_REQUIRED_OPP) { + struct dev_pm_opp_config config = { + .required_dev = pd_dev, + .required_dev_index = i, + }; + + ret = dev_pm_opp_set_config(dev, &config); + if (ret < 0) + goto err_link; + + pds->opp_tokens[i] = ret; + } + + if (link_flags) { + struct device_link *link; + + link = device_link_add(dev, pd_dev, link_flags); + if (!link) { + ret = -ENODEV; + goto err_link; + } + + pds->pd_links[i] = link; + } + + pds->pd_devs[i] = pd_dev; + } + + pds->num_pds = num_pds; + *list = pds; + return num_pds; + +err_link: + dev_pm_opp_clear_config(pds->opp_tokens[i]); + dev_pm_domain_detach(pd_dev, true); +err_attach: + while (--i >= 0) { + dev_pm_opp_clear_config(pds->opp_tokens[i]); + if (pds->pd_links[i]) + device_link_del(pds->pd_links[i]); + dev_pm_domain_detach(pds->pd_devs[i], true); + } + kfree(pds->pd_devs); +free_pds: + kfree(pds); + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list); + +/** + * devm_pm_domain_detach_list - devres-enabled version of dev_pm_domain_detach_list. + * @_list: The list of PM domains to detach. + * + * This function reverse the actions from devm_pm_domain_attach_list(). + * it will be invoked during the remove phase from drivers implicitly if driver + * uses devm_pm_domain_attach_list() to attach the PM domains. + */ +static void devm_pm_domain_detach_list(void *_list) +{ + struct dev_pm_domain_list *list = _list; + + dev_pm_domain_detach_list(list); +} + +/** + * devm_pm_domain_attach_list - devres-enabled version of dev_pm_domain_attach_list + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * NOTE: this will also handle calling devm_pm_domain_detach_list() for + * you during remove phase. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. + */ +int devm_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + int ret, num_pds; + + num_pds = dev_pm_domain_attach_list(dev, data, list); + if (num_pds <= 0) + return num_pds; + + ret = devm_add_action_or_reset(dev, devm_pm_domain_detach_list, *list); + if (ret) + return ret; + + return num_pds; +} +EXPORT_SYMBOL_GPL(devm_pm_domain_attach_list); + +/** * dev_pm_domain_detach - Detach a device from its PM domain. * @dev: Device to detach. * @power_off: Used to indicate whether we should power off the device. @@ -188,6 +365,35 @@ void dev_pm_domain_detach(struct device *dev, bool power_off) EXPORT_SYMBOL_GPL(dev_pm_domain_detach); /** + * dev_pm_domain_detach_list - Detach a list of PM domains. + * @list: The list of PM domains to detach. + * + * This function reverse the actions from dev_pm_domain_attach_list(). + * Typically it should be invoked during the remove phase from drivers. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + */ +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) +{ + int i; + + if (!list) + return; + + for (i = 0; i < list->num_pds; i++) { + dev_pm_opp_clear_config(list->opp_tokens[i]); + if (list->pd_links[i]) + device_link_del(list->pd_links[i]); + dev_pm_domain_detach(list->pd_devs[i], true); + } + + kfree(list->pd_devs); + kfree(list); +} +EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list); + +/** * dev_pm_domain_start - Start the device through its PM domain. * @dev: Device to start. * @@ -228,3 +434,24 @@ void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd) device_pm_check_callbacks(dev); } EXPORT_SYMBOL_GPL(dev_pm_domain_set); + +/** + * dev_pm_domain_set_performance_state - Request a new performance state. + * @dev: The device to make the request for. + * @state: Target performance state for the device. + * + * This function should be called when a new performance state needs to be + * requested for a device that is attached to a PM domain. Note that, the + * support for performance scaling for PM domains is optional. + * + * Returns 0 on success and when performance scaling isn't supported, negative + * error code on failure. + */ +int dev_pm_domain_set_performance_state(struct device *dev, unsigned int state) +{ + if (dev->pm_domain && dev->pm_domain->set_performance_state) + return dev->pm_domain->set_performance_state(dev, state); + + return 0; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_set_performance_state); diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c deleted file mode 100644 index 6471b559230e..000000000000 --- a/drivers/base/power/domain.c +++ /dev/null @@ -1,3407 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * drivers/base/power/domain.c - Common code related to device power domains. - * - * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. - */ -#define pr_fmt(fmt) "PM: " fmt - -#include <linux/delay.h> -#include <linux/kernel.h> -#include <linux/io.h> -#include <linux/platform_device.h> -#include <linux/pm_opp.h> -#include <linux/pm_runtime.h> -#include <linux/pm_domain.h> -#include <linux/pm_qos.h> -#include <linux/pm_clock.h> -#include <linux/slab.h> -#include <linux/err.h> -#include <linux/sched.h> -#include <linux/suspend.h> -#include <linux/export.h> -#include <linux/cpu.h> -#include <linux/debugfs.h> - -#include "power.h" - -#define GENPD_RETRY_MAX_MS 250 /* Approximate */ - -#define GENPD_DEV_CALLBACK(genpd, type, callback, dev) \ -({ \ - type (*__routine)(struct device *__d); \ - type __ret = (type)0; \ - \ - __routine = genpd->dev_ops.callback; \ - if (__routine) { \ - __ret = __routine(dev); \ - } \ - __ret; \ -}) - -static LIST_HEAD(gpd_list); -static DEFINE_MUTEX(gpd_list_lock); - -struct genpd_lock_ops { - void (*lock)(struct generic_pm_domain *genpd); - void (*lock_nested)(struct generic_pm_domain *genpd, int depth); - int (*lock_interruptible)(struct generic_pm_domain *genpd); - void (*unlock)(struct generic_pm_domain *genpd); -}; - -static void genpd_lock_mtx(struct generic_pm_domain *genpd) -{ - mutex_lock(&genpd->mlock); -} - -static void genpd_lock_nested_mtx(struct generic_pm_domain *genpd, - int depth) -{ - mutex_lock_nested(&genpd->mlock, depth); -} - -static int genpd_lock_interruptible_mtx(struct generic_pm_domain *genpd) -{ - return mutex_lock_interruptible(&genpd->mlock); -} - -static void genpd_unlock_mtx(struct generic_pm_domain *genpd) -{ - return mutex_unlock(&genpd->mlock); -} - -static const struct genpd_lock_ops genpd_mtx_ops = { - .lock = genpd_lock_mtx, - .lock_nested = genpd_lock_nested_mtx, - .lock_interruptible = genpd_lock_interruptible_mtx, - .unlock = genpd_unlock_mtx, -}; - -static void genpd_lock_spin(struct generic_pm_domain *genpd) - __acquires(&genpd->slock) -{ - unsigned long flags; - - spin_lock_irqsave(&genpd->slock, flags); - genpd->lock_flags = flags; -} - -static void genpd_lock_nested_spin(struct generic_pm_domain *genpd, - int depth) - __acquires(&genpd->slock) -{ - unsigned long flags; - - spin_lock_irqsave_nested(&genpd->slock, flags, depth); - genpd->lock_flags = flags; -} - -static int genpd_lock_interruptible_spin(struct generic_pm_domain *genpd) - __acquires(&genpd->slock) -{ - unsigned long flags; - - spin_lock_irqsave(&genpd->slock, flags); - genpd->lock_flags = flags; - return 0; -} - -static void genpd_unlock_spin(struct generic_pm_domain *genpd) - __releases(&genpd->slock) -{ - spin_unlock_irqrestore(&genpd->slock, genpd->lock_flags); -} - -static const struct genpd_lock_ops genpd_spin_ops = { - .lock = genpd_lock_spin, - .lock_nested = genpd_lock_nested_spin, - .lock_interruptible = genpd_lock_interruptible_spin, - .unlock = genpd_unlock_spin, -}; - -#define genpd_lock(p) p->lock_ops->lock(p) -#define genpd_lock_nested(p, d) p->lock_ops->lock_nested(p, d) -#define genpd_lock_interruptible(p) p->lock_ops->lock_interruptible(p) -#define genpd_unlock(p) p->lock_ops->unlock(p) - -#define genpd_status_on(genpd) (genpd->status == GENPD_STATE_ON) -#define genpd_is_irq_safe(genpd) (genpd->flags & GENPD_FLAG_IRQ_SAFE) -#define genpd_is_always_on(genpd) (genpd->flags & GENPD_FLAG_ALWAYS_ON) -#define genpd_is_active_wakeup(genpd) (genpd->flags & GENPD_FLAG_ACTIVE_WAKEUP) -#define genpd_is_cpu_domain(genpd) (genpd->flags & GENPD_FLAG_CPU_DOMAIN) -#define genpd_is_rpm_always_on(genpd) (genpd->flags & GENPD_FLAG_RPM_ALWAYS_ON) - -static inline bool irq_safe_dev_in_sleep_domain(struct device *dev, - const struct generic_pm_domain *genpd) -{ - bool ret; - - ret = pm_runtime_is_irq_safe(dev) && !genpd_is_irq_safe(genpd); - - /* - * Warn once if an IRQ safe device is attached to a domain, which - * callbacks are allowed to sleep. This indicates a suboptimal - * configuration for PM, but it doesn't matter for an always on domain. - */ - if (genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd)) - return ret; - - if (ret) - dev_warn_once(dev, "PM domain %s will not be powered off\n", - genpd->name); - - return ret; -} - -static int genpd_runtime_suspend(struct device *dev); - -/* - * Get the generic PM domain for a particular struct device. - * This validates the struct device pointer, the PM domain pointer, - * and checks that the PM domain pointer is a real generic PM domain. - * Any failure results in NULL being returned. - */ -static struct generic_pm_domain *dev_to_genpd_safe(struct device *dev) -{ - if (IS_ERR_OR_NULL(dev) || IS_ERR_OR_NULL(dev->pm_domain)) - return NULL; - - /* A genpd's always have its ->runtime_suspend() callback assigned. */ - if (dev->pm_domain->ops.runtime_suspend == genpd_runtime_suspend) - return pd_to_genpd(dev->pm_domain); - - return NULL; -} - -/* - * This should only be used where we are certain that the pm_domain - * attached to the device is a genpd domain. - */ -static struct generic_pm_domain *dev_to_genpd(struct device *dev) -{ - if (IS_ERR_OR_NULL(dev->pm_domain)) - return ERR_PTR(-EINVAL); - - return pd_to_genpd(dev->pm_domain); -} - -static int genpd_stop_dev(const struct generic_pm_domain *genpd, - struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, stop, dev); -} - -static int genpd_start_dev(const struct generic_pm_domain *genpd, - struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, start, dev); -} - -static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd) -{ - bool ret = false; - - if (!WARN_ON(atomic_read(&genpd->sd_count) == 0)) - ret = !!atomic_dec_and_test(&genpd->sd_count); - - return ret; -} - -static void genpd_sd_counter_inc(struct generic_pm_domain *genpd) -{ - atomic_inc(&genpd->sd_count); - smp_mb__after_atomic(); -} - -#ifdef CONFIG_DEBUG_FS -static struct dentry *genpd_debugfs_dir; - -static void genpd_debug_add(struct generic_pm_domain *genpd); - -static void genpd_debug_remove(struct generic_pm_domain *genpd) -{ - struct dentry *d; - - if (!genpd_debugfs_dir) - return; - - d = debugfs_lookup(genpd->name, genpd_debugfs_dir); - debugfs_remove(d); -} - -static void genpd_update_accounting(struct generic_pm_domain *genpd) -{ - u64 delta, now; - - now = ktime_get_mono_fast_ns(); - if (now <= genpd->accounting_time) - return; - - delta = now - genpd->accounting_time; - - /* - * If genpd->status is active, it means we are just - * out of off and so update the idle time and vice - * versa. - */ - if (genpd->status == GENPD_STATE_ON) - genpd->states[genpd->state_idx].idle_time += delta; - else - genpd->on_time += delta; - - genpd->accounting_time = now; -} -#else -static inline void genpd_debug_add(struct generic_pm_domain *genpd) {} -static inline void genpd_debug_remove(struct generic_pm_domain *genpd) {} -static inline void genpd_update_accounting(struct generic_pm_domain *genpd) {} -#endif - -static int _genpd_reeval_performance_state(struct generic_pm_domain *genpd, - unsigned int state) -{ - struct generic_pm_domain_data *pd_data; - struct pm_domain_data *pdd; - struct gpd_link *link; - - /* New requested state is same as Max requested state */ - if (state == genpd->performance_state) - return state; - - /* New requested state is higher than Max requested state */ - if (state > genpd->performance_state) - return state; - - /* Traverse all devices within the domain */ - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - pd_data = to_gpd_data(pdd); - - if (pd_data->performance_state > state) - state = pd_data->performance_state; - } - - /* - * Traverse all sub-domains within the domain. This can be - * done without any additional locking as the link->performance_state - * field is protected by the parent genpd->lock, which is already taken. - * - * Also note that link->performance_state (subdomain's performance state - * requirement to parent domain) is different from - * link->child->performance_state (current performance state requirement - * of the devices/sub-domains of the subdomain) and so can have a - * different value. - * - * Note that we also take vote from powered-off sub-domains into account - * as the same is done for devices right now. - */ - list_for_each_entry(link, &genpd->parent_links, parent_node) { - if (link->performance_state > state) - state = link->performance_state; - } - - return state; -} - -static int genpd_xlate_performance_state(struct generic_pm_domain *genpd, - struct generic_pm_domain *parent, - unsigned int pstate) -{ - if (!parent->set_performance_state) - return pstate; - - return dev_pm_opp_xlate_performance_state(genpd->opp_table, - parent->opp_table, - pstate); -} - -static int _genpd_set_performance_state(struct generic_pm_domain *genpd, - unsigned int state, int depth) -{ - struct generic_pm_domain *parent; - struct gpd_link *link; - int parent_state, ret; - - if (state == genpd->performance_state) - return 0; - - /* Propagate to parents of genpd */ - list_for_each_entry(link, &genpd->child_links, child_node) { - parent = link->parent; - - /* Find parent's performance state */ - ret = genpd_xlate_performance_state(genpd, parent, state); - if (unlikely(ret < 0)) - goto err; - - parent_state = ret; - - genpd_lock_nested(parent, depth + 1); - - link->prev_performance_state = link->performance_state; - link->performance_state = parent_state; - parent_state = _genpd_reeval_performance_state(parent, - parent_state); - ret = _genpd_set_performance_state(parent, parent_state, depth + 1); - if (ret) - link->performance_state = link->prev_performance_state; - - genpd_unlock(parent); - - if (ret) - goto err; - } - - if (genpd->set_performance_state) { - ret = genpd->set_performance_state(genpd, state); - if (ret) - goto err; - } - - genpd->performance_state = state; - return 0; - -err: - /* Encountered an error, lets rollback */ - list_for_each_entry_continue_reverse(link, &genpd->child_links, - child_node) { - parent = link->parent; - - genpd_lock_nested(parent, depth + 1); - - parent_state = link->prev_performance_state; - link->performance_state = parent_state; - - parent_state = _genpd_reeval_performance_state(parent, - parent_state); - if (_genpd_set_performance_state(parent, parent_state, depth + 1)) { - pr_err("%s: Failed to roll back to %d performance state\n", - parent->name, parent_state); - } - - genpd_unlock(parent); - } - - return ret; -} - -static int genpd_set_performance_state(struct device *dev, unsigned int state) -{ - struct generic_pm_domain *genpd = dev_to_genpd(dev); - struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); - unsigned int prev_state; - int ret; - - prev_state = gpd_data->performance_state; - if (prev_state == state) - return 0; - - gpd_data->performance_state = state; - state = _genpd_reeval_performance_state(genpd, state); - - ret = _genpd_set_performance_state(genpd, state, 0); - if (ret) - gpd_data->performance_state = prev_state; - - return ret; -} - -static int genpd_drop_performance_state(struct device *dev) -{ - unsigned int prev_state = dev_gpd_data(dev)->performance_state; - - if (!genpd_set_performance_state(dev, 0)) - return prev_state; - - return 0; -} - -static void genpd_restore_performance_state(struct device *dev, - unsigned int state) -{ - if (state) - genpd_set_performance_state(dev, state); -} - -/** - * dev_pm_genpd_set_performance_state- Set performance state of device's power - * domain. - * - * @dev: Device for which the performance-state needs to be set. - * @state: Target performance state of the device. This can be set as 0 when the - * device doesn't have any performance state constraints left (And so - * the device wouldn't participate anymore to find the target - * performance state of the genpd). - * - * It is assumed that the users guarantee that the genpd wouldn't be detached - * while this routine is getting called. - * - * Returns 0 on success and negative error values on failures. - */ -int dev_pm_genpd_set_performance_state(struct device *dev, unsigned int state) -{ - struct generic_pm_domain *genpd; - int ret = 0; - - genpd = dev_to_genpd_safe(dev); - if (!genpd) - return -ENODEV; - - if (WARN_ON(!dev->power.subsys_data || - !dev->power.subsys_data->domain_data)) - return -EINVAL; - - genpd_lock(genpd); - if (pm_runtime_suspended(dev)) { - dev_gpd_data(dev)->rpm_pstate = state; - } else { - ret = genpd_set_performance_state(dev, state); - if (!ret) - dev_gpd_data(dev)->rpm_pstate = 0; - } - genpd_unlock(genpd); - - return ret; -} -EXPORT_SYMBOL_GPL(dev_pm_genpd_set_performance_state); - -/** - * dev_pm_genpd_set_next_wakeup - Notify PM framework of an impending wakeup. - * - * @dev: Device to handle - * @next: impending interrupt/wakeup for the device - * - * - * Allow devices to inform of the next wakeup. It's assumed that the users - * guarantee that the genpd wouldn't be detached while this routine is getting - * called. Additionally, it's also assumed that @dev isn't runtime suspended - * (RPM_SUSPENDED)." - * Although devices are expected to update the next_wakeup after the end of - * their usecase as well, it is possible the devices themselves may not know - * about that, so stale @next will be ignored when powering off the domain. - */ -void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next) -{ - struct generic_pm_domain *genpd; - struct gpd_timing_data *td; - - genpd = dev_to_genpd_safe(dev); - if (!genpd) - return; - - td = to_gpd_data(dev->power.subsys_data->domain_data)->td; - if (td) - td->next_wakeup = next; -} -EXPORT_SYMBOL_GPL(dev_pm_genpd_set_next_wakeup); - -static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed) -{ - unsigned int state_idx = genpd->state_idx; - ktime_t time_start; - s64 elapsed_ns; - int ret; - - /* Notify consumers that we are about to power on. */ - ret = raw_notifier_call_chain_robust(&genpd->power_notifiers, - GENPD_NOTIFY_PRE_ON, - GENPD_NOTIFY_OFF, NULL); - ret = notifier_to_errno(ret); - if (ret) - return ret; - - if (!genpd->power_on) - goto out; - - timed = timed && genpd->gd && !genpd->states[state_idx].fwnode; - if (!timed) { - ret = genpd->power_on(genpd); - if (ret) - goto err; - - goto out; - } - - time_start = ktime_get(); - ret = genpd->power_on(genpd); - if (ret) - goto err; - - elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); - if (elapsed_ns <= genpd->states[state_idx].power_on_latency_ns) - goto out; - - genpd->states[state_idx].power_on_latency_ns = elapsed_ns; - genpd->gd->max_off_time_changed = true; - pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n", - genpd->name, "on", elapsed_ns); - -out: - raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_ON, NULL); - return 0; -err: - raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_OFF, - NULL); - return ret; -} - -static int _genpd_power_off(struct generic_pm_domain *genpd, bool timed) -{ - unsigned int state_idx = genpd->state_idx; - ktime_t time_start; - s64 elapsed_ns; - int ret; - - /* Notify consumers that we are about to power off. */ - ret = raw_notifier_call_chain_robust(&genpd->power_notifiers, - GENPD_NOTIFY_PRE_OFF, - GENPD_NOTIFY_ON, NULL); - ret = notifier_to_errno(ret); - if (ret) - return ret; - - if (!genpd->power_off) - goto out; - - timed = timed && genpd->gd && !genpd->states[state_idx].fwnode; - if (!timed) { - ret = genpd->power_off(genpd); - if (ret) - goto busy; - - goto out; - } - - time_start = ktime_get(); - ret = genpd->power_off(genpd); - if (ret) - goto busy; - - elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); - if (elapsed_ns <= genpd->states[state_idx].power_off_latency_ns) - goto out; - - genpd->states[state_idx].power_off_latency_ns = elapsed_ns; - genpd->gd->max_off_time_changed = true; - pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n", - genpd->name, "off", elapsed_ns); - -out: - raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_OFF, - NULL); - return 0; -busy: - raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_ON, NULL); - return ret; -} - -/** - * genpd_queue_power_off_work - Queue up the execution of genpd_power_off(). - * @genpd: PM domain to power off. - * - * Queue up the execution of genpd_power_off() unless it's already been done - * before. - */ -static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) -{ - queue_work(pm_wq, &genpd->power_off_work); -} - -/** - * genpd_power_off - Remove power from a given PM domain. - * @genpd: PM domain to power down. - * @one_dev_on: If invoked from genpd's ->runtime_suspend|resume() callback, the - * RPM status of the releated device is in an intermediate state, not yet turned - * into RPM_SUSPENDED. This means genpd_power_off() must allow one device to not - * be RPM_SUSPENDED, while it tries to power off the PM domain. - * @depth: nesting count for lockdep. - * - * If all of the @genpd's devices have been suspended and all of its subdomains - * have been powered down, remove power from @genpd. - */ -static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, - unsigned int depth) -{ - struct pm_domain_data *pdd; - struct gpd_link *link; - unsigned int not_suspended = 0; - int ret; - - /* - * Do not try to power off the domain in the following situations: - * (1) The domain is already in the "power off" state. - * (2) System suspend is in progress. - */ - if (!genpd_status_on(genpd) || genpd->prepared_count > 0) - return 0; - - /* - * Abort power off for the PM domain in the following situations: - * (1) The domain is configured as always on. - * (2) When the domain has a subdomain being powered on. - */ - if (genpd_is_always_on(genpd) || - genpd_is_rpm_always_on(genpd) || - atomic_read(&genpd->sd_count) > 0) - return -EBUSY; - - /* - * The children must be in their deepest (powered-off) states to allow - * the parent to be powered off. Note that, there's no need for - * additional locking, as powering on a child, requires the parent's - * lock to be acquired first. - */ - list_for_each_entry(link, &genpd->parent_links, parent_node) { - struct generic_pm_domain *child = link->child; - if (child->state_idx < child->state_count - 1) - return -EBUSY; - } - - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - /* - * Do not allow PM domain to be powered off, when an IRQ safe - * device is part of a non-IRQ safe domain. - */ - if (!pm_runtime_suspended(pdd->dev) || - irq_safe_dev_in_sleep_domain(pdd->dev, genpd)) - not_suspended++; - } - - if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on)) - return -EBUSY; - - if (genpd->gov && genpd->gov->power_down_ok) { - if (!genpd->gov->power_down_ok(&genpd->domain)) - return -EAGAIN; - } - - /* Default to shallowest state. */ - if (!genpd->gov) - genpd->state_idx = 0; - - /* Don't power off, if a child domain is waiting to power on. */ - if (atomic_read(&genpd->sd_count) > 0) - return -EBUSY; - - ret = _genpd_power_off(genpd, true); - if (ret) { - genpd->states[genpd->state_idx].rejected++; - return ret; - } - - genpd->status = GENPD_STATE_OFF; - genpd_update_accounting(genpd); - genpd->states[genpd->state_idx].usage++; - - list_for_each_entry(link, &genpd->child_links, child_node) { - genpd_sd_counter_dec(link->parent); - genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); - genpd_unlock(link->parent); - } - - return 0; -} - -/** - * genpd_power_on - Restore power to a given PM domain and its parents. - * @genpd: PM domain to power up. - * @depth: nesting count for lockdep. - * - * Restore power to @genpd and all of its parents so that it is possible to - * resume a device belonging to it. - */ -static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth) -{ - struct gpd_link *link; - int ret = 0; - - if (genpd_status_on(genpd)) - return 0; - - /* - * The list is guaranteed not to change while the loop below is being - * executed, unless one of the parents' .power_on() callbacks fiddles - * with it. - */ - list_for_each_entry(link, &genpd->child_links, child_node) { - struct generic_pm_domain *parent = link->parent; - - genpd_sd_counter_inc(parent); - - genpd_lock_nested(parent, depth + 1); - ret = genpd_power_on(parent, depth + 1); - genpd_unlock(parent); - - if (ret) { - genpd_sd_counter_dec(parent); - goto err; - } - } - - ret = _genpd_power_on(genpd, true); - if (ret) - goto err; - - genpd->status = GENPD_STATE_ON; - genpd_update_accounting(genpd); - - return 0; - - err: - list_for_each_entry_continue_reverse(link, - &genpd->child_links, - child_node) { - genpd_sd_counter_dec(link->parent); - genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); - genpd_unlock(link->parent); - } - - return ret; -} - -static int genpd_dev_pm_start(struct device *dev) -{ - struct generic_pm_domain *genpd = dev_to_genpd(dev); - - return genpd_start_dev(genpd, dev); -} - -static int genpd_dev_pm_qos_notifier(struct notifier_block *nb, - unsigned long val, void *ptr) -{ - struct generic_pm_domain_data *gpd_data; - struct device *dev; - - gpd_data = container_of(nb, struct generic_pm_domain_data, nb); - dev = gpd_data->base.dev; - - for (;;) { - struct generic_pm_domain *genpd = ERR_PTR(-ENODATA); - struct pm_domain_data *pdd; - struct gpd_timing_data *td; - - spin_lock_irq(&dev->power.lock); - - pdd = dev->power.subsys_data ? - dev->power.subsys_data->domain_data : NULL; - if (pdd) { - td = to_gpd_data(pdd)->td; - if (td) { - td->constraint_changed = true; - genpd = dev_to_genpd(dev); - } - } - - spin_unlock_irq(&dev->power.lock); - - if (!IS_ERR(genpd)) { - genpd_lock(genpd); - genpd->gd->max_off_time_changed = true; - genpd_unlock(genpd); - } - - dev = dev->parent; - if (!dev || dev->power.ignore_children) - break; - } - - return NOTIFY_DONE; -} - -/** - * genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0. - * @work: Work structure used for scheduling the execution of this function. - */ -static void genpd_power_off_work_fn(struct work_struct *work) -{ - struct generic_pm_domain *genpd; - - genpd = container_of(work, struct generic_pm_domain, power_off_work); - - genpd_lock(genpd); - genpd_power_off(genpd, false, 0); - genpd_unlock(genpd); -} - -/** - * __genpd_runtime_suspend - walk the hierarchy of ->runtime_suspend() callbacks - * @dev: Device to handle. - */ -static int __genpd_runtime_suspend(struct device *dev) -{ - int (*cb)(struct device *__dev); - - if (dev->type && dev->type->pm) - cb = dev->type->pm->runtime_suspend; - else if (dev->class && dev->class->pm) - cb = dev->class->pm->runtime_suspend; - else if (dev->bus && dev->bus->pm) - cb = dev->bus->pm->runtime_suspend; - else - cb = NULL; - - if (!cb && dev->driver && dev->driver->pm) - cb = dev->driver->pm->runtime_suspend; - - return cb ? cb(dev) : 0; -} - -/** - * __genpd_runtime_resume - walk the hierarchy of ->runtime_resume() callbacks - * @dev: Device to handle. - */ -static int __genpd_runtime_resume(struct device *dev) -{ - int (*cb)(struct device *__dev); - - if (dev->type && dev->type->pm) - cb = dev->type->pm->runtime_resume; - else if (dev->class && dev->class->pm) - cb = dev->class->pm->runtime_resume; - else if (dev->bus && dev->bus->pm) - cb = dev->bus->pm->runtime_resume; - else - cb = NULL; - - if (!cb && dev->driver && dev->driver->pm) - cb = dev->driver->pm->runtime_resume; - - return cb ? cb(dev) : 0; -} - -/** - * genpd_runtime_suspend - Suspend a device belonging to I/O PM domain. - * @dev: Device to suspend. - * - * Carry out a runtime suspend of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int genpd_runtime_suspend(struct device *dev) -{ - struct generic_pm_domain *genpd; - bool (*suspend_ok)(struct device *__dev); - struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); - struct gpd_timing_data *td = gpd_data->td; - bool runtime_pm = pm_runtime_enabled(dev); - ktime_t time_start = 0; - s64 elapsed_ns; - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - /* - * A runtime PM centric subsystem/driver may re-use the runtime PM - * callbacks for other purposes than runtime PM. In those scenarios - * runtime PM is disabled. Under these circumstances, we shall skip - * validating/measuring the PM QoS latency. - */ - suspend_ok = genpd->gov ? genpd->gov->suspend_ok : NULL; - if (runtime_pm && suspend_ok && !suspend_ok(dev)) - return -EBUSY; - - /* Measure suspend latency. */ - if (td && runtime_pm) - time_start = ktime_get(); - - ret = __genpd_runtime_suspend(dev); - if (ret) - return ret; - - ret = genpd_stop_dev(genpd, dev); - if (ret) { - __genpd_runtime_resume(dev); - return ret; - } - - /* Update suspend latency value if the measured time exceeds it. */ - if (td && runtime_pm) { - elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); - if (elapsed_ns > td->suspend_latency_ns) { - td->suspend_latency_ns = elapsed_ns; - dev_dbg(dev, "suspend latency exceeded, %lld ns\n", - elapsed_ns); - genpd->gd->max_off_time_changed = true; - td->constraint_changed = true; - } - } - - /* - * If power.irq_safe is set, this routine may be run with - * IRQs disabled, so suspend only if the PM domain also is irq_safe. - */ - if (irq_safe_dev_in_sleep_domain(dev, genpd)) - return 0; - - genpd_lock(genpd); - gpd_data->rpm_pstate = genpd_drop_performance_state(dev); - genpd_power_off(genpd, true, 0); - genpd_unlock(genpd); - - return 0; -} - -/** - * genpd_runtime_resume - Resume a device belonging to I/O PM domain. - * @dev: Device to resume. - * - * Carry out a runtime resume of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int genpd_runtime_resume(struct device *dev) -{ - struct generic_pm_domain *genpd; - struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); - struct gpd_timing_data *td = gpd_data->td; - bool timed = td && pm_runtime_enabled(dev); - ktime_t time_start = 0; - s64 elapsed_ns; - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - /* - * As we don't power off a non IRQ safe domain, which holds - * an IRQ safe device, we don't need to restore power to it. - */ - if (irq_safe_dev_in_sleep_domain(dev, genpd)) - goto out; - - genpd_lock(genpd); - ret = genpd_power_on(genpd, 0); - if (!ret) - genpd_restore_performance_state(dev, gpd_data->rpm_pstate); - genpd_unlock(genpd); - - if (ret) - return ret; - - out: - /* Measure resume latency. */ - if (timed) - time_start = ktime_get(); - - ret = genpd_start_dev(genpd, dev); - if (ret) - goto err_poweroff; - - ret = __genpd_runtime_resume(dev); - if (ret) - goto err_stop; - - /* Update resume latency value if the measured time exceeds it. */ - if (timed) { - elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); - if (elapsed_ns > td->resume_latency_ns) { - td->resume_latency_ns = elapsed_ns; - dev_dbg(dev, "resume latency exceeded, %lld ns\n", - elapsed_ns); - genpd->gd->max_off_time_changed = true; - td->constraint_changed = true; - } - } - - return 0; - -err_stop: - genpd_stop_dev(genpd, dev); -err_poweroff: - if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) { - genpd_lock(genpd); - gpd_data->rpm_pstate = genpd_drop_performance_state(dev); - genpd_power_off(genpd, true, 0); - genpd_unlock(genpd); - } - - return ret; -} - -static bool pd_ignore_unused; -static int __init pd_ignore_unused_setup(char *__unused) -{ - pd_ignore_unused = true; - return 1; -} -__setup("pd_ignore_unused", pd_ignore_unused_setup); - -/** - * genpd_power_off_unused - Power off all PM domains with no devices in use. - */ -static int __init genpd_power_off_unused(void) -{ - struct generic_pm_domain *genpd; - - if (pd_ignore_unused) { - pr_warn("genpd: Not disabling unused power domains\n"); - return 0; - } - - mutex_lock(&gpd_list_lock); - - list_for_each_entry(genpd, &gpd_list, gpd_list_node) - genpd_queue_power_off_work(genpd); - - mutex_unlock(&gpd_list_lock); - - return 0; -} -late_initcall(genpd_power_off_unused); - -#ifdef CONFIG_PM_SLEEP - -/** - * genpd_sync_power_off - Synchronously power off a PM domain and its parents. - * @genpd: PM domain to power off, if possible. - * @use_lock: use the lock. - * @depth: nesting count for lockdep. - * - * Check if the given PM domain can be powered off (during system suspend or - * hibernation) and do that if so. Also, in that case propagate to its parents. - * - * This function is only called in "noirq" and "syscore" stages of system power - * transitions. The "noirq" callbacks may be executed asynchronously, thus in - * these cases the lock must be held. - */ -static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, - unsigned int depth) -{ - struct gpd_link *link; - - if (!genpd_status_on(genpd) || genpd_is_always_on(genpd)) - return; - - if (genpd->suspended_count != genpd->device_count - || atomic_read(&genpd->sd_count) > 0) - return; - - /* Check that the children are in their deepest (powered-off) state. */ - list_for_each_entry(link, &genpd->parent_links, parent_node) { - struct generic_pm_domain *child = link->child; - if (child->state_idx < child->state_count - 1) - return; - } - - /* Choose the deepest state when suspending */ - genpd->state_idx = genpd->state_count - 1; - if (_genpd_power_off(genpd, false)) - return; - - genpd->status = GENPD_STATE_OFF; - - list_for_each_entry(link, &genpd->child_links, child_node) { - genpd_sd_counter_dec(link->parent); - - if (use_lock) - genpd_lock_nested(link->parent, depth + 1); - - genpd_sync_power_off(link->parent, use_lock, depth + 1); - - if (use_lock) - genpd_unlock(link->parent); - } -} - -/** - * genpd_sync_power_on - Synchronously power on a PM domain and its parents. - * @genpd: PM domain to power on. - * @use_lock: use the lock. - * @depth: nesting count for lockdep. - * - * This function is only called in "noirq" and "syscore" stages of system power - * transitions. The "noirq" callbacks may be executed asynchronously, thus in - * these cases the lock must be held. - */ -static void genpd_sync_power_on(struct generic_pm_domain *genpd, bool use_lock, - unsigned int depth) -{ - struct gpd_link *link; - - if (genpd_status_on(genpd)) - return; - - list_for_each_entry(link, &genpd->child_links, child_node) { - genpd_sd_counter_inc(link->parent); - - if (use_lock) - genpd_lock_nested(link->parent, depth + 1); - - genpd_sync_power_on(link->parent, use_lock, depth + 1); - - if (use_lock) - genpd_unlock(link->parent); - } - - _genpd_power_on(genpd, false); - genpd->status = GENPD_STATE_ON; -} - -/** - * genpd_prepare - Start power transition of a device in a PM domain. - * @dev: Device to start the transition of. - * - * Start a power transition of a device (during a system-wide power transition) - * under the assumption that its pm_domain field points to the domain member of - * an object of type struct generic_pm_domain representing a PM domain - * consisting of I/O devices. - */ -static int genpd_prepare(struct device *dev) -{ - struct generic_pm_domain *genpd; - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - genpd_lock(genpd); - - if (genpd->prepared_count++ == 0) - genpd->suspended_count = 0; - - genpd_unlock(genpd); - - ret = pm_generic_prepare(dev); - if (ret < 0) { - genpd_lock(genpd); - - genpd->prepared_count--; - - genpd_unlock(genpd); - } - - /* Never return 1, as genpd don't cope with the direct_complete path. */ - return ret >= 0 ? 0 : ret; -} - -/** - * genpd_finish_suspend - Completion of suspend or hibernation of device in an - * I/O pm domain. - * @dev: Device to suspend. - * @poweroff: Specifies if this is a poweroff_noirq or suspend_noirq callback. - * - * Stop the device and remove power from the domain if all devices in it have - * been stopped. - */ -static int genpd_finish_suspend(struct device *dev, bool poweroff) -{ - struct generic_pm_domain *genpd; - int ret = 0; - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - if (poweroff) - ret = pm_generic_poweroff_noirq(dev); - else - ret = pm_generic_suspend_noirq(dev); - if (ret) - return ret; - - if (device_wakeup_path(dev) && genpd_is_active_wakeup(genpd)) - return 0; - - if (genpd->dev_ops.stop && genpd->dev_ops.start && - !pm_runtime_status_suspended(dev)) { - ret = genpd_stop_dev(genpd, dev); - if (ret) { - if (poweroff) - pm_generic_restore_noirq(dev); - else - pm_generic_resume_noirq(dev); - return ret; - } - } - - genpd_lock(genpd); - genpd->suspended_count++; - genpd_sync_power_off(genpd, true, 0); - genpd_unlock(genpd); - - return 0; -} - -/** - * genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain. - * @dev: Device to suspend. - * - * Stop the device and remove power from the domain if all devices in it have - * been stopped. - */ -static int genpd_suspend_noirq(struct device *dev) -{ - dev_dbg(dev, "%s()\n", __func__); - - return genpd_finish_suspend(dev, false); -} - -/** - * genpd_resume_noirq - Start of resume of device in an I/O PM domain. - * @dev: Device to resume. - * - * Restore power to the device's PM domain, if necessary, and start the device. - */ -static int genpd_resume_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - if (device_wakeup_path(dev) && genpd_is_active_wakeup(genpd)) - return pm_generic_resume_noirq(dev); - - genpd_lock(genpd); - genpd_sync_power_on(genpd, true, 0); - genpd->suspended_count--; - genpd_unlock(genpd); - - if (genpd->dev_ops.stop && genpd->dev_ops.start && - !pm_runtime_status_suspended(dev)) { - ret = genpd_start_dev(genpd, dev); - if (ret) - return ret; - } - - return pm_generic_resume_noirq(dev); -} - -/** - * genpd_freeze_noirq - Completion of freezing a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Carry out a late freeze of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int genpd_freeze_noirq(struct device *dev) -{ - const struct generic_pm_domain *genpd; - int ret = 0; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - ret = pm_generic_freeze_noirq(dev); - if (ret) - return ret; - - if (genpd->dev_ops.stop && genpd->dev_ops.start && - !pm_runtime_status_suspended(dev)) - ret = genpd_stop_dev(genpd, dev); - - return ret; -} - -/** - * genpd_thaw_noirq - Early thaw of device in an I/O PM domain. - * @dev: Device to thaw. - * - * Start the device, unless power has been removed from the domain already - * before the system transition. - */ -static int genpd_thaw_noirq(struct device *dev) -{ - const struct generic_pm_domain *genpd; - int ret = 0; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - if (genpd->dev_ops.stop && genpd->dev_ops.start && - !pm_runtime_status_suspended(dev)) { - ret = genpd_start_dev(genpd, dev); - if (ret) - return ret; - } - - return pm_generic_thaw_noirq(dev); -} - -/** - * genpd_poweroff_noirq - Completion of hibernation of device in an - * I/O PM domain. - * @dev: Device to poweroff. - * - * Stop the device and remove power from the domain if all devices in it have - * been stopped. - */ -static int genpd_poweroff_noirq(struct device *dev) -{ - dev_dbg(dev, "%s()\n", __func__); - - return genpd_finish_suspend(dev, true); -} - -/** - * genpd_restore_noirq - Start of restore of device in an I/O PM domain. - * @dev: Device to resume. - * - * Make sure the domain will be in the same power state as before the - * hibernation the system is resuming from and start the device if necessary. - */ -static int genpd_restore_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - int ret = 0; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - /* - * At this point suspended_count == 0 means we are being run for the - * first time for the given domain in the present cycle. - */ - genpd_lock(genpd); - if (genpd->suspended_count++ == 0) { - /* - * The boot kernel might put the domain into arbitrary state, - * so make it appear as powered off to genpd_sync_power_on(), - * so that it tries to power it on in case it was really off. - */ - genpd->status = GENPD_STATE_OFF; - } - - genpd_sync_power_on(genpd, true, 0); - genpd_unlock(genpd); - - if (genpd->dev_ops.stop && genpd->dev_ops.start && - !pm_runtime_status_suspended(dev)) { - ret = genpd_start_dev(genpd, dev); - if (ret) - return ret; - } - - return pm_generic_restore_noirq(dev); -} - -/** - * genpd_complete - Complete power transition of a device in a power domain. - * @dev: Device to complete the transition of. - * - * Complete a power transition of a device (during a system-wide power - * transition) under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static void genpd_complete(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return; - - pm_generic_complete(dev); - - genpd_lock(genpd); - - genpd->prepared_count--; - if (!genpd->prepared_count) - genpd_queue_power_off_work(genpd); - - genpd_unlock(genpd); -} - -static void genpd_switch_state(struct device *dev, bool suspend) -{ - struct generic_pm_domain *genpd; - bool use_lock; - - genpd = dev_to_genpd_safe(dev); - if (!genpd) - return; - - use_lock = genpd_is_irq_safe(genpd); - - if (use_lock) - genpd_lock(genpd); - - if (suspend) { - genpd->suspended_count++; - genpd_sync_power_off(genpd, use_lock, 0); - } else { - genpd_sync_power_on(genpd, use_lock, 0); - genpd->suspended_count--; - } - - if (use_lock) - genpd_unlock(genpd); -} - -/** - * dev_pm_genpd_suspend - Synchronously try to suspend the genpd for @dev - * @dev: The device that is attached to the genpd, that can be suspended. - * - * This routine should typically be called for a device that needs to be - * suspended during the syscore suspend phase. It may also be called during - * suspend-to-idle to suspend a corresponding CPU device that is attached to a - * genpd. - */ -void dev_pm_genpd_suspend(struct device *dev) -{ - genpd_switch_state(dev, true); -} -EXPORT_SYMBOL_GPL(dev_pm_genpd_suspend); - -/** - * dev_pm_genpd_resume - Synchronously try to resume the genpd for @dev - * @dev: The device that is attached to the genpd, which needs to be resumed. - * - * This routine should typically be called for a device that needs to be resumed - * during the syscore resume phase. It may also be called during suspend-to-idle - * to resume a corresponding CPU device that is attached to a genpd. - */ -void dev_pm_genpd_resume(struct device *dev) -{ - genpd_switch_state(dev, false); -} -EXPORT_SYMBOL_GPL(dev_pm_genpd_resume); - -#else /* !CONFIG_PM_SLEEP */ - -#define genpd_prepare NULL -#define genpd_suspend_noirq NULL -#define genpd_resume_noirq NULL -#define genpd_freeze_noirq NULL -#define genpd_thaw_noirq NULL -#define genpd_poweroff_noirq NULL -#define genpd_restore_noirq NULL -#define genpd_complete NULL - -#endif /* CONFIG_PM_SLEEP */ - -static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev, - bool has_governor) -{ - struct generic_pm_domain_data *gpd_data; - struct gpd_timing_data *td; - int ret; - - ret = dev_pm_get_subsys_data(dev); - if (ret) - return ERR_PTR(ret); - - gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL); - if (!gpd_data) { - ret = -ENOMEM; - goto err_put; - } - - gpd_data->base.dev = dev; - gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier; - - /* Allocate data used by a governor. */ - if (has_governor) { - td = kzalloc(sizeof(*td), GFP_KERNEL); - if (!td) { - ret = -ENOMEM; - goto err_free; - } - - td->constraint_changed = true; - td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; - td->next_wakeup = KTIME_MAX; - gpd_data->td = td; - } - - spin_lock_irq(&dev->power.lock); - - if (dev->power.subsys_data->domain_data) - ret = -EINVAL; - else - dev->power.subsys_data->domain_data = &gpd_data->base; - - spin_unlock_irq(&dev->power.lock); - - if (ret) - goto err_free; - - return gpd_data; - - err_free: - kfree(gpd_data->td); - kfree(gpd_data); - err_put: - dev_pm_put_subsys_data(dev); - return ERR_PTR(ret); -} - -static void genpd_free_dev_data(struct device *dev, - struct generic_pm_domain_data *gpd_data) -{ - spin_lock_irq(&dev->power.lock); - - dev->power.subsys_data->domain_data = NULL; - - spin_unlock_irq(&dev->power.lock); - - kfree(gpd_data->td); - kfree(gpd_data); - dev_pm_put_subsys_data(dev); -} - -static void genpd_update_cpumask(struct generic_pm_domain *genpd, - int cpu, bool set, unsigned int depth) -{ - struct gpd_link *link; - - if (!genpd_is_cpu_domain(genpd)) - return; - - list_for_each_entry(link, &genpd->child_links, child_node) { - struct generic_pm_domain *parent = link->parent; - - genpd_lock_nested(parent, depth + 1); - genpd_update_cpumask(parent, cpu, set, depth + 1); - genpd_unlock(parent); - } - - if (set) - cpumask_set_cpu(cpu, genpd->cpus); - else - cpumask_clear_cpu(cpu, genpd->cpus); -} - -static void genpd_set_cpumask(struct generic_pm_domain *genpd, int cpu) -{ - if (cpu >= 0) - genpd_update_cpumask(genpd, cpu, true, 0); -} - -static void genpd_clear_cpumask(struct generic_pm_domain *genpd, int cpu) -{ - if (cpu >= 0) - genpd_update_cpumask(genpd, cpu, false, 0); -} - -static int genpd_get_cpu(struct generic_pm_domain *genpd, struct device *dev) -{ - int cpu; - - if (!genpd_is_cpu_domain(genpd)) - return -1; - - for_each_possible_cpu(cpu) { - if (get_cpu_device(cpu) == dev) - return cpu; - } - - return -1; -} - -static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, - struct device *base_dev) -{ - struct genpd_governor_data *gd = genpd->gd; - struct generic_pm_domain_data *gpd_data; - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev)) - return -EINVAL; - - gpd_data = genpd_alloc_dev_data(dev, gd); - if (IS_ERR(gpd_data)) - return PTR_ERR(gpd_data); - - gpd_data->cpu = genpd_get_cpu(genpd, base_dev); - - ret = genpd->attach_dev ? genpd->attach_dev(genpd, dev) : 0; - if (ret) - goto out; - - genpd_lock(genpd); - - genpd_set_cpumask(genpd, gpd_data->cpu); - dev_pm_domain_set(dev, &genpd->domain); - - genpd->device_count++; - if (gd) - gd->max_off_time_changed = true; - - list_add_tail(&gpd_data->base.list_node, &genpd->dev_list); - - genpd_unlock(genpd); - out: - if (ret) - genpd_free_dev_data(dev, gpd_data); - else - dev_pm_qos_add_notifier(dev, &gpd_data->nb, - DEV_PM_QOS_RESUME_LATENCY); - - return ret; -} - -/** - * pm_genpd_add_device - Add a device to an I/O PM domain. - * @genpd: PM domain to add the device to. - * @dev: Device to be added. - */ -int pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev) -{ - int ret; - - mutex_lock(&gpd_list_lock); - ret = genpd_add_device(genpd, dev, dev); - mutex_unlock(&gpd_list_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(pm_genpd_add_device); - -static int genpd_remove_device(struct generic_pm_domain *genpd, - struct device *dev) -{ - struct generic_pm_domain_data *gpd_data; - struct pm_domain_data *pdd; - int ret = 0; - - dev_dbg(dev, "%s()\n", __func__); - - pdd = dev->power.subsys_data->domain_data; - gpd_data = to_gpd_data(pdd); - dev_pm_qos_remove_notifier(dev, &gpd_data->nb, - DEV_PM_QOS_RESUME_LATENCY); - - genpd_lock(genpd); - - if (genpd->prepared_count > 0) { - ret = -EAGAIN; - goto out; - } - - genpd->device_count--; - if (genpd->gd) - genpd->gd->max_off_time_changed = true; - - genpd_clear_cpumask(genpd, gpd_data->cpu); - dev_pm_domain_set(dev, NULL); - - list_del_init(&pdd->list_node); - - genpd_unlock(genpd); - - if (genpd->detach_dev) - genpd->detach_dev(genpd, dev); - - genpd_free_dev_data(dev, gpd_data); - - return 0; - - out: - genpd_unlock(genpd); - dev_pm_qos_add_notifier(dev, &gpd_data->nb, DEV_PM_QOS_RESUME_LATENCY); - - return ret; -} - -/** - * pm_genpd_remove_device - Remove a device from an I/O PM domain. - * @dev: Device to be removed. - */ -int pm_genpd_remove_device(struct device *dev) -{ - struct generic_pm_domain *genpd = dev_to_genpd_safe(dev); - - if (!genpd) - return -EINVAL; - - return genpd_remove_device(genpd, dev); -} -EXPORT_SYMBOL_GPL(pm_genpd_remove_device); - -/** - * dev_pm_genpd_add_notifier - Add a genpd power on/off notifier for @dev - * - * @dev: Device that should be associated with the notifier - * @nb: The notifier block to register - * - * Users may call this function to add a genpd power on/off notifier for an - * attached @dev. Only one notifier per device is allowed. The notifier is - * sent when genpd is powering on/off the PM domain. - * - * It is assumed that the user guarantee that the genpd wouldn't be detached - * while this routine is getting called. - * - * Returns 0 on success and negative error values on failures. - */ -int dev_pm_genpd_add_notifier(struct device *dev, struct notifier_block *nb) -{ - struct generic_pm_domain *genpd; - struct generic_pm_domain_data *gpd_data; - int ret; - - genpd = dev_to_genpd_safe(dev); - if (!genpd) - return -ENODEV; - - if (WARN_ON(!dev->power.subsys_data || - !dev->power.subsys_data->domain_data)) - return -EINVAL; - - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - if (gpd_data->power_nb) - return -EEXIST; - - genpd_lock(genpd); - ret = raw_notifier_chain_register(&genpd->power_notifiers, nb); - genpd_unlock(genpd); - - if (ret) { - dev_warn(dev, "failed to add notifier for PM domain %s\n", - genpd->name); - return ret; - } - - gpd_data->power_nb = nb; - return 0; -} -EXPORT_SYMBOL_GPL(dev_pm_genpd_add_notifier); - -/** - * dev_pm_genpd_remove_notifier - Remove a genpd power on/off notifier for @dev - * - * @dev: Device that is associated with the notifier - * - * Users may call this function to remove a genpd power on/off notifier for an - * attached @dev. - * - * It is assumed that the user guarantee that the genpd wouldn't be detached - * while this routine is getting called. - * - * Returns 0 on success and negative error values on failures. - */ -int dev_pm_genpd_remove_notifier(struct device *dev) -{ - struct generic_pm_domain *genpd; - struct generic_pm_domain_data *gpd_data; - int ret; - - genpd = dev_to_genpd_safe(dev); - if (!genpd) - return -ENODEV; - - if (WARN_ON(!dev->power.subsys_data || - !dev->power.subsys_data->domain_data)) - return -EINVAL; - - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - if (!gpd_data->power_nb) - return -ENODEV; - - genpd_lock(genpd); - ret = raw_notifier_chain_unregister(&genpd->power_notifiers, - gpd_data->power_nb); - genpd_unlock(genpd); - - if (ret) { - dev_warn(dev, "failed to remove notifier for PM domain %s\n", - genpd->name); - return ret; - } - - gpd_data->power_nb = NULL; - return 0; -} -EXPORT_SYMBOL_GPL(dev_pm_genpd_remove_notifier); - -static int genpd_add_subdomain(struct generic_pm_domain *genpd, - struct generic_pm_domain *subdomain) -{ - struct gpd_link *link, *itr; - int ret = 0; - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain) - || genpd == subdomain) - return -EINVAL; - - /* - * If the domain can be powered on/off in an IRQ safe - * context, ensure that the subdomain can also be - * powered on/off in that context. - */ - if (!genpd_is_irq_safe(genpd) && genpd_is_irq_safe(subdomain)) { - WARN(1, "Parent %s of subdomain %s must be IRQ safe\n", - genpd->name, subdomain->name); - return -EINVAL; - } - - link = kzalloc(sizeof(*link), GFP_KERNEL); - if (!link) - return -ENOMEM; - - genpd_lock(subdomain); - genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING); - - if (!genpd_status_on(genpd) && genpd_status_on(subdomain)) { - ret = -EINVAL; - goto out; - } - - list_for_each_entry(itr, &genpd->parent_links, parent_node) { - if (itr->child == subdomain && itr->parent == genpd) { - ret = -EINVAL; - goto out; - } - } - - link->parent = genpd; - list_add_tail(&link->parent_node, &genpd->parent_links); - link->child = subdomain; - list_add_tail(&link->child_node, &subdomain->child_links); - if (genpd_status_on(subdomain)) - genpd_sd_counter_inc(genpd); - - out: - genpd_unlock(genpd); - genpd_unlock(subdomain); - if (ret) - kfree(link); - return ret; -} - -/** - * pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain. - * @genpd: Leader PM domain to add the subdomain to. - * @subdomain: Subdomain to be added. - */ -int pm_genpd_add_subdomain(struct generic_pm_domain *genpd, - struct generic_pm_domain *subdomain) -{ - int ret; - - mutex_lock(&gpd_list_lock); - ret = genpd_add_subdomain(genpd, subdomain); - mutex_unlock(&gpd_list_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(pm_genpd_add_subdomain); - -/** - * pm_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain. - * @genpd: Leader PM domain to remove the subdomain from. - * @subdomain: Subdomain to be removed. - */ -int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, - struct generic_pm_domain *subdomain) -{ - struct gpd_link *l, *link; - int ret = -EINVAL; - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)) - return -EINVAL; - - genpd_lock(subdomain); - genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING); - - if (!list_empty(&subdomain->parent_links) || subdomain->device_count) { - pr_warn("%s: unable to remove subdomain %s\n", - genpd->name, subdomain->name); - ret = -EBUSY; - goto out; - } - - list_for_each_entry_safe(link, l, &genpd->parent_links, parent_node) { - if (link->child != subdomain) - continue; - - list_del(&link->parent_node); - list_del(&link->child_node); - kfree(link); - if (genpd_status_on(subdomain)) - genpd_sd_counter_dec(genpd); - - ret = 0; - break; - } - -out: - genpd_unlock(genpd); - genpd_unlock(subdomain); - - return ret; -} -EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain); - -static void genpd_free_default_power_state(struct genpd_power_state *states, - unsigned int state_count) -{ - kfree(states); -} - -static int genpd_set_default_power_state(struct generic_pm_domain *genpd) -{ - struct genpd_power_state *state; - - state = kzalloc(sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - - genpd->states = state; - genpd->state_count = 1; - genpd->free_states = genpd_free_default_power_state; - - return 0; -} - -static int genpd_alloc_data(struct generic_pm_domain *genpd) -{ - struct genpd_governor_data *gd = NULL; - int ret; - - if (genpd_is_cpu_domain(genpd) && - !zalloc_cpumask_var(&genpd->cpus, GFP_KERNEL)) - return -ENOMEM; - - if (genpd->gov) { - gd = kzalloc(sizeof(*gd), GFP_KERNEL); - if (!gd) { - ret = -ENOMEM; - goto free; - } - - gd->max_off_time_ns = -1; - gd->max_off_time_changed = true; - gd->next_wakeup = KTIME_MAX; - } - - /* Use only one "off" state if there were no states declared */ - if (genpd->state_count == 0) { - ret = genpd_set_default_power_state(genpd); - if (ret) - goto free; - } - - genpd->gd = gd; - return 0; - -free: - if (genpd_is_cpu_domain(genpd)) - free_cpumask_var(genpd->cpus); - kfree(gd); - return ret; -} - -static void genpd_free_data(struct generic_pm_domain *genpd) -{ - if (genpd_is_cpu_domain(genpd)) - free_cpumask_var(genpd->cpus); - if (genpd->free_states) - genpd->free_states(genpd->states, genpd->state_count); - kfree(genpd->gd); -} - -static void genpd_lock_init(struct generic_pm_domain *genpd) -{ - if (genpd->flags & GENPD_FLAG_IRQ_SAFE) { - spin_lock_init(&genpd->slock); - genpd->lock_ops = &genpd_spin_ops; - } else { - mutex_init(&genpd->mlock); - genpd->lock_ops = &genpd_mtx_ops; - } -} - -/** - * pm_genpd_init - Initialize a generic I/O PM domain object. - * @genpd: PM domain object to initialize. - * @gov: PM domain governor to associate with the domain (may be NULL). - * @is_off: Initial value of the domain's power_is_off field. - * - * Returns 0 on successful initialization, else a negative error code. - */ -int pm_genpd_init(struct generic_pm_domain *genpd, - struct dev_power_governor *gov, bool is_off) -{ - int ret; - - if (IS_ERR_OR_NULL(genpd)) - return -EINVAL; - - INIT_LIST_HEAD(&genpd->parent_links); - INIT_LIST_HEAD(&genpd->child_links); - INIT_LIST_HEAD(&genpd->dev_list); - RAW_INIT_NOTIFIER_HEAD(&genpd->power_notifiers); - genpd_lock_init(genpd); - genpd->gov = gov; - INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn); - atomic_set(&genpd->sd_count, 0); - genpd->status = is_off ? GENPD_STATE_OFF : GENPD_STATE_ON; - genpd->device_count = 0; - genpd->provider = NULL; - genpd->has_provider = false; - genpd->accounting_time = ktime_get_mono_fast_ns(); - genpd->domain.ops.runtime_suspend = genpd_runtime_suspend; - genpd->domain.ops.runtime_resume = genpd_runtime_resume; - genpd->domain.ops.prepare = genpd_prepare; - genpd->domain.ops.suspend_noirq = genpd_suspend_noirq; - genpd->domain.ops.resume_noirq = genpd_resume_noirq; - genpd->domain.ops.freeze_noirq = genpd_freeze_noirq; - genpd->domain.ops.thaw_noirq = genpd_thaw_noirq; - genpd->domain.ops.poweroff_noirq = genpd_poweroff_noirq; - genpd->domain.ops.restore_noirq = genpd_restore_noirq; - genpd->domain.ops.complete = genpd_complete; - genpd->domain.start = genpd_dev_pm_start; - - if (genpd->flags & GENPD_FLAG_PM_CLK) { - genpd->dev_ops.stop = pm_clk_suspend; - genpd->dev_ops.start = pm_clk_resume; - } - - /* The always-on governor works better with the corresponding flag. */ - if (gov == &pm_domain_always_on_gov) - genpd->flags |= GENPD_FLAG_RPM_ALWAYS_ON; - - /* Always-on domains must be powered on at initialization. */ - if ((genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd)) && - !genpd_status_on(genpd)) { - pr_err("always-on PM domain %s is not on\n", genpd->name); - return -EINVAL; - } - - /* Multiple states but no governor doesn't make sense. */ - if (!gov && genpd->state_count > 1) - pr_warn("%s: no governor for states\n", genpd->name); - - ret = genpd_alloc_data(genpd); - if (ret) - return ret; - - device_initialize(&genpd->dev); - dev_set_name(&genpd->dev, "%s", genpd->name); - - mutex_lock(&gpd_list_lock); - list_add(&genpd->gpd_list_node, &gpd_list); - mutex_unlock(&gpd_list_lock); - genpd_debug_add(genpd); - - return 0; -} -EXPORT_SYMBOL_GPL(pm_genpd_init); - -static int genpd_remove(struct generic_pm_domain *genpd) -{ - struct gpd_link *l, *link; - - if (IS_ERR_OR_NULL(genpd)) - return -EINVAL; - - genpd_lock(genpd); - - if (genpd->has_provider) { - genpd_unlock(genpd); - pr_err("Provider present, unable to remove %s\n", genpd->name); - return -EBUSY; - } - - if (!list_empty(&genpd->parent_links) || genpd->device_count) { - genpd_unlock(genpd); - pr_err("%s: unable to remove %s\n", __func__, genpd->name); - return -EBUSY; - } - - list_for_each_entry_safe(link, l, &genpd->child_links, child_node) { - list_del(&link->parent_node); - list_del(&link->child_node); - kfree(link); - } - - list_del(&genpd->gpd_list_node); - genpd_unlock(genpd); - genpd_debug_remove(genpd); - cancel_work_sync(&genpd->power_off_work); - genpd_free_data(genpd); - - pr_debug("%s: removed %s\n", __func__, genpd->name); - - return 0; -} - -/** - * pm_genpd_remove - Remove a generic I/O PM domain - * @genpd: Pointer to PM domain that is to be removed. - * - * To remove the PM domain, this function: - * - Removes the PM domain as a subdomain to any parent domains, - * if it was added. - * - Removes the PM domain from the list of registered PM domains. - * - * The PM domain will only be removed, if the associated provider has - * been removed, it is not a parent to any other PM domain and has no - * devices associated with it. - */ -int pm_genpd_remove(struct generic_pm_domain *genpd) -{ - int ret; - - mutex_lock(&gpd_list_lock); - ret = genpd_remove(genpd); - mutex_unlock(&gpd_list_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(pm_genpd_remove); - -#ifdef CONFIG_PM_GENERIC_DOMAINS_OF - -/* - * Device Tree based PM domain providers. - * - * The code below implements generic device tree based PM domain providers that - * bind device tree nodes with generic PM domains registered in the system. - * - * Any driver that registers generic PM domains and needs to support binding of - * devices to these domains is supposed to register a PM domain provider, which - * maps a PM domain specifier retrieved from the device tree to a PM domain. - * - * Two simple mapping functions have been provided for convenience: - * - genpd_xlate_simple() for 1:1 device tree node to PM domain mapping. - * - genpd_xlate_onecell() for mapping of multiple PM domains per node by - * index. - */ - -/** - * struct of_genpd_provider - PM domain provider registration structure - * @link: Entry in global list of PM domain providers - * @node: Pointer to device tree node of PM domain provider - * @xlate: Provider-specific xlate callback mapping a set of specifier cells - * into a PM domain. - * @data: context pointer to be passed into @xlate callback - */ -struct of_genpd_provider { - struct list_head link; - struct device_node *node; - genpd_xlate_t xlate; - void *data; -}; - -/* List of registered PM domain providers. */ -static LIST_HEAD(of_genpd_providers); -/* Mutex to protect the list above. */ -static DEFINE_MUTEX(of_genpd_mutex); - -/** - * genpd_xlate_simple() - Xlate function for direct node-domain mapping - * @genpdspec: OF phandle args to map into a PM domain - * @data: xlate function private data - pointer to struct generic_pm_domain - * - * This is a generic xlate function that can be used to model PM domains that - * have their own device tree nodes. The private data of xlate function needs - * to be a valid pointer to struct generic_pm_domain. - */ -static struct generic_pm_domain *genpd_xlate_simple( - struct of_phandle_args *genpdspec, - void *data) -{ - return data; -} - -/** - * genpd_xlate_onecell() - Xlate function using a single index. - * @genpdspec: OF phandle args to map into a PM domain - * @data: xlate function private data - pointer to struct genpd_onecell_data - * - * This is a generic xlate function that can be used to model simple PM domain - * controllers that have one device tree node and provide multiple PM domains. - * A single cell is used as an index into an array of PM domains specified in - * the genpd_onecell_data struct when registering the provider. - */ -static struct generic_pm_domain *genpd_xlate_onecell( - struct of_phandle_args *genpdspec, - void *data) -{ - struct genpd_onecell_data *genpd_data = data; - unsigned int idx = genpdspec->args[0]; - - if (genpdspec->args_count != 1) - return ERR_PTR(-EINVAL); - - if (idx >= genpd_data->num_domains) { - pr_err("%s: invalid domain index %u\n", __func__, idx); - return ERR_PTR(-EINVAL); - } - - if (!genpd_data->domains[idx]) - return ERR_PTR(-ENOENT); - - return genpd_data->domains[idx]; -} - -/** - * genpd_add_provider() - Register a PM domain provider for a node - * @np: Device node pointer associated with the PM domain provider. - * @xlate: Callback for decoding PM domain from phandle arguments. - * @data: Context pointer for @xlate callback. - */ -static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate, - void *data) -{ - struct of_genpd_provider *cp; - - cp = kzalloc(sizeof(*cp), GFP_KERNEL); - if (!cp) - return -ENOMEM; - - cp->node = of_node_get(np); - cp->data = data; - cp->xlate = xlate; - fwnode_dev_initialized(&np->fwnode, true); - - mutex_lock(&of_genpd_mutex); - list_add(&cp->link, &of_genpd_providers); - mutex_unlock(&of_genpd_mutex); - pr_debug("Added domain provider from %pOF\n", np); - - return 0; -} - -static bool genpd_present(const struct generic_pm_domain *genpd) -{ - bool ret = false; - const struct generic_pm_domain *gpd; - - mutex_lock(&gpd_list_lock); - list_for_each_entry(gpd, &gpd_list, gpd_list_node) { - if (gpd == genpd) { - ret = true; - break; - } - } - mutex_unlock(&gpd_list_lock); - - return ret; -} - -/** - * of_genpd_add_provider_simple() - Register a simple PM domain provider - * @np: Device node pointer associated with the PM domain provider. - * @genpd: Pointer to PM domain associated with the PM domain provider. - */ -int of_genpd_add_provider_simple(struct device_node *np, - struct generic_pm_domain *genpd) -{ - int ret; - - if (!np || !genpd) - return -EINVAL; - - if (!genpd_present(genpd)) - return -EINVAL; - - genpd->dev.of_node = np; - - /* Parse genpd OPP table */ - if (genpd->set_performance_state) { - ret = dev_pm_opp_of_add_table(&genpd->dev); - if (ret) - return dev_err_probe(&genpd->dev, ret, "Failed to add OPP table\n"); - - /* - * Save table for faster processing while setting performance - * state. - */ - genpd->opp_table = dev_pm_opp_get_opp_table(&genpd->dev); - WARN_ON(IS_ERR(genpd->opp_table)); - } - - ret = genpd_add_provider(np, genpd_xlate_simple, genpd); - if (ret) { - if (genpd->set_performance_state) { - dev_pm_opp_put_opp_table(genpd->opp_table); - dev_pm_opp_of_remove_table(&genpd->dev); - } - - return ret; - } - - genpd->provider = &np->fwnode; - genpd->has_provider = true; - - return 0; -} -EXPORT_SYMBOL_GPL(of_genpd_add_provider_simple); - -/** - * of_genpd_add_provider_onecell() - Register a onecell PM domain provider - * @np: Device node pointer associated with the PM domain provider. - * @data: Pointer to the data associated with the PM domain provider. - */ -int of_genpd_add_provider_onecell(struct device_node *np, - struct genpd_onecell_data *data) -{ - struct generic_pm_domain *genpd; - unsigned int i; - int ret = -EINVAL; - - if (!np || !data) - return -EINVAL; - - if (!data->xlate) - data->xlate = genpd_xlate_onecell; - - for (i = 0; i < data->num_domains; i++) { - genpd = data->domains[i]; - - if (!genpd) - continue; - if (!genpd_present(genpd)) - goto error; - - genpd->dev.of_node = np; - - /* Parse genpd OPP table */ - if (genpd->set_performance_state) { - ret = dev_pm_opp_of_add_table_indexed(&genpd->dev, i); - if (ret) { - dev_err_probe(&genpd->dev, ret, - "Failed to add OPP table for index %d\n", i); - goto error; - } - - /* - * Save table for faster processing while setting - * performance state. - */ - genpd->opp_table = dev_pm_opp_get_opp_table(&genpd->dev); - WARN_ON(IS_ERR(genpd->opp_table)); - } - - genpd->provider = &np->fwnode; - genpd->has_provider = true; - } - - ret = genpd_add_provider(np, data->xlate, data); - if (ret < 0) - goto error; - - return 0; - -error: - while (i--) { - genpd = data->domains[i]; - - if (!genpd) - continue; - - genpd->provider = NULL; - genpd->has_provider = false; - - if (genpd->set_performance_state) { - dev_pm_opp_put_opp_table(genpd->opp_table); - dev_pm_opp_of_remove_table(&genpd->dev); - } - } - - return ret; -} -EXPORT_SYMBOL_GPL(of_genpd_add_provider_onecell); - -/** - * of_genpd_del_provider() - Remove a previously registered PM domain provider - * @np: Device node pointer associated with the PM domain provider - */ -void of_genpd_del_provider(struct device_node *np) -{ - struct of_genpd_provider *cp, *tmp; - struct generic_pm_domain *gpd; - - mutex_lock(&gpd_list_lock); - mutex_lock(&of_genpd_mutex); - list_for_each_entry_safe(cp, tmp, &of_genpd_providers, link) { - if (cp->node == np) { - /* - * For each PM domain associated with the - * provider, set the 'has_provider' to false - * so that the PM domain can be safely removed. - */ - list_for_each_entry(gpd, &gpd_list, gpd_list_node) { - if (gpd->provider == &np->fwnode) { - gpd->has_provider = false; - - if (!gpd->set_performance_state) - continue; - - dev_pm_opp_put_opp_table(gpd->opp_table); - dev_pm_opp_of_remove_table(&gpd->dev); - } - } - - fwnode_dev_initialized(&cp->node->fwnode, false); - list_del(&cp->link); - of_node_put(cp->node); - kfree(cp); - break; - } - } - mutex_unlock(&of_genpd_mutex); - mutex_unlock(&gpd_list_lock); -} -EXPORT_SYMBOL_GPL(of_genpd_del_provider); - -/** - * genpd_get_from_provider() - Look-up PM domain - * @genpdspec: OF phandle args to use for look-up - * - * Looks for a PM domain provider under the node specified by @genpdspec and if - * found, uses xlate function of the provider to map phandle args to a PM - * domain. - * - * Returns a valid pointer to struct generic_pm_domain on success or ERR_PTR() - * on failure. - */ -static struct generic_pm_domain *genpd_get_from_provider( - struct of_phandle_args *genpdspec) -{ - struct generic_pm_domain *genpd = ERR_PTR(-ENOENT); - struct of_genpd_provider *provider; - - if (!genpdspec) - return ERR_PTR(-EINVAL); - - mutex_lock(&of_genpd_mutex); - - /* Check if we have such a provider in our array */ - list_for_each_entry(provider, &of_genpd_providers, link) { - if (provider->node == genpdspec->np) - genpd = provider->xlate(genpdspec, provider->data); - if (!IS_ERR(genpd)) - break; - } - - mutex_unlock(&of_genpd_mutex); - - return genpd; -} - -/** - * of_genpd_add_device() - Add a device to an I/O PM domain - * @genpdspec: OF phandle args to use for look-up PM domain - * @dev: Device to be added. - * - * Looks-up an I/O PM domain based upon phandle args provided and adds - * the device to the PM domain. Returns a negative error code on failure. - */ -int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev) -{ - struct generic_pm_domain *genpd; - int ret; - - mutex_lock(&gpd_list_lock); - - genpd = genpd_get_from_provider(genpdspec); - if (IS_ERR(genpd)) { - ret = PTR_ERR(genpd); - goto out; - } - - ret = genpd_add_device(genpd, dev, dev); - -out: - mutex_unlock(&gpd_list_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(of_genpd_add_device); - -/** - * of_genpd_add_subdomain - Add a subdomain to an I/O PM domain. - * @parent_spec: OF phandle args to use for parent PM domain look-up - * @subdomain_spec: OF phandle args to use for subdomain look-up - * - * Looks-up a parent PM domain and subdomain based upon phandle args - * provided and adds the subdomain to the parent PM domain. Returns a - * negative error code on failure. - */ -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) -{ - struct generic_pm_domain *parent, *subdomain; - int ret; - - mutex_lock(&gpd_list_lock); - - parent = genpd_get_from_provider(parent_spec); - if (IS_ERR(parent)) { - ret = PTR_ERR(parent); - goto out; - } - - subdomain = genpd_get_from_provider(subdomain_spec); - if (IS_ERR(subdomain)) { - ret = PTR_ERR(subdomain); - goto out; - } - - ret = genpd_add_subdomain(parent, subdomain); - -out: - mutex_unlock(&gpd_list_lock); - - return ret == -ENOENT ? -EPROBE_DEFER : ret; -} -EXPORT_SYMBOL_GPL(of_genpd_add_subdomain); - -/** - * of_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain. - * @parent_spec: OF phandle args to use for parent PM domain look-up - * @subdomain_spec: OF phandle args to use for subdomain look-up - * - * Looks-up a parent PM domain and subdomain based upon phandle args - * provided and removes the subdomain from the parent PM domain. Returns a - * negative error code on failure. - */ -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) -{ - struct generic_pm_domain *parent, *subdomain; - int ret; - - mutex_lock(&gpd_list_lock); - - parent = genpd_get_from_provider(parent_spec); - if (IS_ERR(parent)) { - ret = PTR_ERR(parent); - goto out; - } - - subdomain = genpd_get_from_provider(subdomain_spec); - if (IS_ERR(subdomain)) { - ret = PTR_ERR(subdomain); - goto out; - } - - ret = pm_genpd_remove_subdomain(parent, subdomain); - -out: - mutex_unlock(&gpd_list_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(of_genpd_remove_subdomain); - -/** - * of_genpd_remove_last - Remove the last PM domain registered for a provider - * @np: Pointer to device node associated with provider - * - * Find the last PM domain that was added by a particular provider and - * remove this PM domain from the list of PM domains. The provider is - * identified by the 'provider' device structure that is passed. The PM - * domain will only be removed, if the provider associated with domain - * has been removed. - * - * Returns a valid pointer to struct generic_pm_domain on success or - * ERR_PTR() on failure. - */ -struct generic_pm_domain *of_genpd_remove_last(struct device_node *np) -{ - struct generic_pm_domain *gpd, *tmp, *genpd = ERR_PTR(-ENOENT); - int ret; - - if (IS_ERR_OR_NULL(np)) - return ERR_PTR(-EINVAL); - - mutex_lock(&gpd_list_lock); - list_for_each_entry_safe(gpd, tmp, &gpd_list, gpd_list_node) { - if (gpd->provider == &np->fwnode) { - ret = genpd_remove(gpd); - genpd = ret ? ERR_PTR(ret) : gpd; - break; - } - } - mutex_unlock(&gpd_list_lock); - - return genpd; -} -EXPORT_SYMBOL_GPL(of_genpd_remove_last); - -static void genpd_release_dev(struct device *dev) -{ - of_node_put(dev->of_node); - kfree(dev); -} - -static struct bus_type genpd_bus_type = { - .name = "genpd", -}; - -/** - * genpd_dev_pm_detach - Detach a device from its PM domain. - * @dev: Device to detach. - * @power_off: Currently not used - * - * Try to locate a corresponding generic PM domain, which the device was - * attached to previously. If such is found, the device is detached from it. - */ -static void genpd_dev_pm_detach(struct device *dev, bool power_off) -{ - struct generic_pm_domain *pd; - unsigned int i; - int ret = 0; - - pd = dev_to_genpd(dev); - if (IS_ERR(pd)) - return; - - dev_dbg(dev, "removing from PM domain %s\n", pd->name); - - /* Drop the default performance state */ - if (dev_gpd_data(dev)->default_pstate) { - dev_pm_genpd_set_performance_state(dev, 0); - dev_gpd_data(dev)->default_pstate = 0; - } - - for (i = 1; i < GENPD_RETRY_MAX_MS; i <<= 1) { - ret = genpd_remove_device(pd, dev); - if (ret != -EAGAIN) - break; - - mdelay(i); - cond_resched(); - } - - if (ret < 0) { - dev_err(dev, "failed to remove from PM domain %s: %d", - pd->name, ret); - return; - } - - /* Check if PM domain can be powered off after removing this device. */ - genpd_queue_power_off_work(pd); - - /* Unregister the device if it was created by genpd. */ - if (dev->bus == &genpd_bus_type) - device_unregister(dev); -} - -static void genpd_dev_pm_sync(struct device *dev) -{ - struct generic_pm_domain *pd; - - pd = dev_to_genpd(dev); - if (IS_ERR(pd)) - return; - - genpd_queue_power_off_work(pd); -} - -static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev, - unsigned int index, bool power_on) -{ - struct of_phandle_args pd_args; - struct generic_pm_domain *pd; - int pstate; - int ret; - - ret = of_parse_phandle_with_args(dev->of_node, "power-domains", - "#power-domain-cells", index, &pd_args); - if (ret < 0) - return ret; - - mutex_lock(&gpd_list_lock); - pd = genpd_get_from_provider(&pd_args); - of_node_put(pd_args.np); - if (IS_ERR(pd)) { - mutex_unlock(&gpd_list_lock); - dev_dbg(dev, "%s() failed to find PM domain: %ld\n", - __func__, PTR_ERR(pd)); - return driver_deferred_probe_check_state(base_dev); - } - - dev_dbg(dev, "adding to PM domain %s\n", pd->name); - - ret = genpd_add_device(pd, dev, base_dev); - mutex_unlock(&gpd_list_lock); - - if (ret < 0) - return dev_err_probe(dev, ret, "failed to add to PM domain %s\n", pd->name); - - dev->pm_domain->detach = genpd_dev_pm_detach; - dev->pm_domain->sync = genpd_dev_pm_sync; - - if (power_on) { - genpd_lock(pd); - ret = genpd_power_on(pd, 0); - genpd_unlock(pd); - } - - if (ret) { - genpd_remove_device(pd, dev); - return -EPROBE_DEFER; - } - - /* Set the default performance state */ - pstate = of_get_required_opp_performance_state(dev->of_node, index); - if (pstate < 0 && pstate != -ENODEV && pstate != -EOPNOTSUPP) { - ret = pstate; - goto err; - } else if (pstate > 0) { - ret = dev_pm_genpd_set_performance_state(dev, pstate); - if (ret) - goto err; - dev_gpd_data(dev)->default_pstate = pstate; - } - return 1; - -err: - dev_err(dev, "failed to set required performance state for power-domain %s: %d\n", - pd->name, ret); - genpd_remove_device(pd, dev); - return ret; -} - -/** - * genpd_dev_pm_attach - Attach a device to its PM domain using DT. - * @dev: Device to attach. - * - * Parse device's OF node to find a PM domain specifier. If such is found, - * attaches the device to retrieved pm_domain ops. - * - * Returns 1 on successfully attached PM domain, 0 when the device don't need a - * PM domain or when multiple power-domains exists for it, else a negative error - * code. Note that if a power-domain exists for the device, but it cannot be - * found or turned on, then return -EPROBE_DEFER to ensure that the device is - * not probed and to re-try again later. - */ -int genpd_dev_pm_attach(struct device *dev) -{ - if (!dev->of_node) - return 0; - - /* - * Devices with multiple PM domains must be attached separately, as we - * can only attach one PM domain per device. - */ - if (of_count_phandle_with_args(dev->of_node, "power-domains", - "#power-domain-cells") != 1) - return 0; - - return __genpd_dev_pm_attach(dev, dev, 0, true); -} -EXPORT_SYMBOL_GPL(genpd_dev_pm_attach); - -/** - * genpd_dev_pm_attach_by_id - Associate a device with one of its PM domains. - * @dev: The device used to lookup the PM domain. - * @index: The index of the PM domain. - * - * Parse device's OF node to find a PM domain specifier at the provided @index. - * If such is found, creates a virtual device and attaches it to the retrieved - * pm_domain ops. To deal with detaching of the virtual device, the ->detach() - * callback in the struct dev_pm_domain are assigned to genpd_dev_pm_detach(). - * - * Returns the created virtual device if successfully attached PM domain, NULL - * when the device don't need a PM domain, else an ERR_PTR() in case of - * failures. If a power-domain exists for the device, but cannot be found or - * turned on, then ERR_PTR(-EPROBE_DEFER) is returned to ensure that the device - * is not probed and to re-try again later. - */ -struct device *genpd_dev_pm_attach_by_id(struct device *dev, - unsigned int index) -{ - struct device *virt_dev; - int num_domains; - int ret; - - if (!dev->of_node) - return NULL; - - /* Verify that the index is within a valid range. */ - num_domains = of_count_phandle_with_args(dev->of_node, "power-domains", - "#power-domain-cells"); - if (index >= num_domains) - return NULL; - - /* Allocate and register device on the genpd bus. */ - virt_dev = kzalloc(sizeof(*virt_dev), GFP_KERNEL); - if (!virt_dev) - return ERR_PTR(-ENOMEM); - - dev_set_name(virt_dev, "genpd:%u:%s", index, dev_name(dev)); - virt_dev->bus = &genpd_bus_type; - virt_dev->release = genpd_release_dev; - virt_dev->of_node = of_node_get(dev->of_node); - - ret = device_register(virt_dev); - if (ret) { - put_device(virt_dev); - return ERR_PTR(ret); - } - - /* Try to attach the device to the PM domain at the specified index. */ - ret = __genpd_dev_pm_attach(virt_dev, dev, index, false); - if (ret < 1) { - device_unregister(virt_dev); - return ret ? ERR_PTR(ret) : NULL; - } - - pm_runtime_enable(virt_dev); - genpd_queue_power_off_work(dev_to_genpd(virt_dev)); - - return virt_dev; -} -EXPORT_SYMBOL_GPL(genpd_dev_pm_attach_by_id); - -/** - * genpd_dev_pm_attach_by_name - Associate a device with one of its PM domains. - * @dev: The device used to lookup the PM domain. - * @name: The name of the PM domain. - * - * Parse device's OF node to find a PM domain specifier using the - * power-domain-names DT property. For further description see - * genpd_dev_pm_attach_by_id(). - */ -struct device *genpd_dev_pm_attach_by_name(struct device *dev, const char *name) -{ - int index; - - if (!dev->of_node) - return NULL; - - index = of_property_match_string(dev->of_node, "power-domain-names", - name); - if (index < 0) - return NULL; - - return genpd_dev_pm_attach_by_id(dev, index); -} - -static const struct of_device_id idle_state_match[] = { - { .compatible = "domain-idle-state", }, - { } -}; - -static int genpd_parse_state(struct genpd_power_state *genpd_state, - struct device_node *state_node) -{ - int err; - u32 residency; - u32 entry_latency, exit_latency; - - err = of_property_read_u32(state_node, "entry-latency-us", - &entry_latency); - if (err) { - pr_debug(" * %pOF missing entry-latency-us property\n", - state_node); - return -EINVAL; - } - - err = of_property_read_u32(state_node, "exit-latency-us", - &exit_latency); - if (err) { - pr_debug(" * %pOF missing exit-latency-us property\n", - state_node); - return -EINVAL; - } - - err = of_property_read_u32(state_node, "min-residency-us", &residency); - if (!err) - genpd_state->residency_ns = 1000 * residency; - - genpd_state->power_on_latency_ns = 1000 * exit_latency; - genpd_state->power_off_latency_ns = 1000 * entry_latency; - genpd_state->fwnode = &state_node->fwnode; - - return 0; -} - -static int genpd_iterate_idle_states(struct device_node *dn, - struct genpd_power_state *states) -{ - int ret; - struct of_phandle_iterator it; - struct device_node *np; - int i = 0; - - ret = of_count_phandle_with_args(dn, "domain-idle-states", NULL); - if (ret <= 0) - return ret == -ENOENT ? 0 : ret; - - /* Loop over the phandles until all the requested entry is found */ - of_for_each_phandle(&it, ret, dn, "domain-idle-states", NULL, 0) { - np = it.node; - if (!of_match_node(idle_state_match, np)) - continue; - - if (!of_device_is_available(np)) - continue; - - if (states) { - ret = genpd_parse_state(&states[i], np); - if (ret) { - pr_err("Parsing idle state node %pOF failed with err %d\n", - np, ret); - of_node_put(np); - return ret; - } - } - i++; - } - - return i; -} - -/** - * of_genpd_parse_idle_states: Return array of idle states for the genpd. - * - * @dn: The genpd device node - * @states: The pointer to which the state array will be saved. - * @n: The count of elements in the array returned from this function. - * - * Returns the device states parsed from the OF node. The memory for the states - * is allocated by this function and is the responsibility of the caller to - * free the memory after use. If any or zero compatible domain idle states is - * found it returns 0 and in case of errors, a negative error code is returned. - */ -int of_genpd_parse_idle_states(struct device_node *dn, - struct genpd_power_state **states, int *n) -{ - struct genpd_power_state *st; - int ret; - - ret = genpd_iterate_idle_states(dn, NULL); - if (ret < 0) - return ret; - - if (!ret) { - *states = NULL; - *n = 0; - return 0; - } - - st = kcalloc(ret, sizeof(*st), GFP_KERNEL); - if (!st) - return -ENOMEM; - - ret = genpd_iterate_idle_states(dn, st); - if (ret <= 0) { - kfree(st); - return ret < 0 ? ret : -EINVAL; - } - - *states = st; - *n = ret; - - return 0; -} -EXPORT_SYMBOL_GPL(of_genpd_parse_idle_states); - -/** - * pm_genpd_opp_to_performance_state - Gets performance state of the genpd from its OPP node. - * - * @genpd_dev: Genpd's device for which the performance-state needs to be found. - * @opp: struct dev_pm_opp of the OPP for which we need to find performance - * state. - * - * Returns performance state encoded in the OPP of the genpd. This calls - * platform specific genpd->opp_to_performance_state() callback to translate - * power domain OPP to performance state. - * - * Returns performance state on success and 0 on failure. - */ -unsigned int pm_genpd_opp_to_performance_state(struct device *genpd_dev, - struct dev_pm_opp *opp) -{ - struct generic_pm_domain *genpd = NULL; - int state; - - genpd = container_of(genpd_dev, struct generic_pm_domain, dev); - - if (unlikely(!genpd->opp_to_performance_state)) - return 0; - - genpd_lock(genpd); - state = genpd->opp_to_performance_state(genpd, opp); - genpd_unlock(genpd); - - return state; -} -EXPORT_SYMBOL_GPL(pm_genpd_opp_to_performance_state); - -static int __init genpd_bus_init(void) -{ - return bus_register(&genpd_bus_type); -} -core_initcall(genpd_bus_init); - -#endif /* CONFIG_PM_GENERIC_DOMAINS_OF */ - - -/*** debugfs support ***/ - -#ifdef CONFIG_DEBUG_FS -/* - * TODO: This function is a slightly modified version of rtpm_status_show - * from sysfs.c, so generalize it. - */ -static void rtpm_status_str(struct seq_file *s, struct device *dev) -{ - static const char * const status_lookup[] = { - [RPM_ACTIVE] = "active", - [RPM_RESUMING] = "resuming", - [RPM_SUSPENDED] = "suspended", - [RPM_SUSPENDING] = "suspending" - }; - const char *p = ""; - - if (dev->power.runtime_error) - p = "error"; - else if (dev->power.disable_depth) - p = "unsupported"; - else if (dev->power.runtime_status < ARRAY_SIZE(status_lookup)) - p = status_lookup[dev->power.runtime_status]; - else - WARN_ON(1); - - seq_printf(s, "%-25s ", p); -} - -static void perf_status_str(struct seq_file *s, struct device *dev) -{ - struct generic_pm_domain_data *gpd_data; - - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - seq_put_decimal_ull(s, "", gpd_data->performance_state); -} - -static int genpd_summary_one(struct seq_file *s, - struct generic_pm_domain *genpd) -{ - static const char * const status_lookup[] = { - [GENPD_STATE_ON] = "on", - [GENPD_STATE_OFF] = "off" - }; - struct pm_domain_data *pm_data; - const char *kobj_path; - struct gpd_link *link; - char state[16]; - int ret; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - if (WARN_ON(genpd->status >= ARRAY_SIZE(status_lookup))) - goto exit; - if (!genpd_status_on(genpd)) - snprintf(state, sizeof(state), "%s-%u", - status_lookup[genpd->status], genpd->state_idx); - else - snprintf(state, sizeof(state), "%s", - status_lookup[genpd->status]); - seq_printf(s, "%-30s %-50s %u", genpd->name, state, genpd->performance_state); - - /* - * Modifications on the list require holding locks on both - * parent and child, so we are safe. - * Also genpd->name is immutable. - */ - list_for_each_entry(link, &genpd->parent_links, parent_node) { - if (list_is_first(&link->parent_node, &genpd->parent_links)) - seq_printf(s, "\n%48s", " "); - seq_printf(s, "%s", link->child->name); - if (!list_is_last(&link->parent_node, &genpd->parent_links)) - seq_puts(s, ", "); - } - - list_for_each_entry(pm_data, &genpd->dev_list, list_node) { - kobj_path = kobject_get_path(&pm_data->dev->kobj, - genpd_is_irq_safe(genpd) ? - GFP_ATOMIC : GFP_KERNEL); - if (kobj_path == NULL) - continue; - - seq_printf(s, "\n %-50s ", kobj_path); - rtpm_status_str(s, pm_data->dev); - perf_status_str(s, pm_data->dev); - kfree(kobj_path); - } - - seq_puts(s, "\n"); -exit: - genpd_unlock(genpd); - - return 0; -} - -static int summary_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd; - int ret = 0; - - seq_puts(s, "domain status children performance\n"); - seq_puts(s, " /device runtime status\n"); - seq_puts(s, "----------------------------------------------------------------------------------------------\n"); - - ret = mutex_lock_interruptible(&gpd_list_lock); - if (ret) - return -ERESTARTSYS; - - list_for_each_entry(genpd, &gpd_list, gpd_list_node) { - ret = genpd_summary_one(s, genpd); - if (ret) - break; - } - mutex_unlock(&gpd_list_lock); - - return ret; -} - -static int status_show(struct seq_file *s, void *data) -{ - static const char * const status_lookup[] = { - [GENPD_STATE_ON] = "on", - [GENPD_STATE_OFF] = "off" - }; - - struct generic_pm_domain *genpd = s->private; - int ret = 0; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - if (WARN_ON_ONCE(genpd->status >= ARRAY_SIZE(status_lookup))) - goto exit; - - if (genpd->status == GENPD_STATE_OFF) - seq_printf(s, "%s-%u\n", status_lookup[genpd->status], - genpd->state_idx); - else - seq_printf(s, "%s\n", status_lookup[genpd->status]); -exit: - genpd_unlock(genpd); - return ret; -} - -static int sub_domains_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd = s->private; - struct gpd_link *link; - int ret = 0; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - list_for_each_entry(link, &genpd->parent_links, parent_node) - seq_printf(s, "%s\n", link->child->name); - - genpd_unlock(genpd); - return ret; -} - -static int idle_states_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd = s->private; - u64 now, delta, idle_time = 0; - unsigned int i; - int ret = 0; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - seq_puts(s, "State Time Spent(ms) Usage Rejected\n"); - - for (i = 0; i < genpd->state_count; i++) { - idle_time += genpd->states[i].idle_time; - - if (genpd->status == GENPD_STATE_OFF && genpd->state_idx == i) { - now = ktime_get_mono_fast_ns(); - if (now > genpd->accounting_time) { - delta = now - genpd->accounting_time; - idle_time += delta; - } - } - - do_div(idle_time, NSEC_PER_MSEC); - seq_printf(s, "S%-13i %-14llu %-14llu %llu\n", i, idle_time, - genpd->states[i].usage, genpd->states[i].rejected); - } - - genpd_unlock(genpd); - return ret; -} - -static int active_time_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd = s->private; - u64 now, on_time, delta = 0; - int ret = 0; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - if (genpd->status == GENPD_STATE_ON) { - now = ktime_get_mono_fast_ns(); - if (now > genpd->accounting_time) - delta = now - genpd->accounting_time; - } - - on_time = genpd->on_time + delta; - do_div(on_time, NSEC_PER_MSEC); - seq_printf(s, "%llu ms\n", on_time); - - genpd_unlock(genpd); - return ret; -} - -static int total_idle_time_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd = s->private; - u64 now, delta, total = 0; - unsigned int i; - int ret = 0; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - for (i = 0; i < genpd->state_count; i++) { - total += genpd->states[i].idle_time; - - if (genpd->status == GENPD_STATE_OFF && genpd->state_idx == i) { - now = ktime_get_mono_fast_ns(); - if (now > genpd->accounting_time) { - delta = now - genpd->accounting_time; - total += delta; - } - } - } - - do_div(total, NSEC_PER_MSEC); - seq_printf(s, "%llu ms\n", total); - - genpd_unlock(genpd); - return ret; -} - - -static int devices_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd = s->private; - struct pm_domain_data *pm_data; - const char *kobj_path; - int ret = 0; - - ret = genpd_lock_interruptible(genpd); - if (ret) - return -ERESTARTSYS; - - list_for_each_entry(pm_data, &genpd->dev_list, list_node) { - kobj_path = kobject_get_path(&pm_data->dev->kobj, - genpd_is_irq_safe(genpd) ? - GFP_ATOMIC : GFP_KERNEL); - if (kobj_path == NULL) - continue; - - seq_printf(s, "%s\n", kobj_path); - kfree(kobj_path); - } - - genpd_unlock(genpd); - return ret; -} - -static int perf_state_show(struct seq_file *s, void *data) -{ - struct generic_pm_domain *genpd = s->private; - - if (genpd_lock_interruptible(genpd)) - return -ERESTARTSYS; - - seq_printf(s, "%u\n", genpd->performance_state); - - genpd_unlock(genpd); - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(summary); -DEFINE_SHOW_ATTRIBUTE(status); -DEFINE_SHOW_ATTRIBUTE(sub_domains); -DEFINE_SHOW_ATTRIBUTE(idle_states); -DEFINE_SHOW_ATTRIBUTE(active_time); -DEFINE_SHOW_ATTRIBUTE(total_idle_time); -DEFINE_SHOW_ATTRIBUTE(devices); -DEFINE_SHOW_ATTRIBUTE(perf_state); - -static void genpd_debug_add(struct generic_pm_domain *genpd) -{ - struct dentry *d; - - if (!genpd_debugfs_dir) - return; - - d = debugfs_create_dir(genpd->name, genpd_debugfs_dir); - - debugfs_create_file("current_state", 0444, - d, genpd, &status_fops); - debugfs_create_file("sub_domains", 0444, - d, genpd, &sub_domains_fops); - debugfs_create_file("idle_states", 0444, - d, genpd, &idle_states_fops); - debugfs_create_file("active_time", 0444, - d, genpd, &active_time_fops); - debugfs_create_file("total_idle_time", 0444, - d, genpd, &total_idle_time_fops); - debugfs_create_file("devices", 0444, - d, genpd, &devices_fops); - if (genpd->set_performance_state) - debugfs_create_file("perf_state", 0444, - d, genpd, &perf_state_fops); -} - -static int __init genpd_debug_init(void) -{ - struct generic_pm_domain *genpd; - - genpd_debugfs_dir = debugfs_create_dir("pm_genpd", NULL); - - debugfs_create_file("pm_genpd_summary", S_IRUGO, genpd_debugfs_dir, - NULL, &summary_fops); - - list_for_each_entry(genpd, &gpd_list, gpd_list_node) - genpd_debug_add(genpd); - - return 0; -} -late_initcall(genpd_debug_init); - -static void __exit genpd_debug_exit(void) -{ - debugfs_remove_recursive(genpd_debugfs_dir); -} -__exitcall(genpd_debug_exit); -#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c deleted file mode 100644 index 282a3a135827..000000000000 --- a/drivers/base/power/domain_governor.c +++ /dev/null @@ -1,411 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * drivers/base/power/domain_governor.c - Governors for device PM domains. - * - * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. - */ -#include <linux/kernel.h> -#include <linux/pm_domain.h> -#include <linux/pm_qos.h> -#include <linux/hrtimer.h> -#include <linux/cpuidle.h> -#include <linux/cpumask.h> -#include <linux/ktime.h> - -static int dev_update_qos_constraint(struct device *dev, void *data) -{ - s64 *constraint_ns_p = data; - s64 constraint_ns; - - if (dev->power.subsys_data && dev->power.subsys_data->domain_data) { - struct gpd_timing_data *td = dev_gpd_data(dev)->td; - - /* - * Only take suspend-time QoS constraints of devices into - * account, because constraints updated after the device has - * been suspended are not guaranteed to be taken into account - * anyway. In order for them to take effect, the device has to - * be resumed and suspended again. - */ - constraint_ns = td ? td->effective_constraint_ns : - PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; - } else { - /* - * The child is not in a domain and there's no info on its - * suspend/resume latencies, so assume them to be negligible and - * take its current PM QoS constraint (that's the only thing - * known at this point anyway). - */ - constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY); - constraint_ns *= NSEC_PER_USEC; - } - - if (constraint_ns < *constraint_ns_p) - *constraint_ns_p = constraint_ns; - - return 0; -} - -/** - * default_suspend_ok - Default PM domain governor routine to suspend devices. - * @dev: Device to check. - */ -static bool default_suspend_ok(struct device *dev) -{ - struct gpd_timing_data *td = dev_gpd_data(dev)->td; - unsigned long flags; - s64 constraint_ns; - - dev_dbg(dev, "%s()\n", __func__); - - spin_lock_irqsave(&dev->power.lock, flags); - - if (!td->constraint_changed) { - bool ret = td->cached_suspend_ok; - - spin_unlock_irqrestore(&dev->power.lock, flags); - return ret; - } - td->constraint_changed = false; - td->cached_suspend_ok = false; - td->effective_constraint_ns = 0; - constraint_ns = __dev_pm_qos_resume_latency(dev); - - spin_unlock_irqrestore(&dev->power.lock, flags); - - if (constraint_ns == 0) - return false; - - constraint_ns *= NSEC_PER_USEC; - /* - * We can walk the children without any additional locking, because - * they all have been suspended at this point and their - * effective_constraint_ns fields won't be modified in parallel with us. - */ - if (!dev->power.ignore_children) - device_for_each_child(dev, &constraint_ns, - dev_update_qos_constraint); - - if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) { - /* "No restriction", so the device is allowed to suspend. */ - td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; - td->cached_suspend_ok = true; - } else if (constraint_ns == 0) { - /* - * This triggers if one of the children that don't belong to a - * domain has a zero PM QoS constraint and it's better not to - * suspend then. effective_constraint_ns is zero already and - * cached_suspend_ok is false, so bail out. - */ - return false; - } else { - constraint_ns -= td->suspend_latency_ns + - td->resume_latency_ns; - /* - * effective_constraint_ns is zero already and cached_suspend_ok - * is false, so if the computed value is not positive, return - * right away. - */ - if (constraint_ns <= 0) - return false; - - td->effective_constraint_ns = constraint_ns; - td->cached_suspend_ok = true; - } - - /* - * The children have been suspended already, so we don't need to take - * their suspend latencies into account here. - */ - return td->cached_suspend_ok; -} - -static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now) -{ - ktime_t domain_wakeup = KTIME_MAX; - ktime_t next_wakeup; - struct pm_domain_data *pdd; - struct gpd_link *link; - - if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY)) - return; - - /* - * Devices that have a predictable wakeup pattern, may specify - * their next wakeup. Let's find the next wakeup from all the - * devices attached to this domain and from all the sub-domains. - * It is possible that component's a next wakeup may have become - * stale when we read that here. We will ignore to ensure the domain - * is able to enter its optimal idle state. - */ - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - next_wakeup = to_gpd_data(pdd)->td->next_wakeup; - if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) - if (ktime_before(next_wakeup, domain_wakeup)) - domain_wakeup = next_wakeup; - } - - list_for_each_entry(link, &genpd->parent_links, parent_node) { - struct genpd_governor_data *cgd = link->child->gd; - - next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX; - if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) - if (ktime_before(next_wakeup, domain_wakeup)) - domain_wakeup = next_wakeup; - } - - genpd->gd->next_wakeup = domain_wakeup; -} - -static bool next_wakeup_allows_state(struct generic_pm_domain *genpd, - unsigned int state, ktime_t now) -{ - ktime_t domain_wakeup = genpd->gd->next_wakeup; - s64 idle_time_ns, min_sleep_ns; - - min_sleep_ns = genpd->states[state].power_off_latency_ns + - genpd->states[state].residency_ns; - - idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); - - return idle_time_ns >= min_sleep_ns; -} - -static bool __default_power_down_ok(struct dev_pm_domain *pd, - unsigned int state) -{ - struct generic_pm_domain *genpd = pd_to_genpd(pd); - struct gpd_link *link; - struct pm_domain_data *pdd; - s64 min_off_time_ns; - s64 off_on_time_ns; - - off_on_time_ns = genpd->states[state].power_off_latency_ns + - genpd->states[state].power_on_latency_ns; - - min_off_time_ns = -1; - /* - * Check if subdomains can be off for enough time. - * - * All subdomains have been powered off already at this point. - */ - list_for_each_entry(link, &genpd->parent_links, parent_node) { - struct genpd_governor_data *cgd = link->child->gd; - - s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1; - - if (sd_max_off_ns < 0) - continue; - - /* - * Check if the subdomain is allowed to be off long enough for - * the current domain to turn off and on (that's how much time - * it will have to wait worst case). - */ - if (sd_max_off_ns <= off_on_time_ns) - return false; - - if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0) - min_off_time_ns = sd_max_off_ns; - } - - /* - * Check if the devices in the domain can be off enough time. - */ - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - struct gpd_timing_data *td; - s64 constraint_ns; - - /* - * Check if the device is allowed to be off long enough for the - * domain to turn off and on (that's how much time it will - * have to wait worst case). - */ - td = to_gpd_data(pdd)->td; - constraint_ns = td->effective_constraint_ns; - /* - * Zero means "no suspend at all" and this runs only when all - * devices in the domain are suspended, so it must be positive. - */ - if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) - continue; - - if (constraint_ns <= off_on_time_ns) - return false; - - if (min_off_time_ns > constraint_ns || min_off_time_ns < 0) - min_off_time_ns = constraint_ns; - } - - /* - * If the computed minimum device off time is negative, there are no - * latency constraints, so the domain can spend arbitrary time in the - * "off" state. - */ - if (min_off_time_ns < 0) - return true; - - /* - * The difference between the computed minimum subdomain or device off - * time and the time needed to turn the domain on is the maximum - * theoretical time this domain can spend in the "off" state. - */ - genpd->gd->max_off_time_ns = min_off_time_ns - - genpd->states[state].power_on_latency_ns; - return true; -} - -/** - * _default_power_down_ok - Default generic PM domain power off governor routine. - * @pd: PM domain to check. - * @now: current ktime. - * - * This routine must be executed under the PM domain's lock. - */ -static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now) -{ - struct generic_pm_domain *genpd = pd_to_genpd(pd); - struct genpd_governor_data *gd = genpd->gd; - int state_idx = genpd->state_count - 1; - struct gpd_link *link; - - /* - * Find the next wakeup from devices that can determine their own wakeup - * to find when the domain would wakeup and do it for every device down - * the hierarchy. It is not worth while to sleep if the state's residency - * cannot be met. - */ - update_domain_next_wakeup(genpd, now); - if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) { - /* Let's find out the deepest domain idle state, the devices prefer */ - while (state_idx >= 0) { - if (next_wakeup_allows_state(genpd, state_idx, now)) { - gd->max_off_time_changed = true; - break; - } - state_idx--; - } - - if (state_idx < 0) { - state_idx = 0; - gd->cached_power_down_ok = false; - goto done; - } - } - - if (!gd->max_off_time_changed) { - genpd->state_idx = gd->cached_power_down_state_idx; - return gd->cached_power_down_ok; - } - - /* - * We have to invalidate the cached results for the parents, so - * use the observation that default_power_down_ok() is not - * going to be called for any parent until this instance - * returns. - */ - list_for_each_entry(link, &genpd->child_links, child_node) { - struct genpd_governor_data *pgd = link->parent->gd; - - if (pgd) - pgd->max_off_time_changed = true; - } - - gd->max_off_time_ns = -1; - gd->max_off_time_changed = false; - gd->cached_power_down_ok = true; - - /* - * Find a state to power down to, starting from the state - * determined by the next wakeup. - */ - while (!__default_power_down_ok(pd, state_idx)) { - if (state_idx == 0) { - gd->cached_power_down_ok = false; - break; - } - state_idx--; - } - -done: - genpd->state_idx = state_idx; - gd->cached_power_down_state_idx = genpd->state_idx; - return gd->cached_power_down_ok; -} - -static bool default_power_down_ok(struct dev_pm_domain *pd) -{ - return _default_power_down_ok(pd, ktime_get()); -} - -#ifdef CONFIG_CPU_IDLE -static bool cpu_power_down_ok(struct dev_pm_domain *pd) -{ - struct generic_pm_domain *genpd = pd_to_genpd(pd); - struct cpuidle_device *dev; - ktime_t domain_wakeup, next_hrtimer; - ktime_t now = ktime_get(); - s64 idle_duration_ns; - int cpu, i; - - /* Validate dev PM QoS constraints. */ - if (!_default_power_down_ok(pd, now)) - return false; - - if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) - return true; - - /* - * Find the next wakeup for any of the online CPUs within the PM domain - * and its subdomains. Note, we only need the genpd->cpus, as it already - * contains a mask of all CPUs from subdomains. - */ - domain_wakeup = ktime_set(KTIME_SEC_MAX, 0); - for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) { - dev = per_cpu(cpuidle_devices, cpu); - if (dev) { - next_hrtimer = READ_ONCE(dev->next_hrtimer); - if (ktime_before(next_hrtimer, domain_wakeup)) - domain_wakeup = next_hrtimer; - } - } - - /* The minimum idle duration is from now - until the next wakeup. */ - idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); - if (idle_duration_ns <= 0) - return false; - - /* - * Find the deepest idle state that has its residency value satisfied - * and by also taking into account the power off latency for the state. - * Start at the state picked by the dev PM QoS constraint validation. - */ - i = genpd->state_idx; - do { - if (idle_duration_ns >= (genpd->states[i].residency_ns + - genpd->states[i].power_off_latency_ns)) { - genpd->state_idx = i; - return true; - } - } while (--i >= 0); - - return false; -} - -struct dev_power_governor pm_domain_cpu_gov = { - .suspend_ok = default_suspend_ok, - .power_down_ok = cpu_power_down_ok, -}; -#endif - -struct dev_power_governor simple_qos_governor = { - .suspend_ok = default_suspend_ok, - .power_down_ok = default_power_down_ok, -}; - -/** - * pm_genpd_gov_always_on - A governor implementing an always-on policy - */ -struct dev_power_governor pm_domain_always_on_gov = { - .suspend_ok = default_suspend_ok, -}; diff --git a/drivers/base/power/generic_ops.c b/drivers/base/power/generic_ops.c index 4fa525668cb7..af99bbcf281c 100644 --- a/drivers/base/power/generic_ops.c +++ b/drivers/base/power/generic_ops.c @@ -8,6 +8,13 @@ #include <linux/pm_runtime.h> #include <linux/export.h> +#define CALL_PM_OP(dev, op) \ +({ \ + struct device *_dev = (dev); \ + const struct dev_pm_ops *pm = _dev->driver ? _dev->driver->pm : NULL; \ + pm && pm->op ? pm->op(_dev) : 0; \ +}) + #ifdef CONFIG_PM /** * pm_generic_runtime_suspend - Generic runtime suspend callback for subsystems. @@ -19,12 +26,7 @@ */ int pm_generic_runtime_suspend(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret; - - ret = pm && pm->runtime_suspend ? pm->runtime_suspend(dev) : 0; - - return ret; + return CALL_PM_OP(dev, runtime_suspend); } EXPORT_SYMBOL_GPL(pm_generic_runtime_suspend); @@ -38,12 +40,7 @@ EXPORT_SYMBOL_GPL(pm_generic_runtime_suspend); */ int pm_generic_runtime_resume(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret; - - ret = pm && pm->runtime_resume ? pm->runtime_resume(dev) : 0; - - return ret; + return CALL_PM_OP(dev, runtime_resume); } EXPORT_SYMBOL_GPL(pm_generic_runtime_resume); #endif /* CONFIG_PM */ @@ -72,9 +69,7 @@ int pm_generic_prepare(struct device *dev) */ int pm_generic_suspend_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend_noirq ? pm->suspend_noirq(dev) : 0; + return CALL_PM_OP(dev, suspend_noirq); } EXPORT_SYMBOL_GPL(pm_generic_suspend_noirq); @@ -84,9 +79,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend_noirq); */ int pm_generic_suspend_late(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend_late ? pm->suspend_late(dev) : 0; + return CALL_PM_OP(dev, suspend_late); } EXPORT_SYMBOL_GPL(pm_generic_suspend_late); @@ -96,9 +89,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend_late); */ int pm_generic_suspend(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend ? pm->suspend(dev) : 0; + return CALL_PM_OP(dev, suspend); } EXPORT_SYMBOL_GPL(pm_generic_suspend); @@ -108,33 +99,17 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend); */ int pm_generic_freeze_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze_noirq ? pm->freeze_noirq(dev) : 0; + return CALL_PM_OP(dev, freeze_noirq); } EXPORT_SYMBOL_GPL(pm_generic_freeze_noirq); /** - * pm_generic_freeze_late - Generic freeze_late callback for subsystems. - * @dev: Device to freeze. - */ -int pm_generic_freeze_late(struct device *dev) -{ - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze_late ? pm->freeze_late(dev) : 0; -} -EXPORT_SYMBOL_GPL(pm_generic_freeze_late); - -/** * pm_generic_freeze - Generic freeze callback for subsystems. * @dev: Device to freeze. */ int pm_generic_freeze(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze ? pm->freeze(dev) : 0; + return CALL_PM_OP(dev, freeze); } EXPORT_SYMBOL_GPL(pm_generic_freeze); @@ -144,9 +119,7 @@ EXPORT_SYMBOL_GPL(pm_generic_freeze); */ int pm_generic_poweroff_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff_noirq ? pm->poweroff_noirq(dev) : 0; + return CALL_PM_OP(dev, poweroff_noirq); } EXPORT_SYMBOL_GPL(pm_generic_poweroff_noirq); @@ -156,9 +129,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff_noirq); */ int pm_generic_poweroff_late(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff_late ? pm->poweroff_late(dev) : 0; + return CALL_PM_OP(dev, poweroff_late); } EXPORT_SYMBOL_GPL(pm_generic_poweroff_late); @@ -168,9 +139,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff_late); */ int pm_generic_poweroff(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff ? pm->poweroff(dev) : 0; + return CALL_PM_OP(dev, poweroff); } EXPORT_SYMBOL_GPL(pm_generic_poweroff); @@ -180,33 +149,17 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff); */ int pm_generic_thaw_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw_noirq ? pm->thaw_noirq(dev) : 0; + return CALL_PM_OP(dev, thaw_noirq); } EXPORT_SYMBOL_GPL(pm_generic_thaw_noirq); /** - * pm_generic_thaw_early - Generic thaw_early callback for subsystems. - * @dev: Device to thaw. - */ -int pm_generic_thaw_early(struct device *dev) -{ - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw_early ? pm->thaw_early(dev) : 0; -} -EXPORT_SYMBOL_GPL(pm_generic_thaw_early); - -/** * pm_generic_thaw - Generic thaw callback for subsystems. * @dev: Device to thaw. */ int pm_generic_thaw(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw ? pm->thaw(dev) : 0; + return CALL_PM_OP(dev, thaw); } EXPORT_SYMBOL_GPL(pm_generic_thaw); @@ -216,9 +169,7 @@ EXPORT_SYMBOL_GPL(pm_generic_thaw); */ int pm_generic_resume_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume_noirq ? pm->resume_noirq(dev) : 0; + return CALL_PM_OP(dev, resume_noirq); } EXPORT_SYMBOL_GPL(pm_generic_resume_noirq); @@ -228,9 +179,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume_noirq); */ int pm_generic_resume_early(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume_early ? pm->resume_early(dev) : 0; + return CALL_PM_OP(dev, resume_early); } EXPORT_SYMBOL_GPL(pm_generic_resume_early); @@ -240,9 +189,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume_early); */ int pm_generic_resume(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume ? pm->resume(dev) : 0; + return CALL_PM_OP(dev, resume); } EXPORT_SYMBOL_GPL(pm_generic_resume); @@ -252,9 +199,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume); */ int pm_generic_restore_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore_noirq ? pm->restore_noirq(dev) : 0; + return CALL_PM_OP(dev, restore_noirq); } EXPORT_SYMBOL_GPL(pm_generic_restore_noirq); @@ -264,9 +209,7 @@ EXPORT_SYMBOL_GPL(pm_generic_restore_noirq); */ int pm_generic_restore_early(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore_early ? pm->restore_early(dev) : 0; + return CALL_PM_OP(dev, restore_early); } EXPORT_SYMBOL_GPL(pm_generic_restore_early); @@ -276,9 +219,7 @@ EXPORT_SYMBOL_GPL(pm_generic_restore_early); */ int pm_generic_restore(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore ? pm->restore(dev) : 0; + return CALL_PM_OP(dev, restore); } EXPORT_SYMBOL_GPL(pm_generic_restore); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index c50139207794..97a8b4fcf471 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -34,16 +34,13 @@ #include <linux/cpufreq.h> #include <linux/devfreq.h> #include <linux/timer.h> +#include <linux/nmi.h> #include "../base.h" #include "power.h" typedef int (*pm_callback_t)(struct device *); -#define list_for_each_entry_rcu_locked(pos, head, member) \ - list_for_each_entry_rcu(pos, head, member, \ - device_links_read_lock_held()) - /* * The entries in the dpm_list list are in a depth first order, simply * because children are guaranteed to be discovered after parents, and @@ -60,12 +57,26 @@ static LIST_HEAD(dpm_suspended_list); static LIST_HEAD(dpm_late_early_list); static LIST_HEAD(dpm_noirq_list); -struct suspend_stats suspend_stats; static DEFINE_MUTEX(dpm_list_mtx); static pm_message_t pm_transition; +static DEFINE_MUTEX(async_wip_mtx); static int async_error; +/** + * pm_hibernate_is_recovering - if recovering from hibernate due to error. + * + * Used to query if dev_pm_ops.thaw() is called for normal hibernation case or + * recovering from some error. + * + * Return: true for error case, false for normal case. + */ +bool pm_hibernate_is_recovering(void) +{ + return pm_transition.event == PM_EVENT_RECOVER; +} +EXPORT_SYMBOL_GPL(pm_hibernate_is_recovering); + static const char *pm_verb(int event) { switch (event) { @@ -85,6 +96,8 @@ static const char *pm_verb(int event) return "restore"; case PM_EVENT_RECOVER: return "recover"; + case PM_EVENT_POWEROFF: + return "poweroff"; default: return "(unknown PM event)"; } @@ -209,7 +222,7 @@ static ktime_t initcall_debug_start(struct device *dev, void *cb) if (!pm_print_times_enabled) return 0; - dev_info(dev, "calling %pS @ %i, parent: %s\n", cb, + dev_info(dev, "calling %ps @ %i, parent: %s\n", cb, task_pid_nr(current), dev->parent ? dev_name(dev->parent) : "none"); return ktime_get(); @@ -224,7 +237,7 @@ static void initcall_debug_report(struct device *dev, ktime_t calltime, return; rettime = ktime_get(); - dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error, + dev_info(dev, "%ps returned %d after %Ld usecs\n", cb, error, (unsigned long long)ktime_us_delta(rettime, calltime)); } @@ -250,7 +263,7 @@ static int dpm_wait_fn(struct device *dev, void *async_ptr) static void dpm_wait_for_children(struct device *dev, bool async) { - device_for_each_child(dev, &async, dpm_wait_fn); + device_for_each_child(dev, &async, dpm_wait_fn); } static void dpm_wait_for_suppliers(struct device *dev, bool async) @@ -267,8 +280,9 @@ static void dpm_wait_for_suppliers(struct device *dev, bool async) * callbacks freeing the link objects for the links in the list we're * walking. */ - list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) - if (READ_ONCE(link->status) != DL_STATE_DORMANT) + dev_for_each_link_to_supplier(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT && + !device_link_flag_is_sync_state_only(link->flags)) dpm_wait(link->supplier, async); device_links_read_unlock(idx); @@ -324,8 +338,9 @@ static void dpm_wait_for_consumers(struct device *dev, bool async) * continue instead of trying to continue in parallel with its * unregistration). */ - list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node) - if (READ_ONCE(link->status) != DL_STATE_DORMANT) + dev_for_each_link_to_consumer(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT && + !device_link_flag_is_sync_state_only(link->flags)) dpm_wait(link->consumer, async); device_links_read_unlock(idx); @@ -355,6 +370,7 @@ static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state) case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff; case PM_EVENT_THAW: @@ -389,6 +405,7 @@ static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops, case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze_late; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff_late; case PM_EVENT_THAW: @@ -423,6 +440,7 @@ static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t stat case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze_noirq; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff_noirq; case PM_EVENT_THAW: @@ -497,11 +515,17 @@ struct dpm_watchdog { struct device *dev; struct task_struct *tsk; struct timer_list timer; + bool fatal; }; #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \ struct dpm_watchdog wd +static bool __read_mostly dpm_watchdog_all_cpu_backtrace; +module_param(dpm_watchdog_all_cpu_backtrace, bool, 0644); +MODULE_PARM_DESC(dpm_watchdog_all_cpu_backtrace, + "Backtrace all CPUs on DPM watchdog timeout"); + /** * dpm_watchdog_handler - Driver suspend / resume watchdog handler. * @t: The timer that PM watchdog depends on. @@ -512,12 +536,28 @@ struct dpm_watchdog { */ static void dpm_watchdog_handler(struct timer_list *t) { - struct dpm_watchdog *wd = from_timer(wd, t, timer); + struct dpm_watchdog *wd = timer_container_of(wd, t, timer); + struct timer_list *timer = &wd->timer; + unsigned int time_left; + + if (wd->fatal) { + unsigned int this_cpu = smp_processor_id(); - dev_emerg(wd->dev, "**** DPM device timeout ****\n"); - show_stack(wd->tsk, NULL, KERN_EMERG); - panic("%s %s: unrecoverable failure\n", - dev_driver_string(wd->dev), dev_name(wd->dev)); + dev_emerg(wd->dev, "**** DPM device timeout ****\n"); + show_stack(wd->tsk, NULL, KERN_EMERG); + if (dpm_watchdog_all_cpu_backtrace) + trigger_allbutcpu_cpu_backtrace(this_cpu); + panic("%s %s: unrecoverable failure\n", + dev_driver_string(wd->dev), dev_name(wd->dev)); + } + + time_left = CONFIG_DPM_WATCHDOG_TIMEOUT - CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT; + dev_warn(wd->dev, "**** DPM device timeout after %u seconds; %u seconds until panic ****\n", + CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT, time_left); + show_stack(wd->tsk, NULL, KERN_WARNING); + + wd->fatal = true; + mod_timer(timer, jiffies + HZ * time_left); } /** @@ -531,10 +571,11 @@ static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev) wd->dev = dev; wd->tsk = current; + wd->fatal = CONFIG_DPM_WATCHDOG_TIMEOUT == CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT; timer_setup_on_stack(timer, dpm_watchdog_handler, 0); /* use same timeout value for both suspend and resume */ - timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT; + timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT; add_timer(timer); } @@ -546,8 +587,8 @@ static void dpm_watchdog_clear(struct dpm_watchdog *wd) { struct timer_list *timer = &wd->timer; - del_timer_sync(timer); - destroy_timer_on_stack(timer); + timer_delete_sync(timer); + timer_destroy_on_stack(timer); } #else #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) @@ -578,6 +619,102 @@ bool dev_pm_skip_resume(struct device *dev) return !dev->power.must_resume; } +static bool is_async(struct device *dev) +{ + return dev->power.async_suspend && pm_async_enabled + && !pm_trace_is_enabled(); +} + +static bool __dpm_async(struct device *dev, async_func_t func) +{ + if (dev->power.work_in_progress) + return true; + + if (!is_async(dev)) + return false; + + dev->power.work_in_progress = true; + + get_device(dev); + + if (async_schedule_dev_nocall(func, dev)) + return true; + + put_device(dev); + + return false; +} + +static bool dpm_async_fn(struct device *dev, async_func_t func) +{ + guard(mutex)(&async_wip_mtx); + + return __dpm_async(dev, func); +} + +static int dpm_async_with_cleanup(struct device *dev, void *fn) +{ + guard(mutex)(&async_wip_mtx); + + if (!__dpm_async(dev, fn)) + dev->power.work_in_progress = false; + + return 0; +} + +static void dpm_async_resume_children(struct device *dev, async_func_t func) +{ + /* + * Prevent racing with dpm_clear_async_state() during initial list + * walks in dpm_noirq_resume_devices(), dpm_resume_early(), and + * dpm_resume(). + */ + guard(mutex)(&dpm_list_mtx); + + /* + * Start processing "async" children of the device unless it's been + * started already for them. + */ + device_for_each_child(dev, func, dpm_async_with_cleanup); +} + +static void dpm_async_resume_subordinate(struct device *dev, async_func_t func) +{ + struct device_link *link; + int idx; + + dpm_async_resume_children(dev, func); + + idx = device_links_read_lock(); + + /* Start processing the device's "async" consumers. */ + dev_for_each_link_to_consumer(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT) + dpm_async_with_cleanup(link->consumer, func); + + device_links_read_unlock(idx); +} + +static void dpm_clear_async_state(struct device *dev) +{ + reinit_completion(&dev->power.completion); + dev->power.work_in_progress = false; +} + +static bool dpm_root_device(struct device *dev) +{ + lockdep_assert_held(&dpm_list_mtx); + + /* + * Since this function is required to run under dpm_list_mtx, the + * list_empty() below will only return true if the device's list of + * consumers is actually empty before calling it. + */ + return !dev->parent && list_empty(&dev->links.suppliers); +} + +static void async_resume_noirq(void *data, async_cookie_t cookie); + /** * device_resume_noirq - Execute a "noirq resume" callback for given device. * @dev: Device to handle. @@ -587,7 +724,7 @@ bool dev_pm_skip_resume(struct device *dev) * The driver of @dev will not receive interrupts while this function is being * executed. */ -static int device_resume_noirq(struct device *dev, pm_message_t state, bool async) +static void device_resume_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -600,8 +737,20 @@ static int device_resume_noirq(struct device *dev, pm_message_t state, bool asyn if (dev->power.syscore || dev->power.direct_complete) goto Out; - if (!dev->power.is_noirq_suspended) + if (!dev->power.is_noirq_suspended) { + /* + * This means that system suspend has been aborted in the noirq + * phase before invoking the noirq suspend callback for the + * device, so if device_suspend_late() has left it in suspend, + * device_resume_early() should leave it in suspend either in + * case the early resume of it depends on the noirq resume that + * has not run. + */ + if (dev_pm_skip_suspend(dev)) + dev->power.must_resume = false; + goto Out; + } if (!dpm_wait_for_superior(dev, async)) goto Out; @@ -614,12 +763,12 @@ static int device_resume_noirq(struct device *dev, pm_message_t state, bool asyn * so change its status accordingly. * * Otherwise, the device is going to be resumed, so set its PM-runtime - * status to "active", but do that only if DPM_FLAG_SMART_SUSPEND is set - * to avoid confusing drivers that don't use it. + * status to "active" unless its power.smart_suspend flag is clear, in + * which case it is not necessary to update its PM-runtime status. */ if (skip_resume) pm_runtime_set_suspended(dev); - else if (dev_pm_skip_suspend(dev)) + else if (dev_pm_smart_suspend(dev)) pm_runtime_set_active(dev); if (dev->pm_domain) { @@ -655,37 +804,21 @@ Skip: Out: complete_all(&dev->power.completion); TRACE_RESUME(error); - return error; -} - -static bool is_async(struct device *dev) -{ - return dev->power.async_suspend && pm_async_enabled - && !pm_trace_is_enabled(); -} - -static bool dpm_async_fn(struct device *dev, async_func_t func) -{ - reinit_completion(&dev->power.completion); - if (is_async(dev)) { - get_device(dev); - async_schedule_dev(func, dev); - return true; + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); } - return false; + dpm_async_resume_subordinate(dev, async_resume_noirq); } static void async_resume_noirq(void *data, async_cookie_t cookie) { - struct device *dev = (struct device *)data; - int error; - - error = device_resume_noirq(dev, pm_transition, true); - if (error) - pm_dev_err(dev, pm_transition, " async", error); + struct device *dev = data; + device_resume_noirq(dev, pm_transition, true); put_device(dev); } @@ -695,43 +828,44 @@ static void dpm_noirq_resume_devices(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true); - mutex_lock(&dpm_list_mtx); + + async_error = 0; pm_transition = state; + mutex_lock(&dpm_list_mtx); + /* - * Advanced the async threads upfront, - * in case the starting of async threads is - * delayed by non-async resuming devices. + * Start processing "async" root devices upfront so they don't wait for + * the "sync" devices they don't depend on. */ - list_for_each_entry(dev, &dpm_noirq_list, power.entry) - dpm_async_fn(dev, async_resume_noirq); + list_for_each_entry(dev, &dpm_noirq_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_root_device(dev)) + dpm_async_with_cleanup(dev, async_resume_noirq); + } while (!list_empty(&dpm_noirq_list)) { dev = to_device(dpm_noirq_list.next); - get_device(dev); list_move_tail(&dev->power.entry, &dpm_late_early_list); - mutex_unlock(&dpm_list_mtx); + if (!dpm_async_fn(dev, async_resume_noirq)) { + get_device(dev); - if (!is_async(dev)) { - int error; + mutex_unlock(&dpm_list_mtx); - error = device_resume_noirq(dev, state, false); - if (error) { - suspend_stats.failed_resume_noirq++; - dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, state, " noirq", error); - } - } + device_resume_noirq(dev, state, false); - put_device(dev); + put_device(dev); - mutex_lock(&dpm_list_mtx); + mutex_lock(&dpm_list_mtx); + } } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, "noirq"); + if (READ_ONCE(async_error)) + dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); + trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false); } @@ -750,6 +884,8 @@ void dpm_resume_noirq(pm_message_t state) device_wakeup_disarm_wake_irqs(); } +static void async_resume_early(void *data, async_cookie_t cookie); + /** * device_resume_early - Execute an "early resume" callback for given device. * @dev: Device to handle. @@ -758,7 +894,7 @@ void dpm_resume_noirq(pm_message_t state) * * Runtime PM is disabled for @dev while this function is being executed. */ -static int device_resume_early(struct device *dev, pm_message_t state, bool async) +static void device_resume_early(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -767,12 +903,15 @@ static int device_resume_early(struct device *dev, pm_message_t state, bool asyn TRACE_DEVICE(dev); TRACE_RESUME(0); - if (dev->power.syscore || dev->power.direct_complete) + if (dev->power.direct_complete) goto Out; if (!dev->power.is_late_suspended) goto Out; + if (dev->power.syscore) + goto Skip; + if (!dpm_wait_for_superior(dev, async)) goto Out; @@ -805,24 +944,27 @@ Run: Skip: dev->power.is_late_suspended = false; + pm_runtime_enable(dev); Out: TRACE_RESUME(error); - pm_runtime_enable(dev); complete_all(&dev->power.completion); - return error; + + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async early" : " early", error); + } + + dpm_async_resume_subordinate(dev, async_resume_early); } static void async_resume_early(void *data, async_cookie_t cookie) { - struct device *dev = (struct device *)data; - int error; - - error = device_resume_early(dev, pm_transition, true); - if (error) - pm_dev_err(dev, pm_transition, " async", error); + struct device *dev = data; + device_resume_early(dev, pm_transition, true); put_device(dev); } @@ -836,43 +978,44 @@ void dpm_resume_early(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume_early"), state.event, true); - mutex_lock(&dpm_list_mtx); + + async_error = 0; pm_transition = state; + mutex_lock(&dpm_list_mtx); + /* - * Advanced the async threads upfront, - * in case the starting of async threads is - * delayed by non-async resuming devices. + * Start processing "async" root devices upfront so they don't wait for + * the "sync" devices they don't depend on. */ - list_for_each_entry(dev, &dpm_late_early_list, power.entry) - dpm_async_fn(dev, async_resume_early); + list_for_each_entry(dev, &dpm_late_early_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_root_device(dev)) + dpm_async_with_cleanup(dev, async_resume_early); + } while (!list_empty(&dpm_late_early_list)) { dev = to_device(dpm_late_early_list.next); - get_device(dev); list_move_tail(&dev->power.entry, &dpm_suspended_list); - mutex_unlock(&dpm_list_mtx); + if (!dpm_async_fn(dev, async_resume_early)) { + get_device(dev); - if (!is_async(dev)) { - int error; + mutex_unlock(&dpm_list_mtx); - error = device_resume_early(dev, state, false); - if (error) { - suspend_stats.failed_resume_early++; - dpm_save_failed_step(SUSPEND_RESUME_EARLY); - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, state, " early", error); - } - } + device_resume_early(dev, state, false); - put_device(dev); + put_device(dev); - mutex_lock(&dpm_list_mtx); + mutex_lock(&dpm_list_mtx); + } } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, "early"); + if (READ_ONCE(async_error)) + dpm_save_failed_step(SUSPEND_RESUME_EARLY); + trace_suspend_resume(TPS("dpm_resume_early"), state.event, false); } @@ -887,13 +1030,15 @@ void dpm_resume_start(pm_message_t state) } EXPORT_SYMBOL_GPL(dpm_resume_start); +static void async_resume(void *data, async_cookie_t cookie); + /** * device_resume - Execute "resume" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being resumed asynchronously. */ -static int device_resume(struct device *dev, pm_message_t state, bool async) +static void device_resume(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -906,8 +1051,20 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) if (dev->power.syscore) goto Complete; + if (!dev->power.is_suspended) + goto Complete; + + dev->power.is_suspended = false; + if (dev->power.direct_complete) { - /* Match the pm_runtime_disable() in __device_suspend(). */ + /* + * Allow new children to be added under the device after this + * point if it has no PM callbacks. + */ + if (dev->power.no_pm_callbacks) + dev->power.is_prepared = false; + + /* Match the pm_runtime_disable() in device_suspend(). */ pm_runtime_enable(dev); goto Complete; } @@ -924,9 +1081,6 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) */ dev->power.is_prepared = false; - if (!dev->power.is_suspended) - goto Unlock; - if (dev->pm_domain) { info = "power domain "; callback = pm_op(&dev->pm_domain->ops, state); @@ -964,9 +1118,7 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) End: error = dpm_run_callback(callback, dev, state, info); - dev->power.is_suspended = false; - Unlock: device_unlock(dev); dpm_watchdog_clear(&wd); @@ -975,17 +1127,20 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) TRACE_RESUME(error); - return error; + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async" : "", error); + } + + dpm_async_resume_subordinate(dev, async_resume); } static void async_resume(void *data, async_cookie_t cookie) { - struct device *dev = (struct device *)data; - int error; + struct device *dev = data; - error = device_resume(dev, pm_transition, true); - if (error) - pm_dev_err(dev, pm_transition, " async", error); + device_resume(dev, pm_transition, true); put_device(dev); } @@ -1002,45 +1157,43 @@ void dpm_resume(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume"), state.event, true); - might_sleep(); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; - list_for_each_entry(dev, &dpm_suspended_list, power.entry) - dpm_async_fn(dev, async_resume); + mutex_lock(&dpm_list_mtx); + + /* + * Start processing "async" root devices upfront so they don't wait for + * the "sync" devices they don't depend on. + */ + list_for_each_entry(dev, &dpm_suspended_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_root_device(dev)) + dpm_async_with_cleanup(dev, async_resume); + } while (!list_empty(&dpm_suspended_list)) { dev = to_device(dpm_suspended_list.next); - get_device(dev); - if (!is_async(dev)) { - int error; + list_move_tail(&dev->power.entry, &dpm_prepared_list); + + if (!dpm_async_fn(dev, async_resume)) { + get_device(dev); mutex_unlock(&dpm_list_mtx); - error = device_resume(dev, state, false); - if (error) { - suspend_stats.failed_resume++; - dpm_save_failed_step(SUSPEND_RESUME); - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, state, "", error); - } + device_resume(dev, state, false); + + put_device(dev); mutex_lock(&dpm_list_mtx); } - if (!list_empty(&dev->power.entry)) - list_move_tail(&dev->power.entry, &dpm_prepared_list); - - mutex_unlock(&dpm_list_mtx); - - put_device(dev); - - mutex_lock(&dpm_list_mtx); } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, NULL); + if (READ_ONCE(async_error)) + dpm_save_failed_step(SUSPEND_RESUME); cpufreq_resume(); devfreq_resume(); @@ -1089,6 +1242,8 @@ static void device_complete(struct device *dev, pm_message_t state) device_unlock(dev); out: + /* If enabling runtime PM for the device is blocked, unblock it. */ + pm_runtime_unblock(dev); pm_runtime_put(dev); } @@ -1104,7 +1259,6 @@ void dpm_complete(pm_message_t state) struct list_head list; trace_suspend_resume(TPS("dpm_complete"), state.event, true); - might_sleep(); INIT_LIST_HEAD(&list); mutex_lock(&dpm_list_mtx); @@ -1143,6 +1297,7 @@ void dpm_complete(pm_message_t state) void dpm_resume_end(pm_message_t state) { dpm_resume(state); + pm_restore_gfp_mask(); dpm_complete(state); } EXPORT_SYMBOL_GPL(dpm_resume_end); @@ -1150,6 +1305,82 @@ EXPORT_SYMBOL_GPL(dpm_resume_end); /*------------------------- Suspend routines -------------------------*/ +static bool dpm_leaf_device(struct device *dev) +{ + struct device *child; + + lockdep_assert_held(&dpm_list_mtx); + + child = device_find_any_child(dev); + if (child) { + put_device(child); + + return false; + } + + /* + * Since this function is required to run under dpm_list_mtx, the + * list_empty() below will only return true if the device's list of + * consumers is actually empty before calling it. + */ + return list_empty(&dev->links.consumers); +} + +static bool dpm_async_suspend_parent(struct device *dev, async_func_t func) +{ + guard(mutex)(&dpm_list_mtx); + + /* + * If the device is suspended asynchronously and the parent's callback + * deletes both the device and the parent itself, the parent object may + * be freed while this function is running, so avoid that by checking + * if the device has been deleted already as the parent cannot be + * deleted before it. + */ + if (!device_pm_initialized(dev)) + return false; + + /* Start processing the device's parent if it is "async". */ + if (dev->parent) + dpm_async_with_cleanup(dev->parent, func); + + return true; +} + +static void dpm_async_suspend_superior(struct device *dev, async_func_t func) +{ + struct device_link *link; + int idx; + + if (!dpm_async_suspend_parent(dev, func)) + return; + + idx = device_links_read_lock(); + + /* Start processing the device's "async" suppliers. */ + dev_for_each_link_to_supplier(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT) + dpm_async_with_cleanup(link->supplier, func); + + device_links_read_unlock(idx); +} + +static void dpm_async_suspend_complete_all(struct list_head *device_list) +{ + struct device *dev; + + guard(mutex)(&async_wip_mtx); + + list_for_each_entry_reverse(dev, device_list, power.entry) { + /* + * In case the device is being waited for and async processing + * has not started for it yet, let the waiters make progress. + */ + if (!dev->power.work_in_progress) + complete_all(&dev->power.completion); + } +} + /** * resume_event - Return a "resume" message for given "suspend" sleep state. * @sleep_state: PM message representing a sleep state. @@ -1181,14 +1412,16 @@ static void dpm_superior_set_must_resume(struct device *dev) idx = device_links_read_lock(); - list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) + dev_for_each_link_to_supplier(link, dev) link->supplier->power.must_resume = true; device_links_read_unlock(idx); } +static void async_suspend_noirq(void *data, async_cookie_t cookie); + /** - * __device_suspend_noirq - Execute a "noirq suspend" callback for given device. + * device_suspend_noirq - Execute a "noirq suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. @@ -1196,7 +1429,7 @@ static void dpm_superior_set_must_resume(struct device *dev) * The driver of @dev will not receive interrupts while this function is being * executed. */ -static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async) +static void device_suspend_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1207,7 +1440,7 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a dpm_wait_for_subordinate(dev, async); - if (async_error) + if (READ_ONCE(async_error)) goto Complete; if (dev->power.syscore || dev->power.direct_complete) @@ -1240,7 +1473,9 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a Run: error = dpm_run_callback(callback, dev, state, info); if (error) { - async_error = error; + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); goto Complete; } @@ -1248,14 +1483,13 @@ Skip: dev->power.is_noirq_suspended = true; /* - * Skipping the resume of devices that were in use right before the - * system suspend (as indicated by their PM-runtime usage counters) - * would be suboptimal. Also resume them if doing that is not allowed - * to be skipped. + * Devices must be resumed unless they are explicitly allowed to be left + * in suspend, but even in that case skipping the resume of devices that + * were in use right before the system suspend (as indicated by their + * runtime PM usage counters and child counters) would be suboptimal. */ - if (atomic_read(&dev->power.usage_count) > 1 || - !(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) && - dev->power.may_skip_resume)) + if (!(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) && + dev->power.may_skip_resume) || !pm_runtime_need_not_resume(dev)) dev->power.must_resume = true; if (dev->power.must_resume) @@ -1264,76 +1498,81 @@ Skip: Complete: complete_all(&dev->power.completion); TRACE_SUSPEND(error); - return error; -} -static void async_suspend_noirq(void *data, async_cookie_t cookie) -{ - struct device *dev = (struct device *)data; - int error; - - error = __device_suspend_noirq(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + if (error || READ_ONCE(async_error)) + return; - put_device(dev); + dpm_async_suspend_superior(dev, async_suspend_noirq); } -static int device_suspend_noirq(struct device *dev) +static void async_suspend_noirq(void *data, async_cookie_t cookie) { - if (dpm_async_fn(dev, async_suspend_noirq)) - return 0; + struct device *dev = data; - return __device_suspend_noirq(dev, pm_transition, false); + device_suspend_noirq(dev, pm_transition, true); + put_device(dev); } static int dpm_noirq_suspend_devices(pm_message_t state) { ktime_t starttime = ktime_get(); - int error = 0; + struct device *dev; + int error; trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true); - mutex_lock(&dpm_list_mtx); + pm_transition = state; async_error = 0; - while (!list_empty(&dpm_late_early_list)) { - struct device *dev = to_device(dpm_late_early_list.prev); + mutex_lock(&dpm_list_mtx); - get_device(dev); - mutex_unlock(&dpm_list_mtx); + /* + * Start processing "async" leaf devices upfront so they don't need to + * wait for the "sync" devices they don't depend on. + */ + list_for_each_entry_reverse(dev, &dpm_late_early_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_leaf_device(dev)) + dpm_async_with_cleanup(dev, async_suspend_noirq); + } - error = device_suspend_noirq(dev); + while (!list_empty(&dpm_late_early_list)) { + dev = to_device(dpm_late_early_list.prev); - mutex_lock(&dpm_list_mtx); + list_move(&dev->power.entry, &dpm_noirq_list); - if (error) { - pm_dev_err(dev, state, " noirq", error); - dpm_save_failed_dev(dev_name(dev)); - } else if (!list_empty(&dev->power.entry)) { - list_move(&dev->power.entry, &dpm_noirq_list); - } + if (dpm_async_fn(dev, async_suspend_noirq)) + continue; + + get_device(dev); mutex_unlock(&dpm_list_mtx); + device_suspend_noirq(dev, state, false); + put_device(dev); mutex_lock(&dpm_list_mtx); - if (error || async_error) + if (READ_ONCE(async_error)) { + dpm_async_suspend_complete_all(&dpm_late_early_list); + /* + * Move all devices to the target list to resume them + * properly. + */ + list_splice_init(&dpm_late_early_list, &dpm_noirq_list); break; + } } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); - if (!error) - error = async_error; - if (error) { - suspend_stats.failed_suspend_noirq++; + error = READ_ONCE(async_error); + if (error) dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ); - } + dpm_show_time(starttime, state, error, "noirq"); trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false); return error; @@ -1375,15 +1614,17 @@ static void dpm_propagate_wakeup_to_parent(struct device *dev) spin_unlock_irq(&parent->power.lock); } +static void async_suspend_late(void *data, async_cookie_t cookie); + /** - * __device_suspend_late - Execute a "late suspend" callback for given device. + * device_suspend_late - Execute a "late suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. * * Runtime PM is disabled for @dev while this function is being executed. */ -static int __device_suspend_late(struct device *dev, pm_message_t state, bool async) +static void device_suspend_late(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1392,21 +1633,28 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as TRACE_DEVICE(dev); TRACE_SUSPEND(0); - __pm_runtime_disable(dev, false); - dpm_wait_for_subordinate(dev, async); - if (async_error) + if (READ_ONCE(async_error)) goto Complete; if (pm_wakeup_pending()) { - async_error = -EBUSY; + WRITE_ONCE(async_error, -EBUSY); goto Complete; } - if (dev->power.syscore || dev->power.direct_complete) + if (dev->power.direct_complete) goto Complete; + /* + * Disable runtime PM for the device without checking if there is a + * pending resume request for it. + */ + __pm_runtime_disable(dev, false); + + if (dev->power.syscore) + goto Skip; + if (dev->pm_domain) { info = "late power domain "; callback = pm_late_early_op(&dev->pm_domain->ops, state); @@ -1434,7 +1682,10 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as Run: error = dpm_run_callback(callback, dev, state, info); if (error) { - async_error = error; + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async late" : " late", error); + pm_runtime_enable(dev); goto Complete; } dpm_propagate_wakeup_to_parent(dev); @@ -1445,28 +1696,19 @@ Skip: Complete: TRACE_SUSPEND(error); complete_all(&dev->power.completion); - return error; -} -static void async_suspend_late(void *data, async_cookie_t cookie) -{ - struct device *dev = (struct device *)data; - int error; + if (error || READ_ONCE(async_error)) + return; - error = __device_suspend_late(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } - put_device(dev); + dpm_async_suspend_superior(dev, async_suspend_late); } -static int device_suspend_late(struct device *dev) +static void async_suspend_late(void *data, async_cookie_t cookie) { - if (dpm_async_fn(dev, async_suspend_late)) - return 0; + struct device *dev = data; - return __device_suspend_late(dev, pm_transition, false); + device_suspend_late(dev, pm_transition, true); + put_device(dev); } /** @@ -1476,48 +1718,63 @@ static int device_suspend_late(struct device *dev) int dpm_suspend_late(pm_message_t state) { ktime_t starttime = ktime_get(); - int error = 0; + struct device *dev; + int error; trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true); - wake_up_all_idle_cpus(); - mutex_lock(&dpm_list_mtx); + pm_transition = state; async_error = 0; - while (!list_empty(&dpm_suspended_list)) { - struct device *dev = to_device(dpm_suspended_list.prev); + wake_up_all_idle_cpus(); - get_device(dev); + mutex_lock(&dpm_list_mtx); - mutex_unlock(&dpm_list_mtx); + /* + * Start processing "async" leaf devices upfront so they don't need to + * wait for the "sync" devices they don't depend on. + */ + list_for_each_entry_reverse(dev, &dpm_suspended_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_leaf_device(dev)) + dpm_async_with_cleanup(dev, async_suspend_late); + } - error = device_suspend_late(dev); + while (!list_empty(&dpm_suspended_list)) { + dev = to_device(dpm_suspended_list.prev); - mutex_lock(&dpm_list_mtx); + list_move(&dev->power.entry, &dpm_late_early_list); - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_late_early_list); + if (dpm_async_fn(dev, async_suspend_late)) + continue; - if (error) { - pm_dev_err(dev, state, " late", error); - dpm_save_failed_dev(dev_name(dev)); - } + get_device(dev); mutex_unlock(&dpm_list_mtx); + device_suspend_late(dev, state, false); + put_device(dev); mutex_lock(&dpm_list_mtx); - if (error || async_error) + if (READ_ONCE(async_error)) { + dpm_async_suspend_complete_all(&dpm_suspended_list); + /* + * Move all devices to the target list to resume them + * properly. + */ + list_splice_init(&dpm_suspended_list, &dpm_late_early_list); break; + } } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); - if (!error) - error = async_error; + + error = READ_ONCE(async_error); if (error) { - suspend_stats.failed_suspend_late++; dpm_save_failed_step(SUSPEND_SUSPEND_LATE); dpm_resume_early(resume_event(state)); } @@ -1588,7 +1845,7 @@ static void dpm_clear_superiors_direct_complete(struct device *dev) idx = device_links_read_lock(); - list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) { + dev_for_each_link_to_supplier(link, dev) { spin_lock_irq(&link->supplier->power.lock); link->supplier->power.direct_complete = false; spin_unlock_irq(&link->supplier->power.lock); @@ -1597,13 +1854,15 @@ static void dpm_clear_superiors_direct_complete(struct device *dev) device_links_read_unlock(idx); } +static void async_suspend(void *data, async_cookie_t cookie); + /** - * __device_suspend - Execute "suspend" callbacks for given device. + * device_suspend - Execute "suspend" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. */ -static int __device_suspend(struct device *dev, pm_message_t state, bool async) +static void device_suspend(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1615,7 +1874,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) dpm_wait_for_subordinate(dev, async); - if (async_error) { + if (READ_ONCE(async_error)) { dev->power.direct_complete = false; goto Complete; } @@ -1635,7 +1894,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) if (pm_wakeup_pending()) { dev->power.direct_complete = false; - async_error = -EBUSY; + WRITE_ONCE(async_error, -EBUSY); goto Complete; } @@ -1651,6 +1910,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) pm_runtime_disable(dev); if (pm_runtime_status_suspended(dev)) { pm_dev_dbg(dev, state, "direct-complete "); + dev->power.is_suspended = true; goto Complete; } @@ -1717,34 +1977,27 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) dpm_watchdog_clear(&wd); Complete: - if (error) - async_error = error; + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async" : "", error); + } complete_all(&dev->power.completion); TRACE_SUSPEND(error); - return error; -} -static void async_suspend(void *data, async_cookie_t cookie) -{ - struct device *dev = (struct device *)data; - int error; - - error = __device_suspend(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + if (error || READ_ONCE(async_error)) + return; - put_device(dev); + dpm_async_suspend_superior(dev, async_suspend); } -static int device_suspend(struct device *dev) +static void async_suspend(void *data, async_cookie_t cookie) { - if (dpm_async_fn(dev, async_suspend)) - return 0; + struct device *dev = data; - return __device_suspend(dev, pm_transition, false); + device_suspend(dev, pm_transition, true); + put_device(dev); } /** @@ -1754,7 +2007,8 @@ static int device_suspend(struct device *dev) int dpm_suspend(pm_message_t state) { ktime_t starttime = ktime_get(); - int error = 0; + struct device *dev; + int error; trace_suspend_resume(TPS("dpm_suspend"), state.event, true); might_sleep(); @@ -1762,49 +2016,103 @@ int dpm_suspend(pm_message_t state) devfreq_suspend(); cpufreq_suspend(); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; - while (!list_empty(&dpm_prepared_list)) { - struct device *dev = to_device(dpm_prepared_list.prev); - get_device(dev); + mutex_lock(&dpm_list_mtx); - mutex_unlock(&dpm_list_mtx); + /* + * Start processing "async" leaf devices upfront so they don't need to + * wait for the "sync" devices they don't depend on. + */ + list_for_each_entry_reverse(dev, &dpm_prepared_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_leaf_device(dev)) + dpm_async_with_cleanup(dev, async_suspend); + } + + while (!list_empty(&dpm_prepared_list)) { + dev = to_device(dpm_prepared_list.prev); - error = device_suspend(dev); + list_move(&dev->power.entry, &dpm_suspended_list); - mutex_lock(&dpm_list_mtx); + if (dpm_async_fn(dev, async_suspend)) + continue; - if (error) { - pm_dev_err(dev, state, "", error); - dpm_save_failed_dev(dev_name(dev)); - } else if (!list_empty(&dev->power.entry)) { - list_move(&dev->power.entry, &dpm_suspended_list); - } + get_device(dev); mutex_unlock(&dpm_list_mtx); + device_suspend(dev, state, false); + put_device(dev); mutex_lock(&dpm_list_mtx); - if (error || async_error) + if (READ_ONCE(async_error)) { + dpm_async_suspend_complete_all(&dpm_prepared_list); + /* + * Move all devices to the target list to resume them + * properly. + */ + list_splice_init(&dpm_prepared_list, &dpm_suspended_list); break; + } } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); - if (!error) - error = async_error; - if (error) { - suspend_stats.failed_suspend++; + + error = READ_ONCE(async_error); + if (error) dpm_save_failed_step(SUSPEND_SUSPEND); - } + dpm_show_time(starttime, state, error, NULL); trace_suspend_resume(TPS("dpm_suspend"), state.event, false); return error; } +static bool device_prepare_smart_suspend(struct device *dev) +{ + struct device_link *link; + bool ret = true; + int idx; + + /* + * The "smart suspend" feature is enabled for devices whose drivers ask + * for it and for devices without PM callbacks. + * + * However, if "smart suspend" is not enabled for the device's parent + * or any of its suppliers that take runtime PM into account, it cannot + * be enabled for the device either. + */ + if (!dev->power.no_pm_callbacks && + !dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND)) + return false; + + if (dev->parent && !dev_pm_smart_suspend(dev->parent) && + !dev->parent->power.ignore_children && !pm_runtime_blocked(dev->parent)) + return false; + + idx = device_links_read_lock(); + + dev_for_each_link_to_supplier(link, dev) { + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) + continue; + + if (!dev_pm_smart_suspend(link->supplier) && + !pm_runtime_blocked(link->supplier)) { + ret = false; + break; + } + } + + device_links_read_unlock(idx); + + return ret; +} + /** * device_prepare - Prepare a device for system power transition. * @dev: Device to handle. @@ -1816,6 +2124,7 @@ int dpm_suspend(pm_message_t state) static int device_prepare(struct device *dev, pm_message_t state) { int (*callback)(struct device *) = NULL; + bool smart_suspend; int ret = 0; /* @@ -1825,6 +2134,13 @@ static int device_prepare(struct device *dev, pm_message_t state) * it again during the complete phase. */ pm_runtime_get_noresume(dev); + /* + * If runtime PM is disabled for the device at this point and it has + * never been enabled so far, it should not be enabled until this system + * suspend-resume cycle is complete, so prepare to trigger a warning on + * subsequent attempts to enable it. + */ + smart_suspend = !pm_runtime_block_if_disabled(dev); if (dev->power.syscore) return 0; @@ -1832,6 +2148,7 @@ static int device_prepare(struct device *dev, pm_message_t state) device_lock(dev); dev->power.wakeup_path = false; + dev->power.out_band_wakeup = false; if (dev->power.no_pm_callbacks) goto unlock; @@ -1859,6 +2176,13 @@ unlock: pm_runtime_put(dev); return ret; } + /* Do not enable "smart suspend" for devices with disabled runtime PM. */ + if (smart_suspend) + smart_suspend = device_prepare_smart_suspend(dev); + + spin_lock_irq(&dev->power.lock); + + dev->power.smart_suspend = smart_suspend; /* * A positive return value from ->prepare() means "this device appears * to be runtime-suspended and its state is fine, so if it really is @@ -1866,11 +2190,12 @@ unlock: * will do the same thing with all of its descendants". This only * applies to suspend transitions, however. */ - spin_lock_irq(&dev->power.lock); dev->power.direct_complete = state.event == PM_EVENT_SUSPEND && (ret > 0 || dev->power.no_pm_callbacks) && !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE); + spin_unlock_irq(&dev->power.lock); + return 0; } @@ -1885,7 +2210,6 @@ int dpm_prepare(pm_message_t state) int error = 0; trace_suspend_resume(TPS("dpm_prepare"), state.event, true); - might_sleep(); /* * Give a chance for the known devices to complete their probes, before @@ -1950,11 +2274,13 @@ int dpm_suspend_start(pm_message_t state) int error; error = dpm_prepare(state); - if (error) { - suspend_stats.failed_prepare++; + if (error) dpm_save_failed_step(SUSPEND_PREPARE); - } else + else { + pm_restrict_gfp_mask(); error = dpm_suspend(state); + } + dpm_show_time(starttime, state, error, "start"); return error; } @@ -1963,7 +2289,7 @@ EXPORT_SYMBOL_GPL(dpm_suspend_start); void __suspend_report_result(const char *function, struct device *dev, void *fn, int ret) { if (ret) - dev_err(dev, "%s(): %pS returns %d\n", function, fn, ret); + dev_err(dev, "%s(): %ps returns %d\n", function, fn, ret); } EXPORT_SYMBOL_GPL(__suspend_report_result); @@ -2034,6 +2360,5 @@ void device_pm_check_callbacks(struct device *dev) bool dev_pm_skip_suspend(struct device *dev) { - return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) && - pm_runtime_status_suspended(dev); + return dev_pm_smart_suspend(dev) && pm_runtime_status_suspended(dev); } diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h index 0eb7f02b3ad5..922ed457db19 100644 --- a/drivers/base/power/power.h +++ b/drivers/base/power/power.h @@ -29,6 +29,7 @@ extern u64 pm_runtime_active_time(struct device *dev); #define WAKE_IRQ_DEDICATED_MASK (WAKE_IRQ_DEDICATED_ALLOCATED | \ WAKE_IRQ_DEDICATED_MANAGED | \ WAKE_IRQ_DEDICATED_REVERSE) +#define WAKE_IRQ_DEDICATED_ENABLED BIT(3) struct wake_irq { struct device *dev; diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 8e93167f1783..ff393cba7649 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -137,6 +137,7 @@ s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type) return ret; } +EXPORT_SYMBOL_GPL(dev_pm_qos_read_value); /** * apply_constraint - Add/modify/remove device PM QoS request. @@ -201,7 +202,7 @@ static int dev_pm_qos_constraints_allocate(struct device *dev) if (!qos) return -ENOMEM; - n = kzalloc(3 * sizeof(*n), GFP_KERNEL); + n = kcalloc(3, sizeof(*n), GFP_KERNEL); if (!n) { kfree(qos); return -ENOMEM; diff --git a/drivers/base/power/runtime-test.c b/drivers/base/power/runtime-test.c new file mode 100644 index 000000000000..1535ad2b0264 --- /dev/null +++ b/drivers/base/power/runtime-test.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2025 Google, Inc. + */ + +#include <linux/cleanup.h> +#include <linux/pm_runtime.h> +#include <kunit/device.h> +#include <kunit/test.h> + +#define DEVICE_NAME "pm_runtime_test_device" + +static void pm_runtime_depth_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_get_sync(dev)); /* "already active" */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +/* Test pm_runtime_put() and friends when already suspended. */ +static void pm_runtime_already_suspended_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + pm_runtime_get_noresume(dev); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, 1, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_request_autosuspend(dev)); + + pm_runtime_get_noresume(dev); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_put_sync_autosuspend(dev)); + + pm_runtime_get_noresume(dev); + pm_runtime_put_autosuspend(dev); + + /* Grab 2 refcounts */ + pm_runtime_get_noresume(dev); + pm_runtime_get_noresume(dev); + /* The first put() sees usage_count 1 */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync_autosuspend(dev)); + /* The second put() sees usage_count 0 but tells us "already suspended". */ + KUNIT_EXPECT_EQ(test, 1, pm_runtime_put_sync_autosuspend(dev)); + + /* Should have remained suspended the whole time. */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +static void pm_runtime_idle_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_idle(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + pm_runtime_put_noidle(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_idle(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_request_idle(dev)); +} + +static void pm_runtime_disabled_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + /* Never called pm_runtime_enable() */ + KUNIT_EXPECT_FALSE(test, pm_runtime_enabled(dev)); + + /* "disabled" is treated as "active" */ + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_FALSE(test, pm_runtime_suspended(dev)); + + /* + * Note: these "fail", but they still acquire/release refcounts, so + * keep them balanced. + */ + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_get(dev)); + pm_runtime_put(dev); + + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_get(dev)); + pm_runtime_put_autosuspend(dev); + + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_resume_and_get(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_request_idle(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_request_resume(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_request_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_resume(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_autosuspend(dev)); + + /* Still disabled */ + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_FALSE(test, pm_runtime_enabled(dev)); +} + +static void pm_runtime_error_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + /* Fake a .runtime_resume() error */ + dev->power.runtime_error = -EIO; + + /* + * Note: these "fail", but they still acquire/release refcounts, so + * keep them balanced. + */ + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get(dev)); + pm_runtime_put(dev); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get(dev)); + pm_runtime_put_autosuspend(dev); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_put_sync_autosuspend(dev)); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_resume_and_get(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_request_idle(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_request_resume(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_request_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_resume(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_autosuspend(dev)); + + /* Error is still pending */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, -EIO, dev->power.runtime_error); + /* Clear error */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_set_suspended(dev)); + KUNIT_EXPECT_EQ(test, 0, dev->power.runtime_error); + /* Still suspended */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get(dev)); + pm_runtime_barrier(dev); + pm_runtime_put(dev); + pm_runtime_suspend(dev); /* flush the put(), to suspend */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + pm_runtime_put_autosuspend(dev); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_resume_and_get(dev)); + + /* + * The following should all return -EAGAIN (usage is non-zero) or 1 + * (already resumed). + */ + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_request_idle(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_request_resume(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_request_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_resume(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_autosuspend(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + + /* Suspended again */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +/* + * Explore a typical probe() sequence in which a device marks itself powered, + * but doesn't hold any runtime PM reference, so it suspends as soon as it goes + * idle. + */ +static void pm_runtime_probe_active_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + KUNIT_EXPECT_TRUE(test, pm_runtime_status_suspended(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_set_active(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + + pm_runtime_enable(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + + /* Nothing to flush. We stay active. */ + pm_runtime_barrier(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + + /* Ask for idle? Now we suspend. */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_idle(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +static struct kunit_case pm_runtime_test_cases[] = { + KUNIT_CASE(pm_runtime_depth_test), + KUNIT_CASE(pm_runtime_already_suspended_test), + KUNIT_CASE(pm_runtime_idle_test), + KUNIT_CASE(pm_runtime_disabled_test), + KUNIT_CASE(pm_runtime_error_test), + KUNIT_CASE(pm_runtime_probe_active_test), + {} +}; + +static struct kunit_suite pm_runtime_test_suite = { + .name = "pm_runtime_test_cases", + .test_cases = pm_runtime_test_cases, +}; + +kunit_test_suite(pm_runtime_test_suite); +MODULE_DESCRIPTION("Runtime power management unit test suite"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index b52049098d4e..84676cc24221 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -11,6 +11,7 @@ #include <linux/export.h> #include <linux/pm_runtime.h> #include <linux/pm_wakeirq.h> +#include <linux/rculist.h> #include <trace/events/rpm.h> #include "../base.h" @@ -18,10 +19,24 @@ typedef int (*pm_callback_t)(struct device *); +static inline pm_callback_t get_callback_ptr(const void *start, size_t offset) +{ + return *(pm_callback_t *)(start + offset); +} + +static pm_callback_t __rpm_get_driver_callback(struct device *dev, + size_t cb_offset) +{ + if (dev->driver && dev->driver->pm) + return get_callback_ptr(dev->driver->pm, cb_offset); + + return NULL; +} + static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset) { - pm_callback_t cb; const struct dev_pm_ops *ops; + pm_callback_t cb = NULL; if (dev->pm_domain) ops = &dev->pm_domain->ops; @@ -35,12 +50,10 @@ static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset) ops = NULL; if (ops) - cb = *(pm_callback_t *)((void *)ops + cb_offset); - else - cb = NULL; + cb = get_callback_ptr(ops, cb_offset); - if (!cb && dev->driver && dev->driver->pm) - cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset); + if (!cb) + cb = __rpm_get_driver_callback(dev, cb_offset); return cb; } @@ -77,7 +90,7 @@ static void update_pm_runtime_accounting(struct device *dev) /* * Because ktime_get_mono_fast_ns() is not monotonic during * timekeeping updates, ensure that 'now' is after the last saved - * timesptamp. + * timestamp. */ if (now < last) return; @@ -93,6 +106,7 @@ static void update_pm_runtime_accounting(struct device *dev) static void __update_runtime_status(struct device *dev, enum rpm_status status) { update_pm_runtime_accounting(dev); + trace_rpm_status(dev, status); dev->power.runtime_status = status; } @@ -203,7 +217,7 @@ static int dev_memalloc_noio(struct device *dev, void *data) * resume/suspend callback of any one of its ancestors(or the * block device itself), the deadlock may be triggered inside the * memory allocation since it might not complete until the block - * device becomes active and the involed page I/O finishes. The + * device becomes active and the involved page I/O finishes. The * situation is pointed out first by Alan Stern. Network device * are involved in iSCSI kind of situation. * @@ -243,8 +257,7 @@ void pm_runtime_set_memalloc_noio(struct device *dev, bool enable) * flag was set by any one of the descendants. */ if (!dev || (!enable && - device_for_each_child(dev, NULL, - dev_memalloc_noio))) + device_for_each_child(dev, NULL, dev_memalloc_noio))) break; } mutex_unlock(&dev_hotplug_mutex); @@ -265,15 +278,13 @@ static int rpm_check_suspend_allowed(struct device *dev) retval = -EACCES; else if (atomic_read(&dev->power.usage_count)) retval = -EAGAIN; - else if (!dev->power.ignore_children && - atomic_read(&dev->power.child_count)) + else if (!dev->power.ignore_children && atomic_read(&dev->power.child_count)) retval = -EBUSY; /* Pending resume requests take precedence over suspends. */ - else if ((dev->power.deferred_resume - && dev->power.runtime_status == RPM_SUSPENDING) - || (dev->power.request_pending - && dev->power.request == RPM_REQ_RESUME)) + else if ((dev->power.deferred_resume && + dev->power.runtime_status == RPM_SUSPENDING) || + (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME)) retval = -EAGAIN; else if (__dev_pm_qos_resume_latency(dev) == 0) retval = -EPERM; @@ -291,7 +302,7 @@ static int rpm_get_suppliers(struct device *dev) device_links_read_lock_held()) { int retval; - if (!(link->flags & DL_FLAG_PM_RUNTIME)) + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) continue; retval = pm_runtime_get_sync(link->supplier); @@ -404,9 +415,9 @@ static int __rpm_callback(int (*cb)(struct device *), struct device *dev) * * Do that if resume fails too. */ - if (use_links - && ((dev->power.runtime_status == RPM_SUSPENDING && !retval) - || (dev->power.runtime_status == RPM_RESUMING && retval))) { + if (use_links && + ((dev->power.runtime_status == RPM_SUSPENDING && !retval) || + (dev->power.runtime_status == RPM_RESUMING && retval))) { idx = device_links_read_lock(); __rpm_put_suppliers(dev, false); @@ -422,6 +433,49 @@ fail: } /** + * rpm_callback - Run a given runtime PM callback for a given device. + * @cb: Runtime PM callback to run. + * @dev: Device to run the callback for. + */ +static int rpm_callback(int (*cb)(struct device *), struct device *dev) +{ + int retval; + + if (dev->power.memalloc_noio) { + unsigned int noio_flag; + + /* + * Deadlock might be caused if memory allocation with + * GFP_KERNEL happens inside runtime_suspend and + * runtime_resume callbacks of one block device's + * ancestor or the block device itself. Network + * device might be thought as part of iSCSI block + * device, so network device and its ancestor should + * be marked as memalloc_noio too. + */ + noio_flag = memalloc_noio_save(); + retval = __rpm_callback(cb, dev); + memalloc_noio_restore(noio_flag); + } else { + retval = __rpm_callback(cb, dev); + } + + /* + * Since -EACCES means that runtime PM is disabled for the given device, + * it should not be returned by runtime PM callbacks. If it is returned + * nevertheless, assume it to be a transient error and convert it to + * -EAGAIN. + */ + if (retval == -EACCES) + retval = -EAGAIN; + + if (retval != -EAGAIN && retval != -EBUSY) + dev->power.runtime_error = retval; + + return retval; +} + +/** * rpm_idle - Notify device bus type if the device can be suspended. * @dev: Device to notify the bus type about. * @rpmflags: Flag bits. @@ -439,11 +493,14 @@ static int rpm_idle(struct device *dev, int rpmflags) int (*callback)(struct device *); int retval; - trace_rpm_idle_rcuidle(dev, rpmflags); + trace_rpm_idle(dev, rpmflags); retval = rpm_check_suspend_allowed(dev); if (retval < 0) ; /* Conditions are wrong. */ + else if ((rpmflags & RPM_GET_PUT) && retval == 1) + ; /* put() is allowed in RPM_SUSPENDED */ + /* Idle notifications are allowed only in the RPM_ACTIVE state. */ else if (dev->power.runtime_status != RPM_ACTIVE) retval = -EAGAIN; @@ -459,6 +516,7 @@ static int rpm_idle(struct device *dev, int rpmflags) /* Act as though RPM_NOWAIT is always set. */ else if (dev->power.idle_notification) retval = -EINPROGRESS; + if (retval) goto out; @@ -478,55 +536,33 @@ static int rpm_idle(struct device *dev, int rpmflags) dev->power.request_pending = true; queue_work(pm_wq, &dev->power.work); } - trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0); + trace_rpm_return_int(dev, _THIS_IP_, 0); return 0; } dev->power.idle_notification = true; - retval = __rpm_callback(callback, dev); + if (dev->power.irq_safe) + spin_unlock(&dev->power.lock); + else + spin_unlock_irq(&dev->power.lock); + + retval = callback(dev); + + if (dev->power.irq_safe) + spin_lock(&dev->power.lock); + else + spin_lock_irq(&dev->power.lock); dev->power.idle_notification = false; wake_up_all(&dev->power.wait_queue); out: - trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); + trace_rpm_return_int(dev, _THIS_IP_, retval); return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO); } /** - * rpm_callback - Run a given runtime PM callback for a given device. - * @cb: Runtime PM callback to run. - * @dev: Device to run the callback for. - */ -static int rpm_callback(int (*cb)(struct device *), struct device *dev) -{ - int retval; - - if (dev->power.memalloc_noio) { - unsigned int noio_flag; - - /* - * Deadlock might be caused if memory allocation with - * GFP_KERNEL happens inside runtime_suspend and - * runtime_resume callbacks of one block device's - * ancestor or the block device itself. Network - * device might be thought as part of iSCSI block - * device, so network device and its ancestor should - * be marked as memalloc_noio too. - */ - noio_flag = memalloc_noio_save(); - retval = __rpm_callback(cb, dev); - memalloc_noio_restore(noio_flag); - } else { - retval = __rpm_callback(cb, dev); - } - - dev->power.runtime_error = retval; - return retval != -EACCES ? retval : -EIO; -} - -/** * rpm_suspend - Carry out runtime suspend of given device. * @dev: Device to suspend. * @rpmflags: Flag bits. @@ -554,7 +590,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) struct device *parent = NULL; int retval; - trace_rpm_suspend_rcuidle(dev, rpmflags); + trace_rpm_suspend(dev, rpmflags); repeat: retval = rpm_check_suspend_allowed(dev); @@ -564,12 +600,12 @@ static int rpm_suspend(struct device *dev, int rpmflags) /* Synchronous suspends are not allowed in the RPM_RESUMING state. */ if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC)) retval = -EAGAIN; + if (retval) goto out; /* If the autosuspend_delay time hasn't expired yet, reschedule. */ - if ((rpmflags & RPM_AUTO) - && dev->power.runtime_status != RPM_SUSPENDING) { + if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) { u64 expires = pm_runtime_autosuspend_expiration(dev); if (expires != 0) { @@ -584,7 +620,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) * rest. */ if (!(dev->power.timer_expires && - dev->power.timer_expires <= expires)) { + dev->power.timer_expires <= expires)) { /* * We add a slack of 25% to gather wakeups * without sacrificing the granularity. @@ -594,9 +630,9 @@ static int rpm_suspend(struct device *dev, int rpmflags) dev->power.timer_expires = expires; hrtimer_start_range_ns(&dev->power.suspend_timer, - ns_to_ktime(expires), - slack, - HRTIMER_MODE_ABS); + ns_to_ktime(expires), + slack, + HRTIMER_MODE_ABS); } dev->power.timer_autosuspends = 1; goto out; @@ -705,7 +741,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) } out: - trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); + trace_rpm_return_int(dev, _THIS_IP_, retval); return retval; @@ -715,21 +751,18 @@ static int rpm_suspend(struct device *dev, int rpmflags) dev->power.deferred_resume = false; wake_up_all(&dev->power.wait_queue); - if (retval == -EAGAIN || retval == -EBUSY) { - dev->power.runtime_error = 0; + /* + * On transient errors, if the callback routine failed an autosuspend, + * and if the last_busy time has been updated so that there is a new + * autosuspend expiration time, automatically reschedule another + * autosuspend. + */ + if (!dev->power.runtime_error && (rpmflags & RPM_AUTO) && + pm_runtime_autosuspend_expiration(dev) != 0) + goto repeat; + + pm_runtime_cancel_pending(dev); - /* - * If the callback routine failed an autosuspend, and - * if the last_busy time has been updated so that there - * is a new autosuspend expiration time, automatically - * reschedule another autosuspend. - */ - if ((rpmflags & RPM_AUTO) && - pm_runtime_autosuspend_expiration(dev) != 0) - goto repeat; - } else { - pm_runtime_cancel_pending(dev); - } goto out; } @@ -757,7 +790,7 @@ static int rpm_resume(struct device *dev, int rpmflags) struct device *parent = NULL; int retval = 0; - trace_rpm_resume_rcuidle(dev, rpmflags); + trace_rpm_resume(dev, rpmflags); repeat: if (dev->power.runtime_error) { @@ -766,6 +799,8 @@ static int rpm_resume(struct device *dev, int rpmflags) if (dev->power.runtime_status == RPM_ACTIVE && dev->power.last_status == RPM_ACTIVE) retval = 1; + else if (rpmflags & RPM_TRANSPARENT) + goto out; else retval = -EACCES; } @@ -787,8 +822,8 @@ static int rpm_resume(struct device *dev, int rpmflags) goto out; } - if (dev->power.runtime_status == RPM_RESUMING - || dev->power.runtime_status == RPM_SUSPENDING) { + if (dev->power.runtime_status == RPM_RESUMING || + dev->power.runtime_status == RPM_SUSPENDING) { DEFINE_WAIT(wait); if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) { @@ -815,8 +850,8 @@ static int rpm_resume(struct device *dev, int rpmflags) for (;;) { prepare_to_wait(&dev->power.wait_queue, &wait, TASK_UNINTERRUPTIBLE); - if (dev->power.runtime_status != RPM_RESUMING - && dev->power.runtime_status != RPM_SUSPENDING) + if (dev->power.runtime_status != RPM_RESUMING && + dev->power.runtime_status != RPM_SUSPENDING) break; spin_unlock_irq(&dev->power.lock); @@ -836,9 +871,9 @@ static int rpm_resume(struct device *dev, int rpmflags) */ if (dev->power.no_callbacks && !parent && dev->parent) { spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING); - if (dev->parent->power.disable_depth > 0 - || dev->parent->power.ignore_children - || dev->parent->power.runtime_status == RPM_ACTIVE) { + if (dev->parent->power.disable_depth > 0 || + dev->parent->power.ignore_children || + dev->parent->power.runtime_status == RPM_ACTIVE) { atomic_inc(&dev->parent->power.child_count); spin_unlock(&dev->parent->power.lock); retval = 1; @@ -867,6 +902,7 @@ static int rpm_resume(struct device *dev, int rpmflags) parent = dev->parent; if (dev->power.irq_safe) goto skip_parent; + spin_unlock(&dev->power.lock); pm_runtime_get_noresume(parent); @@ -876,8 +912,8 @@ static int rpm_resume(struct device *dev, int rpmflags) * Resume the parent if it has runtime PM enabled and not been * set to ignore its children. */ - if (!parent->power.disable_depth - && !parent->power.ignore_children) { + if (!parent->power.disable_depth && + !parent->power.ignore_children) { rpm_resume(parent, 0); if (parent->power.runtime_status != RPM_ACTIVE) retval = -EBUSY; @@ -887,6 +923,7 @@ static int rpm_resume(struct device *dev, int rpmflags) spin_lock(&dev->power.lock); if (retval) goto out; + goto repeat; } skip_parent: @@ -925,7 +962,7 @@ static int rpm_resume(struct device *dev, int rpmflags) spin_lock_irq(&dev->power.lock); } - trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); + trace_rpm_return_int(dev, _THIS_IP_, retval); return retval; } @@ -991,7 +1028,7 @@ static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer) * If 'expires' is after the current time, we've been called * too early. */ - if (expires > 0 && expires < ktime_get_mono_fast_ns()) { + if (expires > 0 && expires <= ktime_get_mono_fast_ns()) { dev->power.timer_expires = 0; rpm_suspend(dev, dev->power.timer_autosuspends ? (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC); @@ -1081,7 +1118,7 @@ int __pm_runtime_idle(struct device *dev, int rpmflags) if (retval < 0) { return retval; } else if (retval > 0) { - trace_rpm_usage_rcuidle(dev, rpmflags); + trace_rpm_usage(dev, rpmflags); return 0; } } @@ -1119,7 +1156,7 @@ int __pm_runtime_suspend(struct device *dev, int rpmflags) if (retval < 0) { return retval; } else if (retval > 0) { - trace_rpm_usage_rcuidle(dev, rpmflags); + trace_rpm_usage(dev, rpmflags); return 0; } } @@ -1165,16 +1202,18 @@ int __pm_runtime_resume(struct device *dev, int rpmflags) EXPORT_SYMBOL_GPL(__pm_runtime_resume); /** - * pm_runtime_get_if_active - Conditionally bump up device usage counter. + * pm_runtime_get_conditional - Conditionally bump up device usage counter. * @dev: Device to handle. * @ign_usage_count: Whether or not to look at the current usage counter value. * * Return -EINVAL if runtime PM is disabled for @dev. * - * Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either - * @ign_usage_count is %true or the runtime PM usage counter of @dev is not - * zero, increment the usage counter of @dev and return 1. Otherwise, return 0 - * without changing the usage counter. + * Otherwise, if its runtime PM status is %RPM_ACTIVE and (1) @ign_usage_count + * is set, or (2) @dev is not ignoring children and its active child count is + * nonzero, or (3) the runtime PM usage counter of @dev is not zero, increment + * the usage counter of @dev and return 1. + * + * Otherwise, return 0 without changing the usage counter. * * If @ign_usage_count is %true, this function can be used to prevent suspending * the device when its runtime PM status is %RPM_ACTIVE. @@ -1186,7 +1225,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_resume); * The caller is responsible for decrementing the runtime PM usage counter of * @dev after this function has returned a positive value for it. */ -int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) +static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count) { unsigned long flags; int retval; @@ -1196,20 +1235,58 @@ int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) retval = -EINVAL; } else if (dev->power.runtime_status != RPM_ACTIVE) { retval = 0; - } else if (ign_usage_count) { + } else if (ign_usage_count || (!dev->power.ignore_children && + atomic_read(&dev->power.child_count) > 0)) { retval = 1; atomic_inc(&dev->power.usage_count); } else { retval = atomic_inc_not_zero(&dev->power.usage_count); } - trace_rpm_usage_rcuidle(dev, 0); + trace_rpm_usage(dev, 0); spin_unlock_irqrestore(&dev->power.lock, flags); return retval; } + +/** + * pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is + * in active state + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE, in which case it returns 1. If the device is in a different + * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the + * device, in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_active(struct device *dev) +{ + return pm_runtime_get_conditional(dev, true); +} EXPORT_SYMBOL_GPL(pm_runtime_get_if_active); /** + * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE and its runtime PM usage counter is greater than 0 or it is not + * ignoring children and its active child count is nonzero. 1 is returned in + * this case. + * + * If @dev is in a different state or it is not in use (that is, its usage + * counter is 0, or it is ignoring children, or its active child count is 0), + * 0 is returned. + * + * -EINVAL is returned if runtime PM is disabled for the device, in which case + * also the usage counter of @dev is not updated. + */ +int pm_runtime_get_if_in_use(struct device *dev) +{ + return pm_runtime_get_conditional(dev, false); +} +EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use); + +/** * __pm_runtime_set_status - Set runtime PM status of a device. * @dev: Device to handle. * @status: New runtime PM status of the device. @@ -1291,9 +1368,9 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status) * not active, has runtime PM enabled and the * 'power.ignore_children' flag unset. */ - if (!parent->power.disable_depth - && !parent->power.ignore_children - && parent->power.runtime_status != RPM_ACTIVE) { + if (!parent->power.disable_depth && + !parent->power.ignore_children && + parent->power.runtime_status != RPM_ACTIVE) { dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n", dev_name(dev), dev_name(parent)); @@ -1358,9 +1435,9 @@ static void __pm_runtime_barrier(struct device *dev) dev->power.request_pending = false; } - if (dev->power.runtime_status == RPM_SUSPENDING - || dev->power.runtime_status == RPM_RESUMING - || dev->power.idle_notification) { + if (dev->power.runtime_status == RPM_SUSPENDING || + dev->power.runtime_status == RPM_RESUMING || + dev->power.idle_notification) { DEFINE_WAIT(wait); /* Suspend, wake-up or idle notification in progress. */ @@ -1390,47 +1467,48 @@ static void __pm_runtime_barrier(struct device *dev) * Next, make sure that all pending requests for the device have been flushed * from pm_wq and wait for all runtime PM operations involving the device in * progress to complete. - * - * Return value: - * 1, if there was a resume request pending and the device had to be woken up, - * 0, otherwise */ -int pm_runtime_barrier(struct device *dev) +void pm_runtime_barrier(struct device *dev) { - int retval = 0; - pm_runtime_get_noresume(dev); spin_lock_irq(&dev->power.lock); if (dev->power.request_pending - && dev->power.request == RPM_REQ_RESUME) { + && dev->power.request == RPM_REQ_RESUME) rpm_resume(dev, 0); - retval = 1; - } __pm_runtime_barrier(dev); spin_unlock_irq(&dev->power.lock); pm_runtime_put_noidle(dev); - - return retval; } EXPORT_SYMBOL_GPL(pm_runtime_barrier); -/** - * __pm_runtime_disable - Disable runtime PM of a device. - * @dev: Device to handle. - * @check_resume: If set, check if there's a resume request for the device. - * - * Increment power.disable_depth for the device and if it was zero previously, - * cancel all pending runtime PM requests for the device and wait for all - * operations in progress to complete. The device can be either active or - * suspended after its runtime PM has been disabled. - * - * If @check_resume is set and there's a resume request pending when - * __pm_runtime_disable() is called and power.disable_depth is zero, the - * function will wake up the device before disabling its runtime PM. - */ +bool pm_runtime_block_if_disabled(struct device *dev) +{ + bool ret; + + spin_lock_irq(&dev->power.lock); + + ret = !pm_runtime_enabled(dev); + if (ret && dev->power.last_status == RPM_INVALID) + dev->power.last_status = RPM_BLOCKED; + + spin_unlock_irq(&dev->power.lock); + + return ret; +} + +void pm_runtime_unblock(struct device *dev) +{ + spin_lock_irq(&dev->power.lock); + + if (dev->power.last_status == RPM_BLOCKED) + dev->power.last_status = RPM_INVALID; + + spin_unlock_irq(&dev->power.lock); +} + void __pm_runtime_disable(struct device *dev, bool check_resume) { spin_lock_irq(&dev->power.lock); @@ -1445,8 +1523,8 @@ void __pm_runtime_disable(struct device *dev, bool check_resume) * means there probably is some I/O to process and disabling runtime PM * shouldn't prevent the device from processing the I/O. */ - if (check_resume && dev->power.request_pending - && dev->power.request == RPM_REQ_RESUME) { + if (check_resume && dev->power.request_pending && + dev->power.request == RPM_REQ_RESUME) { /* * Prevent suspends and idle notifications from being carried * out after we have woken up the device. @@ -1489,6 +1567,10 @@ void pm_runtime_enable(struct device *dev) if (--dev->power.disable_depth > 0) goto out; + if (dev->power.last_status == RPM_BLOCKED) { + dev_warn(dev, "Attempt to enable runtime PM when it is blocked\n"); + dump_stack(); + } dev->power.last_status = RPM_INVALID; dev->power.accounting_timestamp = ktime_get_mono_fast_ns(); @@ -1502,6 +1584,32 @@ out: } EXPORT_SYMBOL_GPL(pm_runtime_enable); +static void pm_runtime_set_suspended_action(void *data) +{ + pm_runtime_set_suspended(data); +} + +/** + * devm_pm_runtime_set_active_enabled - set_active version of devm_pm_runtime_enable. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_set_active_enabled(struct device *dev) +{ + int err; + + err = pm_runtime_set_active(dev); + if (err) + return err; + + err = devm_add_action_or_reset(dev, pm_runtime_set_suspended_action, dev); + if (err) + return err; + + return devm_pm_runtime_enable(dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_set_active_enabled); + static void pm_runtime_disable_action(void *data) { pm_runtime_dont_use_autosuspend(data); @@ -1524,13 +1632,34 @@ int devm_pm_runtime_enable(struct device *dev) } EXPORT_SYMBOL_GPL(devm_pm_runtime_enable); +static void pm_runtime_put_noidle_action(void *data) +{ + pm_runtime_put_noidle(data); +} + +/** + * devm_pm_runtime_get_noresume - devres-enabled version of pm_runtime_get_noresume. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_get_noresume(struct device *dev) +{ + pm_runtime_get_noresume(dev); + + return devm_add_action_or_reset(dev, pm_runtime_put_noidle_action, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_get_noresume); + /** * pm_runtime_forbid - Block runtime PM of a device. * @dev: Device to handle. * - * Increase the device's usage count and clear its power.runtime_auto flag, - * so that it cannot be suspended at run time until pm_runtime_allow() is called - * for it. + * Resume @dev if already suspended and block runtime suspend of @dev in such + * a way that it can be unblocked via the /sys/devices/.../power/control + * interface, or otherwise by calling pm_runtime_allow(). + * + * Calling this function many times in a row has the same effect as calling it + * once. */ void pm_runtime_forbid(struct device *dev) { @@ -1551,7 +1680,13 @@ EXPORT_SYMBOL_GPL(pm_runtime_forbid); * pm_runtime_allow - Unblock runtime PM of a device. * @dev: Device to handle. * - * Decrease the device's usage count and set its power.runtime_auto flag. + * Unblock runtime suspend of @dev after it has been blocked by + * pm_runtime_forbid() (for instance, if it has been blocked via the + * /sys/devices/.../power/control interface), check if @dev can be + * suspended and suspend it in that case. + * + * Calling this function many times in a row has the same effect as calling it + * once. */ void pm_runtime_allow(struct device *dev) { @@ -1566,7 +1701,7 @@ void pm_runtime_allow(struct device *dev) if (ret == 0) rpm_idle(dev, RPM_AUTO | RPM_ASYNC); else if (ret > 0) - trace_rpm_usage_rcuidle(dev, RPM_AUTO | RPM_ASYNC); + trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC); out: spin_unlock_irq(&dev->power.lock); @@ -1606,6 +1741,7 @@ void pm_runtime_irq_safe(struct device *dev) { if (dev->parent) pm_runtime_get_sync(dev->parent); + spin_lock_irq(&dev->power.lock); dev->power.irq_safe = 1; spin_unlock_irq(&dev->power.lock); @@ -1635,7 +1771,7 @@ static void update_autosuspend(struct device *dev, int old_delay, int old_use) atomic_inc(&dev->power.usage_count); rpm_resume(dev, 0); } else { - trace_rpm_usage_rcuidle(dev, 0); + trace_rpm_usage(dev, 0); } } @@ -1716,12 +1852,12 @@ void pm_runtime_init(struct device *dev) dev->power.request_pending = false; dev->power.request = RPM_REQ_NONE; dev->power.deferred_resume = false; - dev->power.needs_force_resume = 0; + dev->power.needs_force_resume = false; INIT_WORK(&dev->power.work, pm_runtime_work); dev->power.timer_expires = 0; - hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - dev->power.suspend_timer.function = pm_suspend_timer_fn; + hrtimer_setup(&dev->power.suspend_timer, pm_suspend_timer_fn, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS); init_waitqueue_head(&dev->power.wait_queue); } @@ -1743,6 +1879,11 @@ void pm_runtime_reinit(struct device *dev) pm_runtime_put(dev->parent); } } + /* + * Clear power.needs_force_resume in case it has been set by + * pm_runtime_force_suspend() invoked from a driver remove callback. + */ + dev->power.needs_force_resume = false; } /** @@ -1766,9 +1907,8 @@ void pm_runtime_get_suppliers(struct device *dev) idx = device_links_read_lock(); - list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, - device_links_read_lock_held()) - if (link->flags & DL_FLAG_PM_RUNTIME) { + dev_for_each_link_to_supplier(link, dev) + if (device_link_test(link, DL_FLAG_PM_RUNTIME)) { link->supplier_preactivated = true; pm_runtime_get_sync(link->supplier); } @@ -1822,7 +1962,7 @@ static void pm_runtime_drop_link_count(struct device *dev) */ void pm_runtime_drop_link(struct device_link *link) { - if (!(link->flags & DL_FLAG_PM_RUNTIME)) + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) return; pm_runtime_drop_link_count(link->consumer); @@ -1830,13 +1970,23 @@ void pm_runtime_drop_link(struct device_link *link) pm_request_idle(link->supplier); } -static bool pm_runtime_need_not_resume(struct device *dev) +static pm_callback_t get_callback(struct device *dev, size_t cb_offset) { - return atomic_read(&dev->power.usage_count) <= 1 && - (atomic_read(&dev->power.child_count) == 0 || - dev->power.ignore_children); + /* + * Setting power.strict_midlayer means that the middle layer + * code does not want its runtime PM callbacks to be invoked via + * pm_runtime_force_suspend() and pm_runtime_force_resume(), so + * return a direct pointer to the driver callback in that case. + */ + if (dev_pm_strict_midlayer_is_set(dev)) + return __rpm_get_driver_callback(dev, cb_offset); + + return __rpm_get_callback(dev, cb_offset); } +#define GET_CALLBACK(dev, callback) \ + get_callback(dev, offsetof(struct dev_pm_ops, callback)) + /** * pm_runtime_force_suspend - Force a device into suspend state if needed. * @dev: Device to suspend. @@ -1860,10 +2010,10 @@ int pm_runtime_force_suspend(struct device *dev) int ret; pm_runtime_disable(dev); - if (pm_runtime_status_suspended(dev)) + if (pm_runtime_status_suspended(dev) || dev->power.needs_force_resume) return 0; - callback = RPM_GET_CALLBACK(dev, runtime_suspend); + callback = GET_CALLBACK(dev, runtime_suspend); dev_pm_enable_wake_irq_check(dev, true); ret = callback ? callback(dev) : 0; @@ -1875,15 +2025,16 @@ int pm_runtime_force_suspend(struct device *dev) /* * If the device can stay in suspend after the system-wide transition * to the working state that will follow, drop the children counter of - * its parent, but set its status to RPM_SUSPENDED anyway in case this - * function will be called again for it in the meantime. + * its parent and the usage counters of its suppliers. Otherwise, set + * power.needs_force_resume to let pm_runtime_force_resume() know that + * the device needs to be taken care of and to prevent this function + * from handling the device again in case the device is passed to it + * once more subsequently. */ - if (pm_runtime_need_not_resume(dev)) { + if (pm_runtime_need_not_resume(dev)) pm_runtime_set_suspended(dev); - } else { - __update_runtime_status(dev, RPM_SUSPENDED); - dev->power.needs_force_resume = 1; - } + else + dev->power.needs_force_resume = true; return 0; @@ -1894,33 +2045,37 @@ err: } EXPORT_SYMBOL_GPL(pm_runtime_force_suspend); +#ifdef CONFIG_PM_SLEEP + /** * pm_runtime_force_resume - Force a device into resume state if needed. * @dev: Device to resume. * - * Prior invoking this function we expect the user to have brought the device - * into low power state by a call to pm_runtime_force_suspend(). Here we reverse - * those actions and bring the device into full power, if it is expected to be - * used on system resume. In the other case, we defer the resume to be managed - * via runtime PM. + * This function expects that either pm_runtime_force_suspend() has put the + * device into a low-power state prior to calling it, or the device had been + * runtime-suspended before the preceding system-wide suspend transition and it + * was left in suspend during that transition. + * + * The actions carried out by pm_runtime_force_suspend(), or by a runtime + * suspend in general, are reversed and the device is brought back into full + * power if it is expected to be used on system resume, which is the case when + * its needs_force_resume flag is set or when its smart_suspend flag is set and + * its runtime PM status is "active". * - * Typically this function may be invoked from a system resume callback. + * In other cases, the resume is deferred to be managed via runtime PM. + * + * Typically, this function may be invoked from a system resume callback. */ int pm_runtime_force_resume(struct device *dev) { int (*callback)(struct device *); int ret = 0; - if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume) + if (!dev->power.needs_force_resume && (!dev_pm_smart_suspend(dev) || + pm_runtime_status_suspended(dev))) goto out; - /* - * The value of the parent's children counter is correct already, so - * just update the status of the device. - */ - __update_runtime_status(dev, RPM_ACTIVE); - - callback = RPM_GET_CALLBACK(dev, runtime_resume); + callback = GET_CALLBACK(dev, runtime_resume); dev_pm_disable_wake_irq_check(dev, false); ret = callback ? callback(dev) : 0; @@ -1931,9 +2086,30 @@ int pm_runtime_force_resume(struct device *dev) } pm_runtime_mark_last_busy(dev); + out: - dev->power.needs_force_resume = 0; + /* + * The smart_suspend flag can be cleared here because it is not going + * to be necessary until the next system-wide suspend transition that + * will update it again. + */ + dev->power.smart_suspend = false; + /* + * Also clear needs_force_resume to make this function skip devices that + * have been seen by it once. + */ + dev->power.needs_force_resume = false; + pm_runtime_enable(dev); return ret; } EXPORT_SYMBOL_GPL(pm_runtime_force_resume); + +bool pm_runtime_need_not_resume(struct device *dev) +{ + return atomic_read(&dev->power.usage_count) <= 1 && + (atomic_read(&dev->power.child_count) == 0 || + dev->power.ignore_children); +} + +#endif /* CONFIG_PM_SLEEP */ diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c index a1474fb67db9..13b31a3adc77 100644 --- a/drivers/base/power/sysfs.c +++ b/drivers/base/power/sysfs.c @@ -6,7 +6,6 @@ #include <linux/export.h> #include <linux/pm_qos.h> #include <linux/pm_runtime.h> -#include <linux/pm_wakeup.h> #include <linux/atomic.h> #include <linux/jiffies.h> #include "power.h" @@ -509,14 +508,6 @@ static ssize_t wakeup_last_time_ms_show(struct device *dev, return sysfs_emit(buf, "%lld\n", msec); } -static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid, - kgid_t kgid) -{ - if (dev->power.wakeup && dev->power.wakeup->dev) - return device_change_owner(dev->power.wakeup->dev, kuid, kgid); - return 0; -} - static DEVICE_ATTR_RO(wakeup_last_time_ms); #ifdef CONFIG_PM_AUTOSLEEP @@ -541,6 +532,15 @@ static ssize_t wakeup_prevent_sleep_time_ms_show(struct device *dev, static DEVICE_ATTR_RO(wakeup_prevent_sleep_time_ms); #endif /* CONFIG_PM_AUTOSLEEP */ + +static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid, + kgid_t kgid) +{ + if (dev->power.wakeup && dev->power.wakeup->dev) + return device_change_owner(dev->power.wakeup->dev, kuid, kgid); + return 0; +} + #else /* CONFIG_PM_SLEEP */ static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid) @@ -611,15 +611,9 @@ static DEVICE_ATTR_RW(async); #endif /* CONFIG_PM_ADVANCED_DEBUG */ static struct attribute *power_attrs[] = { -#ifdef CONFIG_PM_ADVANCED_DEBUG -#ifdef CONFIG_PM_SLEEP +#if defined(CONFIG_PM_ADVANCED_DEBUG) && defined(CONFIG_PM_SLEEP) &dev_attr_async.attr, #endif - &dev_attr_runtime_status.attr, - &dev_attr_runtime_usage.attr, - &dev_attr_runtime_active_kids.attr, - &dev_attr_runtime_enabled.attr, -#endif /* CONFIG_PM_ADVANCED_DEBUG */ NULL, }; static const struct attribute_group pm_attr_group = { @@ -650,13 +644,16 @@ static const struct attribute_group pm_wakeup_attr_group = { }; static struct attribute *runtime_attrs[] = { -#ifndef CONFIG_PM_ADVANCED_DEBUG &dev_attr_runtime_status.attr, -#endif &dev_attr_control.attr, &dev_attr_runtime_suspended_time.attr, &dev_attr_runtime_active_time.attr, &dev_attr_autosuspend_delay_ms.attr, +#ifdef CONFIG_PM_ADVANCED_DEBUG + &dev_attr_runtime_usage.attr, + &dev_attr_runtime_active_kids.attr, + &dev_attr_runtime_enabled.attr, +#endif NULL, }; static const struct attribute_group pm_runtime_attr_group = { diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c index 72b7a92337b1..d8da7195bb00 100644 --- a/drivers/base/power/trace.c +++ b/drivers/base/power/trace.c @@ -120,7 +120,7 @@ static unsigned int read_magic_time(void) struct rtc_time time; unsigned int val; - if (mc146818_get_time(&time) < 0) { + if (mc146818_get_time(&time, 1000) < 0) { pr_err("Unable to read current time from RTC\n"); return 0; } @@ -238,10 +238,8 @@ int show_trace_dev_match(char *buf, size_t size) unsigned int hash = hash_string(DEVSEED, dev_name(dev), DEVHASH); if (hash == value) { - int len = snprintf(buf, size, "%s\n", + int len = scnprintf(buf, size, "%s\n", dev_driver_string(dev)); - if (len > size) - len = size; buf += len; ret += len; size -= len; diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index d487a6bac630..8aa28c08b289 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -103,6 +103,32 @@ void dev_pm_clear_wake_irq(struct device *dev) } EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq); +static void devm_pm_clear_wake_irq(void *dev) +{ + dev_pm_clear_wake_irq(dev); +} + +/** + * devm_pm_set_wake_irq - device-managed variant of dev_pm_set_wake_irq + * @dev: Device entry + * @irq: Device IO interrupt + * + * + * Attach a device IO interrupt as a wake IRQ, same with dev_pm_set_wake_irq, + * but the device will be auto clear wake capability on driver detach. + */ +int devm_pm_set_wake_irq(struct device *dev, int irq) +{ + int ret; + + ret = dev_pm_set_wake_irq(dev, irq); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_pm_clear_wake_irq, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_set_wake_irq); + /** * handle_threaded_wake_irq - Handler for dedicated wake-up interrupts * @irq: Device specific dedicated wake-up interrupt @@ -194,7 +220,6 @@ err_free: return err; } - /** * dev_pm_set_dedicated_wake_irq - Request a dedicated wake-up interrupt * @dev: Device entry @@ -206,11 +231,6 @@ err_free: * Sets up a threaded interrupt handler for a device that has * a dedicated wake-up interrupt in addition to the device IO * interrupt. - * - * The interrupt starts disabled, and needs to be managed for - * the device by the bus code or the device driver using - * dev_pm_enable_wake_irq*() and dev_pm_disable_wake_irq*() - * functions. */ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) { @@ -232,11 +252,6 @@ EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq); * the status of WAKE_IRQ_DEDICATED_REVERSE to tell rpm_suspend() * to enable dedicated wake-up interrupt after running the runtime suspend * callback for @dev. - * - * The interrupt starts disabled, and needs to be managed for - * the device by the bus code or the device driver using - * dev_pm_enable_wake_irq*() and dev_pm_disable_wake_irq*() - * functions. */ int dev_pm_set_dedicated_wake_irq_reverse(struct device *dev, int irq) { @@ -245,44 +260,6 @@ int dev_pm_set_dedicated_wake_irq_reverse(struct device *dev, int irq) EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq_reverse); /** - * dev_pm_enable_wake_irq - Enable device wake-up interrupt - * @dev: Device - * - * Optionally called from the bus code or the device driver for - * runtime_resume() to override the PM runtime core managed wake-up - * interrupt handling to enable the wake-up interrupt. - * - * Note that for runtime_suspend()) the wake-up interrupts - * should be unconditionally enabled unlike for suspend() - * that is conditional. - */ -void dev_pm_enable_wake_irq(struct device *dev) -{ - struct wake_irq *wirq = dev->power.wakeirq; - - if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) - enable_irq(wirq->irq); -} -EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); - -/** - * dev_pm_disable_wake_irq - Disable device wake-up interrupt - * @dev: Device - * - * Optionally called from the bus code or the device driver for - * runtime_suspend() to override the PM runtime core managed wake-up - * interrupt handling to disable the wake-up interrupt. - */ -void dev_pm_disable_wake_irq(struct device *dev) -{ - struct wake_irq *wirq = dev->power.wakeirq; - - if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) - disable_irq_nosync(wirq->irq); -} -EXPORT_SYMBOL_GPL(dev_pm_disable_wake_irq); - -/** * dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt * @dev: Device * @can_change_status: Can change wake-up interrupt status @@ -314,8 +291,10 @@ void dev_pm_enable_wake_irq_check(struct device *dev, return; enable: - if (!can_change_status || !(wirq->status & WAKE_IRQ_DEDICATED_REVERSE)) + if (!can_change_status || !(wirq->status & WAKE_IRQ_DEDICATED_REVERSE)) { enable_irq(wirq->irq); + wirq->status |= WAKE_IRQ_DEDICATED_ENABLED; + } } /** @@ -336,8 +315,10 @@ void dev_pm_disable_wake_irq_check(struct device *dev, bool cond_disable) if (cond_disable && (wirq->status & WAKE_IRQ_DEDICATED_REVERSE)) return; - if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) + if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) { + wirq->status &= ~WAKE_IRQ_DEDICATED_ENABLED; disable_irq_nosync(wirq->irq); + } } /** @@ -358,8 +339,10 @@ void dev_pm_enable_wake_irq_complete(struct device *dev) return; if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED && - wirq->status & WAKE_IRQ_DEDICATED_REVERSE) + wirq->status & WAKE_IRQ_DEDICATED_REVERSE) { enable_irq(wirq->irq); + wirq->status |= WAKE_IRQ_DEDICATED_ENABLED; + } } /** @@ -376,7 +359,7 @@ void dev_pm_arm_wake_irq(struct wake_irq *wirq) if (device_may_wakeup(wirq->dev)) { if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED && - !pm_runtime_status_suspended(wirq->dev)) + !(wirq->status & WAKE_IRQ_DEDICATED_ENABLED)) enable_irq(wirq->irq); enable_irq_wake(wirq->irq); @@ -399,7 +382,7 @@ void dev_pm_disarm_wake_irq(struct wake_irq *wirq) disable_irq_wake(wirq->irq); if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED && - !pm_runtime_status_suspended(wirq->dev)) + !(wirq->status & WAKE_IRQ_DEDICATED_ENABLED)) disable_irq_nosync(wirq->irq); } } diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 7cc0c0cf8eaa..1e1a0e7eeac5 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -19,11 +19,6 @@ #include "power.h" -#ifndef CONFIG_SUSPEND -suspend_state_t pm_suspend_target_state; -#define pm_suspend_target_state (PM_SUSPEND_ON) -#endif - #define list_for_each_entry_rcu_locked(pos, head, member) \ list_for_each_entry_rcu(pos, head, member, \ srcu_read_lock_held(&wakeup_srcu)) @@ -82,7 +77,7 @@ static DEFINE_IDA(wakeup_ida); * wakeup_source_create - Create a struct wakeup_source object. * @name: Name of the new wakeup source. */ -struct wakeup_source *wakeup_source_create(const char *name) +static struct wakeup_source *wakeup_source_create(const char *name) { struct wakeup_source *ws; const char *ws_name; @@ -111,7 +106,6 @@ err_name: err_ws: return NULL; } -EXPORT_SYMBOL_GPL(wakeup_source_create); /* * Record wakeup_source statistics being deleted into a dummy wakeup_source. @@ -154,7 +148,7 @@ static void wakeup_source_free(struct wakeup_source *ws) * * Use only for wakeup source objects created with wakeup_source_create(). */ -void wakeup_source_destroy(struct wakeup_source *ws) +static void wakeup_source_destroy(struct wakeup_source *ws) { if (!ws) return; @@ -163,13 +157,12 @@ void wakeup_source_destroy(struct wakeup_source *ws) wakeup_source_record(ws); wakeup_source_free(ws); } -EXPORT_SYMBOL_GPL(wakeup_source_destroy); /** * wakeup_source_add - Add given object to the list of wakeup sources. * @ws: Wakeup source object to add to the list. */ -void wakeup_source_add(struct wakeup_source *ws) +static void wakeup_source_add(struct wakeup_source *ws) { unsigned long flags; @@ -184,32 +177,29 @@ void wakeup_source_add(struct wakeup_source *ws) list_add_rcu(&ws->entry, &wakeup_sources); raw_spin_unlock_irqrestore(&events_lock, flags); } -EXPORT_SYMBOL_GPL(wakeup_source_add); /** * wakeup_source_remove - Remove given object from the wakeup sources list. * @ws: Wakeup source object to remove from the list. */ -void wakeup_source_remove(struct wakeup_source *ws) +static void wakeup_source_remove(struct wakeup_source *ws) { unsigned long flags; if (WARN_ON(!ws)) return; + /* + * After shutting down the timer, wakeup_source_activate() will warn if + * the given wakeup source is passed to it. + */ + timer_shutdown_sync(&ws->timer); + raw_spin_lock_irqsave(&events_lock, flags); list_del_rcu(&ws->entry); raw_spin_unlock_irqrestore(&events_lock, flags); synchronize_srcu(&wakeup_srcu); - - del_timer_sync(&ws->timer); - /* - * Clear timer.function to make wakeup_source_not_registered() treat - * this wakeup source as not registered. - */ - ws->timer.function = NULL; } -EXPORT_SYMBOL_GPL(wakeup_source_remove); /** * wakeup_source_register - Create wakeup source and add it to the list. @@ -342,7 +332,7 @@ int device_wakeup_enable(struct device *dev) if (!dev || !dev->power.can_wakeup) return -EINVAL; - if (pm_suspend_target_state != PM_SUSPEND_ON) + if (pm_sleep_transition_in_progress()) dev_dbg(dev, "Suspicious %s() during system transition!\n", __func__); ws = wakeup_source_register(dev, dev_name(dev)); @@ -456,16 +446,15 @@ static struct wakeup_source *device_wakeup_detach(struct device *dev) * Detach the @dev's wakeup source object from it, unregister this wakeup source * object and destroy it. */ -int device_wakeup_disable(struct device *dev) +void device_wakeup_disable(struct device *dev) { struct wakeup_source *ws; if (!dev || !dev->power.can_wakeup) - return -EINVAL; + return; ws = device_wakeup_detach(dev); wakeup_source_unregister(ws); - return 0; } EXPORT_SYMBOL_GPL(device_wakeup_disable); @@ -507,19 +496,23 @@ EXPORT_SYMBOL_GPL(device_set_wakeup_capable); */ int device_set_wakeup_enable(struct device *dev, bool enable) { - return enable ? device_wakeup_enable(dev) : device_wakeup_disable(dev); + if (enable) + return device_wakeup_enable(dev); + + device_wakeup_disable(dev); + return 0; } EXPORT_SYMBOL_GPL(device_set_wakeup_enable); /** - * wakeup_source_not_registered - validate the given wakeup source. + * wakeup_source_not_usable - validate the given wakeup source. * @ws: Wakeup source to be validated. */ -static bool wakeup_source_not_registered(struct wakeup_source *ws) +static bool wakeup_source_not_usable(struct wakeup_source *ws) { /* - * Use timer struct to check if the given source is initialized - * by wakeup_source_add. + * Use the timer struct to check if the given wakeup source has been + * initialized by wakeup_source_add() and it is not going away. */ return ws->timer.function != pm_wakeup_timer_fn; } @@ -564,8 +557,7 @@ static void wakeup_source_activate(struct wakeup_source *ws) { unsigned int cec; - if (WARN_ONCE(wakeup_source_not_registered(ws), - "unregistered wakeup source\n")) + if (WARN_ONCE(wakeup_source_not_usable(ws), "unusable wakeup source\n")) return; ws->active = true; @@ -615,7 +607,7 @@ void __pm_stay_awake(struct wakeup_source *ws) spin_lock_irqsave(&ws->lock, flags); wakeup_source_report_event(ws, false); - del_timer(&ws->timer); + timer_delete(&ws->timer); ws->timer_expires = 0; spin_unlock_irqrestore(&ws->lock, flags); @@ -695,7 +687,7 @@ static void wakeup_source_deactivate(struct wakeup_source *ws) ws->max_time = duration; ws->last_time = now; - del_timer(&ws->timer); + timer_delete(&ws->timer); ws->timer_expires = 0; if (ws->autosleep_enabled) @@ -765,7 +757,7 @@ EXPORT_SYMBOL_GPL(pm_relax); */ static void pm_wakeup_timer_fn(struct timer_list *t) { - struct wakeup_source *ws = from_timer(ws, t, timer); + struct wakeup_source *ws = timer_container_of(ws, t, timer); unsigned long flags; spin_lock_irqsave(&ws->lock, flags); diff --git a/drivers/base/power/wakeup_stats.c b/drivers/base/power/wakeup_stats.c index 924fac493c4f..3ffd427248e8 100644 --- a/drivers/base/power/wakeup_stats.c +++ b/drivers/base/power/wakeup_stats.c @@ -34,6 +34,7 @@ wakeup_attr(active_count); wakeup_attr(event_count); wakeup_attr(wakeup_count); wakeup_attr(expire_count); +wakeup_attr(relax_count); static ssize_t active_time_ms_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -119,6 +120,7 @@ static struct attribute *wakeup_source_attrs[] = { &dev_attr_event_count.attr, &dev_attr_wakeup_count.attr, &dev_attr_expire_count.attr, + &dev_attr_relax_count.attr, &dev_attr_active_time_ms.attr, &dev_attr_total_time_ms.attr, &dev_attr_max_time_ms.attr, @@ -210,7 +212,7 @@ void wakeup_source_sysfs_remove(struct wakeup_source *ws) static int __init wakeup_sources_sysfs_init(void) { - wakeup_class = class_create(THIS_MODULE, "wakeup"); + wakeup_class = class_create("wakeup"); return PTR_ERR_OR_ZERO(wakeup_class); } |